Test the visual selector data loads as JSON

changedetectionio/__init__.py

@@ -503,7 +503,7 @@ def changedetection_app(config=None, datastore_o=None):
         from changedetectionio import fetch_site_status
 
         # Get the most recent one
-        newest_history_key = datastore.get_val(uuid, 'newest_history_key')
+        newest_history_key = datastore.data['watching'][uuid].get('newest_history_key')
 
         # 0 means that theres only one, so that there should be no 'unviewed' history available
         if newest_history_key == 0:

changedetectionio/fetch_site_status.py

@@ -63,13 +63,11 @@ class perform_site_check():
 
 
     def run(self, uuid):
-        timestamp = int(time.time())  # used for storage etc too
-
         changed_detected = False
         screenshot = False  # as bytes
         stripped_text_from_html = ""
 
-        watch = self.datastore.data['watching'][uuid]
+        watch = self.datastore.data['watching'].get(uuid)
 
         # Protect against file:// access
         if re.search(r'^file', watch['url'], re.IGNORECASE) and not os.getenv('ALLOW_FILE_URI', False):
@@ -80,7 +78,7 @@ class perform_site_check():
         # Unset any existing notification error
         update_obj = {'last_notification_error': False, 'last_error': False}
 
-        extra_headers = self.datastore.get_val(uuid, 'headers')
+        extra_headers =self.datastore.data['watching'][uuid].get('headers')
 
         # Tweak the base config with the per-watch ones
         request_headers = self.datastore.data['settings']['headers'].copy()
@@ -93,9 +91,9 @@ class perform_site_check():
             request_headers['Accept-Encoding'] = request_headers['Accept-Encoding'].replace(', br', '')
 
         timeout = self.datastore.data['settings']['requests']['timeout']
-        url = self.datastore.get_val(uuid, 'url')
+        url = watch.get('url')
-        request_body = self.datastore.get_val(uuid, 'body')
+        request_body = self.datastore.data['watching'][uuid].get('body')
-        request_method = self.datastore.get_val(uuid, 'method')
+        request_method = self.datastore.data['watching'][uuid].get('method')
         ignore_status_codes = self.datastore.data['watching'][uuid].get('ignore_status_codes', False)
 
         # source: support

changedetectionio/forms.py

@@ -355,6 +355,8 @@ class watchForm(commonSettingsForm):
     filter_failure_notification_send = BooleanField(
         'Send a notification when the filter can no longer be found on the page', default=False)
 
+    notification_use_default = BooleanField('Use default/system notification settings', default=True)
+
     def validate(self, **kwargs):
         if not super().validate():
             return False

changedetectionio/html_tools.py

@@ -4,6 +4,8 @@ from typing import List
 from bs4 import BeautifulSoup
 from jsonpath_ng.ext import parse
 import re
+from inscriptis import get_text
+from inscriptis.model.config import ParserConfig
 
 class FilterNotFoundInResponse(ValueError):
     def __init__(self, msg):
@@ -188,16 +190,9 @@ def strip_ignore_text(content, wordlist, mode="content"):
 
 
 def html_to_text(html_content: str, render_anchor_tag_content=False) -> str:
-    import multiprocessing
-
-    from inscriptis.model.config import ParserConfig
-
     """Converts html string to a string with just the text. If ignoring
     rendering anchor tag content is enable, anchor tag content are also
     included in the text
-    
-    @NOTE: HORRIBLE LXML INDUCED MEMORY LEAK WORKAROUND HERE 
-           https://www.reddit.com/r/Python/comments/j0gl8t/psa_pythonlxml_memory_leaks_and_a_solution/ 
 
     :param html_content: string with html content
     :param render_anchor_tag_content: boolean flag indicating whether to extract
@@ -219,19 +214,8 @@ def html_to_text(html_content: str, render_anchor_tag_content=False) -> str:
     else:
         parser_config = None
 
-
+    # get text and annotations via inscriptis
-    def parse_function(html_content, parser_config, results_queue):
+    text_content = get_text(html_content, config=parser_config)
-        from inscriptis import get_text
-        # get text and annotations via inscriptis
-        text_content = get_text(html_content, config=parser_config)
-        results_queue.put(text_content)
-
-    results_queue = multiprocessing.Queue()
-    parse_process = multiprocessing.Process(target=parse_function, args=(html_content, parser_config, results_queue))
-    parse_process.daemon = True
-    parse_process.start()
-    text_content = results_queue.get()  # blocks until results are available
-    parse_process.terminate()
 
     return text_content
 

changedetectionio/model/Watch.py

@@ -35,6 +35,7 @@ class model(dict):
             'notification_title': default_notification_title,
             'notification_body': default_notification_body,
             'notification_format': default_notification_format,
+            'notification_use_default': True, # Use default for new
             'notification_muted': False,
             'css_filter': '',
             'last_error': False,

changedetectionio/static/js/watch-settings.js

@@ -1,7 +1,7 @@
-$(document).ready(function() {
+$(document).ready(function () {
-    function toggle() {
+    function toggle_fetch_backend() {
         if ($('input[name="fetch_backend"]:checked').val() == 'html_webdriver') {
-            if(playwright_enabled) {
+            if (playwright_enabled) {
                 // playwright supports headers, so hide everything else
                 // See #664
                 $('#requests-override-options #request-method').hide();
@@ -13,12 +13,8 @@ $(document).ready(function() {
                 // selenium/webdriver doesnt support anything afaik, hide it all
                 $('#requests-override-options').hide();
             }
-
-
             $('#webdriver-override-options').show();
-
         } else {
-
             $('#requests-override-options').show();
             $('#requests-override-options *:hidden').show();
             $('#webdriver-override-options').hide();
@@ -26,8 +22,27 @@ $(document).ready(function() {
     }
 
     $('input[name="fetch_backend"]').click(function (e) {
-        toggle();
+        toggle_fetch_backend();
     });
-    toggle();
+    toggle_fetch_backend();
 
+    function toggle_default_notifications() {
+        var n=$('#notification_urls, #notification_title, #notification_body, #notification_format');
+        if ($('#notification_use_default').is(':checked')) {
+            $('#notification-field-group').fadeOut();
+            $(n).each(function (e) {
+                $(this).attr('readonly', true);
+            });
+        } else {
+            $('#notification-field-group').show();
+            $(n).each(function (e) {
+                $(this).attr('readonly', false);
+            });
+        }
+    }
+
+    $('#notification_use_default').click(function (e) {
+        toggle_default_notifications();
+    });
+    toggle_default_notifications();
 });

changedetectionio/store.py

@@ -244,10 +244,6 @@ class ChangeDetectionStore:
 
         return False
 
-    def get_val(self, uuid, val):
-        # Probably their should be dict...
-        return self.data['watching'][uuid].get(val)
-
     # Remove a watchs data but keep the entry (URL etc)
     def clear_watch_history(self, uuid):
         import pathlib
@@ -539,4 +535,28 @@ class ChangeDetectionStore:
                 del(watch['last_changed'])
             except:
                 continue
+        return
+
+
+    def update_5(self):
+
+        from changedetectionio.notification import (
+            default_notification_body,
+            default_notification_format,
+            default_notification_title,
+        )
+
+        for uuid, watch in self.data['watching'].items():
+            try:
+                # If it's all the same to the system settings, then prefer system notification settings
+                # include \r\n -> \n incase they already hit submit and the browser put \r in
+                if watch.get('notification_body').replace('\r\n', '\n') == default_notification_body.replace('\r\n', '\n') and \
+                        watch.get('notification_format') == default_notification_format and \
+                        watch.get('notification_title').replace('\r\n', '\n') == default_notification_title.replace('\r\n', '\n') and \
+                        watch.get('notification_urls') == self.__data['settings']['application']['notification_urls']:
+                        watch['notification_use_default'] = True
+                else:
+                    watch['notification_use_default'] = False
+            except:
+                continue
         return

changedetectionio/templates/edit.html

@@ -135,9 +135,11 @@ User-Agent: wonderbra 1.0") }}
             </div>
 
             <div class="tab-pane-inner" id="notifications">
-                <strong>Note: <i>These settings override the global settings for this watch.</i></strong>
                 <fieldset>
-                    <div class="field-group">
+                    <div  class="pure-control-group inline-radio">
+                      {{ render_checkbox_field(form.notification_use_default) }}
+                    </div>
+                    <div class="field-group" id="notification-field-group">
                         {{ render_common_settings_form(form, current_base_url, emailprefix) }}
                     </div>
                 </fieldset>

changedetectionio/tests/test_notification.py

@@ -71,6 +71,7 @@ def test_check_notification(client, live_server):
         "url": test_url,
         "tag": "my tag",
         "title": "my title",
+        # No 'notification_use_default' here, so it's effectively False/off
         "headers": "",
         "fetch_backend": "html_requests"})
 
@@ -215,3 +216,82 @@ def test_notification_validation(client, live_server):
         url_for("form_delete", uuid="all"),
         follow_redirects=True
     )
+
+# Check that the default VS watch specific notification is hit
+def test_check_notification_use_default(client, live_server):
+    set_original_response()
+    notification_url = url_for('test_notification_endpoint', _external=True).replace('http', 'json')
+    test_url = url_for('test_endpoint', _external=True)
+
+    res = client.post(
+        url_for("form_quick_watch_add"),
+        data={"url": test_url, "tag": ''},
+        follow_redirects=True
+    )
+    assert b"Watch added" in res.data
+
+    ## Setup the local one and enable it
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={"notification_urls": notification_url,
+              "notification_title": "watch-notification",
+              "notification_body": "watch-body",
+              'notification_use_default': "True",
+              "notification_format": "Text",
+              "url": test_url,
+              "tag": "my tag",
+              "title": "my title",
+              "headers": "",
+              "fetch_backend": "html_requests"},
+        follow_redirects=True
+    )
+
+    res = client.post(
+        url_for("settings_page"),
+        data={"application-notification_title": "global-notifications-title",
+              "application-notification_body": "global-notifications-body\n",
+              "application-notification_format": "Text",
+              "application-notification_urls": notification_url,
+              "requests-time_between_check-minutes": 180,
+              "fetch_backend": "html_requests"
+              },
+        follow_redirects=True
+    )
+
+    # A change should by default trigger a notification of the global-notifications
+    time.sleep(1)
+    set_modified_response()
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    time.sleep(2)
+    with open("test-datastore/notification.txt", "r") as f:
+        assert 'global-notifications-title' in f.read()
+
+    ## Setup the local one and enable it
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={"notification_urls": notification_url,
+              "notification_title": "watch-notification",
+              "notification_body": "watch-body",
+              # No 'notification_use_default' here, so it's effectively False/off = "dont use default, use this one"
+              "notification_format": "Text",
+              "url": test_url,
+              "tag": "my tag",
+              "title": "my title",
+              "headers": "",
+              "fetch_backend": "html_requests"},
+        follow_redirects=True
+    )
+    set_original_response()
+
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    time.sleep(2)
+    assert os.path.isfile("test-datastore/notification.txt")
+    with open("test-datastore/notification.txt", "r") as f:
+        assert 'watch-notification' in f.read()
+
+
+    # cleanup for the next
+    client.get(
+        url_for("form_delete", uuid="all"),
+        follow_redirects=True
+    )

changedetectionio/tests/visualselector/test_fetch_data.py

@@ -7,6 +7,7 @@ from ..util import live_server_setup, wait_for_all_checks, extract_UUID_from_cli
 # Add a site in paused mode, add an invalid filter, we should still have visual selector data ready
 def test_visual_selector_content_ready(client, live_server):
     import os
+    import json
 
     assert os.getenv('PLAYWRIGHT_DRIVER_URL'), "Needs PLAYWRIGHT_DRIVER_URL set for this test"
     live_server_setup(live_server)
@@ -33,3 +34,7 @@ def test_visual_selector_content_ready(client, live_server):
     uuid = extract_UUID_from_client(client)
     assert os.path.isfile(os.path.join('test-datastore', uuid, 'last-screenshot.png')), "last-screenshot.png should exist"
     assert os.path.isfile(os.path.join('test-datastore', uuid, 'elements.json')), "xpath elements.json data should exist"
+
+    # Open it and see if it roughly looks correct
+    with open(os.path.join('test-datastore', uuid, 'elements.json'), 'r') as f:
+        json.load(f)

changedetectionio/update_worker.py

@@ -41,7 +41,7 @@ class update_worker(threading.Thread):
             )
 
         # Did it have any notification alerts to hit?
-        if len(watch['notification_urls']):
+        if not watch.get('notification_use_default') and len(watch['notification_urls']):
             print(">>> Notifications queued for UUID from watch {}".format(watch_uuid))
             n_object['notification_urls'] = watch['notification_urls']
             n_object['notification_title'] = watch['notification_title']
@@ -49,7 +49,7 @@ class update_worker(threading.Thread):
             n_object['notification_format'] = watch['notification_format']
 
         # No? maybe theres a global setting, queue them all
-        elif len(self.datastore.data['settings']['application']['notification_urls']):
+        elif watch.get('notification_use_default') and len(self.datastore.data['settings']['application']['notification_urls']):
             print(">>> Watch notification URLs were empty, using GLOBAL notifications for UUID: {}".format(watch_uuid))
             n_object['notification_urls'] = self.datastore.data['settings']['application']['notification_urls']
             n_object['notification_title'] = self.datastore.data['settings']['application']['notification_title']