Some defaults

.github/ISSUE_TEMPLATE/bug_report.md

@@ -7,20 +7,6 @@ assignees: 'dgtlmoon'
 
 ---
 
-**DO NOT USE THIS FORM TO REPORT THAT A PARTICULAR WEBSITE IS NOT SCRAPING/WATCHING AS EXPECTED**
-
-This form is only for direct bugs and feature requests todo directly with the software.
-
-Please report watched websites (full URL and _any_ settings) that do not work with changedetection.io as expected [**IN THE DISCUSSION FORUMS**](https://github.com/dgtlmoon/changedetection.io/discussions) or your report will be deleted
-
-CONSIDER TAKING OUT A SUBSCRIPTION FOR A SMALL PRICE PER MONTH, YOU GET THE BENEFIT OF USING OUR PAID PROXIES AND FURTHERING THE DEVELOPMENT OF CHANGEDETECTION.IO
-
-THANK YOU
-
-
-
-
-
 **Describe the bug**
 A clear and concise description of what the bug is.
 

README.md

@@ -1,6 +1,7 @@
 ## Web Site Change Detection, Monitoring and Notification.
 
-Live your data-life pro-actively, track website content changes and receive notifications via Discord, Email, Slack, Telegram and 70+ more
+[**Try our $6.99/month subscription - Unlimited checks and watches!**](https://lemonade.changedetection.io/start)
+ 
 
 [<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/docs/screenshot.png" style="max-width:100%;" alt="Self-hosted web page change monitoring"  title="Self-hosted web page change monitoring"  />](https://lemonade.changedetection.io/start)
 
@@ -10,7 +11,7 @@ Live your data-life pro-actively, track website content changes and receive noti
 
 Know when important content changes, we support notifications via Discord, Telegram, Home-Assistant, Slack, Email and 70+ more
 
-[**Don't have time? Let us host it for you! try our $6.99/month subscription - use our proxies and support!**](https://lemonade.changedetection.io/start) , _half the price of other website change monitoring services and comes with unlimited watches & checks!_
+[**Try our $6.99/month subscription - unlimited checks and watches!**](https://lemonade.changedetection.io/start) , _half the price of other website change monitoring services and comes with unlimited watches & checks!_
 
 
 
@@ -39,18 +40,7 @@ Know when important content changes, we support notifications via Discord, Teleg
 - Monitor HTML source code for unexpected changes, strengthen your PCI compliance
 - You have a very sensitive list of URLs to watch and you do _not_ want to use the paid alternatives. (Remember, _you_ are the product)
 
-_Need an actual Chrome runner with Javascript support? We support fetching via WebDriver and Playwright!</a>_
-
-#### Key Features
-
-- Lots of trigger filters, such as "Trigger on text", "Remove text by selector", "Ignore text", "Extract text", also using regular-expressions!
-- Target elements with xPath and CSS Selectors, Easily monitor complex JSON with JsonPath rules
-- Switch between fast non-JS and Chrome JS based "fetchers"
-- Easily specify how often a site should be checked
-- Execute JS before extracting text (Good for logging in, see examples in the UI!)
-- Override Request Headers, Specify `POST` or `GET` and other methods
-- Use the "Visual Selector" to help target specific elements
-
+_Need an actual Chrome runner with Javascript support? We support fetching via WebDriver!</a>_
 
 ## Screenshots
 

changedetectionio/__init__.py

@@ -44,7 +44,7 @@ from flask_wtf import CSRFProtect
 from changedetectionio import html_tools
 from changedetectionio.api import api_v1
 
-__version__ = '0.39.18'
+__version__ = '0.39.17.2'
 
 datastore = None
 
@@ -503,7 +503,7 @@ def changedetection_app(config=None, datastore_o=None):
         from changedetectionio import fetch_site_status
 
         # Get the most recent one
-        newest_history_key = datastore.data['watching'][uuid].get('newest_history_key')
+        newest_history_key = datastore.get_val(uuid, 'newest_history_key')
 
         # 0 means that theres only one, so that there should be no 'unviewed' history available
         if newest_history_key == 0:

changedetectionio/fetch_site_status.py

@@ -63,11 +63,13 @@ class perform_site_check():
 
 
     def run(self, uuid):
+        timestamp = int(time.time())  # used for storage etc too
+
         changed_detected = False
         screenshot = False  # as bytes
         stripped_text_from_html = ""
 
-        watch = self.datastore.data['watching'].get(uuid)
+        watch = self.datastore.data['watching'][uuid]
 
         # Protect against file:// access
         if re.search(r'^file', watch['url'], re.IGNORECASE) and not os.getenv('ALLOW_FILE_URI', False):
@@ -78,7 +80,7 @@ class perform_site_check():
         # Unset any existing notification error
         update_obj = {'last_notification_error': False, 'last_error': False}
 
-        extra_headers =self.datastore.data['watching'][uuid].get('headers')
+        extra_headers = self.datastore.get_val(uuid, 'headers')
 
         # Tweak the base config with the per-watch ones
         request_headers = self.datastore.data['settings']['headers'].copy()
@@ -91,9 +93,9 @@ class perform_site_check():
             request_headers['Accept-Encoding'] = request_headers['Accept-Encoding'].replace(', br', '')
 
         timeout = self.datastore.data['settings']['requests']['timeout']
-        url = watch.get('url')
-        request_body = self.datastore.data['watching'][uuid].get('body')
-        request_method = self.datastore.data['watching'][uuid].get('method')
+        url = self.datastore.get_val(uuid, 'url')
+        request_body = self.datastore.get_val(uuid, 'body')
+        request_method = self.datastore.get_val(uuid, 'method')
         ignore_status_codes = self.datastore.data['watching'][uuid].get('ignore_status_codes', False)
 
         # source: support

changedetectionio/forms.py

@@ -355,8 +355,6 @@ class watchForm(commonSettingsForm):
     filter_failure_notification_send = BooleanField(
         'Send a notification when the filter can no longer be found on the page', default=False)
 
-    notification_use_default = BooleanField('Use default/system notification settings', default=True)
-
     def validate(self, **kwargs):
         if not super().validate():
             return False

changedetectionio/model/Watch.py

@@ -32,10 +32,10 @@ class model(dict):
             'ignore_text': [],  # List of text to ignore when calculating the comparison checksum
             # Custom notification content
             'notification_urls': [],  # List of URLs to add to the notification Queue (Usually AppRise)
-            'notification_title': default_notification_title,
-            'notification_body': default_notification_body,
-            'notification_format': default_notification_format,
-            'notification_use_default': True, # Use default for new
+            'notification_use_default': True, # A checkbox to make it easier to understand if we are using this or not
+            'notification_title': None,
+            'notification_body': None,
+            'notification_format': None,
             'notification_muted': False,
             'css_filter': '',
             'last_error': False,

changedetectionio/static/js/watch-settings.js

@@ -1,7 +1,7 @@
-$(document).ready(function () {
-    function toggle_fetch_backend() {
+$(document).ready(function() {
+    function toggle() {
         if ($('input[name="fetch_backend"]:checked').val() == 'html_webdriver') {
-            if (playwright_enabled) {
+            if(playwright_enabled) {
                 // playwright supports headers, so hide everything else
                 // See #664
                 $('#requests-override-options #request-method').hide();
@@ -13,8 +13,12 @@ $(document).ready(function () {
                 // selenium/webdriver doesnt support anything afaik, hide it all
                 $('#requests-override-options').hide();
             }
+
+
             $('#webdriver-override-options').show();
+
         } else {
+
             $('#requests-override-options').show();
             $('#requests-override-options *:hidden').show();
             $('#webdriver-override-options').hide();
@@ -22,27 +26,8 @@ $(document).ready(function () {
     }
 
     $('input[name="fetch_backend"]').click(function (e) {
-        toggle_fetch_backend();
+        toggle();
     });
-    toggle_fetch_backend();
+    toggle();
 
-    function toggle_default_notifications() {
-        var n=$('#notification_urls, #notification_title, #notification_body, #notification_format');
-        if ($('#notification_use_default').is(':checked')) {
-            $('#notification-field-group').fadeOut();
-            $(n).each(function (e) {
-                $(this).attr('readonly', true);
-            });
-        } else {
-            $('#notification-field-group').show();
-            $(n).each(function (e) {
-                $(this).attr('readonly', false);
-            });
-        }
-    }
-
-    $('#notification_use_default').click(function (e) {
-        toggle_default_notifications();
-    });
-    toggle_default_notifications();
 });

changedetectionio/store.py

@@ -244,6 +244,10 @@ class ChangeDetectionStore:
 
         return False
 
+    def get_val(self, uuid, val):
+        # Probably their should be dict...
+        return self.data['watching'][uuid].get(val)
+
     # Remove a watchs data but keep the entry (URL etc)
     def clear_watch_history(self, uuid):
         import pathlib
@@ -537,26 +541,8 @@ class ChangeDetectionStore:
                 continue
         return
 
-
+    # Any watch notification that exactly the same as the default
+    # Then set the 'notification_use_default' to True and the other values to None
     def update_5(self):
-
-        from changedetectionio.notification import (
-            default_notification_body,
-            default_notification_format,
-            default_notification_title,
-        )
-
-        for uuid, watch in self.data['watching'].items():
-            try:
-                # If it's all the same to the system settings, then prefer system notification settings
-                # include \r\n -> \n incase they already hit submit and the browser put \r in
-                if watch.get('notification_body').replace('\r\n', '\n') == default_notification_body.replace('\r\n', '\n') and \
-                        watch.get('notification_format') == default_notification_format and \
-                        watch.get('notification_title').replace('\r\n', '\n') == default_notification_title.replace('\r\n', '\n') and \
-                        watch.get('notification_urls') == self.__data['settings']['application']['notification_urls']:
-                        watch['notification_use_default'] = True
-                else:
-                    watch['notification_use_default'] = False
-            except:
-                continue
+        # @todo
         return

changedetectionio/templates/edit.html

@@ -135,11 +135,9 @@ User-Agent: wonderbra 1.0") }}
             </div>
 
             <div class="tab-pane-inner" id="notifications">
+                <strong>Note: <i>These settings override the global settings for this watch.</i></strong>
                 <fieldset>
-                    <div  class="pure-control-group inline-radio">
-                      {{ render_checkbox_field(form.notification_use_default) }}
-                    </div>
-                    <div class="field-group" id="notification-field-group">
+                    <div class="field-group">
                         {{ render_common_settings_form(form, current_base_url, emailprefix) }}
                     </div>
                 </fieldset>

changedetectionio/templates/preview.html

@@ -57,7 +57,6 @@
          </br>
          {% if is_html_webdriver %}
            {% if screenshot %}
-             <div class="snapshot-age">{{watch.snapshot_screenshot_ctime|format_timestamp_timeago}}</div>
              <img style="max-width: 80%" id="screenshot-img" alt="Current screenshot from most recent request"/>
            {% else %}
               No screenshot available just yet! Try rechecking the page.

changedetectionio/tests/test_notification.py

@@ -71,7 +71,6 @@ def test_check_notification(client, live_server):
         "url": test_url,
         "tag": "my tag",
         "title": "my title",
-        # No 'notification_use_default' here, so it's effectively False/off
         "headers": "",
         "fetch_backend": "html_requests"})
 
@@ -216,82 +215,3 @@ def test_notification_validation(client, live_server):
         url_for("form_delete", uuid="all"),
         follow_redirects=True
     )
-
-# Check that the default VS watch specific notification is hit
-def test_check_notification_use_default(client, live_server):
-    set_original_response()
-    notification_url = url_for('test_notification_endpoint', _external=True).replace('http', 'json')
-    test_url = url_for('test_endpoint', _external=True)
-
-    res = client.post(
-        url_for("form_quick_watch_add"),
-        data={"url": test_url, "tag": ''},
-        follow_redirects=True
-    )
-    assert b"Watch added" in res.data
-
-    ## Setup the local one and enable it
-    res = client.post(
-        url_for("edit_page", uuid="first"),
-        data={"notification_urls": notification_url,
-              "notification_title": "watch-notification",
-              "notification_body": "watch-body",
-              'notification_use_default': "True",
-              "notification_format": "Text",
-              "url": test_url,
-              "tag": "my tag",
-              "title": "my title",
-              "headers": "",
-              "fetch_backend": "html_requests"},
-        follow_redirects=True
-    )
-
-    res = client.post(
-        url_for("settings_page"),
-        data={"application-notification_title": "global-notifications-title",
-              "application-notification_body": "global-notifications-body\n",
-              "application-notification_format": "Text",
-              "application-notification_urls": notification_url,
-              "requests-time_between_check-minutes": 180,
-              "fetch_backend": "html_requests"
-              },
-        follow_redirects=True
-    )
-
-    # A change should by default trigger a notification of the global-notifications
-    time.sleep(1)
-    set_modified_response()
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    time.sleep(2)
-    with open("test-datastore/notification.txt", "r") as f:
-        assert 'global-notifications-title' in f.read()
-
-    ## Setup the local one and enable it
-    res = client.post(
-        url_for("edit_page", uuid="first"),
-        data={"notification_urls": notification_url,
-              "notification_title": "watch-notification",
-              "notification_body": "watch-body",
-              # No 'notification_use_default' here, so it's effectively False/off = "dont use default, use this one"
-              "notification_format": "Text",
-              "url": test_url,
-              "tag": "my tag",
-              "title": "my title",
-              "headers": "",
-              "fetch_backend": "html_requests"},
-        follow_redirects=True
-    )
-    set_original_response()
-
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    time.sleep(2)
-    assert os.path.isfile("test-datastore/notification.txt")
-    with open("test-datastore/notification.txt", "r") as f:
-        assert 'watch-notification' in f.read()
-
-
-    # cleanup for the next
-    client.get(
-        url_for("form_delete", uuid="all"),
-        follow_redirects=True
-    )

changedetectionio/tests/visualselector/test_fetch_data.py

@@ -7,7 +7,6 @@ from ..util import live_server_setup, wait_for_all_checks, extract_UUID_from_cli
 # Add a site in paused mode, add an invalid filter, we should still have visual selector data ready
 def test_visual_selector_content_ready(client, live_server):
     import os
-    import json
 
     assert os.getenv('PLAYWRIGHT_DRIVER_URL'), "Needs PLAYWRIGHT_DRIVER_URL set for this test"
     live_server_setup(live_server)
@@ -34,7 +33,3 @@ def test_visual_selector_content_ready(client, live_server):
     uuid = extract_UUID_from_client(client)
     assert os.path.isfile(os.path.join('test-datastore', uuid, 'last-screenshot.png')), "last-screenshot.png should exist"
     assert os.path.isfile(os.path.join('test-datastore', uuid, 'elements.json')), "xpath elements.json data should exist"
-
-    # Open it and see if it roughly looks correct
-    with open(os.path.join('test-datastore', uuid, 'elements.json'), 'r') as f:
-        json.load(f)

changedetectionio/update_worker.py

@@ -41,7 +41,7 @@ class update_worker(threading.Thread):
             )
 
         # Did it have any notification alerts to hit?
-        if not watch.get('notification_use_default') and len(watch['notification_urls']):
+        if len(watch['notification_urls']):
             print(">>> Notifications queued for UUID from watch {}".format(watch_uuid))
             n_object['notification_urls'] = watch['notification_urls']
             n_object['notification_title'] = watch['notification_title']
@@ -49,7 +49,7 @@ class update_worker(threading.Thread):
             n_object['notification_format'] = watch['notification_format']
 
         # No? maybe theres a global setting, queue them all
-        elif watch.get('notification_use_default') and len(self.datastore.data['settings']['application']['notification_urls']):
+        elif len(self.datastore.data['settings']['application']['notification_urls']):
             print(">>> Watch notification URLs were empty, using GLOBAL notifications for UUID: {}".format(watch_uuid))
             n_object['notification_urls'] = self.datastore.data['settings']['application']['notification_urls']
             n_object['notification_title'] = self.datastore.data['settings']['application']['notification_title']