Re #2742 fixing URL encoding of headers so that '+' in URL is correctly parsed as ' ' (and other url-encodings)

changedetectionio/__init__.py

@@ -2,7 +2,7 @@
 
 # Read more https://github.com/dgtlmoon/changedetection.io/wiki
 
-__version__ = '0.47.01'
+__version__ = '0.47.03'
 
 from changedetectionio.strtobool import strtobool
 from json.decoder import JSONDecodeError

changedetectionio/apprise_plugin/__init__.py

@@ -13,6 +13,7 @@ from loguru import logger
 def apprise_custom_api_call_wrapper(body, title, notify_type, *args, **kwargs):
     import requests
     import json
+    from urllib.parse import unquote_plus
     from apprise.utils import parse_url as apprise_parse_url
     from apprise import URLBase
 
@@ -47,7 +48,7 @@ def apprise_custom_api_call_wrapper(body, title, notify_type, *args, **kwargs):
     if results:
         # Add our headers that the user can potentially over-ride if they wish
         # to to our returned result set and tidy entries by unquoting them
-        headers = {URLBase.unquote(x): URLBase.unquote(y)
+        headers = {unquote_plus(x): unquote_plus(y)
                    for x, y in results['qsd+'].items()}
 
         # https://github.com/caronc/apprise/wiki/Notify_Custom_JSON#get-parameter-manipulation
@@ -55,14 +56,14 @@ def apprise_custom_api_call_wrapper(body, title, notify_type, *args, **kwargs):
         # but here we are making straight requests, so we need todo convert this against apprise's logic
         for k, v in results['qsd'].items():
             if not k.strip('+-') in results['qsd+'].keys():
-                params[URLBase.unquote(k)] = URLBase.unquote(v)
+                params[unquote_plus(k)] = unquote_plus(v)
 
         # Determine Authentication
         auth = ''
         if results.get('user') and results.get('password'):
-            auth = (URLBase.unquote(results.get('user')), URLBase.unquote(results.get('user')))
+            auth = (unquote_plus(results.get('user')), unquote_plus(results.get('user')))
         elif results.get('user'):
-            auth = (URLBase.unquote(results.get('user')))
+            auth = (unquote_plus(results.get('user')))
 
     # Try to auto-guess if it's JSON
     h = 'application/json; charset=utf-8'

changedetectionio/flask_app.py

@@ -67,7 +67,6 @@ FlaskCompress(app)
 
 # Stop browser caching of assets
 app.config['SEND_FILE_MAX_AGE_DEFAULT'] = 0
-
 app.config.exit = Event()
 
 app.config['NEW_VERSION_AVAILABLE'] = False
@@ -470,7 +469,7 @@ def changedetection_app(config=None, datastore_o=None):
                     continue
             if watch.get('last_error'):
                 errored_count += 1
-                
+
             if search_q:
                 if (watch.get('title') and search_q in watch.get('title').lower()) or search_q in watch.get('url', '').lower():
                     sorted_watches.append(watch)
@@ -533,7 +532,7 @@ def changedetection_app(config=None, datastore_o=None):
     @login_optionally_required
     def ajax_callback_send_notification_test(watch_uuid=None):
 
-        # Watch_uuid could be unset in the case its used in tag editor, global setings
+        # Watch_uuid could be unset in the case it`s used in tag editor, global settings
         import apprise
         import random
         from .apprise_asset import asset
@@ -542,13 +541,15 @@ def changedetection_app(config=None, datastore_o=None):
         from changedetectionio.apprise_plugin import apprise_custom_api_call_wrapper
         is_global_settings_form = request.args.get('mode', '') == 'global-settings'
         is_group_settings_form = request.args.get('mode', '') == 'group-settings'
-
         # Use an existing random one on the global/main settings form
-        if not watch_uuid and (is_global_settings_form or is_group_settings_form):
+        if not watch_uuid and (is_global_settings_form or is_group_settings_form) \
+                and datastore.data.get('watching'):
+
             logger.debug(f"Send test notification - Choosing random Watch {watch_uuid}")
             watch_uuid = random.choice(list(datastore.data['watching'].keys()))
-
-        watch = datastore.data['watching'].get(watch_uuid)
+            watch = datastore.data['watching'].get(watch_uuid)
+        else:
+            watch = None
 
         notification_urls = request.form['notification_urls'].strip().splitlines()
 
@@ -1396,7 +1397,7 @@ def changedetection_app(config=None, datastore_o=None):
         url = request.form.get('url').strip()
         if datastore.url_exists(url):
             flash(f'Warning, URL {url} already exists', "notice")
-            
+
         add_paused = request.form.get('edit_and_watch_submit_button') != None
         processor = request.form.get('processor', 'text_json_diff')
         new_uuid = datastore.add_watch(url=url, tag=request.form.get('tags').strip(), extras={'paused': add_paused, 'processor': processor})

changedetectionio/forms.py

@@ -496,7 +496,7 @@ class processor_text_json_diff_form(commonSettingsForm):
     text_should_not_be_present = StringListField('Block change-detection while text matches', [validators.Optional(), ValidateListRegex()])
     webdriver_js_execute_code = TextAreaField('Execute JavaScript before change detection', render_kw={"rows": "5"}, validators=[validators.Optional()])
 
-    save_button = SubmitField('Save', render_kw={"class": "pure-button pure-button-primary"})
+    save_button = SubmitField('Save', render_kw={"class": "pure-button button-small pure-button-primary"})
 
     proxy = RadioField('Proxy')
     filter_failure_notification_send = BooleanField(
@@ -616,7 +616,7 @@ class globalSettingsForm(Form):
 
     requests = FormField(globalSettingsRequestForm)
     application = FormField(globalSettingsApplicationForm)
-    save_button = SubmitField('Save', render_kw={"class": "pure-button pure-button-primary"})
+    save_button = SubmitField('Save', render_kw={"class": "pure-button button-small pure-button-primary"})
 
 
 class extractDataForm(Form):

changedetectionio/processors/text_json_diff/processor.py

@@ -331,13 +331,21 @@ class perform_site_check(difference_detection_processor):
             if result:
                 blocked = True
 
-        # The main thing that all this at the moment comes down to :)
-        if watch.get('previous_md5') != fetched_md5:
-            changed_detected = True
 
         # Looks like something changed, but did it match all the rules?
         if blocked:
             changed_detected = False
+        else:
+            # The main thing that all this at the moment comes down to :)
+            if watch.get('previous_md5') != fetched_md5:
+                changed_detected = True
+
+            # Always record the new checksum
+            update_obj["previous_md5"] = fetched_md5
+
+            # On the first run of a site, watch['previous_md5'] will be None, set it the current one.
+            if not watch.get('previous_md5'):
+                watch['previous_md5'] = fetched_md5
 
         logger.debug(f"Watch UUID {watch.get('uuid')} content check - Previous MD5: {watch.get('previous_md5')}, Fetched MD5 {fetched_md5}")
 
@@ -357,12 +365,6 @@ class perform_site_check(difference_detection_processor):
                 else:
                     logger.debug(f"check_unique_lines: UUID {watch.get('uuid')} had unique content")
 
-        # Always record the new checksum
-        update_obj["previous_md5"] = fetched_md5
-
-        # On the first run of a site, watch['previous_md5'] will be None, set it the current one.
-        if not watch.get('previous_md5'):
-            watch['previous_md5'] = fetched_md5
 
         # stripped_text_from_html - Everything after filters and NO 'ignored' content
         return changed_detected, update_obj, stripped_text_from_html

changedetectionio/static/js/tabs.js

@@ -26,8 +26,7 @@ function set_active_tab() {
     if (tab.length) {
         tab[0].parentElement.className = "active";
     }
-    // hash could move the page down
-    window.scrollTo(0, 0);
+
 }
 
 function focus_error_tab() {

changedetectionio/static/styles/diff.css

@@ -153,7 +153,8 @@ html[data-darkmode="true"] {
     border: 1px solid transparent;
     vertical-align: top;
     font: 1em monospace;
-    text-align: left; }
+    text-align: left;
+    overflow: clip; }
   #diff-ui pre {
     white-space: pre-wrap; }
 
@@ -172,7 +173,9 @@ ins {
   text-decoration: none; }
 
 #result {
-  white-space: pre-wrap; }
+  white-space: pre-wrap;
+  word-break: break-word;
+  overflow-wrap: break-word; }
 
 #settings {
   background: rgba(0, 0, 0, 0.05);
@@ -231,3 +234,12 @@ td#diff-col div {
   border-radius: 5px;
   background: var(--color-background);
   box-shadow: 1px 1px 4px var(--color-shadow-jump); }
+
+.pure-form button.reset-margin {
+  margin: 0px; }
+
+.diff-fieldset {
+  display: flex;
+  align-items: center;
+  gap: 4px;
+  flex-wrap: wrap; }

changedetectionio/static/styles/scss/diff.scss

@@ -24,6 +24,7 @@
     vertical-align: top;
     font: 1em monospace;
     text-align: left;
+    overflow: clip; // clip overflowing contents to cell boundariess
   }
 
   pre {
@@ -50,6 +51,8 @@ ins {
 
 #result {
   white-space: pre-wrap;
+  word-break: break-word;
+  overflow-wrap: break-word;
 
   .change {
     span {}
@@ -134,3 +137,15 @@ td#diff-col div {
   background: var(--color-background);
   box-shadow: 1px 1px 4px var(--color-shadow-jump);
 }
+
+// resets button margin to 0px
+.pure-form button.reset-margin {
+  margin: 0px;
+}
+
+.diff-fieldset {
+  display: flex;
+  align-items: center;
+  gap: 4px;
+  flex-wrap: wrap;
+}

changedetectionio/static/styles/scss/parts/_extra_browsers.scss

@@ -11,7 +11,22 @@ ul#requests-extra_browsers {
   /* each proxy entry is a `table` */
   table {
     tr {
-      display: inline;
+      display: table-row; // default display for small screens
+      input[type=text] {
+        width: 100%;
+      }
+    }
+  }
+  
+  // apply inline display for larger screens
+  @media only screen and (min-width: 1280px) {
+    table {
+      tr {
+        display: inline;
+        input[type=text] {
+          width: 100%;
+        }
+      }
     }
   }
 }

changedetectionio/static/styles/scss/parts/_extra_proxies.scss

@@ -11,7 +11,19 @@ ul#requests-extra_proxies {
   /* each proxy entry is a `table` */
   table {
     tr {
-      display: inline;
+      display: table-row; // default display for small screens
+      input[type=text] {
+        width: 100%;
+      }
+    }
+  }
+  
+  // apply inline display for large screens
+  @media only screen and (min-width: 1024px) {
+    table {
+      tr {
+        display: inline;
+      }
     }
   }
 }

changedetectionio/static/styles/scss/styles.scss

@@ -148,7 +148,7 @@ body.spinner-active {
 }
 
 
-.tabs ul li a {
+.tab-pane-inner {
   // .tab-pane-inner will have the #id that the tab button jumps/anchors to
   scroll-margin-top: 200px;
 }

changedetectionio/static/styles/styles.css

@@ -112,7 +112,12 @@ ul#requests-extra_proxies {
   ul#requests-extra_proxies li > label {
     display: none; }
   ul#requests-extra_proxies table tr {
-    display: inline; }
+    display: table-row; }
+    ul#requests-extra_proxies table tr input[type=text] {
+      width: 100%; }
+  @media only screen and (min-width: 1024px) {
+    ul#requests-extra_proxies table tr {
+      display: inline; } }
 
 #request {
   /* Auto proxy scan/checker */ }
@@ -161,7 +166,14 @@ ul#requests-extra_browsers {
   ul#requests-extra_browsers li > label {
     display: none; }
   ul#requests-extra_browsers table tr {
-    display: inline; }
+    display: table-row; }
+    ul#requests-extra_browsers table tr input[type=text] {
+      width: 100%; }
+  @media only screen and (min-width: 1280px) {
+    ul#requests-extra_browsers table tr {
+      display: inline; }
+      ul#requests-extra_browsers table tr input[type=text] {
+        width: 100%; } }
 
 #extra-browsers-setting {
   border: 1px solid var(--color-grey-800);
@@ -605,8 +617,7 @@ body.spinner-active #pure-menu-horizontal-spinner {
     background-color: var(--color-background-menu-link-hover);
     color: var(--color-text-menu-link-hover); }
 
-
-.tabs ul li a {
+.tab-pane-inner {
   scroll-margin-top: 200px; }
 
 section.content {

changedetectionio/templates/diff.html

@@ -14,7 +14,7 @@
 
 <div id="settings">
     <form class="pure-form " action="" method="GET" id="diff-form">
-        <fieldset>
+        <fieldset class="diff-fieldset">
             {% if versions|length >= 1 %}
                 <strong>Compare</strong>
                 <del class="change"><span>from</span></del>
@@ -33,7 +33,7 @@
                         </option>
                     {% endfor %}
                 </select>
-                <button type="submit" class="pure-button pure-button-primary">Go</button>
+                <button type="submit" class="pure-button pure-button-primary reset-margin">Go</button>
             {% endif %}
         </fieldset>
         <fieldset>

changedetectionio/templates/settings.html

@@ -276,7 +276,7 @@ nav
                 <div class="pure-control-group">
                     {{ render_button(form.save_button) }}
                     <a href="{{url_for('index')}}" class="pure-button button-small button-cancel">Back</a>
-                    <a href="{{url_for('clear_all_history')}}" class="pure-button button-small button-cancel">Clear Snapshot History</a>
+                    <a href="{{url_for('clear_all_history')}}" class="pure-button button-small button-error">Clear Snapshot History</a>
                 </div>
             </div>
         </form>

changedetectionio/tests/test_block_while_text_present.py

@@ -65,11 +65,8 @@ def test_check_block_changedetection_text_NOT_present(client, live_server, measu
     live_server_setup(live_server)
     # Use a mix of case in ZzZ to prove it works case-insensitive.
     ignore_text = "out of stoCk\r\nfoobar"
-
     set_original_ignore_response()
 
-    # Give the endpoint time to spin up
-    time.sleep(1)
 
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)
@@ -127,13 +124,24 @@ def test_check_block_changedetection_text_NOT_present(client, live_server, measu
     assert b'unviewed' not in res.data
     assert b'/test-endpoint' in res.data
 
+    # 2548
+    # Going back to the ORIGINAL should NOT trigger a change
+    set_original_ignore_response()
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    wait_for_all_checks(client)
+    res = client.get(url_for("index"))
+    assert b'unviewed' not in res.data
 
-    # Now we set a change where the text is gone, it should now trigger
+
+    # Now we set a change where the text is gone AND its different content, it should now trigger
     set_modified_response_minus_block_text()
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
     res = client.get(url_for("index"))
     assert b'unviewed' in res.data
 
+
+
+
     res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data

changedetectionio/tests/test_notification.py

@@ -284,7 +284,7 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server, measure_me
     # CUSTOM JSON BODY CHECK for POST://
     set_original_response()
     # https://github.com/caronc/apprise/wiki/Notify_Custom_JSON#header-manipulation
-    test_notification_url = url_for('test_notification_endpoint', _external=True).replace('http://', 'post://')+"?xxx={{ watch_url }}&+custom-header=123"
+    test_notification_url = url_for('test_notification_endpoint', _external=True).replace('http://', 'post://')+"?xxx={{ watch_url }}&+custom-header=123&+second=hello+world%20%22space%22"
 
     res = client.post(
         url_for("settings_page"),
@@ -326,6 +326,7 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server, measure_me
         assert j['secret'] == 444
         assert j['somebug'] == '网站监测 内容更新了'
 
+
     # URL check, this will always be converted to lowercase
     assert os.path.isfile("test-datastore/notification-url.txt")
     with open("test-datastore/notification-url.txt", 'r') as f:
@@ -337,6 +338,7 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server, measure_me
     with open("test-datastore/notification-headers.txt", 'r') as f:
         notification_headers = f.read()
         assert 'custom-header: 123' in notification_headers.lower()
+        assert 'second: hello world "space"' in notification_headers.lower()
 
 
     # Should always be automatically detected as JSON content type even when we set it as 'Text' (default)
@@ -429,3 +431,24 @@ def test_global_send_test_notification(client, live_server, measure_memory_usage
         follow_redirects=True
     )
 
+    #2727 - be sure a test notification when there are zero watches works ( should all be deleted now)
+
+    os.unlink("test-datastore/notification.txt")
+
+
+    ######### Test global/system settings
+    res = client.post(
+        url_for("ajax_callback_send_notification_test")+"?mode=global-settings",
+        data={"notification_urls": test_notification_url},
+        follow_redirects=True
+    )
+
+    assert res.status_code != 400
+    assert res.status_code != 500
+
+    # Give apprise time to fire
+    time.sleep(4)
+
+    with open("test-datastore/notification.txt", 'r') as f:
+        x = f.read()
+        assert 'change detection is cool 网站监测 内容更新了' in x

changedetectionio/update_worker.py

@@ -81,7 +81,8 @@ class update_worker(threading.Thread):
             'watch_url': watch.get('url') if watch else None,
         })
 
-        n_object.update(watch.extra_notification_token_values())
+        if watch:
+            n_object.update(watch.extra_notification_token_values())
 
         logger.trace(f"Main rendered notification placeholders (diff_added etc) calculated in {time.time()-now:.3f}s")
         logger.debug("Queued notification for sending")