Adding help link

changedetectionio/__init__.py

@@ -38,7 +38,7 @@ from flask_paginate import Pagination, get_page_parameter
 from changedetectionio import html_tools
 from changedetectionio.api import api_v1
 
-__version__ = '0.45.7.3'
+__version__ = '0.45.8.1'
 
 from changedetectionio.store import BASE_URL_NOT_SET_TEXT
 

changedetectionio/notification.py

@@ -46,6 +46,9 @@ from apprise.decorators import notify
 @notify(on="puts")
 def apprise_custom_api_call_wrapper(body, title, notify_type, *args, **kwargs):
     import requests
+    from apprise.utils import parse_url as apprise_parse_url
+    from apprise.URLBase import URLBase
+
     url = kwargs['meta'].get('url')
 
     if url.startswith('post'):
@@ -68,16 +71,45 @@ def apprise_custom_api_call_wrapper(body, title, notify_type, *args, **kwargs):
     url = url.replace('delete://', 'http://')
     url = url.replace('deletes://', 'https://')
 
-    # Try to auto-guess if it's JSON
     headers = {}
+    params = {}
+    auth = None
+
+    # Convert /foobar?+some-header=hello to proper header dictionary
+    results = apprise_parse_url(url)
+    if results:
+        # Add our headers that the user can potentially over-ride if they wish
+        # to to our returned result set and tidy entries by unquoting them
+        headers = {URLBase.unquote(x): URLBase.unquote(y)
+                   for x, y in results['qsd+'].items()}
+
+        # https://github.com/caronc/apprise/wiki/Notify_Custom_JSON#get-parameter-manipulation
+        # In Apprise, it relies on prefixing each request arg with "-", because it uses say &method=update as a flag for apprise
+        # but here we are making straight requests, so we need todo convert this against apprise's logic
+        for k, v in results['qsd'].items():
+            if not k.strip('+-') in results['qsd+'].keys():
+                params[URLBase.unquote(k)] = URLBase.unquote(v)
+
+        # Determine Authentication
+        auth = ''
+        if results.get('user') and results.get('password'):
+            auth = (URLBase.unquote(results.get('user')), URLBase.unquote(results.get('user')))
+        elif results.get('user'):
+            auth = (URLBase.unquote(results.get('user')))
+
+    # Try to auto-guess if it's JSON
     try:
         json.loads(body)
-        headers = {'Content-Type': 'application/json; charset=utf-8'}
+        headers['Content-Type'] = 'application/json; charset=utf-8'
     except ValueError as e:
         pass
 
-
-    r(url, headers=headers, data=body)
+    r(results.get('url'),
+      auth=auth,
+      data=body,
+      headers=headers,
+      params=params
+      )
 
 
 def process_notification(n_object, datastore):

changedetectionio/processors/__init__.py

@@ -52,6 +52,11 @@ class difference_detection_processor():
                 prefer_fetch_backend = 'base_html_playwright'
                 browser_connection_url = connection[0].get('browser_connection_url')
 
+        # PDF should be html_requests because playwright will serve it up (so far) in a embedded page
+        # @todo https://github.com/dgtlmoon/changedetection.io/issues/2019
+        # @todo needs test to or a fix
+        if self.watch.is_pdf:
+           prefer_fetch_backend = "html_requests"
 
         # Grab the right kind of 'fetcher', (playwright, requests, etc)
         if hasattr(content_fetcher, prefer_fetch_backend):

changedetectionio/res/stock-not-in-stock.js

@@ -1,6 +1,7 @@
 function isItemInStock() {
   // @todo Pass these in so the same list can be used in non-JS fetchers
   const outOfStockTexts = [
+    ' أخبرني عندما يتوفر',
     '0 in stock',
     'agotado',
     'artikel zurzeit vergriffen',
@@ -16,9 +17,12 @@ function isItemInStock() {
     'currently have any tickets for this',
     'currently unavailable',
     'dostępne wkrótce',
+    'dostępne wkrótce',
     'en rupture de stock',
     'ist derzeit nicht auf lager',
+    'ist derzeit nicht auf lager',
     'item is no longer available',
+    'let me know when it\'s available',
     'message if back in stock',
     'nachricht bei',
     'nicht auf lager',
@@ -42,7 +46,9 @@ function isItemInStock() {
     'unavailable tickets',
     'we do not currently have an estimate of when this product will be back in stock.',
     'zur zeit nicht an lager',
+    '品切れ',
     '已售完',
+    '품절'
   ];
 
 

changedetectionio/static/js/vis.js

@@ -2,21 +2,28 @@ $(document).ready(function () {
 
     // Lazy Hide/Show elements mechanism
     $('[data-visible-for]').hide();
-    $(':radio').on('keyup keypress blur change click', function (e) {
-        $('[data-visible-for]').hide();
-        $('.advanced-options').hide();
-        var n = $(this).attr('name') + "=" + $(this).val();
+    function show_related_elem(e) {
+        var n = $(e).attr('name') + "=" + $(e).val();
         if (n === 'fetch_backend=system') {
             n = "fetch_backend=" + default_system_fetch_backend;
         }
         $(`[data-visible-for~="${n}"]`).show();
-
+    }
+    $(':radio').on('keyup keypress blur change click', function (e) {
+        $(`[data-visible-for]`).hide();
+        $('.advanced-options').hide();
+        show_related_elem(this);
     });
-    $(':radio:checked').change();
+
+    $(':radio:checked').each(function (e) {
+       show_related_elem(this);
+    })
 
 
     // Show advanced
     $('.show-advanced').click(function (e) {
-        $(this).closest('.tab-pane-inner').find('.advanced-options').toggle();
+        $(this).closest('.tab-pane-inner').find('.advanced-options').each(function (e) {
+            $(this).toggle();
+        })
     });
 });

changedetectionio/static/js/watch-settings.js

@@ -1,18 +1,4 @@
 $(document).ready(function () {
-
-    // Lazy Hide/Show elements mechanism
-    $('[data-visible-for]').hide();
-    $(':radio').on('keyup keypress blur change click', function (e){
-        $('[data-visible-for]').hide();
-        var n = $(this).attr('name') + "=" + $(this).val();
-        if (n === 'fetch_backend=system') {
-            n = "fetch_backend=" + default_system_fetch_backend;
-        }
-        $(`[data-visible-for~="${n}"]`).show();
-
-    });
-    $(':radio:checked').change();
-
     $('#notification-setting-reset-to-default').click(function (e) {
         $('#notification_title').val('');
         $('#notification_body').val('');

changedetectionio/static/styles/scss/styles.scss

@@ -402,8 +402,24 @@ label {
   }
 
   #watch-add-wrapper-zone {
-    >div {
-      display: inline-block;
+
+    @media only screen and (min-width: 760px) {
+      display: flex;
+      gap: 0.3rem;
+      flex-direction: row;
+    }
+    /* URL field grows always, other stay static in width */
+    > span {
+      flex-grow: 0;
+
+      input {
+        width: 100%;
+        padding-right: 1em;
+      }
+
+      &:first-child {
+        flex-grow: 1;
+      }
     }
 
     @media only screen and (max-width: 760px) {

changedetectionio/static/styles/styles.css

@@ -683,11 +683,23 @@ label:hover {
   #new-watch-form legend {
     color: var(--color-text-legend);
     font-weight: bold; }
-  #new-watch-form #watch-add-wrapper-zone > div {
-    display: inline-block; }
-  @media only screen and (max-width: 760px) {
-    #new-watch-form #watch-add-wrapper-zone #url {
-      width: 100%; } }
+  #new-watch-form #watch-add-wrapper-zone {
+    /* URL field grows always, other stay static in width */ }
+    @media only screen and (min-width: 760px) {
+      #new-watch-form #watch-add-wrapper-zone {
+        display: flex;
+        gap: 0.3rem;
+        flex-direction: row; } }
+    #new-watch-form #watch-add-wrapper-zone > span {
+      flex-grow: 0; }
+      #new-watch-form #watch-add-wrapper-zone > span input {
+        width: 100%;
+        padding-right: 1em; }
+      #new-watch-form #watch-add-wrapper-zone > span:first-child {
+        flex-grow: 1; }
+    @media only screen and (max-width: 760px) {
+      #new-watch-form #watch-add-wrapper-zone #url {
+        width: 100%; } }
 
 #diff-col {
   padding-left: 40px; }

changedetectionio/templates/_common_fields.jinja

@@ -16,7 +16,7 @@
                                 <li><code><a target=_new href="https://github.com/caronc/apprise/wiki/Notify_discord">discord://</a></code> (or <code>https://discord.com/api/webhooks...</code>)) only supports a maximum <strong>2,000 characters</strong> of notification text, including the title.</li>
                                 <li><code><a target=_new href="https://github.com/caronc/apprise/wiki/Notify_telegram">tgram://</a></code> bots can't send messages to other bots, so you should specify chat ID of non-bot user.</li>
                                 <li><code><a target=_new href="https://github.com/caronc/apprise/wiki/Notify_telegram">tgram://</a></code> only supports very limited HTML and can fail when extra tags are sent, <a href="https://core.telegram.org/bots/api#html-style">read more here</a> (or use plaintext/markdown format)</li>
-                                <li><code>gets://</code>, <code>posts://</code>, <code>puts://</code>, <code>deletes://</code> for direct API calls (or omit the "<code>s</code>" for non-SSL ie <code>get://</code>)</li>
+                                <li><code>gets://</code>, <code>posts://</code>, <code>puts://</code>, <code>deletes://</code> for direct API calls (or omit the "<code>s</code>" for non-SSL ie <code>get://</code>) <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Notification-configuration-notes#postposts">more help here</a></li>
                                   <li>Accepts the <code>{{ '{{token}}' }}</code> placeholders listed below</li>
                               </ul>
                             </div>

changedetectionio/templates/_helpers.jinja

@@ -39,6 +39,24 @@
 {% endmacro %}
 
 
+{% macro render_nolabel_field(field) %}
+    <span>
+    {{ field(**kwargs)|safe }}
+        {% if field.errors %}
+            <span class="error">
+      {% if field.errors %}
+          <ul class=errors>
+        {% for error in field.errors %}
+            <li>{{ error }}</li>
+        {% endfor %}
+        </ul>
+      {% endif %}
+      </span>
+        {% endif %}
+    </span>
+{% endmacro %}
+
+
 {% macro render_button(field) %}
   {{ field(**kwargs)|safe }}
 {% endmacro %}

changedetectionio/templates/edit.html

@@ -127,7 +127,7 @@
                 {% endif %}
 
                 <!-- webdriver always -->
-                <fieldset data-visible-for="fetch_backend=html_webdriver">
+                <fieldset data-visible-for="fetch_backend=html_webdriver"  style="display: none;">
                     <div class="pure-control-group">
                         {{ render_field(form.webdriver_delay) }}
                         <div class="pure-form-message-inline">
@@ -153,15 +153,15 @@
                     </div>
                 </fieldset>
                 <!-- html requests always -->
-                <fieldset data-visible-for="fetch_backend=html_requests" style="display: none;">
+                <fieldset data-visible-for="fetch_backend=html_requests">
                     <div class="pure-control-group">
                         <a class="pure-button button-secondary button-xsmall show-advanced">Show advanced options</a>
                     </div>
                     <div class="advanced-options"  style="display: none;">
-                        <div class="pure-control-group advanced-options" id="request-method" style="display: none;">
+                        <div class="pure-control-group" id="request-method">
                             {{ render_field(form.method) }}
                         </div>
-                        <div class="advanced-options" id="request-body">
+                        <div id="request-body">
                                             {{ render_field(form.body, rows=5, placeholder="Example
 {
    \"name\":\"John\",
@@ -187,8 +187,8 @@ User-Agent: wonderbra 1.0") }}
                             (Not supported by Selenium browser)
                         </div>
                     </div>
-            <fieldset data-visible-for="fetch_backend=html_requests fetch_backend=html_webdriver" style="display: none;">
-                    <div  class="pure-control-group inline-radio advanced-options"  style="display: none;">
+            <fieldset data-visible-for="fetch_backend=html_requests fetch_backend=html_webdriver" >
+                    <div class="pure-control-group inline-radio advanced-options"  style="display: none;">
                     {{ render_checkbox_field(form.ignore_status_codes) }}
                     </div>
             </fieldset>

changedetectionio/templates/settings.html

@@ -236,8 +236,11 @@ nav
                 <span class="pure-form-message-inline">SOCKS5 proxies with authentication are only supported with 'plain requests' fetcher, for other fetchers you should whitelist the IP access instead</span>
                 </div>
                 <div class="pure-control-group" id="extra-browsers-setting">
-                    <span class="pure-form-message-inline"><i>Extra Browsers</i> allow changedetection.io to communicate with a different web-browser.</span><br>
-                  {{ render_field(form.requests.form.extra_browsers) }}
+                    <p>
+                    <span class="pure-form-message-inline"><i>Extra Browsers</i> can be attached to further defeat CAPTCHA's on websites that are particularly hard to scrape.</span><br>
+                    <span class="pure-form-message-inline">Simply paste the connection address into the box, <a href="https://changedetection.io/tutorial/using-bright-datas-scraping-browser-pass-captchas-and-other-protection-when-monitoring">More instructions and examples here</a> </span>
+                    </p>
+                    {{ render_field(form.requests.form.extra_browsers) }}
                 </div>
             </div>
             <div id="actions">

changedetectionio/templates/watch-overview.html

@@ -1,6 +1,6 @@
 {% extends 'base.html' %}
 {% block content %}
-{% from '_helpers.jinja' import render_simple_field, render_field %}
+{% from '_helpers.jinja' import render_simple_field, render_field, render_nolabel_field %}
 <script src="{{url_for('static_content', group='js', filename='jquery-3.6.0.min.js')}}"></script>
 <script src="{{url_for('static_content', group='js', filename='watch-overview.js')}}" defer></script>
 
@@ -11,17 +11,14 @@
         <fieldset>
             <legend>Add a new change detection watch</legend>
             <div id="watch-add-wrapper-zone">
-                <div>
-                    {{ render_simple_field(form.url, placeholder="https://...", required=true) }}
-                    {{ render_simple_field(form.tags, value=tags[active_tag].title if active_tag else '', placeholder="watch label / tag") }}
-                </div>
-                <div>
-                    {{ render_simple_field(form.watch_submit_button, title="Watch this URL!" ) }}
-                    {{ render_simple_field(form.edit_and_watch_submit_button, title="Edit first then Watch") }}
-                </div>
+
+                    {{ render_nolabel_field(form.url, placeholder="https://...", required=true) }}
+                    {{ render_nolabel_field(form.tags, value=tags[active_tag].title if active_tag else '', placeholder="watch label / tag") }}
+                    {{ render_nolabel_field(form.watch_submit_button, title="Watch this URL!" ) }}
+                    {{ render_nolabel_field(form.edit_and_watch_submit_button, title="Edit first then Watch") }}
             </div>
             <div id="quick-watch-processor-type">
-                {{ render_simple_field(form.processor, title="Edit first then Watch") }}
+                {{ render_simple_field(form.processor) }}
             </div>
 
         </fieldset>

changedetectionio/tests/conftest.py

@@ -13,22 +13,17 @@ global app
 
 
 def cleanup(datastore_path):
+    import glob
     # Unlink test output files
-    files = [
-        'count.txt',
-        'endpoint-content.txt'
-        'headers.txt',
-        'headers-testtag.txt',
-        'notification.txt',
-        'secret.txt',
-        'url-watches.json',
-        'output.txt',
-    ]
-    for file in files:
-        try:
-            os.unlink("{}/{}".format(datastore_path, file))
-        except FileNotFoundError:
-            pass
+
+    for g in ["*.txt", "*.json", "*.pdf"]:
+        files = glob.glob(os.path.join(datastore_path, g))
+        for f in files:
+            if 'proxies.json' in f:
+                # Usually mounted by docker container during test time
+                continue
+            if os.path.isfile(f):
+                os.unlink(f)
 
 @pytest.fixture(scope='session')
 def app(request):

changedetectionio/tests/test_notification.py

@@ -281,7 +281,8 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server):
 
     # CUSTOM JSON BODY CHECK for POST://
     set_original_response()
-    test_notification_url = url_for('test_notification_endpoint', _external=True).replace('http://', 'post://')+"?xxx={{ watch_url }}"
+    # https://github.com/caronc/apprise/wiki/Notify_Custom_JSON#header-manipulation
+    test_notification_url = url_for('test_notification_endpoint', _external=True).replace('http://', 'post://')+"?xxx={{ watch_url }}&+custom-header=123"
 
     res = client.post(
         url_for("settings_page"),
@@ -297,10 +298,7 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server):
         follow_redirects=True
     )
     assert b'Settings updated' in res.data
-    client.get(
-        url_for("form_delete", uuid="all"),
-        follow_redirects=True
-    )
+
     # Add a watch and trigger a HTTP POST
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
@@ -315,7 +313,9 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server):
     set_modified_response()
 
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    time.sleep(2)
+    wait_for_all_checks(client)
+
+    time.sleep(2) # plus extra delay for notifications to fire
 
     with open("test-datastore/notification.txt", 'r') as f:
         x = f.read()
@@ -328,6 +328,13 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server):
     with open("test-datastore/notification-url.txt", 'r') as f:
         notification_url = f.read()
         assert 'xxx=http' in notification_url
+        # apprise style headers should be stripped
+        assert 'custom-header' not in notification_url
+
+    with open("test-datastore/notification-headers.txt", 'r') as f:
+        notification_headers = f.read()
+        assert 'custom-header: 123' in notification_headers.lower()
+
 
     # Should always be automatically detected as JSON content type even when we set it as 'Text' (default)
     assert os.path.isfile("test-datastore/notification-content-type.txt")
@@ -335,3 +342,8 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server):
         assert 'application/json' in f.read()
 
     os.unlink("test-datastore/notification-url.txt")
+
+    client.get(
+        url_for("form_delete", uuid="all"),
+        follow_redirects=True
+    )

changedetectionio/tests/util.py

@@ -205,6 +205,9 @@ def live_server_setup(live_server):
         with open("test-datastore/notification-url.txt", "w") as f:
             f.write(request.url)
 
+        with open("test-datastore/notification-headers.txt", "w") as f:
+            f.write(str(request.headers))
+
         if request.content_type:
             with open("test-datastore/notification-content-type.txt", "w") as f:
                 f.write(request.content_type)