Merge branch 'master' into jinja2-notifications

Using Jinja2 templating actually simplifies slightly the code, but, most importantly, enables a much greater flexibility
to customize the notifications' content.

This change is breaking: the tokens like `{base_url}` should instead be written s `{{ base_url }}`. It would be very easy
to make it non-breaking, but I think it would be preferable to stick to the default Jinja2 syntax. To make it non-breaking,
the Jinja2 environment should be configured this way:
```
jinja2_env = Environment(
    loader=BaseLoader,
    variable_start_string='{',
    variable_end_string='}',
)
```

NB: This change could enable a few more follow-up enhancements:
* It could make sense as a later evolution to implement the same for the URL list. This could enable features like:
  - Customizing the sender based on the watch parameters through global settings
  - Sending the notification to a different email/channel/group based on a watch attribute, like its tag
* Jinja2 plays fine with complex variables. It would be relatively easy to expose many more variables that could be used
  in the notification.

Changes
=======
* Use Jinja2 template rendering to parse tokens in the notification body and title.
* Replace the settings validator to check that the body and title are both valid Jinja2 templates
  and that they do not contain any invalid token/variable.
* Update the corresponding documentation, setting pages and tests.

changedetectionio/__init__.py

@@ -95,12 +95,6 @@ def init_app_secret(datastore_path):
 
     return secret
 
-
-@app.template_global()
-def get_darkmode_state():
-    css_dark_mode = request.cookies.get('css_dark_mode', 'false')
-    return 'true' if css_dark_mode and strtobool(css_dark_mode) else 'false'
-
 # We use the whole watch object from the store/JSON so we can see if there's some related status in terms of a thread
 # running or something similar.
 @app.template_filter('format_last_checked_time')
@@ -208,6 +202,10 @@ def changedetection_app(config=None, datastore_o=None):
     watch_api.add_resource(api_v1.SystemInfo, '/api/v1/systeminfo',
                            resource_class_kwargs={'datastore': datastore, 'update_q': update_q})
 
+    def getDarkModeSetting():
+      css_dark_mode = request.cookies.get('css_dark_mode')
+      return True if (css_dark_mode == 'true' or css_dark_mode == True) else False
+
     # Setup cors headers to allow all domains
     # https://flask-cors.readthedocs.io/en/latest/
     #    CORS(app)
@@ -407,6 +405,7 @@ def changedetection_app(config=None, datastore_o=None):
 
         form = forms.quickWatchForm(request.form)
         output = render_template("watch-overview.html",
+                                 dark_mode=getDarkModeSetting(),
                                  form=form,
                                  watches=sorted_watches,
                                  tags=existing_tags,
@@ -665,6 +664,7 @@ def changedetection_app(config=None, datastore_o=None):
                                      browser_steps_config=browser_step_ui_config,
                                      current_base_url=datastore.data['settings']['application']['base_url'],
                                      emailprefix=os.getenv('NOTIFICATION_MAIL_BUTTON_PREFIX', False),
+                                     dark_mode=getDarkModeSetting(),
                                      form=form,
                                      has_default_notification_urls=True if len(datastore.data['settings']['application']['notification_urls']) else False,
                                      has_empty_checktime=using_default_check_time,
@@ -752,6 +752,7 @@ def changedetection_app(config=None, datastore_o=None):
 
         output = render_template("settings.html",
                                  form=form,
+                                 dark_mode=getDarkModeSetting(),
                                  current_base_url = datastore.data['settings']['application']['base_url'],
                                  hide_remove_pass=os.getenv("SALTED_PASS", False),
                                  api_key=datastore.data['settings']['application'].get('api_access_token'),
@@ -792,6 +793,7 @@ def changedetection_app(config=None, datastore_o=None):
 
         # Could be some remaining, or we could be on GET
         output = render_template("import.html",
+                                 dark_mode=getDarkModeSetting(),
                                  import_url_list_remaining="\n".join(remaining_urls),
                                  original_distill_json=''
                                  )
@@ -891,6 +893,7 @@ def changedetection_app(config=None, datastore_o=None):
         output = render_template("diff.html",
                                  current_diff_url=watch['url'],
                                  current_previous_version=str(previous_version),
+                                 dark_mode=getDarkModeSetting(),
                                  extra_stylesheets=extra_stylesheets,
                                  extra_title=" - Diff - {}".format(watch['title'] if watch['title'] else watch['url']),
                                  extract_form=extract_form,
@@ -941,6 +944,7 @@ def changedetection_app(config=None, datastore_o=None):
                                      content=content,
                                      history_n=watch.history_n,
                                      extra_stylesheets=extra_stylesheets,
+                                     dark_mode=getDarkModeSetting(),
 #                                     current_diff_url=watch['url'],
                                      watch=watch,
                                      uuid=uuid,
@@ -987,6 +991,7 @@ def changedetection_app(config=None, datastore_o=None):
                                  content=content,
                                  history_n=watch.history_n,
                                  extra_stylesheets=extra_stylesheets,
+                                 dark_mode=getDarkModeSetting(),
                                  ignored_line_numbers=ignored_line_numbers,
                                  triggered_line_numbers=trigger_line_numbers,
                                  current_diff_url=watch['url'],
@@ -1005,10 +1010,15 @@ def changedetection_app(config=None, datastore_o=None):
     def notification_logs():
         global notification_debug_log
         output = render_template("notification-log.html",
+                                 dark_mode=getDarkModeSetting(),
                                  logs=notification_debug_log if len(notification_debug_log) else ["Notification logs are empty - no notifications sent yet."])
 
         return output
 
+    @app.route("/favicon.ico", methods=['GET'])
+    def favicon():
+        return send_from_directory("static/images", path="favicon.ico")
+
     # We're good but backups are even better!
     @app.route("/backup", methods=['GET'])
     @login_required
@@ -1343,10 +1353,6 @@ def changedetection_app(config=None, datastore_o=None):
     import changedetectionio.blueprint.browser_steps as browser_steps
     app.register_blueprint(browser_steps.construct_blueprint(datastore), url_prefix='/browser-steps')
 
-    import changedetectionio.blueprint.price_data_follower as price_data_follower
-    app.register_blueprint(price_data_follower.construct_blueprint(datastore), url_prefix='/price_data_follower')
-
-
     # @todo handle ctrl break
     ticker_thread = threading.Thread(target=ticker_thread_check_time_launch_checks).start()
     threading.Thread(target=notification_runner).start()

changedetectionio/blueprint/price_data_follower/__init__.py

@@ -1,27 +0,0 @@
-
-from distutils.util import strtobool
-from flask import Blueprint, flash, redirect, url_for
-from flask_login import login_required
-from changedetectionio.store import ChangeDetectionStore
-
-def construct_blueprint(datastore: ChangeDetectionStore):
-
-    price_data_follower_blueprint = Blueprint('price_data_follower', __name__)
-
-    @login_required
-    @price_data_follower_blueprint.route("/<string:uuid>/accept", methods=['GET'])
-    def accept(uuid):
-        datastore.data['watching'][uuid]['track_ldjson_price_data'] = 'accepted'
-        return redirect(url_for("form_watch_checknow", uuid=uuid))
-
-
-    @login_required
-    @price_data_follower_blueprint.route("/<string:uuid>/reject", methods=['GET'])
-    def reject(uuid):
-        datastore.data['watching'][uuid]['track_ldjson_price_data'] = 'rejected'
-        return redirect(url_for("index"))
-
-
-    return price_data_follower_blueprint
-
-

changedetectionio/fetch_site_status.py

@@ -2,6 +2,7 @@ import hashlib
 import logging
 import os
 import re
+import time
 import urllib3
 
 from changedetectionio import content_fetcher, html_tools
@@ -139,7 +140,7 @@ class perform_site_check():
             is_html = False
             is_json = False
 
-        include_filters_rule = deepcopy(watch.get('include_filters', []))
+        include_filters_rule = watch.get('include_filters', [])
         # include_filters_rule = watch['include_filters']
         subtractive_selectors = watch.get(
             "subtractive_selectors", []
@@ -147,10 +148,6 @@ class perform_site_check():
             "global_subtractive_selectors", []
         )
 
-        # Inject a virtual LD+JSON price tracker rule
-        if watch.get('track_ldjson_price_data'):
-            include_filters_rule.append('json:$..price')
-
         has_filter_rule = include_filters_rule and len("".join(include_filters_rule).strip())
         has_subtractive_selectors = subtractive_selectors and len(subtractive_selectors[0].strip())
 
@@ -176,13 +173,9 @@ class perform_site_check():
                 # Don't run get_text or xpath/css filters on plaintext
                 stripped_text_from_html = html_content
             else:
-                # Does it have some ld+json price data? used for easier monitoring
-                update_obj['has_ldjson_price_data'] = html_tools.has_ldjson_product_info(fetcher.content)
-
                 # Then we assume HTML
                 if has_filter_rule:
                     html_content = ""
-
                     for filter_rule in include_filters_rule:
                         # For HTML/XML we offer xpath as an option, just start a regular xPath "/.."
                         if filter_rule[0] == '/' or filter_rule.startswith('xpath:'):

changedetectionio/html_tools.py

@@ -127,10 +127,8 @@ def _get_stripped_text_from_json_match(match):
 
     return stripped_text_from_html
 
-# content - json
-# json_filter - ie json:$..price
-# ensure_is_ldjson_info_type - str "product", optional, "@type == product" (I dont know how to do that as a json selector)
-def extract_json_as_string(content, json_filter, ensure_is_ldjson_info_type=None):
+def extract_json_as_string(content, json_filter):
+
     stripped_text_from_html = False
 
     # Try to parse/filter out the JSON, if we get some parser error, then maybe it's embedded <script type=ldjson>
@@ -141,12 +139,7 @@ def extract_json_as_string(content, json_filter, ensure_is_ldjson_info_type=None
         # Foreach <script json></script> blob.. just return the first that matches json_filter
         s = []
         soup = BeautifulSoup(content, 'html.parser')
-
-        if ensure_is_ldjson_info_type:
-            bs_result = soup.findAll('script', {"type": "application/ld+json"})
-        else:
-            bs_result = soup.findAll('script')
-
+        bs_result = soup.findAll('script')
 
         if not bs_result:
             raise JSONNotFound("No parsable JSON found in this document")
@@ -163,12 +156,7 @@ def extract_json_as_string(content, json_filter, ensure_is_ldjson_info_type=None
                 continue
             else:
                 stripped_text_from_html = _parse_json(json_data, json_filter)
-                if ensure_is_ldjson_info_type:
-                    # Could sometimes be list, string or something else random
-                    if isinstance(json_data, dict):
-                        if json_data.get('@type', False) and json_data.get('@type','').lower() == ensure_is_ldjson_info_type.lower():
-                            break
-                elif stripped_text_from_html:
+                if stripped_text_from_html:
                     break
 
     if not stripped_text_from_html:
@@ -255,18 +243,6 @@ def html_to_text(html_content: str, render_anchor_tag_content=False) -> str:
 
     return text_content
 
-
-# Does LD+JSON exist with a @type=='product' and a .price set anywhere?
-def has_ldjson_product_info(content):
-    try:
-        pricing_data = extract_json_as_string(content=content, json_filter='json:$..price', ensure_is_ldjson_info_type="product")
-    except JSONNotFound as e:
-        # Totally fine
-        return False
-    x=bool(pricing_data)
-    return x
-
-
 def workarounds_for_obfuscations(content):
     """
     Some sites are using sneaky tactics to make prices and other information un-renderable by Inscriptis

changedetectionio/model/Watch.py

@@ -26,8 +26,6 @@ class model(dict):
             'extract_title_as_title': False,
             'fetch_backend': None,
             'filter_failure_notification_send': strtobool(os.getenv('FILTER_FAILURE_NOTIFICATION_SEND_DEFAULT', 'True')),
-            'has_ldjson_price_data': None,
-            'track_ldjson_price_data': None,
             'headers': {},  # Extra headers to send
             'ignore_text': [],  # List of text to ignore when calculating the comparison checksum
             'include_filters': [],

changedetectionio/static/styles/diff.css

@@ -121,19 +121,17 @@ html[data-darkmode="true"] {
   --color-icon-github-hover: var(--color-grey-700);
   --color-watch-table-error: var(--color-light-red);
   --color-watch-table-row-text: var(--color-grey-800); }
+  html[data-darkmode="true"] .watch-controls img {
+    opacity: 0.4; }
+  html[data-darkmode="true"] .watch-table .unviewed {
+    color: #fff; }
+    html[data-darkmode="true"] .watch-table .unviewed.error {
+      color: var(--color-watch-table-error); }
   html[data-darkmode="true"] .icon-spread {
     filter: hue-rotate(-10deg) brightness(1.5); }
   html[data-darkmode="true"] .watch-table .title-col a[target="_blank"]::after,
   html[data-darkmode="true"] .watch-table .current-diff-url::after {
     filter: invert(0.5) hue-rotate(10deg) brightness(2); }
-  html[data-darkmode="true"] .watch-table .watch-controls .state-off img {
-    opacity: 0.3; }
-  html[data-darkmode="true"] .watch-table .watch-controls .state-on img {
-    opacity: 1.0; }
-  html[data-darkmode="true"] .watch-table .unviewed {
-    color: #fff; }
-    html[data-darkmode="true"] .watch-table .unviewed.error {
-      color: var(--color-watch-table-error); }
 
 #diff-ui {
   background: var(--color-background);

changedetectionio/static/styles/scss/parts/_variables.scss

@@ -140,6 +140,18 @@ html[data-darkmode="true"] {
   --color-watch-table-error: var(--color-light-red);
   --color-watch-table-row-text: var(--color-grey-800);
 
+  // Anything that can't be manipulated through variables follows.
+  .watch-controls {
+    img {
+      opacity: 0.4;
+    }
+  }
+  .watch-table .unviewed {
+    color: #fff;
+    &.error {
+      color: var(--color-watch-table-error);
+    }
+  }
 
   .icon-spread {
     filter: hue-rotate(-10deg) brightness(1.5);
@@ -151,25 +163,5 @@ html[data-darkmode="true"] {
     .current-diff-url::after {
       filter: invert(.5) hue-rotate(10deg) brightness(2);
     }
-
-    .watch-controls {
-      .state-off {
-        img {
-          opacity: 0.3;
-        }
-      }
-      .state-on {
-        img {
-          opacity: 1.0;
-        }
-      }
-    }
-
-    .unviewed {
-      color: #fff;
-      &.error {
-        color: var(--color-watch-table-error);
-      }
-    }
   }
 }

changedetectionio/static/styles/scss/styles.scss

@@ -1009,20 +1009,3 @@ ul {
   border-radius: 5px;
   color: var(--color-warning);
 }
-
-/* automatic price following helpers */
-.tracking-ldjson-price-data {
-  background-color: var(--color-background-button-green);
-  color: #000;
-  padding: 3px;
-  border-radius: 3px;
-  white-space: nowrap;
-}
-
-.ldjson-price-track-offer {
-  a.pure-button {
-    border-radius: 3px;
-    padding: 3px;
-    background-color: var(--color-background-button-green);
-  }
-}

changedetectionio/static/styles/styles.css

@@ -124,19 +124,17 @@ html[data-darkmode="true"] {
   --color-icon-github-hover: var(--color-grey-700);
   --color-watch-table-error: var(--color-light-red);
   --color-watch-table-row-text: var(--color-grey-800); }
+  html[data-darkmode="true"] .watch-controls img {
+    opacity: 0.4; }
+  html[data-darkmode="true"] .watch-table .unviewed {
+    color: #fff; }
+    html[data-darkmode="true"] .watch-table .unviewed.error {
+      color: var(--color-watch-table-error); }
   html[data-darkmode="true"] .icon-spread {
     filter: hue-rotate(-10deg) brightness(1.5); }
   html[data-darkmode="true"] .watch-table .title-col a[target="_blank"]::after,
   html[data-darkmode="true"] .watch-table .current-diff-url::after {
     filter: invert(0.5) hue-rotate(10deg) brightness(2); }
-  html[data-darkmode="true"] .watch-table .watch-controls .state-off img {
-    opacity: 0.3; }
-  html[data-darkmode="true"] .watch-table .watch-controls .state-on img {
-    opacity: 1.0; }
-  html[data-darkmode="true"] .watch-table .unviewed {
-    color: #fff; }
-    html[data-darkmode="true"] .watch-table .unviewed.error {
-      color: var(--color-watch-table-error); }
 
 /* spinner */
 .spinner,
@@ -945,16 +943,3 @@ ul {
     display: inline;
     height: 26px;
     vertical-align: middle; }
-
-/* automatic price following helpers */
-.tracking-ldjson-price-data {
-  background-color: var(--color-background-button-green);
-  color: #000;
-  padding: 3px;
-  border-radius: 3px;
-  white-space: nowrap; }
-
-.ldjson-price-track-offer a.pure-button {
-  border-radius: 3px;
-  padding: 3px;
-  background-color: var(--color-background-button-green); }

changedetectionio/store.py

@@ -250,15 +250,12 @@ class ChangeDetectionStore:
     def clear_watch_history(self, uuid):
         import pathlib
 
-        self.__data['watching'][uuid].update({
-                'last_checked': 0,
-                'has_ldjson_price_data': None,
-                'last_error': False,
-                'last_notification_error': False,
-                'last_viewed': 0,
-                'previous_md5': False,
-                'track_ldjson_price_data': None,
-            })
+        self.__data['watching'][uuid].update(
+            {'last_checked': 0,
+             'last_viewed': 0,
+             'previous_md5': False,
+             'last_notification_error': False,
+             'last_error': False})
 
         # JSON Data, Screenshots, Textfiles (history index and snapshots), HTML in the future etc
         for item in pathlib.Path(os.path.join(self.datastore_path, uuid)).rglob("*.*"):
@@ -640,7 +637,7 @@ class ChangeDetectionStore:
 
                 n_title = watch.get('notification_title')
                 if n_title:
-                    watch['notification_title'] = re.sub(r, r'{{\1}}', n_title)
+                    self.data['settings']['application']['notification_title'] = re.sub(r, r'{{\1}}', n_title)
 
                 n_urls = watch.get('notification_urls')
                 if n_urls:

changedetectionio/templates/_common_fields.jinja

@@ -72,7 +72,7 @@
                                     </tr>
                                     <tr>
                                         <td><code>{{ '{{ watch_tag }}' }}</code></td>
-                                        <td>The watch label / tag</td>
+                                        <td>The tag of the watch.</td>
                                     </tr>
                                     <tr>
                                         <td><code>{{ '{{ preview_url }}' }}</code></td>

changedetectionio/templates/base.html

@@ -1,5 +1,5 @@
 <!DOCTYPE html>
-<html lang="en" data-darkmode="{{ get_darkmode_state() }}">
+<html lang="en" data-darkmode="{{ dark_mode|lower }}">
 
   <head>
     <meta charset="utf-8"/>

changedetectionio/templates/diff.html

@@ -125,7 +125,7 @@
                     <p>
                         For example, to extract only the numbers from text &dash;</br>
                         <strong>Raw text</strong>: <code>Temperature <span style="color: red">5.5</span>°C in Sydney</code></br>
-                        <strong>RegEx to extract:</strong> <code>Temperature <span style="color: red">([0-9\.]+)</span></code><br/>
+                        <strong>RegEx to extract:</strong> <code>Temperature ([0-9\.]+)</code><br/>
                     </p>
                     <p>
                         <a href="https://RegExr.com/">Be sure to test your RegEx here.</a>

changedetectionio/templates/watch-overview.html

@@ -14,7 +14,7 @@
             <div id="watch-add-wrapper-zone">
                 <div>
                     {{ render_simple_field(form.url, placeholder="https://...", required=true) }}
-                    {{ render_simple_field(form.tag, value=active_tag if active_tag else '', placeholder="watch label / tag") }}
+                    {{ render_simple_field(form.tag, value=active_tag if active_tag else '', placeholder="watch group") }}
                 </div>
                 <div>
                     {{ render_simple_field(form.watch_submit_button, title="Watch this URL!" ) }}
@@ -98,12 +98,6 @@
                     {% if watch.last_notification_error is defined and watch.last_notification_error != False %}
                     <div class="fetch-error notification-error"><a href="{{url_for('notification_logs')}}">{{ watch.last_notification_error }}</a></div>
                     {% endif %}
-                    {% if watch['has_ldjson_price_data'] and not watch['track_ldjson_price_data']  %}
-                    <div class="ldjson-price-track-offer">Embedded price data detected, follow only price data? <a href="{{url_for('price_data_follower.accept', uuid=watch.uuid)}}" class="pure-button button-xsmall">Yes</a> <a href="{{url_for('price_data_follower.reject', uuid=watch.uuid)}}" class="">No</a></div>
-                    {% endif %}
-                    {% if watch['track_ldjson_price_data'] %}
-                    <span class="tracking-ldjson-price-data">Price</span>
-                    {% endif %}
                     {% if not active_tag %}
                     <span class="watch-tag-list">{{ watch.tag}}</span>
                     {% endif %}

changedetectionio/tests/test_automatic_follow_ldjson_price.py

@@ -1,112 +0,0 @@
-#!/usr/bin/python3
-
-import time
-from flask import url_for
-from .util import live_server_setup, extract_UUID_from_client, extract_api_key_from_UI
-
-from ..html_tools import *
-
-
-def set_response_with_ldjson():
-    test_return_data = """<html>
-       <body>
-     Some initial text</br>
-     <p>Which is across multiple lines</p>
-     </br>
-     So let's see what happens.  </br>
-     <div class="sametext">Some text thats the same</div>
-     <div class="changetext">Some text that will change</div>
-     <script type="application/ld+json">
-        {
-           "@context":"https://schema.org/",
-           "@type":"Product",
-           "@id":"https://www.some-virtual-phone-shop.com/celular-iphone-14/p",
-           "name":"Celular Iphone 14 Pro Max 256Gb E Sim A16 Bionic",
-           "brand":{
-              "@type":"Brand",
-              "name":"APPLE"
-           },
-           "image":"https://www.some-virtual-phone-shop.com/15509426/image.jpg",
-           "description":"You dont need it",
-           "mpn":"111111",
-           "sku":"22222",
-           "offers":{
-              "@type":"AggregateOffer",
-              "lowPrice":8097000,
-              "highPrice":8099900,
-              "priceCurrency":"COP",
-              "offers":[
-                 {
-                    "@type":"Offer",
-                    "price":8097000,
-                    "priceCurrency":"COP",
-                    "availability":"http://schema.org/InStock",
-                    "sku":"102375961",
-                    "itemCondition":"http://schema.org/NewCondition",
-                    "seller":{
-                       "@type":"Organization",
-                       "name":"ajax"
-                    }
-                 }
-              ],
-              "offerCount":1
-           }
-        }
-       </script>
-     </body>
-     </html>
-"""
-
-    with open("test-datastore/endpoint-content.txt", "w") as f:
-        f.write(test_return_data)
-    return None
-
-
-# actually only really used by the distll.io importer, but could be handy too
-def test_check_ldjson_price_autodetect(client, live_server):
-    live_server_setup(live_server)
-
-    # Give the endpoint time to spin up
-    time.sleep(1)
-
-    set_response_with_ldjson()
-
-    # Add our URL to the import page
-    test_url = url_for('test_endpoint', _external=True)
-    res = client.post(
-        url_for("import_page"),
-        data={"urls": test_url},
-        follow_redirects=True
-    )
-    assert b"1 Imported" in res.data
-    time.sleep(3)
-
-    # Should get a notice that it's available
-    res = client.get(url_for("index"))
-    assert b'ldjson-price-track-offer' in res.data
-
-    # Accept it
-    uuid = extract_UUID_from_client(client)
-
-    client.get(url_for('price_data_follower.accept', uuid=uuid, follow_redirects=True))
-    time.sleep(2)
-
-    # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    time.sleep(2)
-    # Offer should be gone
-    res = client.get(url_for("index"))
-    assert b'Embedded price data' not in res.data
-    assert b'tracking-ldjson-price-data' in res.data
-
-    # and last snapshop (via API) should be just the price
-    api_key = extract_api_key_from_UI(client)
-    res = client.get(
-        url_for("watchsinglehistory", uuid=uuid, timestamp='latest'),
-        headers={'x-api-key': api_key},
-    )
-
-    # Should just see the price in the API reply
-    assert res.data == b'8097000'
-
-    client.get(url_for("form_delete", uuid="all"), follow_redirects=True)

changedetectionio/tests/test_element_removal.py

@@ -121,7 +121,7 @@ def test_element_removal_full(client, live_server):
         url_for("import_page"), data={"urls": test_url}, follow_redirects=True
     )
     assert b"1 Imported" in res.data
-    time.sleep(1)
+
     # Goto the edit page, add the filter data
     # Not sure why \r needs to be added - absent of the #changetext this is not necessary
     subtractive_selectors_data = "header\r\nfooter\r\nnav\r\n#changetext"

changedetectionio/tests/test_encoding.py

@@ -38,6 +38,9 @@ def test_check_encoding_detection(client, live_server):
         follow_redirects=True
     )
 
+    # Trigger a check
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+
     # Give the thread time to pick it up
     time.sleep(2)
 

changedetectionio/tests/test_errorhandling.py

@@ -77,8 +77,7 @@ def test_DNS_errors(client, live_server):
     time.sleep(3)
 
     res = client.get(url_for("index"))
-    found_name_resolution_error = b"Temporary failure in name resolution" in res.data or b"Name or service not known" in res.data
-    assert found_name_resolution_error
+    assert b'Name or service not known' in res.data
     # Should always record that we tried
     assert bytes("just now".encode('utf-8')) in res.data
 

changedetectionio/tests/test_ignore_text.py

@@ -101,6 +101,9 @@ def test_check_ignore_text_functionality(client, live_server):
     )
     assert b"1 Imported" in res.data
 
+    # Trigger a check
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
 
@@ -196,6 +199,9 @@ def test_check_global_ignore_text_functionality(client, live_server):
     )
     assert b"1 Imported" in res.data
 
+    # Trigger a check
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
 

changedetectionio/tests/test_ignorestatuscode.py

@@ -69,6 +69,8 @@ def test_normal_page_check_works_with_ignore_status_code(client, live_server):
     assert b"1 Imported" in res.data
 
     time.sleep(sleep_time_for_fetch_thread)
+    # Trigger a check
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
 
     set_some_changed_response()
     time.sleep(sleep_time_for_fetch_thread)
@@ -102,6 +104,9 @@ def test_403_page_check_works_with_ignore_status_code(client, live_server):
     )
     assert b"1 Imported" in res.data
 
+    # Trigger a check
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
 
@@ -114,9 +119,11 @@ def test_403_page_check_works_with_ignore_status_code(client, live_server):
     )
     assert b"Updated watch." in res.data
 
+    # Trigger a check
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
-
     #  Make a change
     set_some_changed_response()
 

changedetectionio/tests/test_notification.py

@@ -237,7 +237,6 @@ def test_check_notification(client, live_server):
     )
 
 def test_notification_validation(client, live_server):
-
     time.sleep(1)
 
     # re #242 - when you edited an existing new entry, it would not correctly show the notification settings
@@ -267,15 +266,43 @@ def test_notification_validation(client, live_server):
 #    )
 #    assert b"Notification Body and Title is required when a Notification URL is used" in res.data
 
+    # Now adding a wrong token should give us an error
+# Disabled for now
+#    res = client.post(
+#        url_for("settings_page"),
+#        data={"application-notification_title": "New ChangeDetection.io Notification - {{watch_url}}",
+#              "application-notification_body": "Rubbish: {{rubbish}}\n",
+#              "application-notification_format": "Text",
+#              "application-notification_urls": "json://localhost/foobar",
+#              "requests-time_between_check-minutes": 180,
+#              "fetch_backend": "html_requests"
+#              },
+#        follow_redirects=True
+#    )
+#    assert bytes("Token 'rubbish' is not a valid token or is unknown".encode('utf-8')) in res.data
+
+    # And trying to define an invalid Jinja2 template should also throw an error
+    res = client.post(
+        url_for("settings_page"),
+        data={"application-notification_title": "New ChangeDetection.io Notification - {{ watch_url }}",
+              "application-notification_body": "Rubbish: {{ rubbish }\n",
+              "application-notification_urls": "json://foobar.com",
+              "application-minutes_between_check": 180,
+              "application-fetch_backend": "html_requests"
+              },
+        follow_redirects=True
+    )
+    assert bytes("This is not a valid Jinja2 template".encode('utf-8')) in res.data
+
+
     # cleanup for the next
     client.get(
         url_for("form_delete", uuid="all"),
         follow_redirects=True
     )
 
-
-
-def test_notification_custom_endpoint_and_jinja2(client, live_server):
+def test_notification_jinja2(client, live_server):
+    #live_server_setup(live_server)
     time.sleep(1)
 
     # test_endpoint - that sends the contents of a file
@@ -314,7 +341,6 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server):
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
     time.sleep(2)
 
-
     with open("test-datastore/notification.txt", 'r') as f:
         x=f.read()
         j = json.loads(x)
@@ -326,6 +352,4 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server):
     with open("test-datastore/notification-url.txt", 'r') as f:
         notification_url = f.read()
         assert 'xxx=http' in notification_url
-
-    os.unlink("test-datastore/notification-url.txt")
-
+    os.unlink("test-datastore/notification-url.txt")

changedetectionio/tests/test_request.py

@@ -20,8 +20,6 @@ def test_headers_in_request(client, live_server):
     )
     assert b"1 Imported" in res.data
 
-    time.sleep(1)
-
     res = client.post(
         url_for("import_page"),
         data={"urls": test_url},
@@ -176,7 +174,6 @@ def test_method_in_request(client, live_server):
     )
     assert b"1 Imported" in res.data
 
-    time.sleep(2)
     res = client.post(
         url_for("import_page"),
         data={"urls": test_url},
@@ -184,8 +181,6 @@ def test_method_in_request(client, live_server):
     )
     assert b"1 Imported" in res.data
 
-    time.sleep(2)
-
     # Attempt to add a method which is not valid
     res = client.post(
         url_for("edit_page", uuid="first"),
@@ -211,7 +206,7 @@ def test_method_in_request(client, live_server):
     assert b"Updated watch." in res.data
 
     # Give the thread time to pick up the first version
-    time.sleep(2)
+    time.sleep(5)
 
     # The service should echo back the request verb
     res = client.get(
@@ -222,7 +217,7 @@ def test_method_in_request(client, live_server):
     # The test call service will return the verb as the body
     assert b"PATCH" in res.data
 
-    time.sleep(2)
+    time.sleep(5)
 
     watches_with_method = 0
     with open('test-datastore/url-watches.json') as f:

runtime.txt

@@ -1 +1 @@
-python-3.9.15
+python-3.8.12