0.39.15

Watch overview list - adding spinner to make it easier to see whats currently being 'Checked'
Preview section - add helpful check
2025-12-01 22:02:37 +00:00 · 2022-06-12 14:05:34 +02:00 · 2022-06-12 12:52:17 +02:00 · 2022-06-12 11:10:06 +02:00 · 2022-06-12 10:47:00 +02:00 · 2022-06-11 23:07:09 +02:00
21 changed files with 406 additions and 31 deletions
--- a/.github/ISSUE_TEMPLATE/bug_report.md
+++ b/.github/ISSUE_TEMPLATE/bug_report.md
@@ -21,7 +21,7 @@ Steps to reproduce the behavior:
 3. Scroll down to '....'
 4. See error

-! ALWAYS INCLUDE AN EXAMPLE URL WHERE IT IS POSSIBLE TO RE-CREATE THE ISSUE !
+! ALWAYS INCLUDE AN EXAMPLE URL WHERE IT IS POSSIBLE TO RE-CREATE THE ISSUE - USE THE 'SHARE WATCH' FEATURE AND PASTE IN THE SHARE-LINK!

 **Expected behavior**
 A clear and concise description of what you expected to happen.
--- a/.gitignore
+++ b/.gitignore
@@ -8,5 +8,6 @@ __pycache__
 build
 dist
 venv
+test-datastore
 *.egg-info*
 .vscode/settings.json
--- a/changedetectionio/init.py
+++ b/changedetectionio/init.py
@@ -20,6 +20,7 @@ from copy import deepcopy
 from threading import Event

 import flask_login
+import logging
 import pytz
 import timeago
 from feedgen.feed import FeedGenerator
@@ -43,7 +44,7 @@ from flask_wtf import CSRFProtect
 from changedetectionio import html_tools
 from changedetectionio.api import api_v1

-__version__ = '0.39.14'
+__version__ = '0.39.15'

 datastore = None

@@ -107,7 +108,7 @@ def _jinja2_filter_datetime(watch_obj, format="%Y-%m-%d %H:%M:%S"):
    # Worker thread tells us which UUID it is currently processing.
    for t in running_update_threads:
        if t.current_uuid == watch_obj['uuid']:
-            return "Checking now.."
+            return '<span class="loader"></span><span> Checking now</span>'

    if watch_obj['last_checked'] == 0:
        return 'Not yet'
@@ -351,9 +352,8 @@ def changedetection_app(config=None, datastore_o=None):
                latest_fname = watch.history[dates[-1]]

                html_diff = diff.render_diff(prev_fname, latest_fname, include_equal=False, line_feed_sep="</br>")
-                fe.description(description="<![CDATA["
-                                           "<html><body><h4>{}</h4>{}</body></html>"
-                                           "]]>".format(watch_title, html_diff))
+                fe.content(content="<html><body><h4>{}</h4>{}</body></html>".format(watch_title, html_diff),
+                           type='CDATA')

                fe.guid(guid, permalink=False)
                dt = datetime.datetime.fromtimestamp(int(watch.newest_history_key))
@@ -458,6 +458,19 @@ def changedetection_app(config=None, datastore_o=None):

        return 'OK'

+
+    @app.route("/scrub/<string:uuid>", methods=['GET'])
+    @login_required
+    def scrub_watch(uuid):
+        try:
+            datastore.scrub_watch(uuid)
+        except KeyError:
+            flash('Watch not found', 'error')
+        else:
+            flash("Scrubbed watch {}".format(uuid))
+
+        return redirect(url_for('index'))
+
    @app.route("/scrub", methods=['GET', 'POST'])
    @login_required
    def scrub_page():
@@ -809,7 +822,13 @@ def changedetection_app(config=None, datastore_o=None):

        screenshot_url = datastore.get_screenshot(uuid)

-        output = render_template("diff.html", watch_a=watch,
+        system_uses_webdriver = datastore.data['settings']['application']['fetch_backend'] == 'html_webdriver'
+
+        is_html_webdriver = True if watch.get('fetch_backend') == 'html_webdriver' or (
+                    watch.get('fetch_backend', None) is None and system_uses_webdriver) else False
+
+        output = render_template("diff.html",
+                                 watch_a=watch,
                                 newest=newest_version_file_contents,
                                 previous=previous_version_file_contents,
                                 extra_stylesheets=extra_stylesheets,
@@ -820,7 +839,8 @@ def changedetection_app(config=None, datastore_o=None):
                                 current_diff_url=watch['url'],
                                 extra_title=" - Diff - {}".format(watch['title'] if watch['title'] else watch['url']),
                                 left_sticky=True,
-                                 screenshot=screenshot_url)
+                                 screenshot=screenshot_url,
+                                 is_html_webdriver=is_html_webdriver)

        return output

@@ -835,6 +855,12 @@ def changedetection_app(config=None, datastore_o=None):
        if uuid == 'first':
            uuid = list(datastore.data['watching'].keys()).pop()

+        # Normally you would never reach this, because the 'preview' button is not available when there's no history
+        # However they may try to scrub and reload the page
+        if datastore.data['watching'][uuid].history_n == 0:
+            flash("Preview unavailable - No fetch/check completed or triggers not reached", "error")
+            return redirect(url_for('index'))
+
        extra_stylesheets = [url_for('static_content', group='styles', filename='diff.css')]

        try:
@@ -881,6 +907,11 @@ def changedetection_app(config=None, datastore_o=None):
            content.append({'line': "No history found", 'classes': ''})

        screenshot_url = datastore.get_screenshot(uuid)
+        system_uses_webdriver = datastore.data['settings']['application']['fetch_backend'] == 'html_webdriver'
+
+        is_html_webdriver = True if watch.get('fetch_backend') == 'html_webdriver' or (
+                watch.get('fetch_backend', None) is None and system_uses_webdriver) else False
+
        output = render_template("preview.html",
                                 content=content,
                                 extra_stylesheets=extra_stylesheets,
@@ -889,8 +920,9 @@ def changedetection_app(config=None, datastore_o=None):
                                 current_diff_url=watch['url'],
                                 screenshot=screenshot_url,
                                 watch=watch,
-                                 uuid=uuid)
-        
+                                 uuid=uuid,
+                                 is_html_webdriver=is_html_webdriver)
+
        return output

    @app.route("/settings/notification-logs", methods=['GET'])
@@ -898,7 +930,7 @@ def changedetection_app(config=None, datastore_o=None):
    def notification_logs():
        global notification_debug_log
        output = render_template("notification-log.html",
-                                 logs=notification_debug_log if len(notification_debug_log) else ["No errors or warnings detected"])
+                                 logs=notification_debug_log if len(notification_debug_log) else ["Notification logs are empty - no notifications sent yet."])

        return output

@@ -1169,7 +1201,8 @@ def changedetection_app(config=None, datastore_o=None):


        except Exception as e:
-            flash("Could not share, something went wrong while communicating with the share server.", 'error')
+            logging.error("Error sharing -{}".format(str(e)))
+            flash("Could not share, something went wrong while communicating with the share server - {}".format(str(e)), 'error')

        # https://changedetection.io/share/VrMv05wpXyQa
        # in the browser - should give you a nice info page - wtf
@@ -1217,6 +1250,9 @@ def check_for_new_version():

 def notification_runner():
    global notification_debug_log
+    from datetime import datetime
+    import json
+
    while not app.config.exit.is_set():
        try:
            # At the moment only one thread runs (single runner)
@@ -1225,13 +1261,16 @@ def notification_runner():
            time.sleep(1)

        else:
-            # Process notifications
+
+            now = datetime.now()
+
            try:
                from changedetectionio import notification
+
                notification.process_notification(n_object, datastore)

            except Exception as e:
-                print("Watch URL: {}  Error {}".format(n_object['watch_url'], str(e)))
+                logging.error("Watch URL: {}  Error {}".format(n_object['watch_url'], str(e)))

                # UUID wont be present when we submit a 'test' from the global settings
                if 'uuid' in n_object:
@@ -1241,9 +1280,10 @@ def notification_runner():
                log_lines = str(e).splitlines()
                notification_debug_log += log_lines

-                # Trim the log length
-                notification_debug_log = notification_debug_log[-100:]
-
+            # Process notifications
+            notification_debug_log+= ["{} - SENDING {}".format(now.strftime("%Y/%m/%d %H:%M:%S,000"), json.dumps(n_object))]
+            # Trim the log length
+            notification_debug_log = notification_debug_log[-100:]

 # Thread runner to check every minute, look for new watches to feed into the Queue.
 def ticker_thread_check_time_launch_checks():
--- a/changedetectionio/fetch_site_status.py
+++ b/changedetectionio/fetch_site_status.py
@@ -204,6 +204,20 @@ class perform_site_check():
        else:
            stripped_text_from_html = stripped_text_from_html.encode('utf8')

+        # 615 Extract text by regex
+        extract_text = watch.get('extract_text', [])
+        if len(extract_text) > 0:
+            regex_matched_output = []
+            for s_re in extract_text:
+                result = re.findall(s_re.encode('utf8'), stripped_text_from_html,
+                                    flags=re.MULTILINE | re.DOTALL | re.LOCALE)
+                if result:
+                    regex_matched_output.append(result[0])
+
+            if regex_matched_output:
+                stripped_text_from_html = b'\n'.join(regex_matched_output)
+                text_content_before_ignored_filter = stripped_text_from_html
+
        # Re #133 - if we should strip whitespaces from triggering the change detected comparison
        if self.datastore.data['settings']['application'].get('ignore_whitespace', False):
            fetched_md5 = hashlib.md5(stripped_text_from_html.translate(None, b'\r\n\t ')).hexdigest()
@@ -221,6 +235,7 @@ class perform_site_check():
            # Yeah, lets block first until something matches
            blocked_by_not_found_trigger_text = True
            # Filter and trigger works the same, so reuse it
+            # It should return the line numbers that match
            result = html_tools.strip_ignore_text(content=str(stripped_text_from_html),
                                                  wordlist=watch['trigger_text'],
                                                  mode="line numbers")
--- a/changedetectionio/forms.py
+++ b/changedetectionio/forms.py
@@ -223,7 +223,7 @@ class validateURL(object):
        except validators.ValidationFailure:
            message = field.gettext('\'%s\' is not a valid URL.' % (field.data.strip()))
            raise ValidationError(message)
-        
+
 class ValidateListRegex(object):
    """
    Validates that anything that looks like a regex passes as a regex
@@ -330,6 +330,9 @@ class watchForm(commonSettingsForm):
    css_filter = StringField('CSS/JSON/XPATH Filter', [ValidateCSSJSONXPATHInput()], default='')

    subtractive_selectors = StringListField('Remove elements', [ValidateCSSJSONXPATHInput(allow_xpath=False, allow_json=False)])
+
+    extract_text = StringListField('Extract text', [ValidateListRegex()])
+
    title = StringField('Title', default='')

    ignore_text = StringListField('Ignore text', [ValidateListRegex()])
--- a/changedetectionio/model/Watch.py
+++ b/changedetectionio/model/Watch.py
@@ -35,7 +35,8 @@ class model(dict):
            'notification_title': default_notification_title,
            'notification_body': default_notification_body,
            'notification_format': default_notification_format,
-            'css_filter': "",
+            'css_filter': '',
+            'extract_text': [],  # Extract text by regex after filters
            'subtractive_selectors': [],
            'trigger_text': [],  # List of text or regex to wait for until a change is detected
            'fetch_backend': None,
--- a/changedetectionio/notification.py
+++ b/changedetectionio/notification.py
@@ -97,6 +97,8 @@ def process_notification(n_object, datastore):
                if log_value and 'WARNING' in log_value or 'ERROR' in log_value:
                    raise Exception(log_value)

+
+
 # Notification title + body content parameters get created here.
 def create_notification_parameters(n_object, datastore):
    from copy import deepcopy
--- a/changedetectionio/static/js/diff-overview.js
+++ b/changedetectionio/static/js/diff-overview.js
@@ -0,0 +1,17 @@
+$(document).ready(function () {
+    // Load it when the #screenshot tab is in use, so we dont give a slow experience when waiting for the text diff to load
+    window.addEventListener('hashchange', function (e) {
+        toggle(location.hash);
+    }, false);
+
+    toggle(location.hash);
+
+    function toggle(hash_name) {
+        if (hash_name === '#screenshot') {
+            $("img#screenshot-img").attr('src', screenshot_url);
+            $("#settings").hide();
+        } else {
+            $("#settings").show();
+        }
+    }
+});
--- a/changedetectionio/static/js/notifications.js
+++ b/changedetectionio/static/js/notifications.js
@@ -40,13 +40,19 @@ $(document).ready(function() {
    $.ajax({
      type: "POST",
      url: notification_base_url,
-      data : data
+      data : data,
+        statusCode: {
+        400: function() {
+            // More than likely the CSRF token was lost when the server restarted
+          alert("There was a problem processing the request, please reload the page.");
+        }
+      }
    }).done(function(data){
      console.log(data);
      alert('Sent');
    }).fail(function(data){
      console.log(data);
-      alert('Error: '+data.responseJSON.error);
+      alert('There was an error communicating with the server.');
    })
  });
 });
--- a/changedetectionio/static/js/visual-selector.js
+++ b/changedetectionio/static/js/visual-selector.js
@@ -49,6 +49,8 @@ $(document).ready(function() {
        }
        state_clicked=false;
        ctx.clearRect(0, 0, c.width, c.height);
+        xctx.clearRect(0, 0, c.width, c.height);
+        $("#css_filter").val('');
    });


--- a/changedetectionio/static/styles/styles.css
+++ b/changedetectionio/static/styles/styles.css
@@ -353,6 +353,8 @@ and also iPads specifically.
    /* Hide table headers (but not display: none;, for accessibility) */ }
    .watch-table thead, .watch-table tbody, .watch-table th, .watch-table td, .watch-table tr {
      display: block; }
+    .watch-table .last-checked > span {
+      vertical-align: middle; }
    .watch-table .last-checked::before {
      color: #555;
      content: "Last Checked "; }
@@ -370,7 +372,8 @@ and also iPads specifically.
    .watch-table td {
      /* Behave  like a "row" */
      border: none;
-      border-bottom: 1px solid #eee; }
+      border-bottom: 1px solid #eee;
+      vertical-align: middle; }
      .watch-table td:before {
        /* Top/left values mimic padding */
        top: 6px;
@@ -490,3 +493,42 @@ ul {

 #api-key-copy {
  color: #0078e7; }
+
+/* spinner */
+.loader,
+.loader:after {
+  border-radius: 50%;
+  width: 10px;
+  height: 10px; }
+
+.loader {
+  margin: 0px auto;
+  font-size: 3px;
+  vertical-align: middle;
+  display: inline-block;
+  text-indent: -9999em;
+  border-top: 1.1em solid rgba(38, 104, 237, 0.2);
+  border-right: 1.1em solid rgba(38, 104, 237, 0.2);
+  border-bottom: 1.1em solid rgba(38, 104, 237, 0.2);
+  border-left: 1.1em solid #2668ed;
+  -webkit-transform: translateZ(0);
+  -ms-transform: translateZ(0);
+  transform: translateZ(0);
+  -webkit-animation: load8 1.1s infinite linear;
+  animation: load8 1.1s infinite linear; }
+
+@-webkit-keyframes load8 {
+  0% {
+    -webkit-transform: rotate(0deg);
+    transform: rotate(0deg); }
+  100% {
+    -webkit-transform: rotate(360deg);
+    transform: rotate(360deg); } }
+
+@keyframes load8 {
+  0% {
+    -webkit-transform: rotate(0deg);
+    transform: rotate(0deg); }
+  100% {
+    -webkit-transform: rotate(360deg);
+    transform: rotate(360deg); } }
--- a/changedetectionio/static/styles/styles.scss
+++ b/changedetectionio/static/styles/styles.scss
@@ -487,6 +487,11 @@ and also iPads specifically.
      display: block;
    }

+    .last-checked {
+      > span {
+        vertical-align: middle;
+      }
+    }
    .last-checked::before {
      color: #555;
      content: "Last Checked ";
@@ -517,7 +522,7 @@ and also iPads specifically.
      /* Behave  like a "row" */
      border: none;
      border-bottom: 1px solid #eee;
-
+      vertical-align: middle;
      &:before {
        /* Top/left values mimic padding */
        top: 6px;
@@ -701,3 +706,48 @@ ul {
 #api-key-copy {
  color: #0078e7;
 }
+
+/* spinner */
+.loader,
+.loader:after {
+  border-radius: 50%;
+  width: 10px;
+  height: 10px;
+}
+.loader {
+  margin: 0px auto;
+  font-size: 3px;
+  vertical-align: middle;
+  display: inline-block;
+  text-indent: -9999em;
+  border-top: 1.1em solid rgba(38,104,237, 0.2);
+  border-right: 1.1em solid rgba(38,104,237, 0.2);
+  border-bottom: 1.1em solid rgba(38,104,237, 0.2);
+  border-left: 1.1em solid #2668ed;
+  -webkit-transform: translateZ(0);
+  -ms-transform: translateZ(0);
+  transform: translateZ(0);
+  -webkit-animation: load8 1.1s infinite linear;
+  animation: load8 1.1s infinite linear;
+}
+@-webkit-keyframes load8 {
+  0% {
+    -webkit-transform: rotate(0deg);
+    transform: rotate(0deg);
+  }
+  100% {
+    -webkit-transform: rotate(360deg);
+    transform: rotate(360deg);
+  }
+}
+@keyframes load8 {
+  0% {
+    -webkit-transform: rotate(0deg);
+    transform: rotate(0deg);
+  }
+  100% {
+    -webkit-transform: rotate(360deg);
+    transform: rotate(360deg);
+  }
+}
+
--- a/changedetectionio/store.py
+++ b/changedetectionio/store.py
@@ -254,12 +254,23 @@ class ChangeDetectionStore:
    def scrub_watch(self, uuid):
        import pathlib

-        self.__data['watching'][uuid].update({'history': {}, 'last_checked': 0, 'last_changed': 0, 'previous_md5': False})
-        self.needs_write_urgent = True
+        self.__data['watching'][uuid].update(
+            {'last_checked': 0,
+             'last_changed': 0,
+             'last_viewed': 0,
+             'previous_md5': False,
+             'last_notification_error': False,
+             'last_error': False})

-        for item in pathlib.Path(self.datastore_path).rglob(uuid+"/*.txt"):
+        # JSON Data, Screenshots, Textfiles (history index and snapshots), HTML in the future etc
+        for item in pathlib.Path(os.path.join(self.datastore_path, uuid)).rglob("*.*"):
            unlink(item)

+        # Force the attr to recalculate
+        bump = self.__data['watching'][uuid].history
+
+        self.needs_write_urgent = True
+
    def add_watch(self, url, tag="", extras=None, write_to_disk_now=True):

        if extras is None:
@@ -287,7 +298,7 @@ class ChangeDetectionStore:
                                   'body', 'method',
                                   'ignore_text', 'css_filter',
                                   'subtractive_selectors', 'trigger_text',
-                                   'extract_title_as_title']:
+                                   'extract_title_as_title', 'extract_text']:
                    if res.get(k):
                        apply_extras[k] = res[k]

--- a/changedetectionio/templates/_common_fields.jinja
+++ b/changedetectionio/templates/_common_fields.jinja
@@ -14,7 +14,7 @@
                                <li>Use <a target=_new href="https://github.com/caronc/apprise">AppRise URLs</a> for notification to just about any service! <i><a target=_new href="https://github.com/dgtlmoon/changedetection.io/wiki/Notification-configuration-notes">Please read the notification services wiki here for important configuration notes</a></i>.</li>
                                <li><code>discord://</code> only supports a maximum <strong>2,000 characters</strong> of notification text, including the title.</li>
                                <li><code>tgram://</code> bots cant send messages to other bots, so you should specify chat ID of non-bot user.</li>
-                                <li>Go here for <a href="{{url_for('notification_logs')}}">notification debug logs</a></li>
+
                              </ul>
                            </div>
                            <br/>
@@ -22,6 +22,7 @@
 {% if emailprefix %}
                            <a id="add-email-helper" class="pure-button button-secondary button-xsmall" style="font-size: 70%">Add email</a>
 {% endif %}
+                            <a href="{{url_for('notification_logs')}}" class="pure-button button-secondary button-xsmall" style="font-size: 70%">Notification debug logs</a>
                        </div>
                        <div id="notification-customisation" class="pure-control-group">
                            <div class="pure-control-group">
--- a/changedetectionio/templates/diff.html
+++ b/changedetectionio/templates/diff.html
@@ -1,6 +1,11 @@
 {% extends 'base.html' %}

 {% block content %}
+<script>
+    const screenshot_url="{{url_for('static_content', group='screenshot', filename=uuid)}}";
+</script>
+<script type="text/javascript" src="{{url_for('static_content', group='js', filename='diff-overview.js')}}" defer></script>
+
 <div id="settings">
    <h1>Differences</h1>
    <form class="pure-form " action="" method="GET">
@@ -39,6 +44,7 @@
 <div class="tabs">
    <ul>
        <li class="tab" id="default-tab"><a href="#text">Text</a></li>
+        <li class="tab" id="screenshot-tab"><a href="#screenshot">Screenshot</a></li>
    </ul>
 </div>

@@ -60,6 +66,21 @@
         </table>
         Diff algorithm from the amazing <a href="https://github.com/kpdecker/jsdiff">github.com/kpdecker/jsdiff</a>
     </div>
+     <div class="tab-pane-inner" id="screenshot">
+         <div class="tip">
+             For now, Differences are performed on text, not graphically, only the latest screenshot is available.
+         </div>
+         </br>
+         {% if is_html_webdriver %}
+           {% if screenshot %}
+             <img style="max-width: 80%" id="screenshot-img" alt="Current screenshot from most recent request"/>
+           {% else %}
+              No screenshot available just yet! Try rechecking the page.
+           {% endif %}
+         {% else %}
+           <strong>Screenshot requires Playwright/WebDriver enabled</strong>
+         {% endif %}
+     </div>
 </div>


--- a/changedetectionio/templates/edit.html
+++ b/changedetectionio/templates/edit.html
@@ -199,6 +199,17 @@ nav
                        </span>
                    </div>
                </fieldset>
+                <fieldset>
+                    <div class="pure-control-group">
+                        {{ render_field(form.extract_text, rows=5, placeholder="\d+ online") }}
+                        <span class="pure-form-message-inline">
+                    <ul>
+                        <li>Extracts text in the final output after other filters using regular expressions, for example <code>\d+ online</code></li>
+                        <li>One line per regular-expression.</li>
+                    </ul>
+                        </span>
+                    </div>
+                </fieldset>
            </div>

            <div class="tab-pane-inner visual-selector-ui" id="visualselector">
@@ -248,6 +259,8 @@ nav

                    <a href="{{url_for('form_delete', uuid=uuid)}}"
                       class="pure-button button-small button-error ">Delete</a>
+                    <a href="{{url_for('scrub_watch', uuid=uuid)}}"
+                       class="pure-button button-small button-error ">Scrub</a>
                    <a href="{{url_for('form_clone', uuid=uuid)}}"
                       class="pure-button button-small ">Create Copy</a>
                </div>
--- a/changedetectionio/templates/notification-log.html
+++ b/changedetectionio/templates/notification-log.html
@@ -4,7 +4,7 @@
 <div class="edit-form">
     <div class="inner">

-         <h4 style="margin-top: 0px;">The following issues were detected when sending notifications</h4>
+         <h4 style="margin-top: 0px;">Notification debug log</h4>
                <div id="notification-error-log">
                <ul style="font-size: 80%; margin:0px; padding: 0 0 0 7px">
                {% for log in logs|reverse %}
--- a/changedetectionio/templates/preview.html
+++ b/changedetectionio/templates/preview.html
@@ -1,6 +1,10 @@
 {% extends 'base.html' %}

 {% block content %}
+<script>
+    const screenshot_url="{{url_for('static_content', group='screenshot', filename=uuid)}}";
+</script>
+<script type="text/javascript" src="{{url_for('static_content', group='js', filename='diff-overview.js')}}" defer></script>

 <div id="settings">
    <h1>Current - {{watch.last_checked|format_timestamp_timeago}}</h1>
@@ -10,6 +14,7 @@
 <div class="tabs">
    <ul>
        <li class="tab" id="default-tab"><a href="#text">Text</a></li>
+        <li class="tab" id="screenshot-tab"><a href="#screenshot">Screenshot</a></li>
    </ul>
 </div>

@@ -28,5 +33,20 @@
            </tbody>
        </table>
    </div>
+     <div class="tab-pane-inner" id="screenshot">
+         <div class="tip">
+             For now, Differences are performed on text, not graphically, only the latest screenshot is available.
+         </div>
+         </br>
+         {% if is_html_webdriver %}
+           {% if screenshot %}
+             <img style="max-width: 80%" id="screenshot-img" alt="Current screenshot from most recent request"/>
+           {% else %}
+              No screenshot available just yet! Try rechecking the page.
+           {% endif %}
+         {% else %}
+           <strong>Screenshot requires Playwright/WebDriver enabled</strong>
+         {% endif %}
+     </div>
 </div>
 {% endblock %}
--- a/changedetectionio/templates/watch-overview.html
+++ b/changedetectionio/templates/watch-overview.html
@@ -67,7 +67,7 @@
                    <span class="watch-tag-list">{{ watch.tag}}</span>
                    {% endif %}
                </td>
-                <td class="last-checked">{{watch|format_last_checked_time}}</td>
+                <td class="last-checked">{{watch|format_last_checked_time|safe}}</td>
                <td class="last-changed">{% if watch.history_n >=2 and watch.last_changed %}
                    {{watch.last_changed|format_timestamp_timeago}}
                    {% else %}
--- a/changedetectionio/tests/test_extract_regex.py
+++ b/changedetectionio/tests/test_extract_regex.py
@@ -0,0 +1,127 @@
+#!/usr/bin/python3
+
+import time
+from flask import url_for
+from .util import live_server_setup
+
+from ..html_tools import *
+
+
+def set_original_response():
+    test_return_data = """<html>
+       <body>
+     Some initial text</br>
+     <p>Which is across multiple lines</p>
+     </br>
+     So let's see what happens.  </br>
+     <div id="sametext">Some text thats the same</div>
+     <div id="changetext">Some text that will change</div>
+     </body>
+     </html>
+    """
+
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write(test_return_data)
+    return None
+
+
+def set_modified_response():
+    test_return_data = """<html>
+       <body>
+     Some initial text</br>
+     <p>which has this one new line</p>
+     </br>
+     So let's see what happens.  </br>
+     <div id="sametext">Some text thats the same</div>
+     <div id="changetext">Some text that did change ( 1000 online <br/> 80 guests)</div>
+     </body>
+     </html>
+    """
+
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write(test_return_data)
+
+    return None
+
+
+def test_check_filter_and_regex_extract(client, live_server):
+    sleep_time_for_fetch_thread = 3
+
+    live_server_setup(live_server)
+    css_filter = "#changetext"
+
+    set_original_response()
+
+    # Give the endpoint time to spin up
+    time.sleep(1)
+
+    # Add our URL to the import page
+    test_url = url_for('test_endpoint', _external=True)
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+    # Trigger a check
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
+
+    # Goto the edit page, add our ignore text
+    # Add our URL to the import page
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={"css_filter": css_filter,
+              'extract_text': '\d+ online\n\d+ guests',
+              "url": test_url,
+              "tag": "",
+              "headers": "",
+              'fetch_backend': "html_requests"
+              },
+        follow_redirects=True
+    )
+
+    assert b"Updated watch." in res.data
+
+    # Check it saved
+    res = client.get(
+        url_for("edit_page", uuid="first"),
+    )
+    assert b'\d+ online' in res.data
+
+    # Trigger a check
+#    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
+
+    #  Make a change
+    set_modified_response()
+
+    # Trigger a check
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
+
+    # It should have 'unviewed' still
+    # Because it should be looking at only that 'sametext' id
+    res = client.get(url_for("index"))
+    assert b'unviewed' in res.data
+
+    # Check HTML conversion detected and workd
+    res = client.get(
+        url_for("preview_page", uuid="first"),
+        follow_redirects=True
+    )
+
+    # Class will be blank for now because the frontend didnt apply the diff
+    assert b'<div class="">1000 online' in res.data
+    
+    # Both regexs should be here
+    assert b'<div class="">80 guests' in res.data
+
+    # Should not be here
+    assert b'Some text that did change' not in res.data
--- a/changedetectionio/update_worker.py
+++ b/changedetectionio/update_worker.py
@@ -56,7 +56,10 @@ class update_worker(threading.Thread):
                    except content_fetcher.ReplyWithContentButNoText as e:
                        # Totally fine, it's by choice - just continue on, nothing more to care about
                        # Page had elements/content but no renderable text
-                        self.datastore.update_watch(uuid=uuid, update_obj={'last_error': "Got HTML content but no text found."})
+                        if self.datastore.data['watching'][uuid].get('css_filter'):
+                            self.datastore.update_watch(uuid=uuid, update_obj={'last_error': "Got HTML content but no text found (CSS / xPath Filter not found in page?)"})
+                        else:
+                            self.datastore.update_watch(uuid=uuid, update_obj={'last_error': "Got HTML content but no text found."})
                        pass
                    except content_fetcher.EmptyReply as e:
                        # Some kind of custom to-str handler in the exception handler that does this?
Author	SHA1	Message	Date
dgtlmoon	ba8cf2c8cf	0.39.15	2022-06-12 14:05:34 +02:00
dgtlmoon	3106b6688e	Watch overview list - adding spinner to make it easier to see whats currently being 'Checked'	2022-06-12 12:52:17 +02:00
dgtlmoon	2c83845dac	Preview section - add helpful check	2022-06-12 11:10:06 +02:00
dgtlmoon	111266d6fa	Send test notification - improved handling of errors	2022-06-12 10:47:00 +02:00
dgtlmoon	ead610151f	Notification log - also log normal requests and make the log easier to find	2022-06-11 23:07:09 +02:00
dgtlmoon	7e1e763989	Update bug_report.md	2022-06-11 00:43:28 +02:00
dgtlmoon	327cc4af34	Use correct RSS CDATA handling (#662 )	2022-06-08 18:40:01 +02:00
dgtlmoon	6008ff516e	Improve logging (#671 )	2022-06-08 18:32:41 +02:00
dgtlmoon	cdcf4b353f	New [scrub] button when editing a watch - scrub single watch history (#672 )	2022-06-08 18:32:25 +02:00
dgtlmoon	1ab70f8e86	Diff + Preview - Hide date selector widget when viewing screenshots as its not yet possible to compare screenshots (but will be soon!)	2022-06-07 19:53:13 +02:00
dgtlmoon	8227c012a7	Diff + Preview - Fixing screenshot behaviour after preference change	2022-06-07 19:51:17 +02:00
dgtlmoon	c113d5fb24	Screenshot handling on the diff/preview section refactor (#630 )	2022-06-07 19:22:42 +02:00
dgtlmoon	8c8d4066d7	Shared watches - include "Extract text" filter	2022-06-07 17:06:05 +02:00
dgtlmoon	277dc9e1c1	Improve error message when filter not found in page result (#666 )	2022-06-07 16:43:57 +02:00
dgtlmoon	fc0fd1ce9d	"Extract text" filter - improve placeholder example	2022-06-06 18:26:47 +02:00
dgtlmoon	bd6127728a	Visual selector - 'clear selection' button should clear the filter also	2022-06-06 17:07:29 +02:00
dgtlmoon	4101ae00c6	New feature - "Extract text" filter ability (#624 )	2022-06-06 16:57:50 +02:00
dgtlmoon	62f14df3cb	Fixing RSS feed HTML content formatting (#662 )	2022-06-06 10:24:39 +02:00