Adding handy "limit to watches with errors" button

changedetectionio/__init__.py

@@ -416,8 +416,14 @@ def changedetection_app(config=None, datastore_o=None):
 
         # Sort by last_changed and add the uuid which is usually the key..
         sorted_watches = []
+        with_errors = request.args.get('with_errors') == "1"
+        errored_count = 0
         search_q = request.args.get('q').strip().lower() if request.args.get('q') else False
         for uuid, watch in datastore.data['watching'].items():
+            if with_errors and not watch.get('last_error'):
+                continue
+            if watch.get('last_error'):
+                errored_count += 1
             if limit_tag and not limit_tag in watch['tags']:
                     continue
 
@@ -442,6 +448,7 @@ def changedetection_app(config=None, datastore_o=None):
                                  active_tag=limit_tag,
                                  app_rss_token=datastore.data['settings']['application']['rss_access_token'],
                                  datastore=datastore,
+                                 errored_count=errored_count,
                                  form=form,
                                  guid=datastore.data['app_guid'],
                                  has_proxies=datastore.proxy_list,
@@ -622,7 +629,6 @@ def changedetection_app(config=None, datastore_o=None):
 
             if request.args.get('unpause_on_save'):
                 extra_update_obj['paused'] = False
-
             # Re #110, if they submit the same as the default value, set it to None, so we continue to follow the default
             # Assume we use the default value, unless something relevant is different, then use the form value
             # values could be None, 0 etc.
@@ -708,7 +714,6 @@ def changedetection_app(config=None, datastore_o=None):
 
             # Only works reliably with Playwright
             visualselector_enabled = os.getenv('PLAYWRIGHT_DRIVER_URL', False) and is_html_webdriver
-
             output = render_template("edit.html",
                                      available_processors=processors.available_processors(),
                                      browser_steps_config=browser_step_ui_config,
@@ -857,7 +862,10 @@ def changedetection_app(config=None, datastore_o=None):
     def mark_all_viewed():
 
         # Save the current newest history as the most recently viewed
+        with_errors = request.args.get('with_errors') == "1"
         for watch_uuid, watch in datastore.data['watching'].items():
+            if with_errors and not watch.get('last_error'):
+                continue
             datastore.set_last_viewed(watch_uuid, int(time.time()))
 
         return redirect(url_for('index'))
@@ -1266,6 +1274,8 @@ def changedetection_app(config=None, datastore_o=None):
         # Forced recheck will skip the 'skip if content is the same' rule (, 'reprocess_existing_data': True})))
         tag = request.args.get('tag')
         uuid = request.args.get('uuid')
+        with_errors = request.args.get('with_errors') == "1"
+
         i = 0
 
         running_uuids = []
@@ -1281,6 +1291,8 @@ def changedetection_app(config=None, datastore_o=None):
             # Items that have this current tag
             for watch_uuid, watch in datastore.data['watching'].items():
                 if tag in watch.get('tags', {}):
+                    if with_errors and not watch.get('last_error'):
+                        continue
                     if watch_uuid not in running_uuids and not datastore.data['watching'][watch_uuid]['paused']:
                         update_q.put(
                             queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': watch_uuid, 'skip_when_checksum_same': False})
@@ -1291,8 +1303,11 @@ def changedetection_app(config=None, datastore_o=None):
             # No tag, no uuid, add everything.
             for watch_uuid, watch in datastore.data['watching'].items():
                 if watch_uuid not in running_uuids and not datastore.data['watching'][watch_uuid]['paused']:
+                    if with_errors and not watch.get('last_error'):
+                        continue
                     update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': watch_uuid, 'skip_when_checksum_same': False}))
                     i += 1
+
         flash("{} watches queued for rechecking.".format(i))
         return redirect(url_for('index', tag=tag))
 

changedetectionio/blueprint/browser_steps/__init__.py

@@ -23,8 +23,10 @@
 
 from distutils.util import strtobool
 from flask import Blueprint, request, make_response
-import os
 import logging
+import os
+import re
+
 from changedetectionio.store import ChangeDetectionStore
 from changedetectionio import login_optionally_required
 
@@ -44,7 +46,7 @@ def construct_blueprint(datastore: ChangeDetectionStore):
 
 
         # We keep the playwright session open for many minutes
-        seconds_keepalive = int(os.getenv('BROWSERSTEPS_MINUTES_KEEPALIVE', 10)) * 60
+        keepalive_seconds = int(os.getenv('BROWSERSTEPS_MINUTES_KEEPALIVE', 10)) * 60
 
         browsersteps_start_session = {'start_time': time.time()}
 
@@ -56,16 +58,18 @@ def construct_blueprint(datastore: ChangeDetectionStore):
             # Start the Playwright context, which is actually a nodejs sub-process and communicates over STDIN/STDOUT pipes
             io_interface_context = io_interface_context.start()
 
+        keepalive_ms = ((keepalive_seconds + 3) * 1000)
+        base_url = os.getenv('PLAYWRIGHT_DRIVER_URL', '')
+        a = "?" if not '?' in base_url else '&'
+        base_url += a + f"timeout={keepalive_ms}"
 
-        # keep it alive for 10 seconds more than we advertise, sometimes it helps to keep it shutting down cleanly
-        keepalive = "&timeout={}".format(((seconds_keepalive + 3) * 1000))
         try:
-            browsersteps_start_session['browser'] = io_interface_context.chromium.connect_over_cdp(
-                os.getenv('PLAYWRIGHT_DRIVER_URL', '') + keepalive)
+            browsersteps_start_session['browser'] = io_interface_context.chromium.connect_over_cdp(base_url)
         except Exception as e:
             if 'ECONNREFUSED' in str(e):
                 return make_response('Unable to start the Playwright Browser session, is it running?', 401)
             else:
+                # Other errors, bad URL syntax, bad reply etc
                 return make_response(str(e), 401)
 
         proxy_id = datastore.get_preferred_proxy_for_watch(uuid=watch_uuid)
@@ -118,6 +122,31 @@ def construct_blueprint(datastore: ChangeDetectionStore):
         print("Starting connection with playwright - done")
         return {'browsersteps_session_id': browsersteps_session_id}
 
+    @login_optionally_required
+    @browser_steps_blueprint.route("/browsersteps_image", methods=['GET'])
+    def browser_steps_fetch_screenshot_image():
+        from flask import (
+            make_response,
+            request,
+            send_from_directory,
+        )
+        uuid = request.args.get('uuid')
+        step_n = int(request.args.get('step_n'))
+
+        watch = datastore.data['watching'].get(uuid)
+        filename = f"step_before-{step_n}.jpeg" if request.args.get('type', '') == 'before' else f"step_{step_n}.jpeg"
+
+        if step_n and watch and os.path.isfile(os.path.join(watch.watch_data_dir, filename)):
+            response = make_response(send_from_directory(directory=watch.watch_data_dir, path=filename))
+            response.headers['Content-type'] = 'image/jpeg'
+            response.headers['Cache-Control'] = 'no-cache, no-store, must-revalidate'
+            response.headers['Pragma'] = 'no-cache'
+            response.headers['Expires'] = 0
+            return response
+
+        else:
+            return make_response('Unable to fetch image, is the URL correct? does the watch exist? does the step_type-n.jpeg exist?', 401)
+
     # A request for an action was received
     @login_optionally_required
     @browser_steps_blueprint.route("/browsersteps_update", methods=['POST'])

changedetectionio/blueprint/browser_steps/browser_steps.py

@@ -138,13 +138,13 @@ class steppable_browser_interface():
     def action_wait_for_text(self, selector, value):
         import json
         v = json.dumps(value)
-        self.page.wait_for_function(f'document.querySelector("body").innerText.includes({v});', timeout=90000)
+        self.page.wait_for_function(f'document.querySelector("body").innerText.includes({v});', timeout=30000)
 
     def action_wait_for_text_in_element(self, selector, value):
         import json
         s = json.dumps(selector)
         v = json.dumps(value)
-        self.page.wait_for_function(f'document.querySelector({s}).innerText.includes({v});', timeout=90000)
+        self.page.wait_for_function(f'document.querySelector({s}).innerText.includes({v});', timeout=30000)
 
     # @todo - in the future make some popout interface to capture what needs to be set
     # https://playwright.dev/python/docs/api/class-keyboard

changedetectionio/html_tools.py

@@ -1,9 +1,12 @@
 
 from bs4 import BeautifulSoup
 from inscriptis import get_text
-from inscriptis.model.config import ParserConfig
 from jsonpath_ng.ext import parse
 from typing import List
+from inscriptis.css_profiles import CSS_PROFILES, HtmlElement
+from inscriptis.html_properties import Display
+from inscriptis.model.config import ParserConfig
+from xml.sax.saxutils import escape as xml_escape
 import json
 import re
 
@@ -68,10 +71,15 @@ def element_removal(selectors: List[str], html_content):
 
 
 # Return str Utf-8 of matched rules
-def xpath_filter(xpath_filter, html_content, append_pretty_line_formatting=False):
+def xpath_filter(xpath_filter, html_content, append_pretty_line_formatting=False, is_rss=False):
     from lxml import etree, html
 
-    tree = html.fromstring(bytes(html_content, encoding='utf-8'))
+    parser = None
+    if is_rss:
+        # So that we can keep CDATA for cdata_in_document_to_text() to process
+        parser = etree.XMLParser(strip_cdata=False)
+
+    tree = html.fromstring(bytes(html_content, encoding='utf-8'), parser=parser)
     html_block = ""
 
     r = tree.xpath(xpath_filter.strip(), namespaces={'re': 'http://exslt.org/regular-expressions'})
@@ -94,7 +102,6 @@ def xpath_filter(xpath_filter, html_content, append_pretty_line_formatting=False
 
     return html_block
 
-
 # Extract/find element
 def extract_element(find='title', html_content=''):
 
@@ -260,8 +267,15 @@ def strip_ignore_text(content, wordlist, mode="content"):
 
     return "\n".encode('utf8').join(output)
 
+def cdata_in_document_to_text(html_content: str, render_anchor_tag_content=False) -> str:
+    pattern = '<!\[CDATA\[(\s*(?:.(?<!\]\]>)\s*)*)\]\]>'
+    def repl(m):
+        text = m.group(1)
+        return xml_escape(html_to_text(html_content=text)).strip()
 
-def html_to_text(html_content: str, render_anchor_tag_content=False) -> str:
+    return re.sub(pattern, repl, html_content)
+
+def html_to_text(html_content: str, render_anchor_tag_content=False, is_rss=False) -> str:
     """Converts html string to a string with just the text. If ignoring
     rendering anchor tag content is enable, anchor tag content are also
     included in the text
@@ -277,16 +291,21 @@ def html_to_text(html_content: str, render_anchor_tag_content=False) -> str:
     #  if anchor tag content flag is set to True define a config for
     #  extracting this content
     if render_anchor_tag_content:
-
         parser_config = ParserConfig(
-            annotation_rules={"a": ["hyperlink"]}, display_links=True
+            annotation_rules={"a": ["hyperlink"]},
+            display_links=True
         )
-
-    # otherwise set config to None
+    # otherwise set config to None/default
     else:
         parser_config = None
 
-    # get text and annotations via inscriptis
+    # RSS Mode - Inscriptis will treat `title` as something else.
+    # Make it as a regular block display element (//item/title)
+    # This is a bit of a hack - the real way it to use XSLT to convert it to HTML #1874
+    if is_rss:
+        html_content = re.sub(r'<title([\s>])', r'<h1\1', html_content)
+        html_content = re.sub(r'</title>', r'</h1>', html_content)
+
     text_content = get_text(html_content, config=parser_config)
 
     return text_content

changedetectionio/model/Watch.py

@@ -4,6 +4,7 @@ import os
 import re
 import time
 import uuid
+from pathlib import Path
 
 # Allowable protocols, protects against javascript: etc
 # file:// is further checked by ALLOW_FILE_URI
@@ -18,6 +19,7 @@ from changedetectionio.notification import (
 
 base_config = {
     'body': None,
+    'browser_steps_last_error_step': None,
     'check_unique_lines': False,  # On change-detected, compare against all history if its something new
     'check_count': 0,
     'date_created': None,
@@ -25,6 +27,7 @@ base_config = {
     'extract_text': [],  # Extract text by regex after filters
     'extract_title_as_title': False,
     'fetch_backend': 'system', # plaintext, playwright etc
+    'fetch_time': 0.0,
     'processor': 'text_json_diff', # could be restock_diff or others from .processors
     'filter_failure_notification_send': strtobool(os.getenv('FILTER_FAILURE_NOTIFICATION_SEND_DEFAULT', 'True')),
     'filter_text_added': True,
@@ -489,3 +492,13 @@ class model(dict):
         filepath = os.path.join(self.watch_data_dir, 'last-fetched.br')
         with open(filepath, 'wb') as f:
             f.write(brotli.compress(contents, mode=brotli.MODE_TEXT))
+
+    @property
+    def get_browsersteps_available_screenshots(self):
+        "For knowing which screenshots are available to show the user in BrowserSteps UI"
+        available = []
+        for f in Path(self.watch_data_dir).glob('step_before-*.jpeg'):
+            step_n=re.search(r'step_before-(\d+)', f.name)
+            if step_n:
+                available.append(step_n.group(1))
+        return available

changedetectionio/processors/text_json_diff.py

@@ -11,7 +11,7 @@ from changedetectionio import content_fetcher, html_tools
 from changedetectionio.blueprint.price_data_follower import PRICE_DATA_TRACK_ACCEPT, PRICE_DATA_TRACK_REJECT
 from copy import deepcopy
 from . import difference_detection_processor
-from ..html_tools import PERL_STYLE_REGEX
+from ..html_tools import PERL_STYLE_REGEX, cdata_in_document_to_text
 
 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 
@@ -153,13 +153,22 @@ class perform_site_check(difference_detection_processor):
 
         is_json = 'application/json' in fetcher.get_all_headers().get('content-type', '').lower()
         is_html = not is_json
+        is_rss = False
+
+        ctype_header = fetcher.get_all_headers().get('content-type', '').lower()
+        # Go into RSS preprocess for converting CDATA/comment to usable text
+        if any(substring in ctype_header for substring in ['application/xml', 'application/rss', 'text/xml']):
+            if '<rss' in fetcher.content[:100].lower():
+                fetcher.content = cdata_in_document_to_text(html_content=fetcher.content)
+                is_rss = True
 
         # source: support, basically treat it as plaintext
         if is_source:
             is_html = False
             is_json = False
 
-        if watch.is_pdf or 'application/pdf' in fetcher.get_all_headers().get('content-type', '').lower():
+        inline_pdf = fetcher.get_all_headers().get('content-disposition', '') and '%PDF-1' in fetcher.content[:10]
+        if watch.is_pdf or 'application/pdf' in fetcher.get_all_headers().get('content-type', '').lower() or inline_pdf:
             from shutil import which
             tool = os.getenv("PDF_TO_HTML_TOOL", "pdftohtml")
             if not which(tool):
@@ -242,7 +251,8 @@ class perform_site_check(difference_detection_processor):
                         if filter_rule[0] == '/' or filter_rule.startswith('xpath:'):
                             html_content += html_tools.xpath_filter(xpath_filter=filter_rule.replace('xpath:', ''),
                                                                     html_content=fetcher.content,
-                                                                    append_pretty_line_formatting=not is_source)
+                                                                    append_pretty_line_formatting=not is_source,
+                                                                    is_rss=is_rss)
                         else:
                             # CSS Filter, extract the HTML that matches and feed that into the existing inscriptis::get_text
                             html_content += html_tools.include_filters(include_filters=filter_rule,
@@ -262,8 +272,9 @@ class perform_site_check(difference_detection_processor):
                     do_anchor = self.datastore.data["settings"]["application"].get("render_anchor_tag_content", False)
                     stripped_text_from_html = \
                         html_tools.html_to_text(
-                            html_content,
-                            render_anchor_tag_content=do_anchor
+                            html_content=html_content,
+                            render_anchor_tag_content=do_anchor,
+                            is_rss=is_rss # #1874 activate the <title workaround hack
                         )
 
         # Re #340 - return the content before the 'ignore text' was applied

changedetectionio/static/js/browser-steps.js

@@ -321,8 +321,14 @@ $(document).ready(function () {
             var s = '<div class="control">' + '<a data-step-index=' + i + ' class="pure-button button-secondary button-green button-xsmall apply" >Apply</a>&nbsp;';
             if (i > 0) {
                 // The first step never gets these (Goto-site)
-                s += '<a data-step-index=' + i + ' class="pure-button button-secondary button-xsmall clear" >Clear</a>&nbsp;' +
-                    '<a data-step-index=' + i + ' class="pure-button button-secondary button-red button-xsmall remove" >Remove</a>';
+                s += `<a data-step-index="${i}" class="pure-button button-secondary button-xsmall clear" >Clear</a>&nbsp;` +
+                    `<a data-step-index="${i}" class="pure-button button-secondary button-red button-xsmall remove" >Remove</a>`;
+
+                // if a screenshot is available
+                if (browser_steps_available_screenshots.includes(i.toString())) {
+                    var d = (browser_steps_last_error_step === i+1) ? 'before' : 'after';
+                    s += `&nbsp;<a data-step-index="${i}" class="pure-button button-secondary button-xsmall show-screenshot" title="Show screenshot from last run" data-type="${d}">Pic</a>&nbsp;`;
+                }
             }
             s += '</div>';
             $(this).append(s)
@@ -437,6 +443,24 @@ $(document).ready(function () {
 
     });
 
+    $('ul#browser_steps li .control .show-screenshot').click(function (element) {
+        var step_n = $(event.currentTarget).data('step-index');
+        w = window.open(this.href, "_blank", "width=640,height=480");
+        const t = $(event.currentTarget).data('type');
+
+        const url = browser_steps_fetch_screenshot_image_url + `&step_n=${step_n}&type=${t}`;
+        w.document.body.innerHTML = `<!DOCTYPE html>
+            <html lang="en">
+                <body>
+                    <img src="${url}" style="width: 100%" alt="Browser Step at step ${step_n} from last run." title="Browser Step at step ${step_n} from last run."/>
+                </body>
+        </html>`;
+        w.document.title = `Browser Step at step ${step_n} from last run.`;
+    });
+
+    if (browser_steps_last_error_step) {
+        $("ul#browser_steps>li:nth-child("+browser_steps_last_error_step+")").addClass("browser-step-with-error");
+    }
 
     $("ul#browser_steps select").change(function () {
         set_greyed_state();

changedetectionio/static/js/diff-render.js

@@ -2,8 +2,7 @@ $(document).ready(function () {
     var a = document.getElementById("a");
     var b = document.getElementById("b");
     var result = document.getElementById("result");
-    var inputs = document.getElementsByClassName("change");
-    inputs.current = 0;
+    var inputs;
 
     $('#jump-next-diff').click(function () {
 
@@ -59,9 +58,6 @@ $(document).ready(function () {
         result.textContent = "";
         result.appendChild(fragment);
 
-        // Jump at start
-        inputs.current = 0;
-
         // For nice mouse-over hover/title information
         const removed_current_option = $('#diff-version option:selected')
         if (removed_current_option) {
@@ -75,8 +71,12 @@ $(document).ready(function () {
                 $(this).prop('title', 'Inserted '+inserted_current_option[0].label);
             });
         }
-
-        next_diff();
+        // Set the list of possible differences to jump to
+        inputs = document.querySelectorAll('#diff-ui .change')
+        // Set the "current" diff pointer
+        inputs.current = 0;
+        // Goto diff
+        $('#jump-next-diff').click();
     }
 
     $('.needs-localtime').each(function () {

changedetectionio/static/js/watch-overview.js

@@ -4,6 +4,14 @@ $(function () {
         $(this).closest('.unviewed').removeClass('unviewed');
     });
 
+    $('td[data-timestamp]').each(function () {
+        $(this).prop('title', new Intl.DateTimeFormat(undefined,
+            {
+                dateStyle: 'full',
+                timeStyle: 'long'
+            }).format($(this).data('timestamp') * 1000));
+    })
+
     $("#checkbox-assign-tag").click(function (e) {
         $('#op_extradata').val(prompt("Enter a tag name"));
     });

changedetectionio/static/styles/scss/parts/_browser-steps.scss

@@ -6,6 +6,10 @@
   }
 
   li {
+    &.browser-step-with-error {
+      background-color: #ffd6d6;
+      border-radius: 4px;
+    }
     &:not(:first-child) {
       &:hover {
         opacity: 1.0;

changedetectionio/static/styles/scss/parts/_visualselector.scss

@@ -0,0 +1,28 @@
+
+#selector-wrapper {
+  height: 100%;
+  max-height: 70vh;
+  overflow-y: scroll;
+  position: relative;
+
+  //width: 100%;
+  >img {
+    position: absolute;
+    z-index: 4;
+    max-width: 100%;
+  }
+
+  >canvas {
+    position: relative;
+    z-index: 5;
+    max-width: 100%;
+
+    &:hover {
+      cursor: pointer;
+    }
+  }
+}
+
+#selector-current-xpath {
+  font-size: 80%;
+}

changedetectionio/static/styles/scss/styles.scss

@@ -943,32 +943,7 @@ ul {
   }
 }
 
-#selector-wrapper {
-  height: 100%;
-  overflow-y: scroll;
-  position: relative;
-
-  //width: 100%;
-  >img {
-    position: absolute;
-    z-index: 4;
-    max-width: 100%;
-  }
-
-  >canvas {
-    position: relative;
-    z-index: 5;
-    max-width: 100%;
-
-    &:hover {
-      cursor: pointer;
-    }
-  }
-}
-
-#selector-current-xpath {
-  font-size: 80%;
-}
+@import "parts/_visualselector";
 
 #webdriver-override-options {
   input[type="number"] {

changedetectionio/static/styles/styles.css

@@ -26,6 +26,9 @@
   #browser_steps li {
     list-style: decimal;
     padding: 5px; }
+    #browser_steps li.browser-step-with-error {
+      background-color: #ffd6d6;
+      border-radius: 4px; }
     #browser_steps li:not(:first-child):hover {
       opacity: 1.0; }
     #browser_steps li .control {
@@ -980,6 +983,7 @@ ul {
 
 #selector-wrapper {
   height: 100%;
+  max-height: 70vh;
   overflow-y: scroll;
   position: relative; }
   #selector-wrapper > img {

changedetectionio/store.py

@@ -244,12 +244,16 @@ class ChangeDetectionStore:
         import pathlib
 
         self.__data['watching'][uuid].update({
-                'last_checked': 0,
+                'browser_steps_last_error_step' : None,
+                'check_count': 0,
+                'fetch_time' : 0.0,
                 'has_ldjson_price_data': None,
+                'last_checked': 0,
                 'last_error': False,
                 'last_notification_error': False,
                 'last_viewed': 0,
                 'previous_md5': False,
+                'previous_md5_before_filters': False,
                 'track_ldjson_price_data': None,
             })
 

changedetectionio/templates/diff.html

@@ -60,7 +60,7 @@
 </div>
 
 <div id="diff-jump">
-    <a id="jump-next-diff">Jump</a>
+    <a id="jump-next-diff" title="Jump to next difference">Jump</a>
 </div>
 
 <script src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>

changedetectionio/templates/edit.html

@@ -4,8 +4,10 @@
 {% from '_common_fields.jinja' import render_common_settings_form %}
 <script src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
 <script>
-
+    const browser_steps_available_screenshots=JSON.parse('{{ watch.get_browsersteps_available_screenshots|tojson }}');
     const browser_steps_config=JSON.parse('{{ browser_steps_config|tojson }}');
+    const browser_steps_fetch_screenshot_image_url="{{url_for('browser_steps.browser_steps_fetch_screenshot_image', uuid=uuid)}}";
+    const browser_steps_last_error_step={{ watch.browser_steps_last_error_step|tojson }};
     const browser_steps_start_url="{{url_for('browser_steps.browsersteps_start_session', uuid=uuid)}}";
     const browser_steps_sync_url="{{url_for('browser_steps.browsersteps_ui_update', uuid=uuid)}}";
 {% if emailprefix %}
@@ -49,6 +51,7 @@
             <li class="tab"><a href="#restock">Restock Detection</a></li>
             {% endif %}
             <li class="tab"><a href="#notifications">Notifications</a></li>
+            <li class="tab"><a href="#stats">Stats</a></li>
         </ul>
     </div>
 
@@ -441,7 +444,35 @@ Unavailable") }}
                 </fieldset>
             </div>
             {% endif %}
-
+            <div class="tab-pane-inner" id="stats">
+                <div class="pure-control-group">
+                    <style>
+                    #stats-table tr > td:first-child {
+                        font-weight: bold;
+                    }
+                    </style>
+                    <table class="pure-table" id="stats-table">
+                        <tbody>
+                        <tr>
+                            <td>Check count</td>
+                            <td>{{ watch.check_count }}</td>
+                        </tr>
+                        <tr>
+                            <td>Consecutive filter failures</td>
+                            <td>{{ watch.consecutive_filter_failures }}</td>
+                        </tr>
+                        <tr>
+                            <td>History length</td>
+                            <td>{{ watch.history|length }}</td>
+                        </tr>
+                        <tr>
+                            <td>Last fetch time</td>
+                            <td>{{ watch.fetch_time }}s</td>
+                        </tr>
+                        </tbody>
+                    </table>
+                </div>
+            </div>
             <div id="actions">
                 <div class="pure-control-group">
                     {{ render_button(form.save_button) }}

changedetectionio/templates/watch-overview.html

@@ -154,8 +154,8 @@
                     {% endfor %}
 
                 </td>
-                <td class="last-checked">{{watch|format_last_checked_time|safe}}</td>
-                <td class="last-changed">{% if watch.history_n >=2 and watch.last_changed >0 %}
+                <td class="last-checked" data-timestamp="{{ watch.last_checked }}">{{watch|format_last_checked_time|safe}}</td>
+                <td class="last-changed" data-timestamp="{{ watch.last_changed }}">{% if watch.history_n >=2 and watch.last_changed >0 %}
                     {{watch.last_changed|format_timestamp_timeago}}
                     {% else %}
                     Not yet
@@ -178,13 +178,18 @@
             </tbody>
         </table>
         <ul id="post-list-buttons">
+            {% if errored_count %}
+            <li>
+                <a href="{{url_for('index', with_errors=1, tag=request.args.get('tag')) }}" class="pure-button button-tag button-error ">With errors ({{ errored_count }})</a>
+            </li>
+            {% endif %}
             {% if has_unviewed %}
             <li>
-                <a href="{{url_for('mark_all_viewed', tag=request.args.get('tag')) }}" class="pure-button button-tag ">Mark all viewed</a>
+                <a href="{{url_for('mark_all_viewed',with_errors=request.args.get('with_errors',0)) }}" class="pure-button button-tag ">Mark all viewed</a>
             </li>
             {% endif %}
             <li>
-               <a href="{{ url_for('form_watch_checknow', tag=active_tag) }}" class="pure-button button-tag ">Recheck
+               <a href="{{ url_for('form_watch_checknow', tag=active_tag, with_errors=request.args.get('with_errors',0)) }}" class="pure-button button-tag ">Recheck
                 all {% if active_tag%} in "{{tags[active_tag].title}}"{%endif%}</a>
             </li>
             <li>

changedetectionio/tests/test_rss.py

@@ -2,12 +2,61 @@
 
 import time
 from flask import url_for
-from .util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks, extract_rss_token_from_UI
+from .util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks, extract_rss_token_from_UI, \
+    extract_UUID_from_client
 
 
+def set_original_cdata_xml():
+    test_return_data = """<rss xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:content="http://purl.org/rss/1.0/modules/content/" xmlns:media="http://search.yahoo.com/mrss/" xmlns:atom="http://www.w3.org/2005/Atom" version="2.0">
+    <channel>
+    <title>Gizi</title>
+    <link>https://test.com</link>
+    <atom:link href="https://testsite.com" rel="self" type="application/rss+xml"/>
+    <description>
+    <![CDATA[ The Future Could Be Here ]]>
+    </description>
+    <language>en</language>
+    <item>
+    <title>
+    <![CDATA[ <img src="https://testsite.com/hacked.jpg"> Hackers can access your computer ]]>
+    </title>
+    <link>https://testsite.com/news/12341234234</link>
+    <description>
+    <![CDATA[ <img class="type:primaryImage" src="https://testsite.com/701c981da04869e.jpg"/><p>The days of Terminator and The Matrix could be closer. But be positive.</p><p><a href="https://testsite.com">Read more link...</a></p> ]]>
+    </description>
+    <category>cybernetics</category>
+    <category>rand corporation</category>
+    <pubDate>Tue, 17 Oct 2023 15:10:00 GMT</pubDate>
+    <guid isPermaLink="false">1850933241</guid>
+    <dc:creator>
+    <![CDATA[ Mr Hacker News ]]>
+    </dc:creator>
+    <media:thumbnail url="https://testsite.com/thumbnail-c224e10d81488e818701c981da04869e.jpg"/>
+    </item>
+
+    <item>
+        <title>    Some other title    </title>
+        <link>https://testsite.com/news/12341234236</link>
+        <description>
+        Some other description
+        </description>
+    </item>    
+    </channel>
+    </rss>
+            """
+
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write(test_return_data)
+
+
+def test_setup(client, live_server):
+    live_server_setup(live_server)
+
 def test_rss_and_token(client, live_server):
+    #    live_server_setup(live_server)
+
     set_original_response()
-    live_server_setup(live_server)
+    rss_token = extract_rss_token_from_UI(client)
 
     # Add our URL to the import page
     res = client.post(
@@ -17,11 +66,11 @@ def test_rss_and_token(client, live_server):
     )
 
     assert b"1 Imported" in res.data
-    rss_token = extract_rss_token_from_UI(client)
 
-    time.sleep(2)
+    wait_for_all_checks(client)
+    set_modified_response()
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    time.sleep(2)
+    wait_for_all_checks(client)
 
     # Add our URL to the import page
     res = client.get(
@@ -37,3 +86,80 @@ def test_rss_and_token(client, live_server):
     )
     assert b"Access denied, bad token" not in res.data
     assert b"Random content" in res.data
+
+    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+
+def test_basic_cdata_rss_markup(client, live_server):
+    #live_server_setup(live_server)
+
+    set_original_cdata_xml()
+
+    test_url = url_for('test_endpoint', content_type="application/xml", _external=True)
+
+    # Add our URL to the import page
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+
+    assert b"1 Imported" in res.data
+
+    wait_for_all_checks(client)
+
+    res = client.get(
+        url_for("preview_page", uuid="first"),
+        follow_redirects=True
+    )
+    assert b'CDATA' not in res.data
+    assert b'<![' not in res.data
+    assert b'Hackers can access your computer' in res.data
+    assert b'The days of Terminator' in res.data
+    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+
+def test_rss_xpath_filtering(client, live_server):
+    #live_server_setup(live_server)
+
+    set_original_cdata_xml()
+
+    test_url = url_for('test_endpoint', content_type="application/xml", _external=True)
+
+    res = client.post(
+        url_for("form_quick_watch_add"),
+        data={"url": test_url, "tags": '', 'edit_and_watch_submit_button': 'Edit > Watch'},
+        follow_redirects=True
+    )
+    assert b"Watch added in Paused state, saving will unpause" in res.data
+
+    uuid = extract_UUID_from_client(client)
+    res = client.post(
+        url_for("edit_page", uuid=uuid, unpause_on_save=1),
+        data={
+                "include_filters": "//item/title",
+                "fetch_backend": "html_requests",
+                "headers": "",
+                "proxy": "no-proxy",
+                "tags": "",
+                "url": test_url,
+              },
+        follow_redirects=True
+    )
+    assert b"unpaused" in res.data
+
+    wait_for_all_checks(client)
+
+    res = client.get(
+        url_for("preview_page", uuid="first"),
+        follow_redirects=True
+    )
+    assert b'CDATA' not in res.data
+    assert b'<![' not in res.data
+    # #1874  All but the first <title was getting selected
+    # Convert any HTML with just a top level <title> to <h1> to be sure title renders
+
+    assert b'Hackers can access your computer' in res.data # Should ONLY be selected by the xpath
+    assert b'Some other title' in res.data  # Should ONLY be selected by the xpath
+    assert b'The days of Terminator' not in res.data # Should NOT be selected by the xpath
+    assert b'Some other description' not in res.data  # Should NOT be selected by the xpath
+
+    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)

changedetectionio/tests/test_xpath_selector.py

@@ -2,7 +2,7 @@
 
 import time
 from flask import url_for
-from . util import live_server_setup
+from .util import live_server_setup, wait_for_all_checks
 
 from ..html_tools import *
 
@@ -86,14 +86,14 @@ def test_check_xpath_filter_utf8(client, live_server):
         follow_redirects=True
     )
     assert b"1 Imported" in res.data
-    time.sleep(1)
+    wait_for_all_checks(client)
     res = client.post(
         url_for("edit_page", uuid="first"),
         data={"include_filters": filter, "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests"},
         follow_redirects=True
     )
     assert b"Updated watch." in res.data
-    time.sleep(3)
+    wait_for_all_checks(client)
     res = client.get(url_for("index"))
     assert b'Unicode strings with encoding declaration are not supported.' not in res.data
     res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
@@ -140,14 +140,14 @@ def test_check_xpath_text_function_utf8(client, live_server):
         follow_redirects=True
     )
     assert b"1 Imported" in res.data
-    time.sleep(1)
+    wait_for_all_checks(client)
     res = client.post(
         url_for("edit_page", uuid="first"),
         data={"include_filters": filter, "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests"},
         follow_redirects=True
     )
     assert b"Updated watch." in res.data
-    time.sleep(3)
+    wait_for_all_checks(client)
     res = client.get(url_for("index"))
     assert b'Unicode strings with encoding declaration are not supported.' not in res.data
 
@@ -164,7 +164,6 @@ def test_check_xpath_text_function_utf8(client, live_server):
     assert b'Deleted' in res.data
 
 def test_check_markup_xpath_filter_restriction(client, live_server):
-    sleep_time_for_fetch_thread = 3
 
     xpath_filter = "//*[contains(@class, 'sametext')]"
 
@@ -183,7 +182,7 @@ def test_check_markup_xpath_filter_restriction(client, live_server):
     assert b"1 Imported" in res.data
 
     # Give the thread time to pick it up
-    time.sleep(sleep_time_for_fetch_thread)
+    wait_for_all_checks(client)
 
     # Goto the edit page, add our ignore text
     # Add our URL to the import page
@@ -195,7 +194,7 @@ def test_check_markup_xpath_filter_restriction(client, live_server):
     assert b"Updated watch." in res.data
 
     # Give the thread time to pick it up
-    time.sleep(sleep_time_for_fetch_thread)
+    wait_for_all_checks(client)
 
     # view it/reset state back to viewed
     client.get(url_for("diff_history_page", uuid="first"), follow_redirects=True)
@@ -206,7 +205,7 @@ def test_check_markup_xpath_filter_restriction(client, live_server):
     # Trigger a check
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
     # Give the thread time to pick it up
-    time.sleep(sleep_time_for_fetch_thread)
+    wait_for_all_checks(client)
 
     res = client.get(url_for("index"))
     assert b'unviewed' not in res.data
@@ -216,9 +215,6 @@ def test_check_markup_xpath_filter_restriction(client, live_server):
 
 def test_xpath_validation(client, live_server):
 
-    # Give the endpoint time to spin up
-    time.sleep(1)
-
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
@@ -227,7 +223,7 @@ def test_xpath_validation(client, live_server):
         follow_redirects=True
     )
     assert b"1 Imported" in res.data
-    time.sleep(2)
+    wait_for_all_checks(client)
 
     res = client.post(
         url_for("edit_page", uuid="first"),
@@ -244,11 +240,8 @@ def test_check_with_prefix_include_filters(client, live_server):
     res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
 
-    # Give the endpoint time to spin up
-    time.sleep(1)
-
     set_original_response()
-
+    wait_for_all_checks(client)
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
@@ -257,7 +250,7 @@ def test_check_with_prefix_include_filters(client, live_server):
         follow_redirects=True
     )
     assert b"1 Imported" in res.data
-    time.sleep(3)
+    wait_for_all_checks(client)
 
     res = client.post(
         url_for("edit_page", uuid="first"),
@@ -266,7 +259,7 @@ def test_check_with_prefix_include_filters(client, live_server):
     )
 
     assert b"Updated watch." in res.data
-    time.sleep(3)
+    wait_for_all_checks(client)
 
     res = client.get(
         url_for("preview_page", uuid="first"),
@@ -277,3 +270,46 @@ def test_check_with_prefix_include_filters(client, live_server):
     assert b"Some text that will change" not in res.data #not in selector
 
     client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+def test_various_rules(client, live_server):
+    # Just check these don't error
+    #live_server_setup(live_server)
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write("""<html>
+       <body>
+     Some initial text<br>
+     <p>Which is across multiple lines</p>
+     <br>
+     So let's see what happens.  <br>
+     <div class="sametext">Some text thats the same</div>
+     <div class="changetext">Some text that will change</div>
+     <a href=''>some linky </a>
+     <a href=''>another some linky </a>
+     <!-- related to https://github.com/dgtlmoon/changedetection.io/pull/1774 -->
+     <input   type="email"   id="email" />
+     </body>
+     </html>
+    """)
+    test_url = url_for('test_endpoint', _external=True)
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+    wait_for_all_checks(client)
+
+
+    for r in ['//div', '//a', 'xpath://div', 'xpath://a']:
+        res = client.post(
+            url_for("edit_page", uuid="first"),
+            data={"include_filters": r,
+                  "url": test_url,
+                  "tags": "",
+                  "headers": "",
+                  'fetch_backend': "html_requests"},
+            follow_redirects=True
+        )
+        wait_for_all_checks(client)
+        assert b"Updated watch." in res.data
+        res = client.get(url_for("index"))
+        assert b'fetch-error' not in res.data, f"Should not see errors after '{r} filter"

changedetectionio/update_worker.py

@@ -238,7 +238,9 @@ class update_worker(threading.Thread):
                             # Used as a default and also by some tests
                             update_handler = text_json_diff.perform_site_check(datastore=self.datastore)
 
+                        self.datastore.data['watching'][uuid]['browser_steps_last_error_step'] = None
                         changed_detected, update_obj, contents = update_handler.run(uuid, skip_when_checksum_same=queued_item_data.item.get('skip_when_checksum_same'))
+
                         # Re #342
                         # In Python 3, all strings are sequences of Unicode characters. There is a bytes type that holds raw bytes.
                         # We then convert/.decode('utf-8') for the notification etc
@@ -324,8 +326,13 @@ class update_worker(threading.Thread):
                         if not self.datastore.data['watching'].get(uuid):
                             continue
 
-                        err_text = "Warning, browser step at position {} could not run, target not found, check the watch, add a delay if necessary.".format(e.step_n+1)
-                        self.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text})
+                        error_step = e.step_n + 1
+                        err_text = f"Warning, browser step at position {error_step} could not run, target not found, check the watch, add a delay if necessary, view Browser Steps to see screenshot at that step"
+                        self.datastore.update_watch(uuid=uuid,
+                                                    update_obj={'last_error': err_text,
+                                                                'browser_steps_last_error_step': error_step
+                                                                }
+                                                    )
 
 
                         if self.datastore.data['watching'][uuid].get('filter_failure_notification_send', False):

requirements.txt

@@ -33,7 +33,7 @@ dnspython<2.3.0
 # jq not available on Windows so must be installed manually
 
 # Notification library
-apprise~=1.5.0
+apprise~=1.6.0
 
 # apprise mqtt https://github.com/dgtlmoon/changedetection.io/issues/315
 paho-mqtt
@@ -63,7 +63,8 @@ jinja2-time
 
 # https://peps.python.org/pep-0508/#environment-markers
 # https://github.com/dgtlmoon/changedetection.io/pull/1009
-jq~=1.3 ;python_version >= "3.8" and sys_platform == "linux"
+jq~=1.3; python_version >= "3.8" and sys_platform == "darwin"
+jq~=1.3; python_version >= "3.8" and sys_platform == "linux"
 
 # Any current modern version, required so far for screenshot PNG->JPEG conversion but will be used more in the future
 pillow