handle when the PDF is given as inline content without a proper mime header

.github/workflows/test-only.yml

@@ -29,8 +29,8 @@ jobs:
           docker network create changedet-network
 
           # Selenium+browserless
-          docker run --network changedet-network -d --hostname selenium  -p 4444:4444 --rm --shm-size="2g"  selenium/standalone-chrome:4.14.1
-          docker run --network changedet-network -d --hostname browserless -e "FUNCTION_BUILT_INS=[\"fs\",\"crypto\"]" -e "DEFAULT_LAUNCH_ARGS=[\"--window-size=1920,1080\"]" --rm  -p 3000:3000  --shm-size="2g"  browserless/chrome:1.60-chrome-stable
+          docker run --network changedet-network -d --hostname selenium  -p 4444:4444 --rm --shm-size="2g"  selenium/standalone-chrome-debug:3.141.59
+          docker run --network changedet-network -d --hostname browserless -e "FUNCTION_BUILT_INS=[\"fs\",\"crypto\"]" -e "DEFAULT_LAUNCH_ARGS=[\"--window-size=1920,1080\"]" --rm  -p 3000:3000  --shm-size="2g"  browserless/chrome:1.53-chrome-stable
 
       - name: Build changedetection.io container for testing
         run: |         

Dockerfile

@@ -20,11 +20,6 @@ WORKDIR /install
 
 COPY requirements.txt /requirements.txt
 
-# Instructing pip to fetch wheels from piwheels.org" on ARMv6 and ARMv7 machines
-RUN if [ "$(dpkg --print-architecture)" = "armhf" ] || [ "$(dpkg --print-architecture)" = "armel" ]; then \
-      printf "[global]\nextra-index-url=https://www.piwheels.org/simple\n" > /etc/pip.conf; \
-    fi;
-
 RUN pip install --target=/dependencies -r /requirements.txt
 
 # Playwright is an alternative to Selenium

changedetectionio/__init__.py

@@ -38,7 +38,7 @@ from flask_paginate import Pagination, get_page_parameter
 from changedetectionio import html_tools
 from changedetectionio.api import api_v1
 
-__version__ = '0.45.5'
+__version__ = '0.45.3'
 
 from changedetectionio.store import BASE_URL_NOT_SET_TEXT
 
@@ -416,18 +416,11 @@ def changedetection_app(config=None, datastore_o=None):
 
         # Sort by last_changed and add the uuid which is usually the key..
         sorted_watches = []
-        with_errors = request.args.get('with_errors') == "1"
-        errored_count = 0
         search_q = request.args.get('q').strip().lower() if request.args.get('q') else False
         for uuid, watch in datastore.data['watching'].items():
-            if with_errors and not watch.get('last_error'):
-                continue
-
             if limit_tag and not limit_tag in watch['tags']:
                     continue
-            if watch.get('last_error'):
-                errored_count += 1
-                
+
             if search_q:
                 if (watch.get('title') and search_q in watch.get('title').lower()) or search_q in watch.get('url', '').lower():
                     sorted_watches.append(watch)
@@ -449,7 +442,6 @@ def changedetection_app(config=None, datastore_o=None):
                                  active_tag=limit_tag,
                                  app_rss_token=datastore.data['settings']['application']['rss_access_token'],
                                  datastore=datastore,
-                                 errored_count=errored_count,
                                  form=form,
                                  guid=datastore.data['app_guid'],
                                  has_proxies=datastore.proxy_list,
@@ -630,6 +622,7 @@ def changedetection_app(config=None, datastore_o=None):
 
             if request.args.get('unpause_on_save'):
                 extra_update_obj['paused'] = False
+
             # Re #110, if they submit the same as the default value, set it to None, so we continue to follow the default
             # Assume we use the default value, unless something relevant is different, then use the form value
             # values could be None, 0 etc.
@@ -715,6 +708,7 @@ def changedetection_app(config=None, datastore_o=None):
 
             # Only works reliably with Playwright
             visualselector_enabled = os.getenv('PLAYWRIGHT_DRIVER_URL', False) and is_html_webdriver
+
             output = render_template("edit.html",
                                      available_processors=processors.available_processors(),
                                      browser_steps_config=browser_step_ui_config,
@@ -863,10 +857,7 @@ def changedetection_app(config=None, datastore_o=None):
     def mark_all_viewed():
 
         # Save the current newest history as the most recently viewed
-        with_errors = request.args.get('with_errors') == "1"
         for watch_uuid, watch in datastore.data['watching'].items():
-            if with_errors and not watch.get('last_error'):
-                continue
             datastore.set_last_viewed(watch_uuid, int(time.time()))
 
         return redirect(url_for('index'))
@@ -1275,8 +1266,6 @@ def changedetection_app(config=None, datastore_o=None):
         # Forced recheck will skip the 'skip if content is the same' rule (, 'reprocess_existing_data': True})))
         tag = request.args.get('tag')
         uuid = request.args.get('uuid')
-        with_errors = request.args.get('with_errors') == "1"
-
         i = 0
 
         running_uuids = []
@@ -1292,8 +1281,6 @@ def changedetection_app(config=None, datastore_o=None):
             # Items that have this current tag
             for watch_uuid, watch in datastore.data['watching'].items():
                 if tag in watch.get('tags', {}):
-                    if with_errors and not watch.get('last_error'):
-                        continue
                     if watch_uuid not in running_uuids and not datastore.data['watching'][watch_uuid]['paused']:
                         update_q.put(
                             queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': watch_uuid, 'skip_when_checksum_same': False})
@@ -1304,11 +1291,8 @@ def changedetection_app(config=None, datastore_o=None):
             # No tag, no uuid, add everything.
             for watch_uuid, watch in datastore.data['watching'].items():
                 if watch_uuid not in running_uuids and not datastore.data['watching'][watch_uuid]['paused']:
-                    if with_errors and not watch.get('last_error'):
-                        continue
                     update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': watch_uuid, 'skip_when_checksum_same': False}))
                     i += 1
-
         flash("{} watches queued for rechecking.".format(i))
         return redirect(url_for('index', tag=tag))
 

changedetectionio/blueprint/browser_steps/__init__.py

@@ -23,10 +23,8 @@
 
 from distutils.util import strtobool
 from flask import Blueprint, request, make_response
-import logging
 import os
-import re
-
+import logging
 from changedetectionio.store import ChangeDetectionStore
 from changedetectionio import login_optionally_required
 
@@ -46,7 +44,7 @@ def construct_blueprint(datastore: ChangeDetectionStore):
 
 
         # We keep the playwright session open for many minutes
-        keepalive_seconds = int(os.getenv('BROWSERSTEPS_MINUTES_KEEPALIVE', 10)) * 60
+        seconds_keepalive = int(os.getenv('BROWSERSTEPS_MINUTES_KEEPALIVE', 10)) * 60
 
         browsersteps_start_session = {'start_time': time.time()}
 
@@ -58,18 +56,16 @@ def construct_blueprint(datastore: ChangeDetectionStore):
             # Start the Playwright context, which is actually a nodejs sub-process and communicates over STDIN/STDOUT pipes
             io_interface_context = io_interface_context.start()
 
-        keepalive_ms = ((keepalive_seconds + 3) * 1000)
-        base_url = os.getenv('PLAYWRIGHT_DRIVER_URL', '')
-        a = "?" if not '?' in base_url else '&'
-        base_url += a + f"timeout={keepalive_ms}"
 
+        # keep it alive for 10 seconds more than we advertise, sometimes it helps to keep it shutting down cleanly
+        keepalive = "&timeout={}".format(((seconds_keepalive + 3) * 1000))
         try:
-            browsersteps_start_session['browser'] = io_interface_context.chromium.connect_over_cdp(base_url)
+            browsersteps_start_session['browser'] = io_interface_context.chromium.connect_over_cdp(
+                os.getenv('PLAYWRIGHT_DRIVER_URL', '') + keepalive)
         except Exception as e:
             if 'ECONNREFUSED' in str(e):
                 return make_response('Unable to start the Playwright Browser session, is it running?', 401)
             else:
-                # Other errors, bad URL syntax, bad reply etc
                 return make_response(str(e), 401)
 
         proxy_id = datastore.get_preferred_proxy_for_watch(uuid=watch_uuid)
@@ -122,31 +118,6 @@ def construct_blueprint(datastore: ChangeDetectionStore):
         print("Starting connection with playwright - done")
         return {'browsersteps_session_id': browsersteps_session_id}
 
-    @login_optionally_required
-    @browser_steps_blueprint.route("/browsersteps_image", methods=['GET'])
-    def browser_steps_fetch_screenshot_image():
-        from flask import (
-            make_response,
-            request,
-            send_from_directory,
-        )
-        uuid = request.args.get('uuid')
-        step_n = int(request.args.get('step_n'))
-
-        watch = datastore.data['watching'].get(uuid)
-        filename = f"step_before-{step_n}.jpeg" if request.args.get('type', '') == 'before' else f"step_{step_n}.jpeg"
-
-        if step_n and watch and os.path.isfile(os.path.join(watch.watch_data_dir, filename)):
-            response = make_response(send_from_directory(directory=watch.watch_data_dir, path=filename))
-            response.headers['Content-type'] = 'image/jpeg'
-            response.headers['Cache-Control'] = 'no-cache, no-store, must-revalidate'
-            response.headers['Pragma'] = 'no-cache'
-            response.headers['Expires'] = 0
-            return response
-
-        else:
-            return make_response('Unable to fetch image, is the URL correct? does the watch exist? does the step_type-n.jpeg exist?', 401)
-
     # A request for an action was received
     @login_optionally_required
     @browser_steps_blueprint.route("/browsersteps_update", methods=['POST'])

changedetectionio/blueprint/browser_steps/browser_steps.py

@@ -77,13 +77,13 @@ class steppable_browser_interface():
     def action_goto_url(self, selector=None, value=None):
         # self.page.set_viewport_size({"width": 1280, "height": 5000})
         now = time.time()
-        response = self.page.goto(value, timeout=0, wait_until='load')
-        # Should be the same as the puppeteer_fetch.js methods, means, load with no timeout set (skip timeout)
-        #and also wait for seconds ?
-        #await page.waitForTimeout(1000);
-        #await page.waitForTimeout(extra_wait_ms);
+        response = self.page.goto(value, timeout=0, wait_until='commit')
+
+        # Wait_until = commit
+        # - `'commit'` - consider operation to be finished when network response is received and the document started loading.
+        # Better to not use any smarts from Playwright and just wait an arbitrary number of seconds
+        # This seemed to solve nearly all 'TimeoutErrors'
         print("Time to goto URL ", time.time() - now)
-        return response
 
     def action_click_element_containing_text(self, selector=None, value=''):
         if not len(value.strip()):
@@ -99,8 +99,7 @@ class steppable_browser_interface():
         self.page.fill(selector, value, timeout=10 * 1000)
 
     def action_execute_js(self, selector, value):
-        response = self.page.evaluate(value)
-        return response
+        self.page.evaluate(value)
 
     def action_click_element(self, selector, value):
         print("Clicking element")
@@ -139,13 +138,13 @@ class steppable_browser_interface():
     def action_wait_for_text(self, selector, value):
         import json
         v = json.dumps(value)
-        self.page.wait_for_function(f'document.querySelector("body").innerText.includes({v});', timeout=30000)
+        self.page.wait_for_function(f'document.querySelector("body").innerText.includes({v});', timeout=90000)
 
     def action_wait_for_text_in_element(self, selector, value):
         import json
         s = json.dumps(selector)
         v = json.dumps(value)
-        self.page.wait_for_function(f'document.querySelector({s}).innerText.includes({v});', timeout=30000)
+        self.page.wait_for_function(f'document.querySelector({s}).innerText.includes({v});', timeout=90000)
 
     # @todo - in the future make some popout interface to capture what needs to be set
     # https://playwright.dev/python/docs/api/class-keyboard

changedetectionio/content_fetcher.py

@@ -159,16 +159,6 @@ class Fetcher():
         """
         return {k.lower(): v for k, v in self.headers.items()}
 
-    def browser_steps_get_valid_steps(self):
-        if self.browser_steps is not None and len(self.browser_steps):
-            valid_steps = filter(
-                lambda s: (s['operation'] and len(s['operation']) and s['operation'] != 'Choose one' and s['operation'] != 'Goto site'),
-                self.browser_steps)
-
-            return valid_steps
-
-        return None
-
     def iterate_browser_steps(self):
         from changedetectionio.blueprint.browser_steps.browser_steps import steppable_browser_interface
         from playwright._impl._api_types import TimeoutError
@@ -180,7 +170,10 @@ class Fetcher():
         if self.browser_steps is not None and len(self.browser_steps):
             interface = steppable_browser_interface()
             interface.page = self.page
-            valid_steps = self.browser_steps_get_valid_steps()
+
+            valid_steps = filter(
+                lambda s: (s['operation'] and len(s['operation']) and s['operation'] != 'Choose one' and s['operation'] != 'Goto site'),
+                self.browser_steps)
 
             for step in valid_steps:
                 step_n += 1
@@ -471,26 +464,39 @@ class base_html_playwright(Fetcher):
             if len(request_headers):
                 context.set_extra_http_headers(request_headers)
 
-            # Listen for all console events and handle errors
-            self.page.on("console", lambda msg: print(f"Playwright console: Watch URL: {url} {msg.type}: {msg.text} {msg.args}"))
+                self.page.set_default_navigation_timeout(90000)
+                self.page.set_default_timeout(90000)
 
-            # Re-use as much code from browser steps as possible so its the same
-            from changedetectionio.blueprint.browser_steps.browser_steps import steppable_browser_interface
-            browsersteps_interface = steppable_browser_interface()
-            browsersteps_interface.page = self.page
+                # Listen for all console events and handle errors
+                self.page.on("console", lambda msg: print(f"Playwright console: Watch URL: {url} {msg.type}: {msg.text} {msg.args}"))
 
-            response = browsersteps_interface.action_goto_url(value=url)
-            self.headers = response.all_headers()
-
-            if response is None:
+            # Goto page
+            try:
+                # Wait_until = commit
+                # - `'commit'` - consider operation to be finished when network response is received and the document started loading.
+                # Better to not use any smarts from Playwright and just wait an arbitrary number of seconds
+                # This seemed to solve nearly all 'TimeoutErrors'
+                response = self.page.goto(url, wait_until='commit')
+            except playwright._impl._api_types.Error as e:
+                # Retry once - https://github.com/browserless/chrome/issues/2485
+                # Sometimes errors related to invalid cert's and other can be random
+                print("Content Fetcher > retrying request got error - ", str(e))
+                time.sleep(1)
+                response = self.page.goto(url, wait_until='commit')
+            except Exception as e:
+                print("Content Fetcher > Other exception when page.goto", str(e))
                 context.close()
                 browser.close()
-                print("Content Fetcher > Response object was none")
-                raise EmptyReply(url=url, status_code=None)
+                raise PageUnloadable(url=url, status_code=None, message=str(e))
 
+            # Execute any browser steps
             try:
+                extra_wait = int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)) + self.render_extract_delay
+                self.page.wait_for_timeout(extra_wait * 1000)
+
                 if self.webdriver_js_execute_code is not None and len(self.webdriver_js_execute_code):
-                    browsersteps_interface.action_execute_js(value=self.webdriver_js_execute_code, selector=None)
+                    self.page.evaluate(self.webdriver_js_execute_code)
+
             except playwright._impl._api_types.TimeoutError as e:
                 context.close()
                 browser.close()
@@ -502,26 +508,28 @@ class base_html_playwright(Fetcher):
                 browser.close()
                 raise PageUnloadable(url=url, status_code=None, message=str(e))
 
+            if response is None:
+                context.close()
+                browser.close()
+                print("Content Fetcher > Response object was none")
+                raise EmptyReply(url=url, status_code=None)
+
+            # Run Browser Steps here
+            self.iterate_browser_steps()
+
             extra_wait = int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)) + self.render_extract_delay
-            self.page.wait_for_timeout(extra_wait * 1000)
-
+            time.sleep(extra_wait)
 
+            self.content = self.page.content()
             self.status_code = response.status
-
-            if self.status_code != 200 and not ignore_status_codes:
-                raise Non200ErrorCodeReceived(url=url, status_code=self.status_code)
-
             if len(self.page.content().strip()) == 0:
                 context.close()
                 browser.close()
                 print("Content Fetcher > Content was empty")
                 raise EmptyReply(url=url, status_code=response.status)
 
-            # Run Browser Steps here
-            if self.browser_steps_get_valid_steps():
-                self.iterate_browser_steps()
-                
-            self.page.wait_for_timeout(extra_wait * 1000)
+            self.status_code = response.status
+            self.headers = response.all_headers()
 
             # So we can find an element on the page where its selector was entered manually (maybe not xPath etc)
             if current_include_filters is not None:
@@ -533,7 +541,6 @@ class base_html_playwright(Fetcher):
                 "async () => {" + self.xpath_element_js.replace('%ELEMENTS%', visualselector_xpath_selectors) + "}")
             self.instock_data = self.page.evaluate("async () => {" + self.instock_data_js + "}")
 
-            self.content = self.page.content()
             # Bug 3 in Playwright screenshot handling
             # Some bug where it gives the wrong screenshot size, but making a request with the clip set first seems to solve it
             # JPEG is better here because the screenshots can be very very large
@@ -548,7 +555,7 @@ class base_html_playwright(Fetcher):
             except Exception as e:
                 context.close()
                 browser.close()
-                raise ScreenshotUnavailable(url=url, status_code=response.status_code)
+                raise ScreenshotUnavailable(url=url, status_code=None)
 
             context.close()
             browser.close()
@@ -607,17 +614,14 @@ class base_html_webdriver(Fetcher):
             is_binary=False):
 
         from selenium import webdriver
-        from selenium.webdriver.chrome.options import Options as ChromeOptions
+        from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
         from selenium.common.exceptions import WebDriverException
         # request_body, request_method unused for now, until some magic in the future happens.
 
-        options = ChromeOptions()
-        if self.proxy:
-            options.proxy = self.proxy
-
         self.driver = webdriver.Remote(
             command_executor=self.command_executor,
-            options=options)
+            desired_capabilities=DesiredCapabilities.CHROME,
+            proxy=self.proxy)
 
         try:
             self.driver.get(url)
@@ -649,11 +653,11 @@ class base_html_webdriver(Fetcher):
     # Does the connection to the webdriver work? run a test connection.
     def is_ready(self):
         from selenium import webdriver
-        from selenium.webdriver.chrome.options import Options as ChromeOptions
+        from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
 
         self.driver = webdriver.Remote(
             command_executor=self.command_executor,
-            options=ChromeOptions())
+            desired_capabilities=DesiredCapabilities.CHROME)
 
         # driver.quit() seems to cause better exceptions
         self.quit()

changedetectionio/forms.py

@@ -22,8 +22,7 @@ from wtforms.validators import ValidationError
 # each select <option data-enabled="enabled-0-0"
 from changedetectionio.blueprint.browser_steps.browser_steps import browser_step_ui_config
 
-from changedetectionio import content_fetcher, html_tools
-
+from changedetectionio import content_fetcher
 from changedetectionio.notification import (
     valid_notification_formats,
 )
@@ -285,10 +284,11 @@ class ValidateListRegex(object):
     def __call__(self, form, field):
 
         for line in field.data:
-            if re.search(html_tools.PERL_STYLE_REGEX, line, re.IGNORECASE):
+            if line[0] == '/' and line[-1] == '/':
+                # Because internally we dont wrap in /
+                line = line.strip('/')
                 try:
-                    regex = html_tools.perl_style_slash_enclosed_regex_to_options(line)
-                    re.compile(regex)
+                    re.compile(line)
                 except re.error:
                     message = field.gettext('RegEx \'%s\' is not a valid regular expression.')
                     raise ValidationError(message % (line))

changedetectionio/html_tools.py

@@ -98,7 +98,10 @@ def xpath_filter(xpath_filter, html_content, append_pretty_line_formatting=False
         elif type(element) == etree._ElementUnicodeResult:
             html_block += str(element)
         else:
-            html_block += etree.tostring(element, pretty_print=True).decode('utf-8')
+            if not is_rss:
+                html_block += etree.tostring(element, pretty_print=True).decode('utf-8')
+            else:
+                html_block += f"<div>{element.text}</div>\n"
 
     return html_block
 
@@ -271,7 +274,7 @@ def cdata_in_document_to_text(html_content: str, render_anchor_tag_content=False
     pattern = '<!\[CDATA\[(\s*(?:.(?<!\]\]>)\s*)*)\]\]>'
     def repl(m):
         text = m.group(1)
-        return xml_escape(html_to_text(html_content=text)).strip()
+        return xml_escape(html_to_text(html_content=text))
 
     return re.sub(pattern, repl, html_content)
 
@@ -292,8 +295,7 @@ def html_to_text(html_content: str, render_anchor_tag_content=False, is_rss=Fals
     #  extracting this content
     if render_anchor_tag_content:
         parser_config = ParserConfig(
-            annotation_rules={"a": ["hyperlink"]},
-            display_links=True
+            annotation_rules={"a": ["hyperlink"]}, display_links=True
         )
     # otherwise set config to None/default
     else:
@@ -301,12 +303,13 @@ def html_to_text(html_content: str, render_anchor_tag_content=False, is_rss=Fals
 
     # RSS Mode - Inscriptis will treat `title` as something else.
     # Make it as a regular block display element (//item/title)
-    # This is a bit of a hack - the real way it to use XSLT to convert it to HTML #1874
     if is_rss:
-        html_content = re.sub(r'<title([\s>])', r'<h1\1', html_content)
-        html_content = re.sub(r'</title>', r'</h1>', html_content)
-
-    text_content = get_text(html_content, config=parser_config)
+        css = CSS_PROFILES['strict'].copy()
+        css['title'] = HtmlElement(display=Display.block)
+        text_content = get_text(html_content, ParserConfig(css=css))
+    else:
+        # get text and annotations via inscriptis
+        text_content = get_text(html_content, config=parser_config)
 
     return text_content
 

changedetectionio/model/Watch.py

@@ -4,7 +4,6 @@ import os
 import re
 import time
 import uuid
-from pathlib import Path
 
 # Allowable protocols, protects against javascript: etc
 # file:// is further checked by ALLOW_FILE_URI
@@ -19,7 +18,6 @@ from changedetectionio.notification import (
 
 base_config = {
     'body': None,
-    'browser_steps_last_error_step': None,
     'check_unique_lines': False,  # On change-detected, compare against all history if its something new
     'check_count': 0,
     'date_created': None,
@@ -27,7 +25,6 @@ base_config = {
     'extract_text': [],  # Extract text by regex after filters
     'extract_title_as_title': False,
     'fetch_backend': 'system', # plaintext, playwright etc
-    'fetch_time': 0.0,
     'processor': 'text_json_diff', # could be restock_diff or others from .processors
     'filter_failure_notification_send': strtobool(os.getenv('FILTER_FAILURE_NOTIFICATION_SEND_DEFAULT', 'True')),
     'filter_text_added': True,
@@ -492,13 +489,3 @@ class model(dict):
         filepath = os.path.join(self.watch_data_dir, 'last-fetched.br')
         with open(filepath, 'wb') as f:
             f.write(brotli.compress(contents, mode=brotli.MODE_TEXT))
-
-    @property
-    def get_browsersteps_available_screenshots(self):
-        "For knowing which screenshots are available to show the user in BrowserSteps UI"
-        available = []
-        for f in Path(self.watch_data_dir).glob('step_before-*.jpeg'):
-            step_n=re.search(r'step_before-(\d+)', f.name)
-            if step_n:
-                available.append(step_n.group(1))
-        return available

changedetectionio/processors/text_json_diff.py

@@ -274,7 +274,7 @@ class perform_site_check(difference_detection_processor):
                         html_tools.html_to_text(
                             html_content=html_content,
                             render_anchor_tag_content=do_anchor,
-                            is_rss=is_rss # #1874 activate the <title workaround hack
+                            is_rss=is_rss
                         )
 
         # Re #340 - return the content before the 'ignore text' was applied

changedetectionio/static/js/browser-steps.js

@@ -321,14 +321,8 @@ $(document).ready(function () {
             var s = '<div class="control">' + '<a data-step-index=' + i + ' class="pure-button button-secondary button-green button-xsmall apply" >Apply</a>&nbsp;';
             if (i > 0) {
                 // The first step never gets these (Goto-site)
-                s += `<a data-step-index="${i}" class="pure-button button-secondary button-xsmall clear" >Clear</a>&nbsp;` +
-                    `<a data-step-index="${i}" class="pure-button button-secondary button-red button-xsmall remove" >Remove</a>`;
-
-                // if a screenshot is available
-                if (browser_steps_available_screenshots.includes(i.toString())) {
-                    var d = (browser_steps_last_error_step === i+1) ? 'before' : 'after';
-                    s += `&nbsp;<a data-step-index="${i}" class="pure-button button-secondary button-xsmall show-screenshot" title="Show screenshot from last run" data-type="${d}">Pic</a>&nbsp;`;
-                }
+                s += '<a data-step-index=' + i + ' class="pure-button button-secondary button-xsmall clear" >Clear</a>&nbsp;' +
+                    '<a data-step-index=' + i + ' class="pure-button button-secondary button-red button-xsmall remove" >Remove</a>';
             }
             s += '</div>';
             $(this).append(s)
@@ -443,24 +437,6 @@ $(document).ready(function () {
 
     });
 
-    $('ul#browser_steps li .control .show-screenshot').click(function (element) {
-        var step_n = $(event.currentTarget).data('step-index');
-        w = window.open(this.href, "_blank", "width=640,height=480");
-        const t = $(event.currentTarget).data('type');
-
-        const url = browser_steps_fetch_screenshot_image_url + `&step_n=${step_n}&type=${t}`;
-        w.document.body.innerHTML = `<!DOCTYPE html>
-            <html lang="en">
-                <body>
-                    <img src="${url}" style="width: 100%" alt="Browser Step at step ${step_n} from last run." title="Browser Step at step ${step_n} from last run."/>
-                </body>
-        </html>`;
-        w.document.title = `Browser Step at step ${step_n} from last run.`;
-    });
-
-    if (browser_steps_last_error_step) {
-        $("ul#browser_steps>li:nth-child("+browser_steps_last_error_step+")").addClass("browser-step-with-error");
-    }
 
     $("ul#browser_steps select").change(function () {
         set_greyed_state();

changedetectionio/static/js/diff-render.js

@@ -2,7 +2,8 @@ $(document).ready(function () {
     var a = document.getElementById("a");
     var b = document.getElementById("b");
     var result = document.getElementById("result");
-    var inputs;
+    var inputs = document.getElementsByClassName("change");
+    inputs.current = 0;
 
     $('#jump-next-diff').click(function () {
 
@@ -58,6 +59,9 @@ $(document).ready(function () {
         result.textContent = "";
         result.appendChild(fragment);
 
+        // Jump at start
+        inputs.current = 0;
+
         // For nice mouse-over hover/title information
         const removed_current_option = $('#diff-version option:selected')
         if (removed_current_option) {
@@ -71,11 +75,7 @@ $(document).ready(function () {
                 $(this).prop('title', 'Inserted '+inserted_current_option[0].label);
             });
         }
-        // Set the list of possible differences to jump to
-        inputs = document.querySelectorAll('#diff-ui .change')
-        // Set the "current" diff pointer
-        inputs.current = 0;
-        // Goto diff
+
         $('#jump-next-diff').click();
     }
 

changedetectionio/static/styles/scss/parts/_browser-steps.scss

@@ -6,10 +6,6 @@
   }
 
   li {
-    &.browser-step-with-error {
-      background-color: #ffd6d6;
-      border-radius: 4px;
-    }
     &:not(:first-child) {
       &:hover {
         opacity: 1.0;

changedetectionio/static/styles/scss/parts/_visualselector.scss

@@ -1,28 +0,0 @@
-
-#selector-wrapper {
-  height: 100%;
-  max-height: 70vh;
-  overflow-y: scroll;
-  position: relative;
-
-  //width: 100%;
-  >img {
-    position: absolute;
-    z-index: 4;
-    max-width: 100%;
-  }
-
-  >canvas {
-    position: relative;
-    z-index: 5;
-    max-width: 100%;
-
-    &:hover {
-      cursor: pointer;
-    }
-  }
-}
-
-#selector-current-xpath {
-  font-size: 80%;
-}

changedetectionio/static/styles/scss/styles.scss

@@ -943,7 +943,32 @@ ul {
   }
 }
 
-@import "parts/_visualselector";
+#selector-wrapper {
+  height: 100%;
+  overflow-y: scroll;
+  position: relative;
+
+  //width: 100%;
+  >img {
+    position: absolute;
+    z-index: 4;
+    max-width: 100%;
+  }
+
+  >canvas {
+    position: relative;
+    z-index: 5;
+    max-width: 100%;
+
+    &:hover {
+      cursor: pointer;
+    }
+  }
+}
+
+#selector-current-xpath {
+  font-size: 80%;
+}
 
 #webdriver-override-options {
   input[type="number"] {

changedetectionio/static/styles/styles.css

@@ -26,9 +26,6 @@
   #browser_steps li {
     list-style: decimal;
     padding: 5px; }
-    #browser_steps li.browser-step-with-error {
-      background-color: #ffd6d6;
-      border-radius: 4px; }
     #browser_steps li:not(:first-child):hover {
       opacity: 1.0; }
     #browser_steps li .control {
@@ -983,7 +980,6 @@ ul {
 
 #selector-wrapper {
   height: 100%;
-  max-height: 70vh;
   overflow-y: scroll;
   position: relative; }
   #selector-wrapper > img {

changedetectionio/store.py

@@ -244,16 +244,12 @@ class ChangeDetectionStore:
         import pathlib
 
         self.__data['watching'][uuid].update({
-                'browser_steps_last_error_step' : None,
-                'check_count': 0,
-                'fetch_time' : 0.0,
-                'has_ldjson_price_data': None,
                 'last_checked': 0,
+                'has_ldjson_price_data': None,
                 'last_error': False,
                 'last_notification_error': False,
                 'last_viewed': 0,
                 'previous_md5': False,
-                'previous_md5_before_filters': False,
                 'track_ldjson_price_data': None,
             })
 

changedetectionio/templates/base.html

@@ -85,7 +85,6 @@
               <a href="{{url_for('logout')}}" class="pure-menu-link">LOG OUT</a>
             </li>
           {% endif %}
-          {% if current_user.is_authenticated or not has_password %}
           <li class="pure-menu-item pure-form" id="search-menu-item">
             <!-- We use GET here so it offers people a chance to set bookmarks etc -->
             <form name="searchForm" action="" method="GET">
@@ -96,7 +95,6 @@
               </button>
             </form>
           </li>
-          {% endif %}
           <li class="pure-menu-item">
             <button class="toggle-button" id ="toggle-light-mode" type="button" title="Toggle Light/Dark Mode">
               <span class="visually-hidden">Toggle light/dark mode</span>

changedetectionio/templates/diff.html

@@ -60,7 +60,7 @@
 </div>
 
 <div id="diff-jump">
-    <a id="jump-next-diff" title="Jump to next difference">Jump</a>
+    <a id="jump-next-diff">Jump</a>
 </div>
 
 <script src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>

changedetectionio/templates/edit.html

@@ -4,10 +4,8 @@
 {% from '_common_fields.jinja' import render_common_settings_form %}
 <script src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
 <script>
-    const browser_steps_available_screenshots=JSON.parse('{{ watch.get_browsersteps_available_screenshots|tojson }}');
+
     const browser_steps_config=JSON.parse('{{ browser_steps_config|tojson }}');
-    const browser_steps_fetch_screenshot_image_url="{{url_for('browser_steps.browser_steps_fetch_screenshot_image', uuid=uuid)}}";
-    const browser_steps_last_error_step={{ watch.browser_steps_last_error_step|tojson }};
     const browser_steps_start_url="{{url_for('browser_steps.browsersteps_start_session', uuid=uuid)}}";
     const browser_steps_sync_url="{{url_for('browser_steps.browsersteps_ui_update', uuid=uuid)}}";
 {% if emailprefix %}
@@ -51,7 +49,6 @@
             <li class="tab"><a href="#restock">Restock Detection</a></li>
             {% endif %}
             <li class="tab"><a href="#notifications">Notifications</a></li>
-            <li class="tab"><a href="#stats">Stats</a></li>
         </ul>
     </div>
 
@@ -444,35 +441,7 @@ Unavailable") }}
                 </fieldset>
             </div>
             {% endif %}
-            <div class="tab-pane-inner" id="stats">
-                <div class="pure-control-group">
-                    <style>
-                    #stats-table tr > td:first-child {
-                        font-weight: bold;
-                    }
-                    </style>
-                    <table class="pure-table" id="stats-table">
-                        <tbody>
-                        <tr>
-                            <td>Check count</td>
-                            <td>{{ watch.check_count }}</td>
-                        </tr>
-                        <tr>
-                            <td>Consecutive filter failures</td>
-                            <td>{{ watch.consecutive_filter_failures }}</td>
-                        </tr>
-                        <tr>
-                            <td>History length</td>
-                            <td>{{ watch.history|length }}</td>
-                        </tr>
-                        <tr>
-                            <td>Last fetch time</td>
-                            <td>{{ watch.fetch_time }}s</td>
-                        </tr>
-                        </tbody>
-                    </table>
-                </div>
-            </div>
+
             <div id="actions">
                 <div class="pure-control-group">
                     {{ render_button(form.save_button) }}

changedetectionio/templates/watch-overview.html

@@ -178,18 +178,13 @@
             </tbody>
         </table>
         <ul id="post-list-buttons">
-            {% if errored_count %}
-            <li>
-                <a href="{{url_for('index', with_errors=1, tag=request.args.get('tag')) }}" class="pure-button button-tag button-error ">With errors ({{ errored_count }})</a>
-            </li>
-            {% endif %}
             {% if has_unviewed %}
             <li>
-                <a href="{{url_for('mark_all_viewed',with_errors=request.args.get('with_errors',0)) }}" class="pure-button button-tag ">Mark all viewed</a>
+                <a href="{{url_for('mark_all_viewed', tag=request.args.get('tag')) }}" class="pure-button button-tag ">Mark all viewed</a>
             </li>
             {% endif %}
             <li>
-               <a href="{{ url_for('form_watch_checknow', tag=active_tag, with_errors=request.args.get('with_errors',0)) }}" class="pure-button button-tag ">Recheck
+               <a href="{{ url_for('form_watch_checknow', tag=active_tag) }}" class="pure-button button-tag ">Recheck
                 all {% if active_tag%} in "{{tags[active_tag].title}}"{%endif%}</a>
             </li>
             <li>

changedetectionio/tests/test_extract_regex.py

@@ -202,35 +202,3 @@ def test_check_filter_and_regex_extract(client, live_server):
 
     # Should not be here
     assert b'Some text that did change' not in res.data
-
-
-
-def test_regex_error_handling(client, live_server):
-
-    #live_server_setup(live_server)
-
-    # Add our URL to the import page
-    test_url = url_for('test_endpoint', _external=True)
-    res = client.post(
-        url_for("import_page"),
-        data={"urls": test_url},
-        follow_redirects=True
-    )
-    assert b"1 Imported" in res.data
-
-    ### test regex error handling
-    res = client.post(
-        url_for("edit_page", uuid="first"),
-        data={"extract_text": '/something bad\d{3/XYZ',
-              "url": test_url,
-              "fetch_backend": "html_requests"},
-        follow_redirects=True
-    )
-
-    with open('/tmp/fuck.html', 'wb') as f:
-        f.write(res.data)
-
-    assert b'is not a valid regular expression.' in res.data
-
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
-    assert b'Deleted' in res.data

changedetectionio/tests/test_rss.py

@@ -118,7 +118,7 @@ def test_basic_cdata_rss_markup(client, live_server):
     res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
 
 def test_rss_xpath_filtering(client, live_server):
-    #live_server_setup(live_server)
+#    live_server_setup(live_server)
 
     set_original_cdata_xml()
 
@@ -154,9 +154,6 @@ def test_rss_xpath_filtering(client, live_server):
     )
     assert b'CDATA' not in res.data
     assert b'<![' not in res.data
-    # #1874  All but the first <title was getting selected
-    # Convert any HTML with just a top level <title> to <h1> to be sure title renders
-
     assert b'Hackers can access your computer' in res.data # Should ONLY be selected by the xpath
     assert b'Some other title' in res.data  # Should ONLY be selected by the xpath
     assert b'The days of Terminator' not in res.data # Should NOT be selected by the xpath

changedetectionio/tests/visualselector/test_fetch_data.py

@@ -1,19 +1,18 @@
 #!/usr/bin/python3
 
 import time
-import os
 from flask import url_for
 from ..util import live_server_setup, wait_for_all_checks, extract_UUID_from_client
 
-def test_setup(client, live_server):
-    live_server_setup(live_server)
-
 # Add a site in paused mode, add an invalid filter, we should still have visual selector data ready
 def test_visual_selector_content_ready(client, live_server):
     import os
     import json
 
     assert os.getenv('PLAYWRIGHT_DRIVER_URL'), "Needs PLAYWRIGHT_DRIVER_URL set for this test"
+    time.sleep(1)
+    live_server_setup(live_server)
+
 
     # Add our URL to the import page, because the docker container (playwright/selenium) wont be able to connect to our usual test url
     test_url = "https://changedetection.io/ci-test/test-runjs.html"
@@ -61,75 +60,4 @@ def test_visual_selector_content_ready(client, live_server):
         follow_redirects=True
     )
     assert b'notification_screenshot' in res.data
-    client.get(
-        url_for("form_delete", uuid="all"),
-        follow_redirects=True
-    )
 
-def test_basic_browserstep(client, live_server):
-
-    assert os.getenv('PLAYWRIGHT_DRIVER_URL'), "Needs PLAYWRIGHT_DRIVER_URL set for this test"
-    #live_server_setup(live_server)
-
-    # Add our URL to the import page, because the docker container (playwright/selenium) wont be able to connect to our usual test url
-    test_url = "https://changedetection.io/ci-test/test-runjs.html"
-
-    res = client.post(
-        url_for("form_quick_watch_add"),
-        data={"url": test_url, "tags": '', 'edit_and_watch_submit_button': 'Edit > Watch'},
-        follow_redirects=True
-    )
-    assert b"Watch added in Paused state, saving will unpause" in res.data
-
-    res = client.post(
-        url_for("edit_page", uuid="first", unpause_on_save=1),
-        data={
-              "url": test_url,
-              "tags": "",
-              "headers": "",
-              'fetch_backend': "html_webdriver",
-              'browser_steps-0-operation': 'Goto site',
-              'browser_steps-1-operation': 'Click element',
-              'browser_steps-1-selector': 'button[name=test-button]',
-              'browser_steps-1-optional_value': ''
-        },
-        follow_redirects=True
-    )
-    assert b"unpaused" in res.data
-    wait_for_all_checks(client)
-
-    uuid = extract_UUID_from_client(client)
-
-    # Check HTML conversion detected and workd
-    res = client.get(
-        url_for("preview_page", uuid=uuid),
-        follow_redirects=True
-    )
-    assert b"This text should be removed" not in res.data
-    assert b"I smell JavaScript because the button was pressed" in res.data
-
-    # now test for 404 errors
-    res = client.post(
-        url_for("edit_page", uuid=uuid, unpause_on_save=1),
-        data={
-              "url": "https://changedetection.io/404",
-              "tags": "",
-              "headers": "",
-              'fetch_backend': "html_webdriver",
-              'browser_steps-0-operation': 'Goto site',
-              'browser_steps-1-operation': 'Click element',
-              'browser_steps-1-selector': 'button[name=test-button]',
-              'browser_steps-1-optional_value': ''
-        },
-        follow_redirects=True
-    )
-    assert b"unpaused" in res.data
-    wait_for_all_checks(client)
-
-    res = client.get(url_for("index"))
-    assert b'Error - 404' in res.data
-
-    client.get(
-        url_for("form_delete", uuid="all"),
-        follow_redirects=True
-    )

changedetectionio/update_worker.py

@@ -238,9 +238,7 @@ class update_worker(threading.Thread):
                             # Used as a default and also by some tests
                             update_handler = text_json_diff.perform_site_check(datastore=self.datastore)
 
-                        self.datastore.data['watching'][uuid]['browser_steps_last_error_step'] = None
                         changed_detected, update_obj, contents = update_handler.run(uuid, skip_when_checksum_same=queued_item_data.item.get('skip_when_checksum_same'))
-
                         # Re #342
                         # In Python 3, all strings are sequences of Unicode characters. There is a bytes type that holds raw bytes.
                         # We then convert/.decode('utf-8') for the notification etc
@@ -326,13 +324,8 @@ class update_worker(threading.Thread):
                         if not self.datastore.data['watching'].get(uuid):
                             continue
 
-                        error_step = e.step_n + 1
-                        err_text = f"Warning, browser step at position {error_step} could not run, target not found, check the watch, add a delay if necessary, view Browser Steps to see screenshot at that step"
-                        self.datastore.update_watch(uuid=uuid,
-                                                    update_obj={'last_error': err_text,
-                                                                'browser_steps_last_error_step': error_step
-                                                                }
-                                                    )
+                        err_text = "Warning, browser step at position {} could not run, target not found, check the watch, add a delay if necessary.".format(e.step_n+1)
+                        self.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text})
 
 
                         if self.datastore.data['watching'][uuid].get('filter_failure_notification_send', False):

requirements.txt

@@ -49,7 +49,8 @@ beautifulsoup4
 # XPath filtering, lxml is required by bs4 anyway, but put it here to be safe.
 lxml
 
-selenium~=4.14.0
+# 3.141 was missing socksVersion, 3.150 was not in pypi, so we try 4.1.0
+selenium~=4.1.0
 
 # https://stackoverflow.com/questions/71652965/importerror-cannot-import-name-safe-str-cmp-from-werkzeug-security/71653849#71653849
 # ImportError: cannot import name 'safe_str_cmp' from 'werkzeug.security'
@@ -62,8 +63,7 @@ jinja2-time
 
 # https://peps.python.org/pep-0508/#environment-markers
 # https://github.com/dgtlmoon/changedetection.io/pull/1009
-jq~=1.3; python_version >= "3.8" and sys_platform == "darwin"
-jq~=1.3; python_version >= "3.8" and sys_platform == "linux"
+jq~=1.3 ;python_version >= "3.8" and sys_platform == "linux"
 
 # Any current modern version, required so far for screenshot PNG->JPEG conversion but will be used more in the future
 pillow