More W3C validation fixes

.github/workflows/pypi-release.yml

@@ -45,12 +45,8 @@ jobs:
     - name: Test that the basic pip built package runs without error
       run: |
         set -ex
-        ls -alR 
-        
-        # Find and install the first .whl file
-        find dist -type f -name "*.whl" -exec pip3 install {} \; -quit
+        pip3 install dist/changedetection.io*.whl
         changedetection.io -d /tmp -p 10000 &
-        
         sleep 3
         curl --retry-connrefused --retry 6 http://127.0.0.1:10000/static/styles/pure-min.css >/dev/null
         curl --retry-connrefused --retry 6 http://127.0.0.1:10000/ >/dev/null

changedetectionio/__init__.py

@@ -2,7 +2,7 @@
 
 # Read more https://github.com/dgtlmoon/changedetection.io/wiki
 
-__version__ = '0.49.3'
+__version__ = '0.49.1'
 
 from changedetectionio.strtobool import strtobool
 from json.decoder import JSONDecodeError

changedetectionio/blueprint/browser_steps/__init__.py

@@ -22,10 +22,7 @@ from loguru import logger
 
 browsersteps_sessions = {}
 io_interface_context = None
-import json
-import base64
-import hashlib
-from flask import Response
+
 
 def construct_blueprint(datastore: ChangeDetectionStore):
     browser_steps_blueprint = Blueprint('browser_steps', __name__, template_folder="templates")
@@ -163,13 +160,14 @@ def construct_blueprint(datastore: ChangeDetectionStore):
         if not browsersteps_sessions.get(browsersteps_session_id):
             return make_response('No session exists under that ID', 500)
 
-        is_last_step = False
+
         # Actions - step/apply/etc, do the thing and return state
         if request.method == 'POST':
             # @todo - should always be an existing session
             step_operation = request.form.get('operation')
             step_selector = request.form.get('selector')
             step_optional_value = request.form.get('optional_value')
+            step_n = int(request.form.get('step_n'))
             is_last_step = strtobool(request.form.get('is_last_step'))
 
             # @todo try.. accept.. nice errors not popups..
@@ -184,6 +182,16 @@ def construct_blueprint(datastore: ChangeDetectionStore):
                 # Try to find something of value to give back to the user
                 return make_response(str(e).splitlines()[0], 401)
 
+            # Get visual selector ready/update its data (also use the current filter info from the page?)
+            # When the last 'apply' button was pressed
+            # @todo this adds overhead because the xpath selection is happening twice
+            u = browsersteps_sessions[browsersteps_session_id]['browserstepper'].page.url
+            if is_last_step and u:
+                (screenshot, xpath_data) = browsersteps_sessions[browsersteps_session_id]['browserstepper'].request_visualselector_data()
+                watch = datastore.data['watching'].get(uuid)
+                if watch:
+                    watch.save_screenshot(screenshot=screenshot)
+                    watch.save_xpath_data(data=xpath_data)
 
 #        if not this_session.page:
 #            cleanup_playwright_session()
@@ -191,35 +199,31 @@ def construct_blueprint(datastore: ChangeDetectionStore):
 
         # Screenshots and other info only needed on requesting a step (POST)
         try:
-            (screenshot, xpath_data) = browsersteps_sessions[browsersteps_session_id]['browserstepper'].get_current_state()
-            if is_last_step:
-                watch = datastore.data['watching'].get(uuid)
-                u = browsersteps_sessions[browsersteps_session_id]['browserstepper'].page.url
-                if watch and u:
-                    watch.save_screenshot(screenshot=screenshot)
-                    watch.save_xpath_data(data=xpath_data)
-
+            state = browsersteps_sessions[browsersteps_session_id]['browserstepper'].get_current_state()
         except playwright._impl._api_types.Error as e:
             return make_response("Browser session ran out of time :( Please reload this page."+str(e), 401)
-        except Exception as e:
-            return make_response("Error fetching screenshot and element data - " + str(e), 401)
 
-        # SEND THIS BACK TO THE BROWSER
+        # Use send_file() which is way faster than read/write loop on bytes
+        import json
+        from tempfile import mkstemp
+        from flask import send_file
+        tmp_fd, tmp_file = mkstemp(text=True, suffix=".json", prefix="changedetectionio-")
 
-        output = {
-            "screenshot": f"data:image/jpeg;base64,{base64.b64encode(screenshot).decode('ascii')}",
-            "xpath_data": xpath_data,
-            "session_age_start": browsersteps_sessions[browsersteps_session_id]['browserstepper'].age_start,
-            "browser_time_remaining": round(remaining)
-        }
-        json_data = json.dumps(output)
+        output = json.dumps({'screenshot': "data:image/jpeg;base64,{}".format(
+            base64.b64encode(state[0]).decode('ascii')),
+            'xpath_data': state[1],
+            'session_age_start': browsersteps_sessions[browsersteps_session_id]['browserstepper'].age_start,
+            'browser_time_remaining': round(remaining)
+        })
 
-        # Generate an ETag (hash of the response body)
-        etag_hash = hashlib.md5(json_data.encode('utf-8')).hexdigest()
+        with os.fdopen(tmp_fd, 'w') as f:
+            f.write(output)
 
-        # Create the response with ETag
-        response = Response(json_data, mimetype="application/json; charset=UTF-8")
-        response.set_etag(etag_hash)
+        response = make_response(send_file(path_or_file=tmp_file,
+                                           mimetype='application/json; charset=UTF-8',
+                                           etag=True))
+        # No longer needed
+        os.unlink(tmp_file)
 
         return response
 

changedetectionio/blueprint/browser_steps/browser_steps.py

@@ -1,15 +1,14 @@
+#!/usr/bin/env python3
+
 import os
 import time
 import re
 from random import randint
 from loguru import logger
 
-from changedetectionio.content_fetchers.helpers import capture_stitched_together_full_page, SCREENSHOT_SIZE_STITCH_THRESHOLD
 from changedetectionio.content_fetchers.base import manage_user_agent
 from changedetectionio.safe_jinja import render as jinja_render
 
-
-
 # Two flags, tell the JS which of the "Selector" or "Value" field should be enabled in the front end
 # 0- off, 1- on
 browser_step_ui_config = {'Choose one': '0 0',
@@ -32,7 +31,6 @@ browser_step_ui_config = {'Choose one': '0 0',
 #                          'Extract text and use as filter': '1 0',
                           'Goto site': '0 0',
                           'Goto URL': '0 1',
-                          'Make all child elements visible': '1 0',
                           'Press Enter': '0 0',
                           'Select by label': '1 1',
                           'Scroll down': '0 0',
@@ -40,7 +38,6 @@ browser_step_ui_config = {'Choose one': '0 0',
                           'Wait for seconds': '0 1',
                           'Wait for text': '0 1',
                           'Wait for text in element': '1 1',
-                          'Remove elements': '1 0',
                           #                          'Press Page Down': '0 0',
                           #                          'Press Page Up': '0 0',
                           # weird bug, come back to it later
@@ -195,24 +192,6 @@ class steppable_browser_interface():
     def action_uncheck_checkbox(self, selector, value):
         self.page.locator(selector).uncheck(timeout=self.action_timeout)
 
-    def action_remove_elements(self, selector, value):
-        """Removes all elements matching the given selector from the DOM."""
-        self.page.locator(selector).evaluate_all("els => els.forEach(el => el.remove())")
-
-    def action_make_all_child_elements_visible(self, selector, value):
-        """Recursively makes all child elements inside the given selector fully visible."""
-        self.page.locator(selector).locator("*").evaluate_all("""
-            els => els.forEach(el => {
-                el.style.display = 'block';   // Forces it to be displayed
-                el.style.visibility = 'visible';   // Ensures it's not hidden
-                el.style.opacity = '1';   // Fully opaque
-                el.style.position = 'relative';   // Avoids 'absolute' hiding
-                el.style.height = 'auto';   // Expands collapsed elements
-                el.style.width = 'auto';   // Ensures full visibility
-                el.removeAttribute('hidden');   // Removes hidden attribute
-                el.classList.remove('hidden', 'd-none');  // Removes common CSS hidden classes
-            })
-        """)
 
 # Responsible for maintaining a live 'context' with the chrome CDP
 # @todo - how long do contexts live for anyway?
@@ -280,7 +259,6 @@ class browsersteps_live_ui(steppable_browser_interface):
         logger.debug(f"Time to browser setup {time.time()-now:.2f}s")
         self.page.wait_for_timeout(1 * 1000)
 
-
     def mark_as_closed(self):
         logger.debug("Page closed, cleaning up..")
 
@@ -298,30 +276,39 @@ class browsersteps_live_ui(steppable_browser_interface):
         now = time.time()
         self.page.wait_for_timeout(1 * 1000)
 
+        # The actual screenshot
+        screenshot = self.page.screenshot(type='jpeg', full_page=True, quality=40)
 
-        full_height = self.page.evaluate("document.documentElement.scrollHeight")
-
-        if full_height >= SCREENSHOT_SIZE_STITCH_THRESHOLD:
-            logger.warning(f"Page full Height: {full_height}px longer than {SCREENSHOT_SIZE_STITCH_THRESHOLD}px, using 'stitched screenshot method'.")
-            screenshot = capture_stitched_together_full_page(self.page)
-        else:
-            screenshot = self.page.screenshot(type='jpeg', full_page=True, quality=40)
-
-        logger.debug(f"Time to get screenshot from browser {time.time() - now:.2f}s")
-
-        now = time.time()
         self.page.evaluate("var include_filters=''")
         # Go find the interactive elements
         # @todo in the future, something smarter that can scan for elements with .click/focus etc event handlers?
         elements = 'a,button,input,select,textarea,i,th,td,p,li,h1,h2,h3,h4,div,span'
         xpath_element_js = xpath_element_js.replace('%ELEMENTS%', elements)
-
         xpath_data = self.page.evaluate("async () => {" + xpath_element_js + "}")
         # So the JS will find the smallest one first
         xpath_data['size_pos'] = sorted(xpath_data['size_pos'], key=lambda k: k['width'] * k['height'], reverse=True)
-        logger.debug(f"Time to scrape xpath element data in browser {time.time()-now:.2f}s")
-
+        logger.debug(f"Time to complete get_current_state of browser {time.time()-now:.2f}s")
+        # except
         # playwright._impl._api_types.Error: Browser closed.
         # @todo show some countdown timer?
         return (screenshot, xpath_data)
 
+    def request_visualselector_data(self):
+        """
+        Does the same that the playwright operation in content_fetcher does
+        This is used to just bump the VisualSelector data so it' ready to go if they click on the tab
+        @todo refactor and remove duplicate code, add include_filters
+        :param xpath_data:
+        :param screenshot:
+        :param current_include_filters:
+        :return:
+        """
+        import importlib.resources
+        self.page.evaluate("var include_filters=''")
+        xpath_element_js = importlib.resources.files("changedetectionio.content_fetchers.res").joinpath('xpath_element_scraper.js').read_text()
+        from changedetectionio.content_fetchers import visualselector_xpath_selectors
+        xpath_element_js = xpath_element_js.replace('%ELEMENTS%', visualselector_xpath_selectors)
+        xpath_data = self.page.evaluate("async () => {" + xpath_element_js + "}")
+        screenshot = self.page.screenshot(type='jpeg', full_page=True, quality=int(os.getenv("SCREENSHOT_QUALITY", 72)))
+
+        return (screenshot, xpath_data)

changedetectionio/content_fetchers/helpers.py

@@ -1,104 +0,0 @@
-
-# Pages with a vertical height longer than this will use the 'stitch together' method.
-
-# - Many GPUs have a max texture size of 16384x16384px (or lower on older devices).
-# - If a page is taller than ~8000–10000px, it risks exceeding GPU memory limits.
-# - This is especially important on headless Chromium, where Playwright may fail to allocate a massive full-page buffer.
-
-
-# The size at which we will switch to stitching method
-SCREENSHOT_SIZE_STITCH_THRESHOLD=8000
-
-from loguru import logger
-
-def capture_stitched_together_full_page(page):
-    import io
-    import os
-    import time
-    from PIL import Image, ImageDraw, ImageFont
-
-    MAX_TOTAL_HEIGHT = SCREENSHOT_SIZE_STITCH_THRESHOLD*4  # Maximum total height for the final image (When in stitch mode)
-    MAX_CHUNK_HEIGHT = 4000  # Height per screenshot chunk
-    WARNING_TEXT_HEIGHT = 20  # Height of the warning text overlay
-
-    # Save the original viewport size
-    original_viewport = page.viewport_size
-    now = time.time()
-
-    try:
-        viewport = page.viewport_size
-        page_height = page.evaluate("document.documentElement.scrollHeight")
-
-        # Limit the total capture height
-        capture_height = min(page_height, MAX_TOTAL_HEIGHT)
-
-        images = []
-        total_captured_height = 0
-
-        for offset in range(0, capture_height, MAX_CHUNK_HEIGHT):
-            # Ensure we do not exceed the total height limit
-            chunk_height = min(MAX_CHUNK_HEIGHT, MAX_TOTAL_HEIGHT - total_captured_height)
-
-            # Adjust viewport size for this chunk
-            page.set_viewport_size({"width": viewport["width"], "height": chunk_height})
-
-            # Scroll to the correct position
-            page.evaluate(f"window.scrollTo(0, {offset})")
-
-            # Capture screenshot chunk
-            screenshot_bytes = page.screenshot(type='jpeg', quality=int(os.getenv("SCREENSHOT_QUALITY", 30)))
-            images.append(Image.open(io.BytesIO(screenshot_bytes)))
-
-            total_captured_height += chunk_height
-
-            # Stop if we reached the maximum total height
-            if total_captured_height >= MAX_TOTAL_HEIGHT:
-                break
-
-        # Create the final stitched image
-        stitched_image = Image.new('RGB', (viewport["width"], total_captured_height))
-        y_offset = 0
-
-        # Stitch the screenshot chunks together
-        for img in images:
-            stitched_image.paste(img, (0, y_offset))
-            y_offset += img.height
-
-        logger.debug(f"Screenshot stitched together in {time.time()-now:.2f}s")
-
-        # Overlay warning text if the screenshot was trimmed
-        if page_height > MAX_TOTAL_HEIGHT:
-            draw = ImageDraw.Draw(stitched_image)
-            warning_text = f"WARNING: Screenshot was {page_height}px but trimmed to {MAX_TOTAL_HEIGHT}px because it was too long"
-
-            # Load font (default system font if Arial is unavailable)
-            try:
-                font = ImageFont.truetype("arial.ttf", WARNING_TEXT_HEIGHT)  # Arial (Windows/Mac)
-            except IOError:
-                font = ImageFont.load_default()  # Default font if Arial not found
-
-            # Get text bounding box (correct method for newer Pillow versions)
-            text_bbox = draw.textbbox((0, 0), warning_text, font=font)
-            text_width = text_bbox[2] - text_bbox[0]  # Calculate text width
-            text_height = text_bbox[3] - text_bbox[1]  # Calculate text height
-
-            # Define background rectangle (top of the image)
-            draw.rectangle([(0, 0), (viewport["width"], WARNING_TEXT_HEIGHT)], fill="white")
-
-            # Center text horizontally within the warning area
-            text_x = (viewport["width"] - text_width) // 2
-            text_y = (WARNING_TEXT_HEIGHT - text_height) // 2
-
-            # Draw the warning text in red
-            draw.text((text_x, text_y), warning_text, fill="red", font=font)
-
-        # Save or return the final image
-        output = io.BytesIO()
-        stitched_image.save(output, format="JPEG", quality=int(os.getenv("SCREENSHOT_QUALITY", 30)))
-        screenshot = output.getvalue()
-
-    finally:
-        # Restore the original viewport size
-        page.set_viewport_size(original_viewport)
-
-    return screenshot

changedetectionio/content_fetchers/playwright.py

@@ -4,7 +4,6 @@ from urllib.parse import urlparse
 
 from loguru import logger
 
-from changedetectionio.content_fetchers.helpers import capture_stitched_together_full_page, SCREENSHOT_SIZE_STITCH_THRESHOLD
 from changedetectionio.content_fetchers.base import Fetcher, manage_user_agent
 from changedetectionio.content_fetchers.exceptions import PageUnloadable, Non200ErrorCodeReceived, EmptyReply, ScreenshotUnavailable
 
@@ -90,7 +89,6 @@ class fetcher(Fetcher):
         from playwright.sync_api import sync_playwright
         import playwright._impl._errors
         from changedetectionio.content_fetchers import visualselector_xpath_selectors
-        import time
         self.delete_browser_steps_screenshots()
         response = None
 
@@ -181,7 +179,6 @@ class fetcher(Fetcher):
 
             self.page.wait_for_timeout(extra_wait * 1000)
 
-            now = time.time()
             # So we can find an element on the page where its selector was entered manually (maybe not xPath etc)
             if current_include_filters is not None:
                 self.page.evaluate("var include_filters={}".format(json.dumps(current_include_filters)))
@@ -193,8 +190,6 @@ class fetcher(Fetcher):
             self.instock_data = self.page.evaluate("async () => {" + self.instock_data_js + "}")
 
             self.content = self.page.content()
-            logger.debug(f"Time to scrape xpath element data in browser {time.time() - now:.2f}s")
-
             # Bug 3 in Playwright screenshot handling
             # Some bug where it gives the wrong screenshot size, but making a request with the clip set first seems to solve it
             # JPEG is better here because the screenshots can be very very large
@@ -204,15 +199,10 @@ class fetcher(Fetcher):
             # acceptable screenshot quality here
             try:
                 # The actual screenshot - this always base64 and needs decoding! horrible! huge CPU usage
-                full_height = self.page.evaluate("document.documentElement.scrollHeight")
-
-                if full_height >= SCREENSHOT_SIZE_STITCH_THRESHOLD:
-                    logger.warning(
-                        f"Page full Height: {full_height}px longer than {SCREENSHOT_SIZE_STITCH_THRESHOLD}px, using 'stitched screenshot method'.")
-                    self.screenshot = capture_stitched_together_full_page(self.page)
-                else:
-                    self.screenshot = self.page.screenshot(type='jpeg', full_page=True, quality=int(os.getenv("SCREENSHOT_QUALITY", 30)))
-
+                self.screenshot = self.page.screenshot(type='jpeg',
+                                                       full_page=True,
+                                                       quality=int(os.getenv("SCREENSHOT_QUALITY", 72)),
+                                                       )
             except Exception as e:
                 # It's likely the screenshot was too long/big and something crashed
                 raise ScreenshotUnavailable(url=url, status_code=self.status_code)

changedetectionio/content_fetchers/res/xpath_element_scraper.js

@@ -41,7 +41,7 @@ const findUpTag = (el) => {
 
     //  Strategy 1: If it's an input, with name, and there's only one, prefer that
     if (el.name !== undefined && el.name.length) {
-        var proposed = el.tagName + "[name=\"" + CSS.escape(el.name) + "\"]";
+        var proposed = el.tagName + "[name=" + el.name + "]";
         var proposed_element = window.document.querySelectorAll(proposed);
         if (proposed_element.length) {
             if (proposed_element.length === 1) {
@@ -102,15 +102,13 @@ function collectVisibleElements(parent, visibleElements) {
     const children = parent.children;
     for (let i = 0; i < children.length; i++) {
         const child = children[i];
-        const computedStyle = window.getComputedStyle(child);
-
         if (
             child.nodeType === Node.ELEMENT_NODE &&
-            computedStyle.display !== 'none' &&
-            computedStyle.visibility !== 'hidden' &&
+            window.getComputedStyle(child).display !== 'none' &&
+            window.getComputedStyle(child).visibility !== 'hidden' &&
             child.offsetWidth >= 0 &&
             child.offsetHeight >= 0 &&
-            computedStyle.contentVisibility !== 'hidden'
+            window.getComputedStyle(child).contentVisibility !== 'hidden'
         ) {
             // If the child is an element and is visible, recursively collect visible elements
             collectVisibleElements(child, visibleElements);
@@ -175,7 +173,6 @@ visibleElementsArray.forEach(function (element) {
 
     // Try to identify any possible currency amounts "Sale: 4000" or "Sale now 3000 Kc", can help with the training.
     const hasDigitCurrency = (/\d/.test(text.slice(0, 6)) || /\d/.test(text.slice(-6)) ) &&  /([€£$¥₩₹]|USD|AUD|EUR|Kč|kr|SEK|,–)/.test(text) ;
-    const computedStyle = window.getComputedStyle(element);
 
     size_pos.push({
         xpath: xpath_result,
@@ -187,10 +184,10 @@ visibleElementsArray.forEach(function (element) {
         tagName: (element.tagName) ? element.tagName.toLowerCase() : '',
         // tagtype used by Browser Steps
         tagtype: (element.tagName.toLowerCase() === 'input' && element.type) ? element.type.toLowerCase() : '',
-        isClickable: computedStyle.cursor === "pointer",
+        isClickable: window.getComputedStyle(element).cursor === "pointer",
         // Used by the keras trainer
-        fontSize: computedStyle.getPropertyValue('font-size'),
-        fontWeight: computedStyle.getPropertyValue('font-weight'),
+        fontSize: window.getComputedStyle(element).getPropertyValue('font-size'),
+        fontWeight: window.getComputedStyle(element).getPropertyValue('font-weight'),
         hasDigitCurrency: hasDigitCurrency,
         label: label,
     });

changedetectionio/flask_app.py

@@ -875,14 +875,14 @@ def changedetection_app(config=None, datastore_o=None):
 
             system_uses_webdriver = datastore.data['settings']['application']['fetch_backend'] == 'html_webdriver'
 
-            watch_uses_webdriver = False
+            is_html_webdriver = False
             if (watch.get('fetch_backend') == 'system' and system_uses_webdriver) or watch.get('fetch_backend') == 'html_webdriver' or watch.get('fetch_backend', '').startswith('extra_browser_'):
-                watch_uses_webdriver = True
+                is_html_webdriver = True
 
             from zoneinfo import available_timezones
 
             # Only works reliably with Playwright
-
+            visualselector_enabled = os.getenv('PLAYWRIGHT_DRIVER_URL', False) and is_html_webdriver
             template_args = {
                 'available_processors': processors.available_processors(),
                 'available_timezones': sorted(available_timezones()),
@@ -895,13 +895,14 @@ def changedetection_app(config=None, datastore_o=None):
                 'has_default_notification_urls': True if len(datastore.data['settings']['application']['notification_urls']) else False,
                 'has_extra_headers_file': len(datastore.get_all_headers_in_textfile_for_watch(uuid=uuid)) > 0,
                 'has_special_tag_options': _watch_has_tag_options_set(watch=watch),
-                'watch_uses_webdriver': watch_uses_webdriver,
+                'is_html_webdriver': is_html_webdriver,
                 'jq_support': jq_support,
                 'playwright_enabled': os.getenv('PLAYWRIGHT_DRIVER_URL', False),
                 'settings_application': datastore.data['settings']['application'],
                 'timezone_default_config': datastore.data['settings']['application'].get('timezone'),
                 'using_global_webdriver_wait': not default['webdriver_delay'],
                 'uuid': uuid,
+                'visualselector_enabled': visualselector_enabled,
                 'watch': watch
             }
 

changedetectionio/forms.py

@@ -171,7 +171,7 @@ class validateTimeZoneName(object):
 
 class ScheduleLimitDaySubForm(Form):
     enabled = BooleanField("not set", default=True)
-    start_time = TimeStringField("Start At", default="00:00", validators=[validators.Optional()])
+    start_time = TimeStringField("Start At", default="00:00", render_kw={"placeholder": "HH:MM"}, validators=[validators.Optional()])
     duration = FormField(TimeDurationForm, label="Run duration")
 
 class ScheduleLimitForm(Form):

changedetectionio/static/images/copy.svg

@@ -1,7 +1,7 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
 <svg
    version="1.1"
-   id="copy"
+   id="Layer_1"
    x="0px"
    y="0px"
    viewBox="0 0 115.77 122.88"

changedetectionio/static/images/email.svg

@@ -6,7 +6,7 @@
    height="7.5005589"
    width="11.248507"
    version="1.1"
-   id="email"
+   id="Layer_1"
    viewBox="0 0 7.1975545 4.7993639"
    xml:space="preserve"
    xmlns="http://www.w3.org/2000/svg"

changedetectionio/static/images/schedule.svg

@@ -1,7 +1,7 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
 <svg
    version="1.1"
-   id="schedule"
+   id="Layer_1"
    x="0px"
    y="0px"
    viewBox="0 0 661.20001 665.40002"

changedetectionio/static/js/browser-steps.js

@@ -221,7 +221,7 @@ $(document).ready(function () {
                     // If you switch to "Click X,y" after an element here is setup, it will give the last co-ords anyway
                     //if (x['isClickable'] || x['tagName'].startsWith('h') || x['tagName'] === 'a' || x['tagName'] === 'button' || x['tagtype'] === 'submit' || x['tagtype'] === 'checkbox' || x['tagtype'] === 'radio' || x['tagtype'] === 'li') {
                         $('select', first_available).val('Click element').change();
-                        $('input[type=text]', first_available).first().val(x['xpath']).focus();
+                        $('input[type=text]', first_available).first().val(x['xpath']);
                         found_something = true;
                     //}
                 }
@@ -305,7 +305,7 @@ $(document).ready(function () {
 
         if ($(this).val() === 'Click X,Y' && last_click_xy['x'] > 0 && $(elem_value).val().length === 0) {
             // @todo handle scale
-            $(elem_value).val(last_click_xy['x'] + ',' + last_click_xy['y']).focus();
+            $(elem_value).val(last_click_xy['x'] + ',' + last_click_xy['y']);
         }
     }).change();
 

changedetectionio/static/js/tabs.js

@@ -1,48 +1,66 @@
-// Rewrite this is a plugin.. is all this JS really 'worth it?'
+(function ($) {
+    $.fn.hashTabs = function (options) {
+        var settings = $.extend({
+            tabContainer: ".tabs ul",
+            tabSelector: "li a",
+            tabContent: ".tab-pane-inner",
+            activeClass: "active",
+            errorClass: ".messages .error",
+            bodyClassToggle: "full-width"
+        }, options);
 
-window.addEventListener('hashchange', function () {
-    var tabs = document.getElementsByClassName('active');
-    while (tabs[0]) {
-        tabs[0].classList.remove('active');
-        document.body.classList.remove('full-width');
-    }
-    set_active_tab();
-}, false);
+        var $tabs = $(settings.tabContainer).find(settings.tabSelector);
 
-var has_errors = document.querySelectorAll(".messages .error");
-if (!has_errors.length) {
-    if (document.location.hash == "") {
-        location.replace(document.querySelector(".tabs ul li:first-child a").hash);
-    } else {
-        set_active_tab();
-    }
-} else {
-    focus_error_tab();
-}
+        function setActiveTab() {
+            var hash = window.location.hash;
+            var $activeTab = $tabs.filter("[href='" + hash + "']");
 
-function set_active_tab() {
-    document.body.classList.remove('full-width');
-    var tab = document.querySelectorAll("a[href='" + location.hash + "']");
-    if (tab.length) {
-        tab[0].parentElement.className = "active";
-    }
+            // Remove active class from all tabs
+            $(settings.tabContainer).find("li").removeClass(settings.activeClass);
 
-}
+            // Add active class to selected tab
+            if ($activeTab.length) {
+                $activeTab.parent().addClass(settings.activeClass);
+            }
 
-function focus_error_tab() {
-    // time to use jquery or vuejs really,
-    // activate the tab with the error
-    var tabs = document.querySelectorAll('.tabs li a'), i;
-    for (i = 0; i < tabs.length; ++i) {
-        var tab_name = tabs[i].hash.replace('#', '');
-        var pane_errors = document.querySelectorAll('#' + tab_name + ' .error')
-        if (pane_errors.length) {
-            document.location.hash = '#' + tab_name;
-            return true;
+            // Show the correct content
+            $(settings.tabContent).hide();
+            if (hash) {
+                $(hash).show();
+            }
         }
-    }
-    return false;
-}
-
+
+        function focusErrorTab() {
+            $tabs.each(function () {
+                var tabName = this.hash.replace("#", "");
+                if ($("#" + tabName).find(settings.errorClass).length) {
+                    window.location.hash = "#" + tabName;
+                    return false; // Stop loop on first error tab
+                }
+            });
+        }
+
+        function initializeTabs() {
+            if ($(settings.errorClass).length) {
+                focusErrorTab();
+            } else if (!window.location.hash) {
+                window.location.replace($tabs.first().attr("href"));
+            } else {
+                setActiveTab();
+            }
+        }
+
+        // Listen for hash changes
+        $(window).on("hashchange", setActiveTab);
+
+        // Initialize on page load
+        initializeTabs();
+
+        return this; // Enable jQuery chaining
+    };
+})(jQuery);
 
 
+$(document).ready(function () {
+    $(".tabs").hashTabs();
+});

changedetectionio/static/styles/scss/parts/_browser-steps.scss

@@ -40,22 +40,19 @@
   }
 }
 
-@media only screen and (min-width: 760px) {
 
-  #browser-steps .flex-wrapper {
-    display: flex;
-    flex-flow: row;
-    height: 70vh;
-    font-size: 80%;
+#browser-steps .flex-wrapper {
+  display: flex;
+  flex-flow: row;
+  height: 70vh;
+  font-size: 80%;
+  #browser-steps-ui {
+    flex-grow: 1;      /* Allow it to grow and fill the available space */
+    flex-shrink: 1;    /* Allow it to shrink if needed */
+    flex-basis: 0;     /* Start with 0 base width so it stretches as much as possible */
+    background-color: #eee;
+    border-radius: 5px;
 
-    #browser-steps-ui {
-      flex-grow: 1; /* Allow it to grow and fill the available space */
-      flex-shrink: 1; /* Allow it to shrink if needed */
-      flex-basis: 0; /* Start with 0 base width so it stretches as much as possible */
-      background-color: #eee;
-      border-radius: 5px;
-
-    }
   }
 
   #browser-steps-fieldlist {
@@ -66,21 +63,15 @@
     padding-left: 1rem;
     overflow-y: scroll;
   }
-
-  /*  this is duplicate :( */
-  #browsersteps-selector-wrapper {
-    height: 100% !important;
-  }
 }
 
 /*  this is duplicate :( */
 #browsersteps-selector-wrapper {
-
+  height: 100%;
   width: 100%;
   overflow-y: scroll;
   position: relative;
-  height: 80vh;
-
+  //width: 100%;
   > img {
     position: absolute;
     max-width: 100%;
@@ -100,6 +91,7 @@
     left: 50%;
     top: 50%;
     transform: translate(-50%, -50%);
+    margin-left: -40px;
     z-index: 100;
     max-width: 350px;
     text-align: center;

changedetectionio/static/styles/scss/styles.scss

@@ -945,15 +945,7 @@ $form-edge-padding: 20px;
 }
 
 .tab-pane-inner {
-
-  &:not(:target) {
-    display: none;
-  }
-
-  &:target {
-    display: block;
-  }
-
+  display: none;
   // doesnt need padding because theres another row of buttons/activity
   padding: 0px;
 }

changedetectionio/static/styles/styles.css

@@ -46,22 +46,21 @@
     #browser_steps li > label {
       display: none; }
 
-@media only screen and (min-width: 760px) {
-  #browser-steps .flex-wrapper {
-    display: flex;
-    flex-flow: row;
-    height: 70vh;
-    font-size: 80%; }
-    #browser-steps .flex-wrapper #browser-steps-ui {
-      flex-grow: 1;
-      /* Allow it to grow and fill the available space */
-      flex-shrink: 1;
-      /* Allow it to shrink if needed */
-      flex-basis: 0;
-      /* Start with 0 base width so it stretches as much as possible */
-      background-color: #eee;
-      border-radius: 5px; }
-  #browser-steps-fieldlist {
+#browser-steps .flex-wrapper {
+  display: flex;
+  flex-flow: row;
+  height: 70vh;
+  font-size: 80%; }
+  #browser-steps .flex-wrapper #browser-steps-ui {
+    flex-grow: 1;
+    /* Allow it to grow and fill the available space */
+    flex-shrink: 1;
+    /* Allow it to shrink if needed */
+    flex-basis: 0;
+    /* Start with 0 base width so it stretches as much as possible */
+    background-color: #eee;
+    border-radius: 5px; }
+  #browser-steps .flex-wrapper #browser-steps-fieldlist {
     flex-grow: 0;
     /* Don't allow it to grow */
     flex-shrink: 0;
@@ -72,16 +71,13 @@
     /* Set a max width to prevent overflow */
     padding-left: 1rem;
     overflow-y: scroll; }
-  /*  this is duplicate :( */
-  #browsersteps-selector-wrapper {
-    height: 100% !important; } }
 
 /*  this is duplicate :( */
 #browsersteps-selector-wrapper {
+  height: 100%;
   width: 100%;
   overflow-y: scroll;
   position: relative;
-  height: 80vh;
   /* nice tall skinny one */ }
   #browsersteps-selector-wrapper > img {
     position: absolute;
@@ -96,6 +92,7 @@
     left: 50%;
     top: 50%;
     transform: translate(-50%, -50%);
+    margin-left: -40px;
     z-index: 100;
     max-width: 350px;
     text-align: center; }
@@ -1162,11 +1159,8 @@ textarea::placeholder {
   border-radius: 5px; }
 
 .tab-pane-inner {
+  display: none;
   padding: 0px; }
-  .tab-pane-inner:not(:target) {
-    display: none; }
-  .tab-pane-inner:target {
-    display: block; }
 
 .beta-logo {
   height: 50px;

changedetectionio/templates/_common_fields.html

@@ -40,7 +40,7 @@
                             </div>
                             <div class="pure-control-group">
                                 {{ render_field(form.notification_body , rows=5, class="notification-body", placeholder=settings_application['notification_body']) }}
-                                <span class="pure-form-message-inline">Body for all notifications &dash; You can use <a target="newwindow" href="https://jinja.palletsprojects.com/en/3.0.x/templates/">Jinja2</a> templating in the notification title, body and URL, and tokens from below.
+                                <span class="pure-form-message-inline">Body for all notifications &dash; You can use <a target="_new" href="https://jinja.palletsprojects.com/en/3.0.x/templates/">Jinja2</a> templating in the notification title, body and URL, and tokens from below.
                                 </span>
 
                             </div>

changedetectionio/templates/_helpers.html

@@ -61,18 +61,6 @@
   {{ field(**kwargs)|safe }}
 {% endmacro %}
 
-{% macro playwright_warning() %}
-    <p><strong>Error - Playwright support for Chrome based fetching is not enabled.</strong> Alternatively try our <a href="https://changedetection.io">very affordable subscription based service which has all this setup for you</a>.</p>
-    <p>You may need to <a href="https://github.com/dgtlmoon/changedetection.io/blob/09ebc6ec6338545bdd694dc6eee57f2e9d2b8075/docker-compose.yml#L31">Enable playwright environment variable</a> and uncomment the <strong>sockpuppetbrowser</strong> in the <a href="https://github.com/dgtlmoon/changedetection.io/blob/master/docker-compose.yml">docker-compose.yml</a> file.</p>
-    <br>
-    <p>(Also Selenium/WebDriver can not extract full page screenshots reliably so Playwright is recommended here)</p>
-
-{% endmacro %}
-
-{% macro only_webdriver_type_watches_warning() %}
-    <p><strong>Sorry, this functionality only works with Playwright/Chrome enabled watches.<br>You need to <a href="#request">Set the fetch method to Playwright/Chrome mode and resave</a> and have the Playwright connection enabled.</strong></p><br>
-{% endmacro %}
-
 {% macro render_time_schedule_form(form, available_timezones, timezone_default_config) %}
     <style>
     .day-schedule *, .day-schedule select {

changedetectionio/templates/edit.html

@@ -1,6 +1,6 @@
 {% extends 'base.html' %}
 {% block content %}
-{% from '_helpers.html' import render_field, render_checkbox_field, render_button, render_time_schedule_form, playwright_warning, only_webdriver_type_watches_warning %}
+{% from '_helpers.html' import render_field, render_checkbox_field, render_button, render_time_schedule_form %}
 {% from '_common_fields.html' import render_common_settings_form %}
 <script src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
 <script src="{{url_for('static_content', group='js', filename='vis.js')}}" defer></script>
@@ -40,7 +40,7 @@
 
     <div class="tabs collapsable">
         <ul>
-            <li class="tab"><a href="#general">General</a></li>
+            <li class="tab" id=""><a href="#general">General</a></li>
             <li class="tab"><a href="#request">Request</a></li>
             {% if extra_tab_content %}
             <li class="tab"><a href="#extras_tab">{{ extra_tab_content }}</a></li>
@@ -200,7 +200,7 @@ Math: {{ 1 + 1 }}") }}
             </div>
 
             <div class="tab-pane-inner" id="browser-steps">
-            {% if playwright_enabled and watch_uses_webdriver %}
+            {% if playwright_enabled %}
                 <img class="beta-logo" src="{{url_for('static_content', group='images', filename='beta-logo.png')}}" alt="New beta functionality">
                 <fieldset>
                     <div class="pure-control-group">
@@ -224,7 +224,7 @@ Math: {{ 1 + 1 }}") }}
                                     <span class="loader" >
                                         <span id="browsersteps-click-start">
                                             <h2 >Click here to Start</h2>
-                                            <svg style="height: 3.5rem;" version="1.1" viewBox="0 0 32 32"  xml:space="preserve" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink"><g id="start"/><g id="play_x5F_alt"><path d="M16,0C7.164,0,0,7.164,0,16s7.164,16,16,16s16-7.164,16-16S24.836,0,16,0z M10,24V8l16.008,8L10,24z" style="fill: var(--color-grey-400);"/></g></svg><br>
+                                            <svg style="height: 3.5rem;" version="1.1" viewBox="0 0 32 32"  xml:space="preserve" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink"><g id="Layer_1"/><g id="play_x5F_alt"><path d="M16,0C7.164,0,0,7.164,0,16s7.164,16,16,16s16-7.164,16-16S24.836,0,16,0z M10,24V8l16.008,8L10,24z" style="fill: var(--color-grey-400);"/></g></svg><br>
                                             Please allow 10-15 seconds for the browser to connect.<br>
                                         </span>
                                         <div class="spinner"  style="display: none;"></div>
@@ -242,12 +242,10 @@ Math: {{ 1 + 1 }}") }}
                 </fieldset>
                 {% else %}
                     <span class="pure-form-message-inline">
-                        {% if not watch_uses_webdriver %}
-                            {{ only_webdriver_type_watches_warning() }}
-                        {% endif %}
-                        {%  if not playwright_enabled %}
-                            {{ playwright_warning() }}
-                        {% endif %}
+                        <p>Sorry, this functionality only works with Playwright/Chrome enabled watches.</p>
+                        <p>Enable the Playwright Chrome fetcher, or alternatively try our <a href="https://lemonade.changedetection.io/start">very affordable subscription based service</a>.</p>
+                        <p>This is because Selenium/WebDriver can not extract full page screenshots reliably.</p>
+                        <p>You may need to <a href="https://github.com/dgtlmoon/changedetection.io/blob/09ebc6ec6338545bdd694dc6eee57f2e9d2b8075/docker-compose.yml#L31">Enable playwright environment variable</a> and uncomment the <strong>sockpuppetbrowser</strong> from the docker-compose.yml file.</p>
                     </span>
                 {% endif %}
             </div>
@@ -258,7 +256,7 @@ Math: {{ 1 + 1 }}") }}
                     <div  class="pure-control-group inline-radio">
                       {{ render_checkbox_field(form.notification_muted) }}
                     </div>
-                    {% if watch_uses_webdriver %}
+                    {% if is_html_webdriver %}
                     <div class="pure-control-group inline-radio">
                       {{ render_checkbox_field(form.notification_screenshot) }}
                         <span class="pure-form-message-inline">
@@ -481,7 +479,7 @@ keyword") }}
 
                 <fieldset>
                     <div class="pure-control-group">
-                        {% if playwright_enabled and watch_uses_webdriver %}
+                        {% if visualselector_enabled %}
                             <span class="pure-form-message-inline" id="visual-selector-heading">
                                 The Visual Selector tool lets you select the <i>text</i> elements that will be used for the change detection. It automatically fills-in the filters in the "CSS/JSONPath/JQ/XPath Filters" box of the <a href="#filters-and-triggers">Filters & Triggers</a> tab. Use <strong>Shift+Click</strong> to select multiple items.
                             </span>
@@ -499,12 +497,12 @@ keyword") }}
                             </div>
                             <div id="selector-current-xpath" style="overflow-x: hidden"><strong>Currently:</strong>&nbsp;<span class="text">Loading...</span></div>
                         {% else %}
-                            {% if not watch_uses_webdriver %}
-                                {{ only_webdriver_type_watches_warning() }}
-                            {% endif %}
-                            {% if not playwright_enabled %}
-                                {{ playwright_warning() }}
-                            {% endif %}
+                            <span class="pure-form-message-inline">
+                                <p>Sorry, this functionality only works with Playwright/Chrome enabled watches.</p>
+                                <p>Enable the Playwright Chrome fetcher, or alternatively try our <a href="https://lemonade.changedetection.io/start">very affordable subscription based service</a>.</p>
+                                <p>This is because Selenium/WebDriver can not extract full page screenshots reliably.</p>
+                                <p>You may need to <a href="https://github.com/dgtlmoon/changedetection.io/blob/09ebc6ec6338545bdd694dc6eee57f2e9d2b8075/docker-compose.yml#L31">Enable playwright environment variable</a> and uncomment the <strong>sockpuppetbrowser</strong> from the docker-compose.yml file.</p>
+                            </span>
                         {% endif %}
                     </div>
                 </fieldset>

changedetectionio/templates/svgs/search-icon.svg

@@ -1 +1 @@
-<svg version="1.1" id="search" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 0 122.879 119.799" enable-background="new 0 0 122.879 119.799" xml:space="preserve"><g><path d="M49.988,0h0.016v0.007C63.803,0.011,76.298,5.608,85.34,14.652c9.027,9.031,14.619,21.515,14.628,35.303h0.007v0.033v0.04 h-0.007c-0.005,5.557-0.917,10.905-2.594,15.892c-0.281,0.837-0.575,1.641-0.877,2.409v0.007c-1.446,3.66-3.315,7.12-5.547,10.307 l29.082,26.139l0.018,0.016l0.157,0.146l0.011,0.011c1.642,1.563,2.536,3.656,2.649,5.78c0.11,2.1-0.543,4.248-1.979,5.971 l-0.011,0.016l-0.175,0.203l-0.035,0.035l-0.146,0.16l-0.016,0.021c-1.565,1.642-3.654,2.534-5.78,2.646 c-2.097,0.111-4.247-0.54-5.971-1.978l-0.015-0.011l-0.204-0.175l-0.029-0.024L78.761,90.865c-0.88,0.62-1.778,1.209-2.687,1.765 c-1.233,0.755-2.51,1.466-3.813,2.115c-6.699,3.342-14.269,5.222-22.272,5.222v0.007h-0.016v-0.007 c-13.799-0.004-26.296-5.601-35.338-14.645C5.605,76.291,0.016,63.805,0.007,50.021H0v-0.033v-0.016h0.007 c0.004-13.799,5.601-26.296,14.645-35.338C23.683,5.608,36.167,0.016,49.955,0.007V0H49.988L49.988,0z M50.004,11.21v0.007h-0.016 h-0.033V11.21c-10.686,0.007-20.372,4.35-27.384,11.359C15.56,29.578,11.213,39.274,11.21,49.973h0.007v0.016v0.033H11.21 c0.007,10.686,4.347,20.367,11.359,27.381c7.009,7.012,16.705,11.359,27.403,11.361v-0.007h0.016h0.033v0.007 c10.686-0.007,20.368-4.348,27.382-11.359c7.011-7.009,11.358-16.702,11.36-27.4h-0.006v-0.016v-0.033h0.006 c-0.006-10.686-4.35-20.372-11.358-27.384C70.396,15.56,60.703,11.213,50.004,11.21L50.004,11.21z"/></g></svg>
+<svg version="1.1" id="Layer_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 0 122.879 119.799" enable-background="new 0 0 122.879 119.799" xml:space="preserve"><g><path d="M49.988,0h0.016v0.007C63.803,0.011,76.298,5.608,85.34,14.652c9.027,9.031,14.619,21.515,14.628,35.303h0.007v0.033v0.04 h-0.007c-0.005,5.557-0.917,10.905-2.594,15.892c-0.281,0.837-0.575,1.641-0.877,2.409v0.007c-1.446,3.66-3.315,7.12-5.547,10.307 l29.082,26.139l0.018,0.016l0.157,0.146l0.011,0.011c1.642,1.563,2.536,3.656,2.649,5.78c0.11,2.1-0.543,4.248-1.979,5.971 l-0.011,0.016l-0.175,0.203l-0.035,0.035l-0.146,0.16l-0.016,0.021c-1.565,1.642-3.654,2.534-5.78,2.646 c-2.097,0.111-4.247-0.54-5.971-1.978l-0.015-0.011l-0.204-0.175l-0.029-0.024L78.761,90.865c-0.88,0.62-1.778,1.209-2.687,1.765 c-1.233,0.755-2.51,1.466-3.813,2.115c-6.699,3.342-14.269,5.222-22.272,5.222v0.007h-0.016v-0.007 c-13.799-0.004-26.296-5.601-35.338-14.645C5.605,76.291,0.016,63.805,0.007,50.021H0v-0.033v-0.016h0.007 c0.004-13.799,5.601-26.296,14.645-35.338C23.683,5.608,36.167,0.016,49.955,0.007V0H49.988L49.988,0z M50.004,11.21v0.007h-0.016 h-0.033V11.21c-10.686,0.007-20.372,4.35-27.384,11.359C15.56,29.578,11.213,39.274,11.21,49.973h0.007v0.016v0.033H11.21 c0.007,10.686,4.347,20.367,11.359,27.381c7.009,7.012,16.705,11.359,27.403,11.361v-0.007h0.016h0.033v0.007 c10.686-0.007,20.368-4.348,27.382-11.359c7.011-7.009,11.358-16.702,11.36-27.4h-0.006v-0.016v-0.033h0.006 c-0.006-10.686-4.35-20.372-11.358-27.384C70.396,15.56,60.703,11.213,50.004,11.21L50.004,11.21z"/></g></svg>

docker-compose.yml

@@ -81,7 +81,7 @@ services:
 
 
      # Sockpuppetbrowser is basically chrome wrapped in an API for allowing fast fetching of web-pages.
-     # RECOMMENDED FOR FETCHING PAGES WITH CHROME, be sure to enable the "PLAYWRIGHT_DRIVER_URL" env variable in the main changedetection container
+     # RECOMMENDED FOR FETCHING PAGES WITH CHROME
 #    sockpuppetbrowser:
 #        hostname: sockpuppetbrowser
 #        image: dgtlmoon/sockpuppetbrowser:latest