diff --git a/.github/workflows/pypi-release.yml b/.github/workflows/pypi-release.yml index 3c27c2dab..4210c9b57 100644 --- a/.github/workflows/pypi-release.yml +++ b/.github/workflows/pypi-release.yml @@ -45,8 +45,12 @@ jobs: - name: Test that the basic pip built package runs without error run: | set -ex - pip3 install dist/changedetection.io*.whl + ls -alR + + # Find and install the first .whl file + find dist -type f -name "*.whl" -exec pip3 install {} \; -quit changedetection.io -d /tmp -p 10000 & + sleep 3 curl --retry-connrefused --retry 6 http://127.0.0.1:10000/static/styles/pure-min.css >/dev/null curl --retry-connrefused --retry 6 http://127.0.0.1:10000/ >/dev/null diff --git a/changedetectionio/__init__.py b/changedetectionio/__init__.py index 852a2def4..f1b0daec6 100644 --- a/changedetectionio/__init__.py +++ b/changedetectionio/__init__.py @@ -2,7 +2,7 @@ # Read more https://github.com/dgtlmoon/changedetection.io/wiki -__version__ = '0.49.1' +__version__ = '0.49.4' from changedetectionio.strtobool import strtobool from json.decoder import JSONDecodeError diff --git a/changedetectionio/blueprint/browser_steps/__init__.py b/changedetectionio/blueprint/browser_steps/__init__.py index a472ba4b0..c7767c114 100644 --- a/changedetectionio/blueprint/browser_steps/__init__.py +++ b/changedetectionio/blueprint/browser_steps/__init__.py @@ -22,7 +22,10 @@ from loguru import logger browsersteps_sessions = {} io_interface_context = None - +import json +import base64 +import hashlib +from flask import Response def construct_blueprint(datastore: ChangeDetectionStore): browser_steps_blueprint = Blueprint('browser_steps', __name__, template_folder="templates") @@ -85,7 +88,7 @@ def construct_blueprint(datastore: ChangeDetectionStore): browsersteps_start_session['browserstepper'] = browser_steps.browsersteps_live_ui( playwright_browser=browsersteps_start_session['browser'], proxy=proxy, - start_url=datastore.data['watching'][watch_uuid].get('url'), + start_url=datastore.data['watching'][watch_uuid].link, headers=datastore.data['watching'][watch_uuid].get('headers') ) @@ -160,14 +163,13 @@ def construct_blueprint(datastore: ChangeDetectionStore): if not browsersteps_sessions.get(browsersteps_session_id): return make_response('No session exists under that ID', 500) - + is_last_step = False # Actions - step/apply/etc, do the thing and return state if request.method == 'POST': # @todo - should always be an existing session step_operation = request.form.get('operation') step_selector = request.form.get('selector') step_optional_value = request.form.get('optional_value') - step_n = int(request.form.get('step_n')) is_last_step = strtobool(request.form.get('is_last_step')) # @todo try.. accept.. nice errors not popups.. @@ -182,16 +184,6 @@ def construct_blueprint(datastore: ChangeDetectionStore): # Try to find something of value to give back to the user return make_response(str(e).splitlines()[0], 401) - # Get visual selector ready/update its data (also use the current filter info from the page?) - # When the last 'apply' button was pressed - # @todo this adds overhead because the xpath selection is happening twice - u = browsersteps_sessions[browsersteps_session_id]['browserstepper'].page.url - if is_last_step and u: - (screenshot, xpath_data) = browsersteps_sessions[browsersteps_session_id]['browserstepper'].request_visualselector_data() - watch = datastore.data['watching'].get(uuid) - if watch: - watch.save_screenshot(screenshot=screenshot) - watch.save_xpath_data(data=xpath_data) # if not this_session.page: # cleanup_playwright_session() @@ -199,31 +191,35 @@ def construct_blueprint(datastore: ChangeDetectionStore): # Screenshots and other info only needed on requesting a step (POST) try: - state = browsersteps_sessions[browsersteps_session_id]['browserstepper'].get_current_state() + (screenshot, xpath_data) = browsersteps_sessions[browsersteps_session_id]['browserstepper'].get_current_state() + if is_last_step: + watch = datastore.data['watching'].get(uuid) + u = browsersteps_sessions[browsersteps_session_id]['browserstepper'].page.url + if watch and u: + watch.save_screenshot(screenshot=screenshot) + watch.save_xpath_data(data=xpath_data) + except playwright._impl._api_types.Error as e: return make_response("Browser session ran out of time :( Please reload this page."+str(e), 401) + except Exception as e: + return make_response("Error fetching screenshot and element data - " + str(e), 401) - # Use send_file() which is way faster than read/write loop on bytes - import json - from tempfile import mkstemp - from flask import send_file - tmp_fd, tmp_file = mkstemp(text=True, suffix=".json", prefix="changedetectionio-") + # SEND THIS BACK TO THE BROWSER - output = json.dumps({'screenshot': "data:image/jpeg;base64,{}".format( - base64.b64encode(state[0]).decode('ascii')), - 'xpath_data': state[1], - 'session_age_start': browsersteps_sessions[browsersteps_session_id]['browserstepper'].age_start, - 'browser_time_remaining': round(remaining) - }) + output = { + "screenshot": f"data:image/jpeg;base64,{base64.b64encode(screenshot).decode('ascii')}", + "xpath_data": xpath_data, + "session_age_start": browsersteps_sessions[browsersteps_session_id]['browserstepper'].age_start, + "browser_time_remaining": round(remaining) + } + json_data = json.dumps(output) - with os.fdopen(tmp_fd, 'w') as f: - f.write(output) + # Generate an ETag (hash of the response body) + etag_hash = hashlib.md5(json_data.encode('utf-8')).hexdigest() - response = make_response(send_file(path_or_file=tmp_file, - mimetype='application/json; charset=UTF-8', - etag=True)) - # No longer needed - os.unlink(tmp_file) + # Create the response with ETag + response = Response(json_data, mimetype="application/json; charset=UTF-8") + response.set_etag(etag_hash) return response diff --git a/changedetectionio/blueprint/browser_steps/browser_steps.py b/changedetectionio/blueprint/browser_steps/browser_steps.py index b9765bacd..00a30a36c 100644 --- a/changedetectionio/blueprint/browser_steps/browser_steps.py +++ b/changedetectionio/blueprint/browser_steps/browser_steps.py @@ -1,14 +1,15 @@ -#!/usr/bin/env python3 - import os import time import re from random import randint from loguru import logger +from changedetectionio.content_fetchers.helpers import capture_stitched_together_full_page, SCREENSHOT_SIZE_STITCH_THRESHOLD from changedetectionio.content_fetchers.base import manage_user_agent from changedetectionio.safe_jinja import render as jinja_render + + # Two flags, tell the JS which of the "Selector" or "Value" field should be enabled in the front end # 0- off, 1- on browser_step_ui_config = {'Choose one': '0 0', @@ -31,6 +32,7 @@ browser_step_ui_config = {'Choose one': '0 0', # 'Extract text and use as filter': '1 0', 'Goto site': '0 0', 'Goto URL': '0 1', + 'Make all child elements visible': '1 0', 'Press Enter': '0 0', 'Select by label': '1 1', 'Scroll down': '0 0', @@ -38,6 +40,7 @@ browser_step_ui_config = {'Choose one': '0 0', 'Wait for seconds': '0 1', 'Wait for text': '0 1', 'Wait for text in element': '1 1', + 'Remove elements': '1 0', # 'Press Page Down': '0 0', # 'Press Page Up': '0 0', # weird bug, come back to it later @@ -52,6 +55,8 @@ class steppable_browser_interface(): page = None start_url = None + action_timeout = 10 * 1000 + def __init__(self, start_url): self.start_url = start_url @@ -102,7 +107,7 @@ class steppable_browser_interface(): return elem = self.page.get_by_text(value) if elem.count(): - elem.first.click(delay=randint(200, 500), timeout=3000) + elem.first.click(delay=randint(200, 500), timeout=self.action_timeout) def action_click_element_containing_text_if_exists(self, selector=None, value=''): logger.debug("Clicking element containing text if exists") @@ -111,7 +116,7 @@ class steppable_browser_interface(): elem = self.page.get_by_text(value) logger.debug(f"Clicking element containing text - {elem.count()} elements found") if elem.count(): - elem.first.click(delay=randint(200, 500), timeout=3000) + elem.first.click(delay=randint(200, 500), timeout=self.action_timeout) else: return @@ -119,7 +124,7 @@ class steppable_browser_interface(): if not len(selector.strip()): return - self.page.fill(selector, value, timeout=10 * 1000) + self.page.fill(selector, value, timeout=self.action_timeout) def action_execute_js(self, selector, value): response = self.page.evaluate(value) @@ -130,7 +135,7 @@ class steppable_browser_interface(): if not len(selector.strip()): return - self.page.click(selector=selector, timeout=30 * 1000, delay=randint(200, 500)) + self.page.click(selector=selector, timeout=self.action_timeout + 20 * 1000, delay=randint(200, 500)) def action_click_element_if_exists(self, selector, value): import playwright._impl._errors as _api_types @@ -138,7 +143,7 @@ class steppable_browser_interface(): if not len(selector.strip()): return try: - self.page.click(selector, timeout=10 * 1000, delay=randint(200, 500)) + self.page.click(selector, timeout=self.action_timeout, delay=randint(200, 500)) except _api_types.TimeoutError as e: return except _api_types.Error as e: @@ -185,11 +190,29 @@ class steppable_browser_interface(): self.page.keyboard.press("PageDown", delay=randint(200, 500)) def action_check_checkbox(self, selector, value): - self.page.locator(selector).check(timeout=1000) + self.page.locator(selector).check(timeout=self.action_timeout) def action_uncheck_checkbox(self, selector, value): - self.page.locator(selector, timeout=1000).uncheck(timeout=1000) + self.page.locator(selector).uncheck(timeout=self.action_timeout) + def action_remove_elements(self, selector, value): + """Removes all elements matching the given selector from the DOM.""" + self.page.locator(selector).evaluate_all("els => els.forEach(el => el.remove())") + + def action_make_all_child_elements_visible(self, selector, value): + """Recursively makes all child elements inside the given selector fully visible.""" + self.page.locator(selector).locator("*").evaluate_all(""" + els => els.forEach(el => { + el.style.display = 'block'; // Forces it to be displayed + el.style.visibility = 'visible'; // Ensures it's not hidden + el.style.opacity = '1'; // Fully opaque + el.style.position = 'relative'; // Avoids 'absolute' hiding + el.style.height = 'auto'; // Expands collapsed elements + el.style.width = 'auto'; // Ensures full visibility + el.removeAttribute('hidden'); // Removes hidden attribute + el.classList.remove('hidden', 'd-none'); // Removes common CSS hidden classes + }) + """) # Responsible for maintaining a live 'context' with the chrome CDP # @todo - how long do contexts live for anyway? @@ -257,6 +280,7 @@ class browsersteps_live_ui(steppable_browser_interface): logger.debug(f"Time to browser setup {time.time()-now:.2f}s") self.page.wait_for_timeout(1 * 1000) + def mark_as_closed(self): logger.debug("Page closed, cleaning up..") @@ -274,39 +298,30 @@ class browsersteps_live_ui(steppable_browser_interface): now = time.time() self.page.wait_for_timeout(1 * 1000) - # The actual screenshot - screenshot = self.page.screenshot(type='jpeg', full_page=True, quality=40) + full_height = self.page.evaluate("document.documentElement.scrollHeight") + + if full_height >= SCREENSHOT_SIZE_STITCH_THRESHOLD: + logger.warning(f"Page full Height: {full_height}px longer than {SCREENSHOT_SIZE_STITCH_THRESHOLD}px, using 'stitched screenshot method'.") + screenshot = capture_stitched_together_full_page(self.page) + else: + screenshot = self.page.screenshot(type='jpeg', full_page=True, quality=40) + + logger.debug(f"Time to get screenshot from browser {time.time() - now:.2f}s") + + now = time.time() self.page.evaluate("var include_filters=''") # Go find the interactive elements # @todo in the future, something smarter that can scan for elements with .click/focus etc event handlers? elements = 'a,button,input,select,textarea,i,th,td,p,li,h1,h2,h3,h4,div,span' xpath_element_js = xpath_element_js.replace('%ELEMENTS%', elements) + xpath_data = self.page.evaluate("async () => {" + xpath_element_js + "}") # So the JS will find the smallest one first xpath_data['size_pos'] = sorted(xpath_data['size_pos'], key=lambda k: k['width'] * k['height'], reverse=True) - logger.debug(f"Time to complete get_current_state of browser {time.time()-now:.2f}s") - # except + logger.debug(f"Time to scrape xpath element data in browser {time.time()-now:.2f}s") + # playwright._impl._api_types.Error: Browser closed. # @todo show some countdown timer? return (screenshot, xpath_data) - def request_visualselector_data(self): - """ - Does the same that the playwright operation in content_fetcher does - This is used to just bump the VisualSelector data so it' ready to go if they click on the tab - @todo refactor and remove duplicate code, add include_filters - :param xpath_data: - :param screenshot: - :param current_include_filters: - :return: - """ - import importlib.resources - self.page.evaluate("var include_filters=''") - xpath_element_js = importlib.resources.files("changedetectionio.content_fetchers.res").joinpath('xpath_element_scraper.js').read_text() - from changedetectionio.content_fetchers import visualselector_xpath_selectors - xpath_element_js = xpath_element_js.replace('%ELEMENTS%', visualselector_xpath_selectors) - xpath_data = self.page.evaluate("async () => {" + xpath_element_js + "}") - screenshot = self.page.screenshot(type='jpeg', full_page=True, quality=int(os.getenv("SCREENSHOT_QUALITY", 72))) - - return (screenshot, xpath_data) diff --git a/changedetectionio/content_fetchers/helpers.py b/changedetectionio/content_fetchers/helpers.py new file mode 100644 index 000000000..79826dccf --- /dev/null +++ b/changedetectionio/content_fetchers/helpers.py @@ -0,0 +1,104 @@ + +# Pages with a vertical height longer than this will use the 'stitch together' method. + +# - Many GPUs have a max texture size of 16384x16384px (or lower on older devices). +# - If a page is taller than ~8000–10000px, it risks exceeding GPU memory limits. +# - This is especially important on headless Chromium, where Playwright may fail to allocate a massive full-page buffer. + + +# The size at which we will switch to stitching method +SCREENSHOT_SIZE_STITCH_THRESHOLD=8000 + +from loguru import logger + +def capture_stitched_together_full_page(page): + import io + import os + import time + from PIL import Image, ImageDraw, ImageFont + + MAX_TOTAL_HEIGHT = SCREENSHOT_SIZE_STITCH_THRESHOLD*4 # Maximum total height for the final image (When in stitch mode) + MAX_CHUNK_HEIGHT = 4000 # Height per screenshot chunk + WARNING_TEXT_HEIGHT = 20 # Height of the warning text overlay + + # Save the original viewport size + original_viewport = page.viewport_size + now = time.time() + + try: + viewport = page.viewport_size + page_height = page.evaluate("document.documentElement.scrollHeight") + + # Limit the total capture height + capture_height = min(page_height, MAX_TOTAL_HEIGHT) + + images = [] + total_captured_height = 0 + + for offset in range(0, capture_height, MAX_CHUNK_HEIGHT): + # Ensure we do not exceed the total height limit + chunk_height = min(MAX_CHUNK_HEIGHT, MAX_TOTAL_HEIGHT - total_captured_height) + + # Adjust viewport size for this chunk + page.set_viewport_size({"width": viewport["width"], "height": chunk_height}) + + # Scroll to the correct position + page.evaluate(f"window.scrollTo(0, {offset})") + + # Capture screenshot chunk + screenshot_bytes = page.screenshot(type='jpeg', quality=int(os.getenv("SCREENSHOT_QUALITY", 30))) + images.append(Image.open(io.BytesIO(screenshot_bytes))) + + total_captured_height += chunk_height + + # Stop if we reached the maximum total height + if total_captured_height >= MAX_TOTAL_HEIGHT: + break + + # Create the final stitched image + stitched_image = Image.new('RGB', (viewport["width"], total_captured_height)) + y_offset = 0 + + # Stitch the screenshot chunks together + for img in images: + stitched_image.paste(img, (0, y_offset)) + y_offset += img.height + + logger.debug(f"Screenshot stitched together in {time.time()-now:.2f}s") + + # Overlay warning text if the screenshot was trimmed + if page_height > MAX_TOTAL_HEIGHT: + draw = ImageDraw.Draw(stitched_image) + warning_text = f"WARNING: Screenshot was {page_height}px but trimmed to {MAX_TOTAL_HEIGHT}px because it was too long" + + # Load font (default system font if Arial is unavailable) + try: + font = ImageFont.truetype("arial.ttf", WARNING_TEXT_HEIGHT) # Arial (Windows/Mac) + except IOError: + font = ImageFont.load_default() # Default font if Arial not found + + # Get text bounding box (correct method for newer Pillow versions) + text_bbox = draw.textbbox((0, 0), warning_text, font=font) + text_width = text_bbox[2] - text_bbox[0] # Calculate text width + text_height = text_bbox[3] - text_bbox[1] # Calculate text height + + # Define background rectangle (top of the image) + draw.rectangle([(0, 0), (viewport["width"], WARNING_TEXT_HEIGHT)], fill="white") + + # Center text horizontally within the warning area + text_x = (viewport["width"] - text_width) // 2 + text_y = (WARNING_TEXT_HEIGHT - text_height) // 2 + + # Draw the warning text in red + draw.text((text_x, text_y), warning_text, fill="red", font=font) + + # Save or return the final image + output = io.BytesIO() + stitched_image.save(output, format="JPEG", quality=int(os.getenv("SCREENSHOT_QUALITY", 30))) + screenshot = output.getvalue() + + finally: + # Restore the original viewport size + page.set_viewport_size(original_viewport) + + return screenshot diff --git a/changedetectionio/content_fetchers/playwright.py b/changedetectionio/content_fetchers/playwright.py index 53be33f1d..861cea60e 100644 --- a/changedetectionio/content_fetchers/playwright.py +++ b/changedetectionio/content_fetchers/playwright.py @@ -4,6 +4,7 @@ from urllib.parse import urlparse from loguru import logger +from changedetectionio.content_fetchers.helpers import capture_stitched_together_full_page, SCREENSHOT_SIZE_STITCH_THRESHOLD from changedetectionio.content_fetchers.base import Fetcher, manage_user_agent from changedetectionio.content_fetchers.exceptions import PageUnloadable, Non200ErrorCodeReceived, EmptyReply, ScreenshotUnavailable @@ -89,6 +90,7 @@ class fetcher(Fetcher): from playwright.sync_api import sync_playwright import playwright._impl._errors from changedetectionio.content_fetchers import visualselector_xpath_selectors + import time self.delete_browser_steps_screenshots() response = None @@ -179,6 +181,7 @@ class fetcher(Fetcher): self.page.wait_for_timeout(extra_wait * 1000) + now = time.time() # So we can find an element on the page where its selector was entered manually (maybe not xPath etc) if current_include_filters is not None: self.page.evaluate("var include_filters={}".format(json.dumps(current_include_filters))) @@ -190,6 +193,8 @@ class fetcher(Fetcher): self.instock_data = self.page.evaluate("async () => {" + self.instock_data_js + "}") self.content = self.page.content() + logger.debug(f"Time to scrape xpath element data in browser {time.time() - now:.2f}s") + # Bug 3 in Playwright screenshot handling # Some bug where it gives the wrong screenshot size, but making a request with the clip set first seems to solve it # JPEG is better here because the screenshots can be very very large @@ -199,10 +204,15 @@ class fetcher(Fetcher): # acceptable screenshot quality here try: # The actual screenshot - this always base64 and needs decoding! horrible! huge CPU usage - self.screenshot = self.page.screenshot(type='jpeg', - full_page=True, - quality=int(os.getenv("SCREENSHOT_QUALITY", 72)), - ) + full_height = self.page.evaluate("document.documentElement.scrollHeight") + + if full_height >= SCREENSHOT_SIZE_STITCH_THRESHOLD: + logger.warning( + f"Page full Height: {full_height}px longer than {SCREENSHOT_SIZE_STITCH_THRESHOLD}px, using 'stitched screenshot method'.") + self.screenshot = capture_stitched_together_full_page(self.page) + else: + self.screenshot = self.page.screenshot(type='jpeg', full_page=True, quality=int(os.getenv("SCREENSHOT_QUALITY", 30))) + except Exception as e: # It's likely the screenshot was too long/big and something crashed raise ScreenshotUnavailable(url=url, status_code=self.status_code) diff --git a/changedetectionio/content_fetchers/res/stock-not-in-stock.js b/changedetectionio/content_fetchers/res/stock-not-in-stock.js index c204ee679..b93c0556b 100644 --- a/changedetectionio/content_fetchers/res/stock-not-in-stock.js +++ b/changedetectionio/content_fetchers/res/stock-not-in-stock.js @@ -29,8 +29,11 @@ function isItemInStock() { 'currently unavailable', 'dieser artikel ist bald wieder verfügbar', 'dostępne wkrótce', + 'en rupture', 'en rupture de stock', + 'épuisé', 'esgotado', + 'indisponible', 'indisponível', 'isn\'t in stock right now', 'isnt in stock right now', @@ -53,6 +56,7 @@ function isItemInStock() { 'niet op voorraad', 'no disponible', 'non disponibile', + 'non disponible', 'no longer in stock', 'no tickets available', 'not available', @@ -65,8 +69,10 @@ function isItemInStock() { 'não estamos a aceitar encomendas', 'out of stock', 'out-of-stock', + 'plus disponible', 'prodotto esaurito', 'produkt niedostępny', + 'rupture', 'sold out', 'sold-out', 'stokta yok', diff --git a/changedetectionio/content_fetchers/res/xpath_element_scraper.js b/changedetectionio/content_fetchers/res/xpath_element_scraper.js index ccd894369..182a9b1df 100644 --- a/changedetectionio/content_fetchers/res/xpath_element_scraper.js +++ b/changedetectionio/content_fetchers/res/xpath_element_scraper.js @@ -41,7 +41,7 @@ const findUpTag = (el) => { // Strategy 1: If it's an input, with name, and there's only one, prefer that if (el.name !== undefined && el.name.length) { - var proposed = el.tagName + "[name=" + el.name + "]"; + var proposed = el.tagName + "[name=\"" + CSS.escape(el.name) + "\"]"; var proposed_element = window.document.querySelectorAll(proposed); if (proposed_element.length) { if (proposed_element.length === 1) { @@ -102,13 +102,15 @@ function collectVisibleElements(parent, visibleElements) { const children = parent.children; for (let i = 0; i < children.length; i++) { const child = children[i]; + const computedStyle = window.getComputedStyle(child); + if ( child.nodeType === Node.ELEMENT_NODE && - window.getComputedStyle(child).display !== 'none' && - window.getComputedStyle(child).visibility !== 'hidden' && + computedStyle.display !== 'none' && + computedStyle.visibility !== 'hidden' && child.offsetWidth >= 0 && child.offsetHeight >= 0 && - window.getComputedStyle(child).contentVisibility !== 'hidden' + computedStyle.contentVisibility !== 'hidden' ) { // If the child is an element and is visible, recursively collect visible elements collectVisibleElements(child, visibleElements); @@ -173,6 +175,7 @@ visibleElementsArray.forEach(function (element) { // Try to identify any possible currency amounts "Sale: 4000" or "Sale now 3000 Kc", can help with the training. const hasDigitCurrency = (/\d/.test(text.slice(0, 6)) || /\d/.test(text.slice(-6)) ) && /([€£$¥₩₹]|USD|AUD|EUR|Kč|kr|SEK|,–)/.test(text) ; + const computedStyle = window.getComputedStyle(element); size_pos.push({ xpath: xpath_result, @@ -184,10 +187,10 @@ visibleElementsArray.forEach(function (element) { tagName: (element.tagName) ? element.tagName.toLowerCase() : '', // tagtype used by Browser Steps tagtype: (element.tagName.toLowerCase() === 'input' && element.type) ? element.type.toLowerCase() : '', - isClickable: window.getComputedStyle(element).cursor === "pointer", + isClickable: computedStyle.cursor === "pointer", // Used by the keras trainer - fontSize: window.getComputedStyle(element).getPropertyValue('font-size'), - fontWeight: window.getComputedStyle(element).getPropertyValue('font-weight'), + fontSize: computedStyle.getPropertyValue('font-size'), + fontWeight: computedStyle.getPropertyValue('font-weight'), hasDigitCurrency: hasDigitCurrency, label: label, }); diff --git a/changedetectionio/flask_app.py b/changedetectionio/flask_app.py index e6c368f4f..6930a79ce 100644 --- a/changedetectionio/flask_app.py +++ b/changedetectionio/flask_app.py @@ -902,14 +902,14 @@ def changedetection_app(config=None, datastore_o=None): system_uses_webdriver = datastore.data['settings']['application']['fetch_backend'] == 'html_webdriver' - is_html_webdriver = False + watch_uses_webdriver = False if (watch.get('fetch_backend') == 'system' and system_uses_webdriver) or watch.get('fetch_backend') == 'html_webdriver' or watch.get('fetch_backend', '').startswith('extra_browser_'): - is_html_webdriver = True + watch_uses_webdriver = True from zoneinfo import available_timezones # Only works reliably with Playwright - visualselector_enabled = os.getenv('PLAYWRIGHT_DRIVER_URL', False) and is_html_webdriver + template_args = { 'available_processors': processors.available_processors(), 'available_timezones': sorted(available_timezones()), @@ -922,14 +922,13 @@ def changedetection_app(config=None, datastore_o=None): 'has_default_notification_urls': True if len(datastore.data['settings']['application']['notification_urls']) else False, 'has_extra_headers_file': len(datastore.get_all_headers_in_textfile_for_watch(uuid=uuid)) > 0, 'has_special_tag_options': _watch_has_tag_options_set(watch=watch), - 'is_html_webdriver': is_html_webdriver, + 'watch_uses_webdriver': watch_uses_webdriver, 'jq_support': jq_support, 'playwright_enabled': os.getenv('PLAYWRIGHT_DRIVER_URL', False), 'settings_application': datastore.data['settings']['application'], 'timezone_default_config': datastore.data['settings']['application'].get('timezone'), 'using_global_webdriver_wait': not default['webdriver_delay'], 'uuid': uuid, - 'visualselector_enabled': visualselector_enabled, 'watch': watch } diff --git a/changedetectionio/forms.py b/changedetectionio/forms.py index d31cdd2b6..118679f44 100644 --- a/changedetectionio/forms.py +++ b/changedetectionio/forms.py @@ -172,7 +172,7 @@ class validateTimeZoneName(object): class ScheduleLimitDaySubForm(Form): enabled = BooleanField("not set", default=True) - start_time = TimeStringField("Start At", default="00:00", render_kw={"placeholder": "HH:MM"}, validators=[validators.Optional()]) + start_time = TimeStringField("Start At", default="00:00", validators=[validators.Optional()]) duration = FormField(TimeDurationForm, label="Run duration") class ScheduleLimitForm(Form): diff --git a/changedetectionio/model/Watch.py b/changedetectionio/model/Watch.py index 2fc82616b..de9e8aa27 100644 --- a/changedetectionio/model/Watch.py +++ b/changedetectionio/model/Watch.py @@ -352,7 +352,7 @@ class model(watch_base): # Iterate over all history texts and see if something new exists # Always applying .strip() to start/end but optionally replace any other whitespace def lines_contain_something_unique_compared_to_history(self, lines: list, ignore_whitespace=False): - local_lines = [] + local_lines = set([]) if lines: if ignore_whitespace: if isinstance(lines[0], str): # Can be either str or bytes depending on what was on the disk @@ -527,7 +527,7 @@ class model(watch_base): def save_error_text(self, contents): self.ensure_data_dir_exists() target_path = os.path.join(self.watch_data_dir, "last-error.txt") - with open(target_path, 'w') as f: + with open(target_path, 'w', encoding='utf-8') as f: f.write(contents) def save_xpath_data(self, data, as_error=False): diff --git a/changedetectionio/static/images/copy.svg b/changedetectionio/static/images/copy.svg index e3f791575..b2758f608 100644 --- a/changedetectionio/static/images/copy.svg +++ b/changedetectionio/static/images/copy.svg @@ -1,7 +1,7 @@ 0 && $(elem_value).val().length === 0) { // @todo handle scale - $(elem_value).val(last_click_xy['x'] + ',' + last_click_xy['y']); + $(elem_value).val(last_click_xy['x'] + ',' + last_click_xy['y']).focus(); } }).change(); diff --git a/changedetectionio/static/styles/scss/parts/_browser-steps.scss b/changedetectionio/static/styles/scss/parts/_browser-steps.scss index a5a03de68..2c1dc6557 100644 --- a/changedetectionio/static/styles/scss/parts/_browser-steps.scss +++ b/changedetectionio/static/styles/scss/parts/_browser-steps.scss @@ -40,19 +40,22 @@ } } +@media only screen and (min-width: 760px) { -#browser-steps .flex-wrapper { - display: flex; - flex-flow: row; - height: 70vh; - font-size: 80%; - #browser-steps-ui { - flex-grow: 1; /* Allow it to grow and fill the available space */ - flex-shrink: 1; /* Allow it to shrink if needed */ - flex-basis: 0; /* Start with 0 base width so it stretches as much as possible */ - background-color: #eee; - border-radius: 5px; + #browser-steps .flex-wrapper { + display: flex; + flex-flow: row; + height: 70vh; + font-size: 80%; + #browser-steps-ui { + flex-grow: 1; /* Allow it to grow and fill the available space */ + flex-shrink: 1; /* Allow it to shrink if needed */ + flex-basis: 0; /* Start with 0 base width so it stretches as much as possible */ + background-color: #eee; + border-radius: 5px; + + } } #browser-steps-fieldlist { @@ -63,15 +66,21 @@ padding-left: 1rem; overflow-y: scroll; } + + /* this is duplicate :( */ + #browsersteps-selector-wrapper { + height: 100% !important; + } } /* this is duplicate :( */ #browsersteps-selector-wrapper { - height: 100%; + width: 100%; overflow-y: scroll; position: relative; - //width: 100%; + height: 80vh; + > img { position: absolute; max-width: 100%; @@ -91,7 +100,6 @@ left: 50%; top: 50%; transform: translate(-50%, -50%); - margin-left: -40px; z-index: 100; max-width: 350px; text-align: center; diff --git a/changedetectionio/static/styles/styles.css b/changedetectionio/static/styles/styles.css index d49506dc8..2e8a64079 100644 --- a/changedetectionio/static/styles/styles.css +++ b/changedetectionio/static/styles/styles.css @@ -46,21 +46,22 @@ #browser_steps li > label { display: none; } -#browser-steps .flex-wrapper { - display: flex; - flex-flow: row; - height: 70vh; - font-size: 80%; } - #browser-steps .flex-wrapper #browser-steps-ui { - flex-grow: 1; - /* Allow it to grow and fill the available space */ - flex-shrink: 1; - /* Allow it to shrink if needed */ - flex-basis: 0; - /* Start with 0 base width so it stretches as much as possible */ - background-color: #eee; - border-radius: 5px; } - #browser-steps .flex-wrapper #browser-steps-fieldlist { +@media only screen and (min-width: 760px) { + #browser-steps .flex-wrapper { + display: flex; + flex-flow: row; + height: 70vh; + font-size: 80%; } + #browser-steps .flex-wrapper #browser-steps-ui { + flex-grow: 1; + /* Allow it to grow and fill the available space */ + flex-shrink: 1; + /* Allow it to shrink if needed */ + flex-basis: 0; + /* Start with 0 base width so it stretches as much as possible */ + background-color: #eee; + border-radius: 5px; } + #browser-steps-fieldlist { flex-grow: 0; /* Don't allow it to grow */ flex-shrink: 0; @@ -71,13 +72,16 @@ /* Set a max width to prevent overflow */ padding-left: 1rem; overflow-y: scroll; } + /* this is duplicate :( */ + #browsersteps-selector-wrapper { + height: 100% !important; } } /* this is duplicate :( */ #browsersteps-selector-wrapper { - height: 100%; width: 100%; overflow-y: scroll; position: relative; + height: 80vh; /* nice tall skinny one */ } #browsersteps-selector-wrapper > img { position: absolute; @@ -92,7 +96,6 @@ left: 50%; top: 50%; transform: translate(-50%, -50%); - margin-left: -40px; z-index: 100; max-width: 350px; text-align: center; } diff --git a/changedetectionio/templates/_common_fields.html b/changedetectionio/templates/_common_fields.html index 53d27a50b..9a1cd1285 100644 --- a/changedetectionio/templates/_common_fields.html +++ b/changedetectionio/templates/_common_fields.html @@ -12,13 +12,13 @@ }}

- Tip: Use AppRise Notification URLs for notification to just about any service! Please read the notification services wiki here for important configuration notes.
+ Tip: Use AppRise Notification URLs for notification to just about any service! Please read the notification services wiki here for important configuration notes.

Show advanced help and tips
@@ -40,7 +40,7 @@
{{ render_field(form.notification_body , rows=5, class="notification-body", placeholder=settings_application['notification_body']) }} - Body for all notifications ‐ You can use Jinja2 templating in the notification title, body and URL, and tokens from below. + Body for all notifications ‐ You can use Jinja2 templating in the notification title, body and URL, and tokens from below.
@@ -126,7 +126,7 @@

Warning: Contents of {{ '{{diff}}' }}, {{ '{{diff_removed}}' }}, and {{ '{{diff_added}}' }} depend on how the difference algorithm perceives the change.
- For example, an addition or removal could be perceived as a change in some cases. More Here
+ For example, an addition or removal could be perceived as a change in some cases. More Here

For JSON payloads, use |tojson without quotes for automatic escaping, for example - { "name": {{ '{{ watch_title|tojson }}' }} } diff --git a/changedetectionio/templates/_helpers.html b/changedetectionio/templates/_helpers.html index 85fb5969e..8dd16ff30 100644 --- a/changedetectionio/templates/_helpers.html +++ b/changedetectionio/templates/_helpers.html @@ -61,6 +61,18 @@ {{ field(**kwargs)|safe }} {% endmacro %} +{% macro playwright_warning() %} +

Error - Playwright support for Chrome based fetching is not enabled. Alternatively try our very affordable subscription based service which has all this setup for you.

+

You may need to Enable playwright environment variable and uncomment the sockpuppetbrowser in the docker-compose.yml file.

+
+

(Also Selenium/WebDriver can not extract full page screenshots reliably so Playwright is recommended here)

+ +{% endmacro %} + +{% macro only_webdriver_type_watches_warning() %} +

Sorry, this functionality only works with Playwright/Chrome enabled watches.
You need to Set the fetch method to Playwright/Chrome mode and resave and have the Playwright connection enabled.


+{% endmacro %} + {% macro render_time_schedule_form(form, available_timezones, timezone_default_config) %}