From a8b3918fca9fbc8e9f5a7f3539cd13e4a05096b0 Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Mon, 10 Feb 2025 10:49:40 +0100 Subject: [PATCH 01/23] Browser Steps - Fixing 'Uncheck checkbox' #2958 --- changedetectionio/blueprint/browser_steps/browser_steps.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changedetectionio/blueprint/browser_steps/browser_steps.py b/changedetectionio/blueprint/browser_steps/browser_steps.py index b9765bacd..d4f3a9862 100644 --- a/changedetectionio/blueprint/browser_steps/browser_steps.py +++ b/changedetectionio/blueprint/browser_steps/browser_steps.py @@ -188,7 +188,7 @@ class steppable_browser_interface(): self.page.locator(selector).check(timeout=1000) def action_uncheck_checkbox(self, selector, value): - self.page.locator(selector, timeout=1000).uncheck(timeout=1000) + self.page.locator(selector).uncheck(timeout=1000) # Responsible for maintaining a live 'context' with the chrome CDP From 5d9380609cb6d180368bbd5045172e4df8369355 Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Mon, 10 Feb 2025 10:56:44 +0100 Subject: [PATCH 02/23] Browser Steps - Increasing timeout for actions and unifying timeout values --- .../blueprint/browser_steps/browser_steps.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/changedetectionio/blueprint/browser_steps/browser_steps.py b/changedetectionio/blueprint/browser_steps/browser_steps.py index d4f3a9862..d2141eb41 100644 --- a/changedetectionio/blueprint/browser_steps/browser_steps.py +++ b/changedetectionio/blueprint/browser_steps/browser_steps.py @@ -52,6 +52,8 @@ class steppable_browser_interface(): page = None start_url = None + action_timeout = 10 * 1000 + def __init__(self, start_url): self.start_url = start_url @@ -102,7 +104,7 @@ class steppable_browser_interface(): return elem = self.page.get_by_text(value) if elem.count(): - elem.first.click(delay=randint(200, 500), timeout=3000) + elem.first.click(delay=randint(200, 500), timeout=self.action_timeout) def action_click_element_containing_text_if_exists(self, selector=None, value=''): logger.debug("Clicking element containing text if exists") @@ -111,7 +113,7 @@ class steppable_browser_interface(): elem = self.page.get_by_text(value) logger.debug(f"Clicking element containing text - {elem.count()} elements found") if elem.count(): - elem.first.click(delay=randint(200, 500), timeout=3000) + elem.first.click(delay=randint(200, 500), timeout=self.action_timeout) else: return @@ -119,7 +121,7 @@ class steppable_browser_interface(): if not len(selector.strip()): return - self.page.fill(selector, value, timeout=10 * 1000) + self.page.fill(selector, value, timeout=self.action_timeout) def action_execute_js(self, selector, value): response = self.page.evaluate(value) @@ -130,7 +132,7 @@ class steppable_browser_interface(): if not len(selector.strip()): return - self.page.click(selector=selector, timeout=30 * 1000, delay=randint(200, 500)) + self.page.click(selector=selector, timeout=self.action_timeout + 20 * 1000, delay=randint(200, 500)) def action_click_element_if_exists(self, selector, value): import playwright._impl._errors as _api_types @@ -138,7 +140,7 @@ class steppable_browser_interface(): if not len(selector.strip()): return try: - self.page.click(selector, timeout=10 * 1000, delay=randint(200, 500)) + self.page.click(selector, timeout=self.action_timeout, delay=randint(200, 500)) except _api_types.TimeoutError as e: return except _api_types.Error as e: @@ -185,10 +187,10 @@ class steppable_browser_interface(): self.page.keyboard.press("PageDown", delay=randint(200, 500)) def action_check_checkbox(self, selector, value): - self.page.locator(selector).check(timeout=1000) + self.page.locator(selector).check(timeout=self.action_timeout) def action_uncheck_checkbox(self, selector, value): - self.page.locator(selector).uncheck(timeout=1000) + self.page.locator(selector).uncheck(timeout=self.action_timeout) # Responsible for maintaining a live 'context' with the chrome CDP From 82211eef820024e4f20d7d2206b922e6307ee1eb Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Tue, 11 Feb 2025 11:15:13 +0100 Subject: [PATCH 03/23] Update settings.html --- changedetectionio/templates/settings.html | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/changedetectionio/templates/settings.html b/changedetectionio/templates/settings.html index 7a41a2625..e13bbe60c 100644 --- a/changedetectionio/templates/settings.html +++ b/changedetectionio/templates/settings.html @@ -280,9 +280,7 @@ nav -

- Your proxy provider may need to whitelist our IP of 204.15.192.195 -

+

Tip: "Residential" and "Mobile" proxy type can be more successfull than "Data Center" for blocked websites.

From ad08219d03b90310d9d204e110be7fd95792bca2 Mon Sep 17 00:00:00 2001 From: panzli <48616247+panzli@users.noreply.github.com> Date: Mon, 17 Feb 2025 22:05:09 +0100 Subject: [PATCH 04/23] Removing deprecated docker-compose.yml version attribute (#2967) --- docker-compose.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 1d7c82673..29b60b16c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,4 +1,3 @@ -version: '3.2' services: changedetection: image: ghcr.io/dgtlmoon/changedetection.io From 4d5535d72c4bf9ebeb18c70ab7b82269d52fb78c Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Mon, 17 Feb 2025 22:15:01 +0100 Subject: [PATCH 05/23] UI - Sometimes the DOM wasnt ready when tab selection triggered via CSS, which displayed empty tabs on some browsers --- changedetectionio/static/js/tabs.js | 96 +++++++++++-------- .../static/styles/scss/styles.scss | 10 +- changedetectionio/static/styles/styles.css | 5 +- 3 files changed, 59 insertions(+), 52 deletions(-) diff --git a/changedetectionio/static/js/tabs.js b/changedetectionio/static/js/tabs.js index b4ba6e9fc..45061d76c 100644 --- a/changedetectionio/static/js/tabs.js +++ b/changedetectionio/static/js/tabs.js @@ -1,48 +1,66 @@ -// Rewrite this is a plugin.. is all this JS really 'worth it?' +(function ($) { + $.fn.hashTabs = function (options) { + var settings = $.extend({ + tabContainer: ".tabs ul", + tabSelector: "li a", + tabContent: ".tab-pane-inner", + activeClass: "active", + errorClass: ".messages .error", + bodyClassToggle: "full-width" + }, options); -window.addEventListener('hashchange', function () { - var tabs = document.getElementsByClassName('active'); - while (tabs[0]) { - tabs[0].classList.remove('active'); - document.body.classList.remove('full-width'); - } - set_active_tab(); -}, false); + var $tabs = $(settings.tabContainer).find(settings.tabSelector); -var has_errors = document.querySelectorAll(".messages .error"); -if (!has_errors.length) { - if (document.location.hash == "") { - location.replace(document.querySelector(".tabs ul li:first-child a").hash); - } else { - set_active_tab(); - } -} else { - focus_error_tab(); -} + function setActiveTab() { + var hash = window.location.hash; + var $activeTab = $tabs.filter("[href='" + hash + "']"); -function set_active_tab() { - document.body.classList.remove('full-width'); - var tab = document.querySelectorAll("a[href='" + location.hash + "']"); - if (tab.length) { - tab[0].parentElement.className = "active"; - } + // Remove active class from all tabs + $(settings.tabContainer).find("li").removeClass(settings.activeClass); -} + // Add active class to selected tab + if ($activeTab.length) { + $activeTab.parent().addClass(settings.activeClass); + } -function focus_error_tab() { - // time to use jquery or vuejs really, - // activate the tab with the error - var tabs = document.querySelectorAll('.tabs li a'), i; - for (i = 0; i < tabs.length; ++i) { - var tab_name = tabs[i].hash.replace('#', ''); - var pane_errors = document.querySelectorAll('#' + tab_name + ' .error') - if (pane_errors.length) { - document.location.hash = '#' + tab_name; - return true; + // Show the correct content + $(settings.tabContent).hide(); + if (hash) { + $(hash).show(); + } } - } - return false; -} + function focusErrorTab() { + $tabs.each(function () { + var tabName = this.hash.replace("#", ""); + if ($("#" + tabName).find(settings.errorClass).length) { + window.location.hash = "#" + tabName; + return false; // Stop loop on first error tab + } + }); + } + + function initializeTabs() { + if ($(settings.errorClass).length) { + focusErrorTab(); + } else if (!window.location.hash) { + window.location.replace($tabs.first().attr("href")); + } else { + setActiveTab(); + } + } + + // Listen for hash changes + $(window).on("hashchange", setActiveTab); + + // Initialize on page load + initializeTabs(); + + return this; // Enable jQuery chaining + }; +})(jQuery); +$(document).ready(function () { + $(".tabs").hashTabs(); +}); \ No newline at end of file diff --git a/changedetectionio/static/styles/scss/styles.scss b/changedetectionio/static/styles/scss/styles.scss index 4c698088d..ab55ae824 100644 --- a/changedetectionio/static/styles/scss/styles.scss +++ b/changedetectionio/static/styles/scss/styles.scss @@ -945,15 +945,7 @@ $form-edge-padding: 20px; } .tab-pane-inner { - - &:not(:target) { - display: none; - } - - &:target { - display: block; - } - + display: none; // doesnt need padding because theres another row of buttons/activity padding: 0px; } diff --git a/changedetectionio/static/styles/styles.css b/changedetectionio/static/styles/styles.css index d49506dc8..cdfba5628 100644 --- a/changedetectionio/static/styles/styles.css +++ b/changedetectionio/static/styles/styles.css @@ -1159,11 +1159,8 @@ textarea::placeholder { border-radius: 5px; } .tab-pane-inner { + display: none; padding: 0px; } - .tab-pane-inner:not(:target) { - display: none; } - .tab-pane-inner:target { - display: block; } .beta-logo { height: 50px; From b87c92b9e086fdaacf1b78f771f474d32f1d2468 Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Mon, 17 Feb 2025 22:27:55 +0100 Subject: [PATCH 06/23] Filter - "Unique lines" could possibly crash if history was empty or cleared on the disk --- changedetectionio/model/Watch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changedetectionio/model/Watch.py b/changedetectionio/model/Watch.py index 2fc82616b..6fd5d2e7f 100644 --- a/changedetectionio/model/Watch.py +++ b/changedetectionio/model/Watch.py @@ -352,7 +352,7 @@ class model(watch_base): # Iterate over all history texts and see if something new exists # Always applying .strip() to start/end but optionally replace any other whitespace def lines_contain_something_unique_compared_to_history(self, lines: list, ignore_whitespace=False): - local_lines = [] + local_lines = set([]) if lines: if ignore_whitespace: if isinstance(lines[0], str): # Can be either str or bytes depending on what was on the disk From 939fa86582ed13f6cf0f920410b949c61bc8397e Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Tue, 18 Feb 2025 10:17:19 +0100 Subject: [PATCH 07/23] UI - Tweaks for HTML validation --- changedetectionio/templates/edit.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changedetectionio/templates/edit.html b/changedetectionio/templates/edit.html index 3b3dbbe26..21971e171 100644 --- a/changedetectionio/templates/edit.html +++ b/changedetectionio/templates/edit.html @@ -306,7 +306,7 @@ xpath://body/div/span[contains(@class, 'example-class')]", Note!: //text() function does not work where the <element> contains <![CDATA[]]>
{% endif %} One CSS, xPath, JSON Path/JQ selector per line, any rules that matches will be used.
-

Show advanced help and tips

+ Show advanced help and tips
diff --git a/changedetectionio/templates/settings.html b/changedetectionio/templates/settings.html index e13bbe60c..752ff27e8 100644 --- a/changedetectionio/templates/settings.html +++ b/changedetectionio/templates/settings.html @@ -214,7 +214,7 @@ nav - Chrome + Chrome store icon Chrome Webstore

diff --git a/changedetectionio/templates/watch-overview.html b/changedetectionio/templates/watch-overview.html index 3cdaaf5b7..3e248ea2e 100644 --- a/changedetectionio/templates/watch-overview.html +++ b/changedetectionio/templates/watch-overview.html @@ -119,7 +119,7 @@ or ( watch.get_fetch_backend == "system" and system_default_fetcher == 'html_webdriver' ) or "extra_browser_" in watch.get_fetch_backend %} - + Using a Chrome browser {% endif %} {%if watch.is_pdf %}{% endif %} From e37467f6497b544545e437d8380348d8bc28560c Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Wed, 19 Feb 2025 10:44:54 +0100 Subject: [PATCH 09/23] UI - More W3C HTML validation fixes --- changedetectionio/forms.py | 2 +- changedetectionio/static/images/copy.svg | 2 +- changedetectionio/static/images/email.svg | 2 +- changedetectionio/static/images/schedule.svg | 2 +- changedetectionio/templates/_common_fields.html | 2 +- changedetectionio/templates/edit.html | 4 ++-- changedetectionio/templates/svgs/search-icon.svg | 2 +- 7 files changed, 8 insertions(+), 8 deletions(-) diff --git a/changedetectionio/forms.py b/changedetectionio/forms.py index 11792d622..d4b8e3425 100644 --- a/changedetectionio/forms.py +++ b/changedetectionio/forms.py @@ -171,7 +171,7 @@ class validateTimeZoneName(object): class ScheduleLimitDaySubForm(Form): enabled = BooleanField("not set", default=True) - start_time = TimeStringField("Start At", default="00:00", render_kw={"placeholder": "HH:MM"}, validators=[validators.Optional()]) + start_time = TimeStringField("Start At", default="00:00", validators=[validators.Optional()]) duration = FormField(TimeDurationForm, label="Run duration") class ScheduleLimitForm(Form): diff --git a/changedetectionio/static/images/copy.svg b/changedetectionio/static/images/copy.svg index e3f791575..b2758f608 100644 --- a/changedetectionio/static/images/copy.svg +++ b/changedetectionio/static/images/copy.svg @@ -1,7 +1,7 @@
{{ render_field(form.notification_body , rows=5, class="notification-body", placeholder=settings_application['notification_body']) }} - Body for all notifications ‐ You can use Jinja2 templating in the notification title, body and URL, and tokens from below. + Body for all notifications ‐ You can use Jinja2 templating in the notification title, body and URL, and tokens from below.
diff --git a/changedetectionio/templates/edit.html b/changedetectionio/templates/edit.html index 2560cf665..03b9aec39 100644 --- a/changedetectionio/templates/edit.html +++ b/changedetectionio/templates/edit.html @@ -40,7 +40,7 @@
    -
  • General
    • +
  • General
  • Request
    • {% if extra_tab_content %}
  • {{ extra_tab_content }}
    • @@ -224,7 +224,7 @@ Math: {{ 1 + 1 }}") }}

    Click here to Start

    -
    +
    Please allow 10-15 seconds for the browser to connect.
     diff --git a/changedetectionio/templates/svgs/search-icon.svg b/changedetectionio/templates/svgs/search-icon.svg index d1e3248c3..93860c180 100644 --- a/changedetectionio/templates/svgs/search-icon.svg +++ b/changedetectionio/templates/svgs/search-icon.svg @@ -1 +1 @@ - \ No newline at end of file + \ No newline at end of file From bdd9760f3c9f81554ae5ee824fe60c5de65db9dd Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Wed, 19 Feb 2025 10:46:22 +0100 Subject: [PATCH 10/23] Update docker-compose.yml --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 29b60b16c..bf21f65ad 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -81,7 +81,7 @@ services: # Sockpuppetbrowser is basically chrome wrapped in an API for allowing fast fetching of web-pages. - # RECOMMENDED FOR FETCHING PAGES WITH CHROME + # RECOMMENDED FOR FETCHING PAGES WITH CHROME, be sure to enable the "PLAYWRIGHT_DRIVER_URL" env variable in the main changedetection container # sockpuppetbrowser: # hostname: sockpuppetbrowser # image: dgtlmoon/sockpuppetbrowser:latest From 24c3bfe5ad75649fbee9faf99b529cd14500c7e4 Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Wed, 19 Feb 2025 14:18:18 +0100 Subject: [PATCH 11/23] UI - Make the setup and error messages for Visual Selector and Browser Steps a lot more meaningful (#2977) --- changedetectionio/flask_app.py | 9 +++---- changedetectionio/templates/_helpers.html | 12 +++++++++ changedetectionio/templates/edit.html | 30 ++++++++++++----------- 3 files changed, 32 insertions(+), 19 deletions(-) diff --git a/changedetectionio/flask_app.py b/changedetectionio/flask_app.py index f603e0121..a88df4761 100644 --- a/changedetectionio/flask_app.py +++ b/changedetectionio/flask_app.py @@ -875,14 +875,14 @@ def changedetection_app(config=None, datastore_o=None): system_uses_webdriver = datastore.data['settings']['application']['fetch_backend'] == 'html_webdriver' - is_html_webdriver = False + watch_uses_webdriver = False if (watch.get('fetch_backend') == 'system' and system_uses_webdriver) or watch.get('fetch_backend') == 'html_webdriver' or watch.get('fetch_backend', '').startswith('extra_browser_'): - is_html_webdriver = True + watch_uses_webdriver = True from zoneinfo import available_timezones # Only works reliably with Playwright - visualselector_enabled = os.getenv('PLAYWRIGHT_DRIVER_URL', False) and is_html_webdriver + template_args = { 'available_processors': processors.available_processors(), 'available_timezones': sorted(available_timezones()), @@ -895,14 +895,13 @@ def changedetection_app(config=None, datastore_o=None): 'has_default_notification_urls': True if len(datastore.data['settings']['application']['notification_urls']) else False, 'has_extra_headers_file': len(datastore.get_all_headers_in_textfile_for_watch(uuid=uuid)) > 0, 'has_special_tag_options': _watch_has_tag_options_set(watch=watch), - 'is_html_webdriver': is_html_webdriver, + 'watch_uses_webdriver': watch_uses_webdriver, 'jq_support': jq_support, 'playwright_enabled': os.getenv('PLAYWRIGHT_DRIVER_URL', False), 'settings_application': datastore.data['settings']['application'], 'timezone_default_config': datastore.data['settings']['application'].get('timezone'), 'using_global_webdriver_wait': not default['webdriver_delay'], 'uuid': uuid, - 'visualselector_enabled': visualselector_enabled, 'watch': watch } diff --git a/changedetectionio/templates/_helpers.html b/changedetectionio/templates/_helpers.html index 85fb5969e..8dd16ff30 100644 --- a/changedetectionio/templates/_helpers.html +++ b/changedetectionio/templates/_helpers.html @@ -61,6 +61,18 @@ {{ field(**kwargs)|safe }} {% endmacro %} +{% macro playwright_warning() %} +

    Error - Playwright support for Chrome based fetching is not enabled. Alternatively try our very affordable subscription based service which has all this setup for you.

    +

    You may need to Enable playwright environment variable and uncomment the sockpuppetbrowser in the docker-compose.yml file.

    +
    +

    (Also Selenium/WebDriver can not extract full page screenshots reliably so Playwright is recommended here)

    + +{% endmacro %} + +{% macro only_webdriver_type_watches_warning() %} +

    Sorry, this functionality only works with Playwright/Chrome enabled watches.
    You need to Set the fetch method to Playwright/Chrome mode and resave and have the Playwright connection enabled.


    +{% endmacro %} + {% macro render_time_schedule_form(form, available_timezones, timezone_default_config) %}