Merge branch 'master' into browsersteps-dont-init-unless-steps-available

.github/workflows/test-only.yml

@@ -29,7 +29,7 @@ jobs:
           docker network create changedet-network
 
           # Selenium+browserless
-          docker run --network changedet-network -d --hostname selenium  -p 4444:4444 --rm --shm-size="2g"  selenium/standalone-chrome:4
+          docker run --network changedet-network -d --hostname selenium  -p 4444:4444 --rm --shm-size="2g"  selenium/standalone-chrome:4.14.1
           docker run --network changedet-network -d --hostname browserless -e "FUNCTION_BUILT_INS=[\"fs\",\"crypto\"]" -e "DEFAULT_LAUNCH_ARGS=[\"--window-size=1920,1080\"]" --rm  -p 3000:3000  --shm-size="2g"  browserless/chrome:1.60-chrome-stable
 
       - name: Build changedetection.io container for testing

Dockerfile

@@ -1,5 +1,5 @@
 # pip dependencies install stage
-FROM python:3.11-slim-bookworm as builder
+FROM python:3.11-slim-bullseye as builder
 
 # See `cryptography` pin comment in requirements.txt
 ARG CRYPTOGRAPHY_DONT_BUILD_RUST=1
@@ -34,9 +34,10 @@ RUN pip install --target=/dependencies playwright~=1.27.1 \
     || echo "WARN: Failed to install Playwright. The application can still run, but the Playwright option will be disabled."
 
 # Final image stage
-FROM python:3.11-slim-bookworm
+FROM python:3.11-slim-bullseye
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
+    libssl1.1 \
     libxslt1.1 \
     # For pdftohtml
     poppler-utils \

changedetectionio/__init__.py

@@ -38,7 +38,7 @@ from flask_paginate import Pagination, get_page_parameter
 from changedetectionio import html_tools
 from changedetectionio.api import api_v1
 
-__version__ = '0.45.5'
+__version__ = '0.45.3'
 
 from changedetectionio.store import BASE_URL_NOT_SET_TEXT
 

changedetectionio/content_fetcher.py

@@ -333,8 +333,9 @@ class base_html_playwright(Fetcher):
             # Remove username/password if it exists in the URL or you will receive "ERR_NO_SUPPORTED_PROXIES" error
             # Actual authentication handled by Puppeteer/node
             o = urlparse(self.proxy.get('server'))
-            proxy_url = urllib.parse.quote(o._replace(netloc="{}:{}".format(o.hostname, o.port)).geturl())
-            browserless_function_url = f"{browserless_function_url}&--proxy-server={proxy_url}"
+            # Remove scheme, socks5:// doesnt always work and it will autodetect anyway
+            proxy_url = urllib.parse.quote(o._replace(netloc="{}:{}".format(o.hostname, o.port)).geturl().replace(f"{o.scheme}://", '', 1))
+            browserless_function_url = f"{browserless_function_url}&--proxy-server={proxy_url}&dumpio=true"
 
         try:
             amp = '&' if '?' in browserless_function_url else '?'
@@ -504,7 +505,7 @@ class base_html_playwright(Fetcher):
             extra_wait = int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)) + self.render_extract_delay
             self.page.wait_for_timeout(extra_wait * 1000)
 
-
+            self.content = self.page.content()
             self.status_code = response.status
 
             if self.status_code != 200 and not ignore_status_codes:
@@ -532,7 +533,6 @@ class base_html_playwright(Fetcher):
                 "async () => {" + self.xpath_element_js.replace('%ELEMENTS%', visualselector_xpath_selectors) + "}")
             self.instock_data = self.page.evaluate("async () => {" + self.instock_data_js + "}")
 
-            self.content = self.page.content()
             # Bug 3 in Playwright screenshot handling
             # Some bug where it gives the wrong screenshot size, but making a request with the clip set first seems to solve it
             # JPEG is better here because the screenshots can be very very large

changedetectionio/templates/edit.html

@@ -455,15 +455,15 @@ Unavailable") }}
                         <tbody>
                         <tr>
                             <td>Check count</td>
-                            <td>{{ "{:,}".format( watch.check_count) }}</td>
+                            <td>{{ watch.check_count }}</td>
                         </tr>
                         <tr>
                             <td>Consecutive filter failures</td>
-                            <td>{{ "{:,}".format( watch.consecutive_filter_failures) }}</td>
+                            <td>{{ watch.consecutive_filter_failures }}</td>
                         </tr>
                         <tr>
                             <td>History length</td>
-                            <td>{{ "{:,}".format(watch.history|length) }}</td>
+                            <td>{{ watch.history|length }}</td>
                         </tr>
                         <tr>
                             <td>Last fetch time</td>

changedetectionio/tests/visualselector/test_fetch_data.py

@@ -91,22 +91,14 @@ def test_basic_browserstep(client, live_server):
               'browser_steps-0-operation': 'Goto site',
               'browser_steps-1-operation': 'Click element',
               'browser_steps-1-selector': 'button[name=test-button]',
-              'browser_steps-1-optional_value': ''
+              'browser_steps-1-value': ''
         },
         follow_redirects=True
     )
     assert b"unpaused" in res.data
     wait_for_all_checks(client)
-
     uuid = extract_UUID_from_client(client)
 
-    # Check HTML conversion detected and workd
-    res = client.get(
-        url_for("preview_page", uuid=uuid),
-        follow_redirects=True
-    )
-    assert b"This text should be removed" not in res.data
-    assert b"I smell JavaScript because the button was pressed" in res.data
 
     # now test for 404 errors
     res = client.post(
@@ -119,7 +111,7 @@ def test_basic_browserstep(client, live_server):
               'browser_steps-0-operation': 'Goto site',
               'browser_steps-1-operation': 'Click element',
               'browser_steps-1-selector': 'button[name=test-button]',
-              'browser_steps-1-optional_value': ''
+              'browser_steps-1-value': ''
         },
         follow_redirects=True
     )

docker-compose.yml

@@ -68,7 +68,7 @@ services:
 
 #    browser-chrome:
 #        hostname: browser-chrome
-#        image: selenium/standalone-chrome:4
+#        image: selenium/standalone-chrome-debug:3.141.59
 #        environment:
 #            - VNC_NO_PASSWORD=1
 #            - SCREEN_WIDTH=1920