wait should be after going to the URL

.github/workflows/test-only.yml

@@ -29,8 +29,8 @@ jobs:
           docker network create changedet-network
 
           # Selenium+browserless
-          docker run --network changedet-network -d --hostname selenium  -p 4444:4444 --rm --shm-size="2g"  selenium/standalone-chrome:4.14.1
+          docker run --network changedet-network -d --hostname selenium  -p 4444:4444 --rm --shm-size="2g"  selenium/standalone-chrome-debug:3.141.59
-          docker run --network changedet-network -d --hostname browserless -e "FUNCTION_BUILT_INS=[\"fs\",\"crypto\"]" -e "DEFAULT_LAUNCH_ARGS=[\"--window-size=1920,1080\"]" --rm  -p 3000:3000  --shm-size="2g"  browserless/chrome:1.60-chrome-stable
+          docker run --network changedet-network -d --hostname browserless -e "FUNCTION_BUILT_INS=[\"fs\",\"crypto\"]" -e "DEFAULT_LAUNCH_ARGS=[\"--window-size=1920,1080\"]" --rm  -p 3000:3000  --shm-size="2g"  browserless/chrome:1.53-chrome-stable
 
       - name: Build changedetection.io container for testing
         run: |         

Dockerfile

@@ -20,11 +20,6 @@ WORKDIR /install
 
 COPY requirements.txt /requirements.txt
 
-# Instructing pip to fetch wheels from piwheels.org" on ARMv6 and ARMv7 machines
-RUN if [ "$(dpkg --print-architecture)" = "armhf" ] || [ "$(dpkg --print-architecture)" = "armel" ]; then \
-      printf "[global]\nextra-index-url=https://www.piwheels.org/simple\n" > /etc/pip.conf; \
-    fi;
-
 RUN pip install --target=/dependencies -r /requirements.txt
 
 # Playwright is an alternative to Selenium

changedetectionio/__init__.py

@@ -38,7 +38,7 @@ from flask_paginate import Pagination, get_page_parameter
 from changedetectionio import html_tools
 from changedetectionio.api import api_v1
 
-__version__ = '0.45.4'
+__version__ = '0.45.3'
 
 from changedetectionio.store import BASE_URL_NOT_SET_TEXT
 
@@ -422,12 +422,11 @@ def changedetection_app(config=None, datastore_o=None):
         for uuid, watch in datastore.data['watching'].items():
             if with_errors and not watch.get('last_error'):
                 continue
-
-            if limit_tag and not limit_tag in watch['tags']:
-                    continue
             if watch.get('last_error'):
                 errored_count += 1
-                
+            if limit_tag and not limit_tag in watch['tags']:
+                    continue
+
             if search_q:
                 if (watch.get('title') and search_q in watch.get('title').lower()) or search_q in watch.get('url', '').lower():
                     sorted_watches.append(watch)

changedetectionio/content_fetcher.py

@@ -159,16 +159,6 @@ class Fetcher():
         """
         return {k.lower(): v for k, v in self.headers.items()}
 
-    def browser_steps_get_valid_steps(self):
-        if self.browser_steps is not None and len(self.browser_steps):
-            valid_steps = filter(
-                lambda s: (s['operation'] and len(s['operation']) and s['operation'] != 'Choose one' and s['operation'] != 'Goto site'),
-                self.browser_steps)
-
-            return valid_steps
-
-        return None
-
     def iterate_browser_steps(self):
         from changedetectionio.blueprint.browser_steps.browser_steps import steppable_browser_interface
         from playwright._impl._api_types import TimeoutError
@@ -180,7 +170,10 @@ class Fetcher():
         if self.browser_steps is not None and len(self.browser_steps):
             interface = steppable_browser_interface()
             interface.page = self.page
-            valid_steps = self.browser_steps_get_valid_steps()
+
+            valid_steps = filter(
+                lambda s: (s['operation'] and len(s['operation']) and s['operation'] != 'Choose one' and s['operation'] != 'Goto site'),
+                self.browser_steps)
 
             for step in valid_steps:
                 step_n += 1
@@ -479,18 +472,12 @@ class base_html_playwright(Fetcher):
             browsersteps_interface = steppable_browser_interface()
             browsersteps_interface.page = self.page
 
-            response = browsersteps_interface.action_goto_url(value=url)
-            self.headers = response.all_headers()
-
-            if response is None:
-                context.close()
-                browser.close()
-                print("Content Fetcher > Response object was none")
-                raise EmptyReply(url=url, status_code=None)
-
             try:
+                response = browsersteps_interface.action_goto_url(value=url)
+
                 if self.webdriver_js_execute_code is not None and len(self.webdriver_js_execute_code):
                     browsersteps_interface.action_execute_js(value=self.webdriver_js_execute_code, selector=None)
+
             except playwright._impl._api_types.TimeoutError as e:
                 context.close()
                 browser.close()
@@ -502,26 +489,31 @@ class base_html_playwright(Fetcher):
                 browser.close()
                 raise PageUnloadable(url=url, status_code=None, message=str(e))
 
+            if response is None:
+                context.close()
+                browser.close()
+                print("Content Fetcher > Response object was none")
+                raise EmptyReply(url=url, status_code=None)
+
             extra_wait = int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)) + self.render_extract_delay
             self.page.wait_for_timeout(extra_wait * 1000)
 
+            # Run Browser Steps here
+            self.iterate_browser_steps()
 
+            extra_wait = int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)) + self.render_extract_delay
+            self.page.wait_for_timeout(extra_wait * 1000)
+
+            self.content = self.page.content()
             self.status_code = response.status
-
-            if self.status_code != 200 and not ignore_status_codes:
-                raise Non200ErrorCodeReceived(url=url, status_code=self.status_code)
-
             if len(self.page.content().strip()) == 0:
                 context.close()
                 browser.close()
                 print("Content Fetcher > Content was empty")
                 raise EmptyReply(url=url, status_code=response.status)
 
-            # Run Browser Steps here
+            self.status_code = response.status
-            if self.browser_steps_get_valid_steps():
+            self.headers = response.all_headers()
-                self.iterate_browser_steps()
-                
-            self.page.wait_for_timeout(extra_wait * 1000)
 
             # So we can find an element on the page where its selector was entered manually (maybe not xPath etc)
             if current_include_filters is not None:
@@ -533,7 +525,6 @@ class base_html_playwright(Fetcher):
                 "async () => {" + self.xpath_element_js.replace('%ELEMENTS%', visualselector_xpath_selectors) + "}")
             self.instock_data = self.page.evaluate("async () => {" + self.instock_data_js + "}")
 
-            self.content = self.page.content()
             # Bug 3 in Playwright screenshot handling
             # Some bug where it gives the wrong screenshot size, but making a request with the clip set first seems to solve it
             # JPEG is better here because the screenshots can be very very large
@@ -548,7 +539,7 @@ class base_html_playwright(Fetcher):
             except Exception as e:
                 context.close()
                 browser.close()
-                raise ScreenshotUnavailable(url=url, status_code=response.status_code)
+                raise ScreenshotUnavailable(url=url, status_code=None)
 
             context.close()
             browser.close()
@@ -607,17 +598,14 @@ class base_html_webdriver(Fetcher):
             is_binary=False):
 
         from selenium import webdriver
-        from selenium.webdriver.chrome.options import Options as ChromeOptions
+        from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
         from selenium.common.exceptions import WebDriverException
         # request_body, request_method unused for now, until some magic in the future happens.
 
-        options = ChromeOptions()
-        if self.proxy:
-            options.proxy = self.proxy
-
         self.driver = webdriver.Remote(
             command_executor=self.command_executor,
-            options=options)
+            desired_capabilities=DesiredCapabilities.CHROME,
+            proxy=self.proxy)
 
         try:
             self.driver.get(url)
@@ -649,11 +637,11 @@ class base_html_webdriver(Fetcher):
     # Does the connection to the webdriver work? run a test connection.
     def is_ready(self):
         from selenium import webdriver
-        from selenium.webdriver.chrome.options import Options as ChromeOptions
+        from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
 
         self.driver = webdriver.Remote(
             command_executor=self.command_executor,
-            options=ChromeOptions())
+            desired_capabilities=DesiredCapabilities.CHROME)
 
         # driver.quit() seems to cause better exceptions
         self.quit()

changedetectionio/forms.py

@@ -22,8 +22,7 @@ from wtforms.validators import ValidationError
 # each select <option data-enabled="enabled-0-0"
 from changedetectionio.blueprint.browser_steps.browser_steps import browser_step_ui_config
 
-from changedetectionio import content_fetcher, html_tools
+from changedetectionio import content_fetcher
-
 from changedetectionio.notification import (
     valid_notification_formats,
 )
@@ -285,10 +284,11 @@ class ValidateListRegex(object):
     def __call__(self, form, field):
 
         for line in field.data:
-            if re.search(html_tools.PERL_STYLE_REGEX, line, re.IGNORECASE):
+            if line[0] == '/' and line[-1] == '/':
+                # Because internally we dont wrap in /
+                line = line.strip('/')
                 try:
-                    regex = html_tools.perl_style_slash_enclosed_regex_to_options(line)
+                    re.compile(line)
-                    re.compile(regex)
                 except re.error:
                     message = field.gettext('RegEx \'%s\' is not a valid regular expression.')
                     raise ValidationError(message % (line))

changedetectionio/templates/base.html

@@ -85,7 +85,6 @@
               <a href="{{url_for('logout')}}" class="pure-menu-link">LOG OUT</a>
             </li>
           {% endif %}
-          {% if current_user.is_authenticated or not has_password %}
           <li class="pure-menu-item pure-form" id="search-menu-item">
             <!-- We use GET here so it offers people a chance to set bookmarks etc -->
             <form name="searchForm" action="" method="GET">
@@ -96,7 +95,6 @@
               </button>
             </form>
           </li>
-          {% endif %}
           <li class="pure-menu-item">
             <button class="toggle-button" id ="toggle-light-mode" type="button" title="Toggle Light/Dark Mode">
               <span class="visually-hidden">Toggle light/dark mode</span>

changedetectionio/tests/test_extract_regex.py

@@ -202,35 +202,3 @@ def test_check_filter_and_regex_extract(client, live_server):
 
     # Should not be here
     assert b'Some text that did change' not in res.data
-
-
-
-def test_regex_error_handling(client, live_server):
-
-    #live_server_setup(live_server)
-
-    # Add our URL to the import page
-    test_url = url_for('test_endpoint', _external=True)
-    res = client.post(
-        url_for("import_page"),
-        data={"urls": test_url},
-        follow_redirects=True
-    )
-    assert b"1 Imported" in res.data
-
-    ### test regex error handling
-    res = client.post(
-        url_for("edit_page", uuid="first"),
-        data={"extract_text": '/something bad\d{3/XYZ',
-              "url": test_url,
-              "fetch_backend": "html_requests"},
-        follow_redirects=True
-    )
-
-    with open('/tmp/fuck.html', 'wb') as f:
-        f.write(res.data)
-
-    assert b'is not a valid regular expression.' in res.data
-
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
-    assert b'Deleted' in res.data

changedetectionio/tests/visualselector/test_fetch_data.py

@@ -1,19 +1,18 @@
 #!/usr/bin/python3
 
 import time
-import os
 from flask import url_for
 from ..util import live_server_setup, wait_for_all_checks, extract_UUID_from_client
 
-def test_setup(client, live_server):
-    live_server_setup(live_server)
-
 # Add a site in paused mode, add an invalid filter, we should still have visual selector data ready
 def test_visual_selector_content_ready(client, live_server):
     import os
     import json
 
     assert os.getenv('PLAYWRIGHT_DRIVER_URL'), "Needs PLAYWRIGHT_DRIVER_URL set for this test"
+    time.sleep(1)
+    live_server_setup(live_server)
+
 
     # Add our URL to the import page, because the docker container (playwright/selenium) wont be able to connect to our usual test url
     test_url = "https://changedetection.io/ci-test/test-runjs.html"
@@ -61,75 +60,4 @@ def test_visual_selector_content_ready(client, live_server):
         follow_redirects=True
     )
     assert b'notification_screenshot' in res.data
-    client.get(
-        url_for("form_delete", uuid="all"),
-        follow_redirects=True
-    )
 
-def test_basic_browserstep(client, live_server):
-
-    assert os.getenv('PLAYWRIGHT_DRIVER_URL'), "Needs PLAYWRIGHT_DRIVER_URL set for this test"
-    #live_server_setup(live_server)
-
-    # Add our URL to the import page, because the docker container (playwright/selenium) wont be able to connect to our usual test url
-    test_url = "https://changedetection.io/ci-test/test-runjs.html"
-
-    res = client.post(
-        url_for("form_quick_watch_add"),
-        data={"url": test_url, "tags": '', 'edit_and_watch_submit_button': 'Edit > Watch'},
-        follow_redirects=True
-    )
-    assert b"Watch added in Paused state, saving will unpause" in res.data
-
-    res = client.post(
-        url_for("edit_page", uuid="first", unpause_on_save=1),
-        data={
-              "url": test_url,
-              "tags": "",
-              "headers": "",
-              'fetch_backend': "html_webdriver",
-              'browser_steps-0-operation': 'Goto site',
-              'browser_steps-1-operation': 'Click element',
-              'browser_steps-1-selector': 'button[name=test-button]',
-              'browser_steps-1-optional_value': ''
-        },
-        follow_redirects=True
-    )
-    assert b"unpaused" in res.data
-    wait_for_all_checks(client)
-
-    uuid = extract_UUID_from_client(client)
-
-    # Check HTML conversion detected and workd
-    res = client.get(
-        url_for("preview_page", uuid=uuid),
-        follow_redirects=True
-    )
-    assert b"This text should be removed" not in res.data
-    assert b"I smell JavaScript because the button was pressed" in res.data
-
-    # now test for 404 errors
-    res = client.post(
-        url_for("edit_page", uuid=uuid, unpause_on_save=1),
-        data={
-              "url": "https://changedetection.io/404",
-              "tags": "",
-              "headers": "",
-              'fetch_backend': "html_webdriver",
-              'browser_steps-0-operation': 'Goto site',
-              'browser_steps-1-operation': 'Click element',
-              'browser_steps-1-selector': 'button[name=test-button]',
-              'browser_steps-1-optional_value': ''
-        },
-        follow_redirects=True
-    )
-    assert b"unpaused" in res.data
-    wait_for_all_checks(client)
-
-    res = client.get(url_for("index"))
-    assert b'Error - 404' in res.data
-
-    client.get(
-        url_for("form_delete", uuid="all"),
-        follow_redirects=True
-    )

requirements.txt

@@ -49,7 +49,8 @@ beautifulsoup4
 # XPath filtering, lxml is required by bs4 anyway, but put it here to be safe.
 lxml
 
-selenium~=4.14.0
+# 3.141 was missing socksVersion, 3.150 was not in pypi, so we try 4.1.0
+selenium~=4.1.0
 
 # https://stackoverflow.com/questions/71652965/importerror-cannot-import-name-safe-str-cmp-from-werkzeug-security/71653849#71653849
 # ImportError: cannot import name 'safe_str_cmp' from 'werkzeug.security'