Merge branch 'master' into 1800-selenium-socks5-auth

.github/workflows/test-only.yml

@@ -29,7 +29,8 @@ jobs:
           docker network create changedet-network
 
           # Selenium+browserless
-          docker run --network changedet-network -d --hostname selenium  -p 4444:4444 --rm --shm-size="2g"  selenium/standalone-chrome-debug:3.141.59
+          # 117.0 didnt work.. try dev
+          docker run --network changedet-network -d --hostname selenium  -p 4444:4444 --rm --shm-size="2g" selenium/standalone-chrome:117.0
           docker run --network changedet-network -d --hostname browserless -e "FUNCTION_BUILT_INS=[\"fs\",\"crypto\"]" -e "DEFAULT_LAUNCH_ARGS=[\"--window-size=1920,1080\"]" --rm  -p 3000:3000  --shm-size="2g"  browserless/chrome:1.53-chrome-stable
 
       - name: Build changedetection.io container for testing

changedetectionio/content_fetcher.py

@@ -601,6 +601,24 @@ class base_html_webdriver(Fetcher):
             proxy_args['httpProxy'] = proxy_override
 
         if proxy_args:
+
+            # Convert socks5:// to selenium SOCKS5 correct config
+            o = urlparse(proxy_args.get('httpProxy', ''))
+            # https://github.com/tebeka/selenium/issues/167
+            # https://github.com/SeleniumHQ/selenium/issues/6917
+            if o.scheme == 'socks5':
+                if proxy_args.get('httpProxy'):
+                    del(proxy_args['httpProxy'])
+                path = "/" + o.path if o.path else ''
+                if not o.port:
+                    raise Exception("No Port number specific in this proxy config URL")
+                proxy_args['socksProxy'] = f"{o.hostname}:{o.port}{path}"
+                proxy_args['socksVersion'] = 5
+                proxy_args['proxyType'] = 'MANUAL'
+                if o.username:
+                    proxy_args['socksUsername'] = o.username
+                    proxy_args['socksPassword'] = o.password
+
             self.proxy = SeleniumProxy(raw=proxy_args)
 
     def run(self,

requirements.txt

@@ -49,8 +49,8 @@ beautifulsoup4
 # XPath filtering, lxml is required by bs4 anyway, but put it here to be safe.
 lxml
 
-# 3.141 was missing socksVersion, 3.150 was not in pypi, so we try 4.1.0
-selenium~=4.1.0
+# 3.141 was missing socksVersion, 3.150 was not in pypi, so we try 4.13.0 which should also fix socks authentication problems
+selenium~=4.13.0
 
 # https://stackoverflow.com/questions/71652965/importerror-cannot-import-name-safe-str-cmp-from-werkzeug-security/71653849#71653849
 # ImportError: cannot import name 'safe_str_cmp' from 'werkzeug.security'