test these files

.github/workflows/pypi-release.yml

@@ -11,7 +11,7 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v5
       with:
-        python-version: "3.11"
+        python-version: "3.x"
     - name: Install pypa/build
       run: >-
         python3 -m
@@ -38,14 +38,9 @@ jobs:
       with:
         name: python-package-distributions
         path: dist/
-    - name: Set up Python 3.11
-      uses: actions/setup-python@v5
-      with:
-        python-version: '3.11'
     - name: Test that the basic pip built package runs without error
       run: |
         set -ex
-        sudo pip3 install --upgrade pip 
         pip3 install dist/changedetection.io*.whl
         changedetection.io -d /tmp -p 10000 &
         sleep 3

.github/workflows/test-only.yml

@@ -28,12 +28,12 @@ jobs:
           
           docker network create changedet-network
           
-          # Selenium
+          # Selenium and sockpuppetbrowser
           docker run --network changedet-network -d --hostname selenium  -p 4444:4444 --rm --shm-size="2g"  selenium/standalone-chrome:4
+          docker run --network changedet-network -d --cap-add=SYS_ADMIN --name sockpuppetbrowser --hostname sockpuppetbrowser --rm -p 3000:3000 dgtlmoon/sockpuppetbrowser:latest
           
-          # SocketPuppetBrowser + Extra for custom browser test
-          docker run --network changedet-network -d -e "LOG_LEVEL=TRACE" --cap-add=SYS_ADMIN --name sockpuppetbrowser --hostname sockpuppetbrowser --rm -p 3000:3000 dgtlmoon/sockpuppetbrowser:latest                    
-          docker run --network changedet-network -d -e "LOG_LEVEL=TRACE" --cap-add=SYS_ADMIN --name sockpuppetbrowser-custom-url --hostname sockpuppetbrowser-custom-url  -p 3001:3000 --rm dgtlmoon/sockpuppetbrowser:latest
+          # For accessing custom browser tests
+          docker run --network changedet-network -d --cap-add=SYS_ADMIN --name sockpuppetbrowser-custom-url --hostname sockpuppetbrowser-custom-url --rm dgtlmoon/sockpuppetbrowser:latest
 
       - name: Build changedetection.io container for testing
         run: |         
@@ -65,76 +65,42 @@ jobs:
           # The default pytest logger_level is TRACE
           # To change logger_level for pytest(test/conftest.py),
           # append the docker option. e.g. '-e LOGGER_LEVEL=DEBUG'
-          docker run --name test-cdio-basic-tests --network changedet-network  test-changedetectionio  bash -c 'cd changedetectionio && ./run_basic_tests.sh'
+          docker run --network changedet-network  test-changedetectionio  bash -c 'cd changedetectionio && ./run_basic_tests.sh'
 
-# PLAYWRIGHT/NODE-> CDP
-      - name: Playwright and SocketPuppetBrowser - Specific tests in built container
-        run: |
-          # Playwright via Sockpuppetbrowser fetch
-          # tests/visualselector/test_fetch_data.py will do browser steps  
-          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/fetchers/test_content.py'
-          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/test_errorhandling.py'
-          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/visualselector/test_fetch_data.py'
-          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/fetchers/test_custom_js_before_content.py'
-
-
-      - name: Playwright and SocketPuppetBrowser - Headers and requests
-        run: |       
-          # Settings headers playwright tests - Call back in from Sockpuppetbrowser, check headers
-          docker run --name "changedet" --hostname changedet --rm -e "FLASK_SERVER_NAME=changedet" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000?dumpio=true" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0  --live-server-port=5004 tests/test_request.py'
-
-      - name: Playwright and SocketPuppetBrowser - Restock detection
-        run: |                            
-          # restock detection via playwright - added name=changedet here so that playwright and sockpuppetbrowser can connect to it
-          docker run --rm --name "changedet" -e "FLASK_SERVER_NAME=changedet" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-port=5004 --live-server-host=0.0.0.0 tests/restock/test_restock.py'
-
-# STRAIGHT TO CDP
-      - name: Pyppeteer and SocketPuppetBrowser - Specific tests in built container
-        run: |
-          # Playwright via Sockpuppetbrowser fetch 
-          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/fetchers/test_content.py'
-          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/test_errorhandling.py'
-          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/visualselector/test_fetch_data.py'
-          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/fetchers/test_custom_js_before_content.py'
-
-      - name: Pyppeteer and SocketPuppetBrowser - Headers and requests checks
-        run: |       
-          # Settings headers playwright tests - Call back in from Sockpuppetbrowser, check headers
-          docker run --name "changedet" --hostname changedet --rm  -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "FLASK_SERVER_NAME=changedet" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000?dumpio=true" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0  --live-server-port=5004 tests/test_request.py'
-
-      - name: Pyppeteer and SocketPuppetBrowser - Restock detection
-        run: |                            
-          # restock detection via playwright - added name=changedet here so that playwright and sockpuppetbrowser can connect to it
-          docker run --rm --name "changedet" -e "FLASK_SERVER_NAME=changedet"  -e "FAST_PUPPETEER_CHROME_FETCHER=True"  -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-port=5004 --live-server-host=0.0.0.0 tests/restock/test_restock.py'
-
-# SELENIUM
       - name: Specific tests in built container for Selenium
         run: |
           # Selenium fetch
           docker run --rm -e "WEBDRIVER_URL=http://selenium:4444/wd/hub" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio;pytest tests/fetchers/test_content.py && pytest tests/test_errorhandling.py'
 
+      - name: Specific tests in built container for Playwright and SocketPuppetBrowser
+        run: |
+          # Playwright via Sockpuppetbrowser fetch
+          docker run --rm -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio;pytest tests/fetchers/test_content.py && pytest tests/test_errorhandling.py && pytest tests/visualselector/test_fetch_data.py'
+
+      - name: Specific tests in built container for headers and requests checks with Playwright
+        run: |       
+          # Settings headers playwright tests - Call back in from Sockpuppetbrowser, check headers
+          docker run --name "changedet" --hostname changedet --rm -e "FLASK_SERVER_NAME=changedet" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000?dumpio=true" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0  --live-server-port=5004 tests/test_request.py'
+
       - name: Specific tests in built container for headers and requests checks with Selenium
         run: |
           docker run --name "changedet" --hostname changedet --rm -e "FLASK_SERVER_NAME=changedet" -e "WEBDRIVER_URL=http://selenium:4444/wd/hub" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0  --live-server-port=5004 tests/test_request.py'
 
-# OTHER STUFF
+      - name: Test built container restock detection via Playwright
+        run: |                            
+          # restock detection via playwright - added name=changedet here so that playwright and sockpuppetbrowser can connect to it
+          docker run --rm --name "changedet" -e "FLASK_SERVER_NAME=changedet" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-port=5004 --live-server-host=0.0.0.0 tests/restock/test_restock.py'
+
       - name: Test SMTP notification mime types
         run: |
           # SMTP content types - needs the 'Debug SMTP server/echo message back server' container from above
           docker run --rm  --network changedet-network test-changedetectionio bash -c 'cd changedetectionio;pytest tests/smtp/test_notification_smtp.py'
 
-      # @todo Add a test via playwright/puppeteer
-      # squid with auth is tested in run_proxy_tests.sh -> tests/proxy_list/test_select_custom_proxy.py
-      - name: Test proxy squid style interaction
+      - name: Test proxy interaction
         run: |
           cd changedetectionio
           ./run_proxy_tests.sh
-          cd ..
-
-      - name: Test proxy SOCKS5 style interaction
-        run: |
-          cd changedetectionio
-          ./run_socks_proxy_tests.sh
+          # And again with PLAYWRIGHT_DRIVER_URL=..
           cd ..
 
       - name: Test custom browser URL
@@ -208,16 +174,6 @@ jobs:
           # @todo - scan the container log to see the right "graceful shutdown" text exists           
           docker rm sig-test
 
-      - name: Dump container log
-        if: always()
-        run: |
-          mkdir output-logs
-          docker logs test-cdio-basic-tests > output-logs/test-cdio-basic-tests-stdout.txt
-          docker logs test-cdio-basic-tests 2> output-logs/test-cdio-basic-tests-stderr.txt
-
-      - name: Store container log
-        if: always()
-        uses: actions/upload-artifact@v4
-        with:
-          name: test-cdio-basic-tests-output
-          path: output-logs
+#export WEBDRIVER_URL=http://localhost:4444/wd/hub
+#pytest tests/fetchers/test_content.py
+#pytest tests/test_errorhandling.py

Dockerfile

@@ -1,8 +1,5 @@
 # pip dependencies install stage
-
-# @NOTE! I would love to move to 3.11 but it breaks the async handler in changedetectionio/content_fetchers/puppeteer.py
-#        If you know how to fix it, please do! and test it for both 3.10 and 3.11
-FROM python:3.10-slim-bookworm as builder
+FROM python:3.11-slim-bookworm as builder
 
 # See `cryptography` pin comment in requirements.txt
 ARG CRYPTOGRAPHY_DONT_BUILD_RUST=1
@@ -28,11 +25,11 @@ RUN pip install --target=/dependencies -r /requirements.txt
 # Playwright is an alternative to Selenium
 # Excluded this package from requirements.txt to prevent arm/v6 and arm/v7 builds from failing
 # https://github.com/dgtlmoon/changedetection.io/pull/1067 also musl/alpine (not supported)
-RUN pip install --target=/dependencies playwright~=1.41.2 \
+RUN pip install --target=/dependencies playwright~=1.40 \
     || echo "WARN: Failed to install Playwright. The application can still run, but the Playwright option will be disabled."
 
 # Final image stage
-FROM python:3.10-slim-bookworm
+FROM python:3.11-slim-bookworm
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
     libxslt1.1 \

changedetectionio/__init__.py

@@ -6,11 +6,11 @@ __version__ = '0.45.14'
 
 from distutils.util import strtobool
 from json.decoder import JSONDecodeError
-import os
-#os.environ['EVENTLET_NO_GREENDNS'] = 'yes'
+
 import eventlet
 import eventlet.wsgi
 import getopt
+import os
 import signal
 import socket
 import sys

changedetectionio/blueprint/browser_steps/TODO.txt

@@ -1,7 +0,0 @@
-- This needs an abstraction to directly handle the puppeteer connection methods
-- Then remove the playwright stuff
-- Remove hack redirect at line 65 changedetectionio/processors/__init__.py
-
-The screenshots are base64 encoded/decoded which is very CPU intensive for large screenshots (in playwright) but not
-in the direct puppeteer connection (they are binary end to end)
-

changedetectionio/blueprint/browser_steps/browser_steps.py

@@ -6,8 +6,6 @@ import re
 from random import randint
 from loguru import logger
 
-from changedetectionio.content_fetchers.base import manage_user_agent
-
 # Two flags, tell the JS which of the "Selector" or "Value" field should be enabled in the front end
 # 0- off, 1- on
 browser_step_ui_config = {'Choose one': '0 0',
@@ -180,7 +178,6 @@ class browsersteps_live_ui(steppable_browser_interface):
     stale = False
     # bump and kill this if idle after X sec
     age_start = 0
-    headers = {}
 
     # use a special driver, maybe locally etc
     command_executor = os.getenv(
@@ -195,8 +192,7 @@ class browsersteps_live_ui(steppable_browser_interface):
 
     browser_type = os.getenv("PLAYWRIGHT_BROWSER_TYPE", 'chromium').strip('"')
 
-    def __init__(self, playwright_browser, proxy=None, headers=None):
-        self.headers = headers or {}
+    def __init__(self, playwright_browser, proxy=None):
         self.age_start = time.time()
         self.playwright_browser = playwright_browser
         if self.context is None:
@@ -210,17 +206,16 @@ class browsersteps_live_ui(steppable_browser_interface):
 
         # @todo handle multiple contexts, bind a unique id from the browser on each req?
         self.context = self.playwright_browser.new_context(
-            accept_downloads=False,  # Should never be needed
-            bypass_csp=True,  # This is needed to enable JavaScript execution on GitHub and others
-            extra_http_headers=self.headers,
-            ignore_https_errors=True,
-            proxy=proxy,
-            service_workers=os.getenv('PLAYWRIGHT_SERVICE_WORKERS', 'allow'),
-            # Should be `allow` or `block` - sites like YouTube can transmit large amounts of data via Service Workers
-            user_agent=manage_user_agent(headers=self.headers),
+            # @todo
+            #                user_agent=request_headers['User-Agent'] if request_headers.get('User-Agent') else 'Mozilla/5.0',
+            #               proxy=self.proxy,
+            # This is needed to enable JavaScript execution on GitHub and others
+            bypass_csp=True,
+            # Should never be needed
+            accept_downloads=False,
+            proxy=proxy
         )
 
-
         self.page = self.context.new_page()
 
         # self.page.set_default_navigation_timeout(keep_open)
@@ -248,7 +243,7 @@ class browsersteps_live_ui(steppable_browser_interface):
     def get_current_state(self):
         """Return the screenshot and interactive elements mapping, generally always called after action_()"""
         from pkg_resources import resource_string
-        xpath_element_js = resource_string(__name__, "../../content_fetchers/res/xpath_element_scraper.js").decode('utf-8')
+        xpath_element_js = resource_string(__name__, "../../res/xpath_element_scraper.js").decode('utf-8')
         now = time.time()
         self.page.wait_for_timeout(1 * 1000)
 
@@ -283,10 +278,10 @@ class browsersteps_live_ui(steppable_browser_interface):
         self.page.evaluate("var include_filters=''")
         from pkg_resources import resource_string
         # The code that scrapes elements and makes a list of elements/size/position to click on in the VisualSelector
-        xpath_element_js = resource_string(__name__, "../../content_fetchers/res/xpath_element_scraper.js").decode('utf-8')
-        from changedetectionio.content_fetchers import visualselector_xpath_selectors
+        xpath_element_js = resource_string(__name__, "../../res/xpath_element_scraper.js").decode('utf-8')
+        from changedetectionio.content_fetcher import visualselector_xpath_selectors
         xpath_element_js = xpath_element_js.replace('%ELEMENTS%', visualselector_xpath_selectors)
         xpath_data = self.page.evaluate("async () => {" + xpath_element_js + "}")
-        screenshot = self.page.screenshot(type='jpeg', full_page=True, quality=int(os.getenv("SCREENSHOT_QUALITY", 72)))
+        screenshot = self.page.screenshot(type='jpeg', full_page=True, quality=int(os.getenv("PLAYWRIGHT_SCREENSHOT_QUALITY", 72)))
 
         return (screenshot, xpath_data)

changedetectionio/blueprint/browser_steps/nonContext.py

@@ -1,4 +1,5 @@
 from playwright.sync_api import PlaywrightContextManager
+import asyncio
 
 # So playwright wants to run as a context manager, but we do something horrible and hacky
 # we are holding the session open for as long as possible, then shutting it down, and opening a new one

changedetectionio/blueprint/check_proxies/__init__.py

@@ -1,11 +1,14 @@
 from concurrent.futures import ThreadPoolExecutor
-from changedetectionio.store import ChangeDetectionStore
 
 from functools import wraps
 
 from flask import Blueprint
 from flask_login import login_required
 
+from changedetectionio.processors import text_json_diff
+from changedetectionio.store import ChangeDetectionStore
+
+
 STATUS_CHECKING = 0
 STATUS_FAILED = 1
 STATUS_OK = 2
@@ -29,8 +32,7 @@ def construct_blueprint(datastore: ChangeDetectionStore):
     @threadpool
     def long_task(uuid, preferred_proxy):
         import time
-        from changedetectionio.content_fetchers import exceptions as content_fetcher_exceptions
-        from changedetectionio.processors import text_json_diff
+        from changedetectionio import content_fetcher
 
         status = {'status': '', 'length': 0, 'text': ''}
         from jinja2 import Environment, BaseLoader
@@ -41,7 +43,7 @@ def construct_blueprint(datastore: ChangeDetectionStore):
             update_handler = text_json_diff.perform_site_check(datastore=datastore, watch_uuid=uuid)
             update_handler.call_browser()
         # title, size is len contents not len xfer
-        except content_fetcher_exceptions.Non200ErrorCodeReceived as e:
+        except content_fetcher.Non200ErrorCodeReceived as e:
             if e.status_code == 404:
                 status.update({'status': 'OK', 'length': len(contents), 'text': f"OK but 404 (page not found)"})
             elif e.status_code == 403 or e.status_code == 401:
@@ -50,12 +52,12 @@ def construct_blueprint(datastore: ChangeDetectionStore):
                 status.update({'status': 'ERROR', 'length': len(contents), 'text': f"Status code: {e.status_code}"})
         except text_json_diff.FilterNotFoundInResponse:
             status.update({'status': 'OK', 'length': len(contents), 'text': f"OK but CSS/xPath filter not found (page changed layout?)"})
-        except content_fetcher_exceptions.EmptyReply as e:
+        except content_fetcher.EmptyReply as e:
             if e.status_code == 403 or e.status_code == 401:
                 status.update({'status': 'ERROR OTHER', 'length': len(contents), 'text': f"Got empty reply with code {e.status_code} - Access denied"})
             else:
                 status.update({'status': 'ERROR OTHER', 'length': len(contents) if contents else 0, 'text': f"Empty reply with code {e.status_code}, needs chrome?"})
-        except content_fetcher_exceptions.ReplyWithContentButNoText as e:
+        except content_fetcher.ReplyWithContentButNoText as e:
             txt = f"Got reply but with no content - Status code {e.status_code} - It's possible that the filters were found, but contained no usable text (or contained only an image)."
             status.update({'status': 'ERROR', 'text': txt})
         except Exception as e:

changedetectionio/blueprint/tags/__init__.py

@@ -11,10 +11,9 @@ def construct_blueprint(datastore: ChangeDetectionStore):
     def tags_overview_page():
         from .form import SingleTag
         add_form = SingleTag(request.form)
-        sorted_tags = sorted(datastore.data['settings']['application'].get('tags').items(), key=lambda x: x[1]['title'])
         output = render_template("groups-overview.html",
                                  form=add_form,
-                                 available_tags=sorted_tags,
+                                 available_tags=datastore.data['settings']['application'].get('tags', {}),
                                  )
 
         return output

changedetectionio/blueprint/tags/templates/groups-overview.html

@@ -40,7 +40,7 @@
                 <td colspan="3">No website organisational tags/groups configured</td>
             </tr>
             {% endif %}
-            {% for uuid, tag in available_tags  %}
+            {% for uuid, tag in available_tags.items()  %}
             <tr id="{{ uuid }}" class="{{ loop.cycle('pure-table-odd', 'pure-table-even') }}">
                 <td class="watch-controls">
                     <a class="link-mute state-{{'on' if tag.notification_muted else 'off'}}" href="{{url_for('tags.mute', uuid=tag.uuid)}}"><img src="{{url_for('static_content', group='images', filename='bell-off.svg')}}" alt="Mute notifications" title="Mute notifications" class="icon icon-mute" ></a>

changedetectionio/content_fetcher.py

@@ -0,0 +1,658 @@
+from abc import abstractmethod
+from distutils.util import strtobool
+from urllib.parse import urlparse
+import chardet
+import hashlib
+import json
+import os
+import requests
+import sys
+import time
+import urllib.parse
+from loguru import logger
+
+visualselector_xpath_selectors = 'div,span,form,table,tbody,tr,td,a,p,ul,li,h1,h2,h3,h4, header, footer, section, article, aside, details, main, nav, section, summary'
+
+
+class Non200ErrorCodeReceived(Exception):
+    def __init__(self, status_code, url, screenshot=None, xpath_data=None, page_html=None):
+        # Set this so we can use it in other parts of the app
+        self.status_code = status_code
+        self.url = url
+        self.screenshot = screenshot
+        self.xpath_data = xpath_data
+        self.page_text = None
+
+        if page_html:
+            from changedetectionio import html_tools
+            self.page_text = html_tools.html_to_text(page_html)
+        return
+
+
+class checksumFromPreviousCheckWasTheSame(Exception):
+    def __init__(self):
+        return
+
+
+class JSActionExceptions(Exception):
+    def __init__(self, status_code, url, screenshot, message=''):
+        self.status_code = status_code
+        self.url = url
+        self.screenshot = screenshot
+        self.message = message
+        return
+
+
+class BrowserStepsStepException(Exception):
+    def __init__(self, step_n, original_e):
+        self.step_n = step_n
+        self.original_e = original_e
+        logger.debug(f"Browser Steps exception at step {self.step_n} {str(original_e)}")
+        return
+
+
+# @todo - make base Exception class that announces via logger()
+class PageUnloadable(Exception):
+    def __init__(self, status_code=None, url='', message='', screenshot=False):
+        # Set this so we can use it in other parts of the app
+        self.status_code = status_code
+        self.url = url
+        self.screenshot = screenshot
+        self.message = message
+        return
+
+class BrowserStepsInUnsupportedFetcher(Exception):
+    def __init__(self, url):
+        self.url = url
+        return
+
+class EmptyReply(Exception):
+    def __init__(self, status_code, url, screenshot=None):
+        # Set this so we can use it in other parts of the app
+        self.status_code = status_code
+        self.url = url
+        self.screenshot = screenshot
+        return
+
+
+class ScreenshotUnavailable(Exception):
+    def __init__(self, status_code, url, page_html=None):
+        # Set this so we can use it in other parts of the app
+        self.status_code = status_code
+        self.url = url
+        if page_html:
+            from html_tools import html_to_text
+            self.page_text = html_to_text(page_html)
+        return
+
+
+class ReplyWithContentButNoText(Exception):
+    def __init__(self, status_code, url, screenshot=None, has_filters=False, html_content=''):
+        # Set this so we can use it in other parts of the app
+        self.status_code = status_code
+        self.url = url
+        self.screenshot = screenshot
+        self.has_filters = has_filters
+        self.html_content = html_content
+        return
+
+
+class Fetcher():
+    browser_connection_is_custom = None
+    browser_connection_url = None
+    browser_steps = None
+    browser_steps_screenshot_path = None
+    content = None
+    error = None
+    fetcher_description = "No description"
+    headers = {}
+    instock_data = None
+    instock_data_js = ""
+    status_code = None
+    webdriver_js_execute_code = None
+    xpath_data = None
+    xpath_element_js = ""
+
+    # Will be needed in the future by the VisualSelector, always get this where possible.
+    screenshot = False
+    system_http_proxy = os.getenv('HTTP_PROXY')
+    system_https_proxy = os.getenv('HTTPS_PROXY')
+
+    # Time ONTOP of the system defined env minimum time
+    render_extract_delay = 0
+
+    def __init__(self):
+        from pkg_resources import resource_string
+        # The code that scrapes elements and makes a list of elements/size/position to click on in the VisualSelector
+        self.xpath_element_js = resource_string(__name__, "res/xpath_element_scraper.js").decode('utf-8')
+        self.instock_data_js = resource_string(__name__, "res/stock-not-in-stock.js").decode('utf-8')
+
+    @abstractmethod
+    def get_error(self):
+        return self.error
+
+    @abstractmethod
+    def run(self,
+            url,
+            timeout,
+            request_headers,
+            request_body,
+            request_method,
+            ignore_status_codes=False,
+            current_include_filters=None,
+            is_binary=False):
+        # Should set self.error, self.status_code and self.content
+        pass
+
+    @abstractmethod
+    def quit(self):
+        return
+
+    @abstractmethod
+    def get_last_status_code(self):
+        return self.status_code
+
+    @abstractmethod
+    def screenshot_step(self, step_n):
+        return None
+
+    @abstractmethod
+    # Return true/false if this checker is ready to run, in the case it needs todo some special config check etc
+    def is_ready(self):
+        return True
+
+    def get_all_headers(self):
+        """
+        Get all headers but ensure all keys are lowercase
+        :return:
+        """
+        return {k.lower(): v for k, v in self.headers.items()}
+
+    def browser_steps_get_valid_steps(self):
+        if self.browser_steps is not None and len(self.browser_steps):
+            valid_steps = filter(
+                lambda s: (s['operation'] and len(s['operation']) and s['operation'] != 'Choose one' and s['operation'] != 'Goto site'),
+                self.browser_steps)
+
+            return valid_steps
+
+        return None
+
+    def iterate_browser_steps(self):
+        from changedetectionio.blueprint.browser_steps.browser_steps import steppable_browser_interface
+        from playwright._impl._errors import TimeoutError, Error
+        from jinja2 import Environment
+        jinja2_env = Environment(extensions=['jinja2_time.TimeExtension'])
+
+        step_n = 0
+
+        if self.browser_steps is not None and len(self.browser_steps):
+            interface = steppable_browser_interface()
+            interface.page = self.page
+            valid_steps = self.browser_steps_get_valid_steps()
+
+            for step in valid_steps:
+                step_n += 1
+                logger.debug(f">> Iterating check - browser Step n {step_n} - {step['operation']}...")
+                self.screenshot_step("before-" + str(step_n))
+                self.save_step_html("before-" + str(step_n))
+                try:
+                    optional_value = step['optional_value']
+                    selector = step['selector']
+                    # Support for jinja2 template in step values, with date module added
+                    if '{%' in step['optional_value'] or '{{' in step['optional_value']:
+                        optional_value = str(jinja2_env.from_string(step['optional_value']).render())
+                    if '{%' in step['selector'] or '{{' in step['selector']:
+                        selector = str(jinja2_env.from_string(step['selector']).render())
+
+                    getattr(interface, "call_action")(action_name=step['operation'],
+                                                      selector=selector,
+                                                      optional_value=optional_value)
+                    self.screenshot_step(step_n)
+                    self.save_step_html(step_n)
+                except (Error, TimeoutError) as e:
+                    logger.debug(str(e))
+                    # Stop processing here
+                    raise BrowserStepsStepException(step_n=step_n, original_e=e)
+
+    # It's always good to reset these
+    def delete_browser_steps_screenshots(self):
+        import glob
+        if self.browser_steps_screenshot_path is not None:
+            dest = os.path.join(self.browser_steps_screenshot_path, 'step_*.jpeg')
+            files = glob.glob(dest)
+            for f in files:
+                if os.path.isfile(f):
+                    os.unlink(f)
+
+
+#   Maybe for the future, each fetcher provides its own diff output, could be used for text, image
+#   the current one would return javascript output (as we use JS to generate the diff)
+#
+def available_fetchers():
+    # See the if statement at the bottom of this file for how we switch between playwright and webdriver
+    import inspect
+    p = []
+    for name, obj in inspect.getmembers(sys.modules[__name__], inspect.isclass):
+        if inspect.isclass(obj):
+            # @todo html_ is maybe better as fetcher_ or something
+            # In this case, make sure to edit the default one in store.py and fetch_site_status.py
+            if name.startswith('html_'):
+                t = tuple([name, obj.fetcher_description])
+                p.append(t)
+
+    return p
+
+
+class base_html_playwright(Fetcher):
+    fetcher_description = "Playwright {}/Javascript".format(
+        os.getenv("PLAYWRIGHT_BROWSER_TYPE", 'chromium').capitalize()
+    )
+    if os.getenv("PLAYWRIGHT_DRIVER_URL"):
+        fetcher_description += " via '{}'".format(os.getenv("PLAYWRIGHT_DRIVER_URL"))
+
+    browser_type = ''
+    command_executor = ''
+
+    # Configs for Proxy setup
+    # In the ENV vars, is prefixed with "playwright_proxy_", so it is for example "playwright_proxy_server"
+    playwright_proxy_settings_mappings = ['bypass', 'server', 'username', 'password']
+
+    proxy = None
+
+    def __init__(self, proxy_override=None, custom_browser_connection_url=None):
+        super().__init__()
+
+        self.browser_type = os.getenv("PLAYWRIGHT_BROWSER_TYPE", 'chromium').strip('"')
+
+        if custom_browser_connection_url:
+            self.browser_connection_is_custom = True
+            self.browser_connection_url = custom_browser_connection_url
+        else:
+            # Fallback to fetching from system
+            # .strip('"') is going to save someone a lot of time when they accidently wrap the env value
+            self.browser_connection_url = os.getenv("PLAYWRIGHT_DRIVER_URL", 'ws://playwright-chrome:3000').strip('"')
+
+
+        # If any proxy settings are enabled, then we should setup the proxy object
+        proxy_args = {}
+        for k in self.playwright_proxy_settings_mappings:
+            v = os.getenv('playwright_proxy_' + k, False)
+            if v:
+                proxy_args[k] = v.strip('"')
+
+        if proxy_args:
+            self.proxy = proxy_args
+
+        # allow per-watch proxy selection override
+        if proxy_override:
+            self.proxy = {'server': proxy_override}
+
+        if self.proxy:
+            # Playwright needs separate username and password values
+            parsed = urlparse(self.proxy.get('server'))
+            if parsed.username:
+                self.proxy['username'] = parsed.username
+                self.proxy['password'] = parsed.password
+
+    def screenshot_step(self, step_n=''):
+        screenshot = self.page.screenshot(type='jpeg', full_page=True, quality=85)
+
+        if self.browser_steps_screenshot_path is not None:
+            destination = os.path.join(self.browser_steps_screenshot_path, 'step_{}.jpeg'.format(step_n))
+            logger.debug(f"Saving step screenshot to {destination}")
+            with open(destination, 'wb') as f:
+                f.write(screenshot)
+
+    def save_step_html(self, step_n):
+        content = self.page.content()
+        destination = os.path.join(self.browser_steps_screenshot_path, 'step_{}.html'.format(step_n))
+        logger.debug(f"Saving step HTML to {destination}")
+        with open(destination, 'w') as f:
+            f.write(content)
+
+    def run(self,
+            url,
+            timeout,
+            request_headers,
+            request_body,
+            request_method,
+            ignore_status_codes=False,
+            current_include_filters=None,
+            is_binary=False):
+
+
+        from playwright.sync_api import sync_playwright
+        import playwright._impl._errors
+
+        self.delete_browser_steps_screenshots()
+        response = None
+
+        with sync_playwright() as p:
+            browser_type = getattr(p, self.browser_type)
+
+            # Seemed to cause a connection Exception even tho I can see it connect
+            # self.browser = browser_type.connect(self.command_executor, timeout=timeout*1000)
+            # 60,000 connection timeout only
+            browser = browser_type.connect_over_cdp(self.browser_connection_url, timeout=60000)
+
+            # SOCKS5 with authentication is not supported (yet)
+            # https://github.com/microsoft/playwright/issues/10567
+
+            # Set user agent to prevent Cloudflare from blocking the browser
+            # Use the default one configured in the App.py model that's passed from fetch_site_status.py
+            context = browser.new_context(
+                user_agent={k.lower(): v for k, v in request_headers.items()}.get('user-agent', None),
+                proxy=self.proxy,
+                # This is needed to enable JavaScript execution on GitHub and others
+                bypass_csp=True,
+                # Should be `allow` or `block` - sites like YouTube can transmit large amounts of data via Service Workers
+                service_workers=os.getenv('PLAYWRIGHT_SERVICE_WORKERS', 'allow'),
+                # Should never be needed
+                accept_downloads=False
+            )
+
+            self.page = context.new_page()
+            if len(request_headers):
+                context.set_extra_http_headers(request_headers)
+
+            # Listen for all console events and handle errors
+            self.page.on("console", lambda msg: print(f"Playwright console: Watch URL: {url} {msg.type}: {msg.text} {msg.args}"))
+
+            # Re-use as much code from browser steps as possible so its the same
+            from changedetectionio.blueprint.browser_steps.browser_steps import steppable_browser_interface
+            browsersteps_interface = steppable_browser_interface()
+            browsersteps_interface.page = self.page
+
+            response = browsersteps_interface.action_goto_url(value=url)
+            self.headers = response.all_headers()
+
+            if response is None:
+                context.close()
+                browser.close()
+                logger.debug("Content Fetcher > Response object was none")
+                raise EmptyReply(url=url, status_code=None)
+
+            try:
+                if self.webdriver_js_execute_code is not None and len(self.webdriver_js_execute_code):
+                    browsersteps_interface.action_execute_js(value=self.webdriver_js_execute_code, selector=None)
+            except playwright._impl._errors.TimeoutError as e:
+                context.close()
+                browser.close()
+                # This can be ok, we will try to grab what we could retrieve
+                pass
+            except Exception as e:
+                logger.debug(f"Content Fetcher > Other exception when executing custom JS code {str(e)}")
+                context.close()
+                browser.close()
+                raise PageUnloadable(url=url, status_code=None, message=str(e))
+
+            extra_wait = int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)) + self.render_extract_delay
+            self.page.wait_for_timeout(extra_wait * 1000)
+
+            try:
+                self.status_code = response.status
+            except Exception as e:
+                # https://github.com/dgtlmoon/changedetection.io/discussions/2122#discussioncomment-8241962
+                logger.critical(f"Response from the browser/Playwright did not have a status_code! Response follows.")
+                logger.critical(response)
+                context.close()
+                browser.close()
+                raise PageUnloadable(url=url, status_code=None, message=str(e))
+
+            if self.status_code != 200 and not ignore_status_codes:
+
+                screenshot=self.page.screenshot(type='jpeg', full_page=True,
+                                     quality=int(os.getenv("PLAYWRIGHT_SCREENSHOT_QUALITY", 72)))
+
+                raise Non200ErrorCodeReceived(url=url, status_code=self.status_code, screenshot=screenshot)
+
+            if len(self.page.content().strip()) == 0:
+                context.close()
+                browser.close()
+                logger.debug("Content Fetcher > Content was empty")
+                raise EmptyReply(url=url, status_code=response.status)
+
+            # Run Browser Steps here
+            if self.browser_steps_get_valid_steps():
+                self.iterate_browser_steps()
+                
+            self.page.wait_for_timeout(extra_wait * 1000)
+
+            # So we can find an element on the page where its selector was entered manually (maybe not xPath etc)
+            if current_include_filters is not None:
+                self.page.evaluate("var include_filters={}".format(json.dumps(current_include_filters)))
+            else:
+                self.page.evaluate("var include_filters=''")
+
+            self.xpath_data = self.page.evaluate(
+                "async () => {" + self.xpath_element_js.replace('%ELEMENTS%', visualselector_xpath_selectors) + "}")
+            self.instock_data = self.page.evaluate("async () => {" + self.instock_data_js + "}")
+
+            self.content = self.page.content()
+            # Bug 3 in Playwright screenshot handling
+            # Some bug where it gives the wrong screenshot size, but making a request with the clip set first seems to solve it
+            # JPEG is better here because the screenshots can be very very large
+
+            # Screenshots also travel via the ws:// (websocket) meaning that the binary data is base64 encoded
+            # which will significantly increase the IO size between the server and client, it's recommended to use the lowest
+            # acceptable screenshot quality here
+            try:
+                # The actual screenshot - this always base64 and needs decoding! horrible! huge CPU usage
+                self.screenshot = self.page.screenshot(type='jpeg',
+                                                       full_page=True,
+                                                       quality=int(os.getenv("PLAYWRIGHT_SCREENSHOT_QUALITY", 72)),
+                                                       )
+            except Exception as e:
+                # It's likely the screenshot was too long/big and something crashed
+                raise ScreenshotUnavailable(url=url, status_code=self.status_code)
+            finally:
+                context.close()
+                browser.close()
+
+
+class base_html_webdriver(Fetcher):
+    if os.getenv("WEBDRIVER_URL"):
+        fetcher_description = "WebDriver Chrome/Javascript via '{}'".format(os.getenv("WEBDRIVER_URL"))
+    else:
+        fetcher_description = "WebDriver Chrome/Javascript"
+
+    # Configs for Proxy setup
+    # In the ENV vars, is prefixed with "webdriver_", so it is for example "webdriver_sslProxy"
+    selenium_proxy_settings_mappings = ['proxyType', 'ftpProxy', 'httpProxy', 'noProxy',
+                                        'proxyAutoconfigUrl', 'sslProxy', 'autodetect',
+                                        'socksProxy', 'socksVersion', 'socksUsername', 'socksPassword']
+    proxy = None
+
+    def __init__(self, proxy_override=None, custom_browser_connection_url=None):
+        super().__init__()
+        from selenium.webdriver.common.proxy import Proxy as SeleniumProxy
+
+        # .strip('"') is going to save someone a lot of time when they accidently wrap the env value
+        if not custom_browser_connection_url:
+            self.browser_connection_url = os.getenv("WEBDRIVER_URL", 'http://browser-chrome:4444/wd/hub').strip('"')
+        else:
+            self.browser_connection_is_custom = True
+            self.browser_connection_url = custom_browser_connection_url
+
+        # If any proxy settings are enabled, then we should setup the proxy object
+        proxy_args = {}
+        for k in self.selenium_proxy_settings_mappings:
+            v = os.getenv('webdriver_' + k, False)
+            if v:
+                proxy_args[k] = v.strip('"')
+
+        # Map back standard HTTP_ and HTTPS_PROXY to webDriver httpProxy/sslProxy
+        if not proxy_args.get('webdriver_httpProxy') and self.system_http_proxy:
+            proxy_args['httpProxy'] = self.system_http_proxy
+        if not proxy_args.get('webdriver_sslProxy') and self.system_https_proxy:
+            proxy_args['httpsProxy'] = self.system_https_proxy
+
+        # Allows override the proxy on a per-request basis
+        if proxy_override is not None:
+            proxy_args['httpProxy'] = proxy_override
+
+        if proxy_args:
+            self.proxy = SeleniumProxy(raw=proxy_args)
+
+    def run(self,
+            url,
+            timeout,
+            request_headers,
+            request_body,
+            request_method,
+            ignore_status_codes=False,
+            current_include_filters=None,
+            is_binary=False):
+
+        from selenium import webdriver
+        from selenium.webdriver.chrome.options import Options as ChromeOptions
+        from selenium.common.exceptions import WebDriverException
+        # request_body, request_method unused for now, until some magic in the future happens.
+
+        options = ChromeOptions()
+        if self.proxy:
+            options.proxy = self.proxy
+
+        self.driver = webdriver.Remote(
+            command_executor=self.browser_connection_url,
+            options=options)
+
+        try:
+            self.driver.get(url)
+        except WebDriverException as e:
+            # Be sure we close the session window
+            self.quit()
+            raise
+
+        self.driver.set_window_size(1280, 1024)
+        self.driver.implicitly_wait(int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)))
+
+        if self.webdriver_js_execute_code is not None:
+            self.driver.execute_script(self.webdriver_js_execute_code)
+            # Selenium doesn't automatically wait for actions as good as Playwright, so wait again
+            self.driver.implicitly_wait(int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)))
+
+        # @todo - how to check this? is it possible?
+        self.status_code = 200
+        # @todo somehow we should try to get this working for WebDriver
+        # raise EmptyReply(url=url, status_code=r.status_code)
+
+        # @todo - dom wait loaded?
+        time.sleep(int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)) + self.render_extract_delay)
+        self.content = self.driver.page_source
+        self.headers = {}
+
+        self.screenshot = self.driver.get_screenshot_as_png()
+
+    # Does the connection to the webdriver work? run a test connection.
+    def is_ready(self):
+        from selenium import webdriver
+        from selenium.webdriver.chrome.options import Options as ChromeOptions
+
+        self.driver = webdriver.Remote(
+            command_executor=self.command_executor,
+            options=ChromeOptions())
+
+        # driver.quit() seems to cause better exceptions
+        self.quit()
+        return True
+
+    def quit(self):
+        if self.driver:
+            try:
+                self.driver.quit()
+            except Exception as e:
+                logger.debug(f"Content Fetcher > Exception in chrome shutdown/quit {str(e)}")
+
+
+# "html_requests" is listed as the default fetcher in store.py!
+class html_requests(Fetcher):
+    fetcher_description = "Basic fast Plaintext/HTTP Client"
+
+    def __init__(self, proxy_override=None, custom_browser_connection_url=None):
+        super().__init__()
+        self.proxy_override = proxy_override
+        # browser_connection_url is none because its always 'launched locally'
+
+    def run(self,
+            url,
+            timeout,
+            request_headers,
+            request_body,
+            request_method,
+            ignore_status_codes=False,
+            current_include_filters=None,
+            is_binary=False):
+
+        if self.browser_steps_get_valid_steps():
+            raise BrowserStepsInUnsupportedFetcher(url=url)
+
+        # Make requests use a more modern looking user-agent
+        if not {k.lower(): v for k, v in request_headers.items()}.get('user-agent', None):
+            request_headers['User-Agent'] = os.getenv("DEFAULT_SETTINGS_HEADERS_USERAGENT",
+                                                      'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.66 Safari/537.36')
+
+        proxies = {}
+
+        # Allows override the proxy on a per-request basis
+
+        # https://requests.readthedocs.io/en/latest/user/advanced/#socks
+        # Should also work with `socks5://user:pass@host:port` type syntax.
+
+        if self.proxy_override:
+            proxies = {'http': self.proxy_override, 'https': self.proxy_override, 'ftp': self.proxy_override}
+        else:
+            if self.system_http_proxy:
+                proxies['http'] = self.system_http_proxy
+            if self.system_https_proxy:
+                proxies['https'] = self.system_https_proxy
+
+        r = requests.request(method=request_method,
+                             data=request_body,
+                             url=url,
+                             headers=request_headers,
+                             timeout=timeout,
+                             proxies=proxies,
+                             verify=False)
+
+        # If the response did not tell us what encoding format to expect, Then use chardet to override what `requests` thinks.
+        # For example - some sites don't tell us it's utf-8, but return utf-8 content
+        # This seems to not occur when using webdriver/selenium, it seems to detect the text encoding more reliably.
+        # https://github.com/psf/requests/issues/1604 good info about requests encoding detection
+        if not is_binary:
+            # Don't run this for PDF (and requests identified as binary) takes a _long_ time
+            if not r.headers.get('content-type') or not 'charset=' in r.headers.get('content-type'):
+                encoding = chardet.detect(r.content)['encoding']
+                if encoding:
+                    r.encoding = encoding
+
+        self.headers = r.headers
+
+        if not r.content or not len(r.content):
+            raise EmptyReply(url=url, status_code=r.status_code)
+
+        # @todo test this
+        # @todo maybe you really want to test zero-byte return pages?
+        if r.status_code != 200 and not ignore_status_codes:
+            # maybe check with content works?
+            raise Non200ErrorCodeReceived(url=url, status_code=r.status_code, page_html=r.text)
+
+        self.status_code = r.status_code
+        if is_binary:
+            # Binary files just return their checksum until we add something smarter
+            self.content = hashlib.md5(r.content).hexdigest()
+        else:
+            self.content = r.text
+
+
+        self.raw_content = r.content
+
+
+# Decide which is the 'real' HTML webdriver, this is more a system wide config
+# rather than site-specific.
+use_playwright_as_chrome_fetcher = os.getenv('PLAYWRIGHT_DRIVER_URL', False)
+if use_playwright_as_chrome_fetcher:
+    html_webdriver = base_html_playwright
+else:
+    html_webdriver = base_html_webdriver

changedetectionio/content_fetchers/__init__.py

@@ -1,43 +0,0 @@
-import sys
-from distutils.util import strtobool
-from loguru import logger
-from changedetectionio.content_fetchers.exceptions import BrowserStepsStepException
-import os
-
-visualselector_xpath_selectors = 'div,span,form,table,tbody,tr,td,a,p,ul,li,h1,h2,h3,h4,header,footer,section,article,aside,details,main,nav,section,summary'
-
-# available_fetchers() will scan this implementation looking for anything starting with html_
-# this information is used in the form selections
-from changedetectionio.content_fetchers.requests import fetcher as html_requests
-
-def available_fetchers():
-    # See the if statement at the bottom of this file for how we switch between playwright and webdriver
-    import inspect
-    p = []
-    for name, obj in inspect.getmembers(sys.modules[__name__], inspect.isclass):
-        if inspect.isclass(obj):
-            # @todo html_ is maybe better as fetcher_ or something
-            # In this case, make sure to edit the default one in store.py and fetch_site_status.py
-            if name.startswith('html_'):
-                t = tuple([name, obj.fetcher_description])
-                p.append(t)
-
-    return p
-
-
-# Decide which is the 'real' HTML webdriver, this is more a system wide config
-# rather than site-specific.
-use_playwright_as_chrome_fetcher = os.getenv('PLAYWRIGHT_DRIVER_URL', False)
-if use_playwright_as_chrome_fetcher:
-    # @note - For now, browser steps always uses playwright
-    if not strtobool(os.getenv('FAST_PUPPETEER_CHROME_FETCHER', 'False')) or False:
-        logger.debug('Using Playwright library as fetcher')
-        from .playwright import fetcher as html_webdriver
-    else:
-        logger.debug('Using direct Python Puppeteer library as fetcher')
-        from .puppeteer import fetcher as html_webdriver
-
-else:
-    logger.debug("Falling back to selenium as fetcher")
-    from .webdriver_selenium import fetcher as html_webdriver
-

changedetectionio/content_fetchers/base.py

@@ -1,171 +0,0 @@
-import os
-from abc import abstractmethod
-from loguru import logger
-
-from changedetectionio.content_fetchers import BrowserStepsStepException
-
-
-def manage_user_agent(headers, current_ua=''):
-    """
-    Basic setting of user-agent
-
-    NOTE!!!!!! The service that does the actual Chrome fetching should handle any anti-robot techniques
-    THERE ARE MANY WAYS THAT IT CAN BE DETECTED AS A ROBOT!!
-    This does not take care of
-    - Scraping of 'navigator' (platform, productSub, vendor, oscpu etc etc) browser object (navigator.appVersion) etc
-    - TCP/IP fingerprint JA3 etc
-    - Graphic rendering fingerprinting
-    - Your IP being obviously in a pool of bad actors
-    - Too many requests
-    - Scraping of SCH-UA browser replies (thanks google!!)
-    - Scraping of ServiceWorker, new window calls etc
-
-    See https://filipvitas.medium.com/how-to-set-user-agent-header-with-puppeteer-js-and-not-fail-28c7a02165da
-    Puppeteer requests https://github.com/dgtlmoon/pyppeteerstealth
-
-    :param page:
-    :param headers:
-    :return:
-    """
-    # Ask it what the user agent is, if its obviously ChromeHeadless, switch it to the default
-    ua_in_custom_headers = next((v for k, v in headers.items() if k.lower() == "user-agent"), None)
-    if ua_in_custom_headers:
-        return ua_in_custom_headers
-
-    if not ua_in_custom_headers and current_ua:
-        current_ua = current_ua.replace('HeadlessChrome', 'Chrome')
-        return current_ua
-
-    return None
-
-
-class Fetcher():
-    browser_connection_is_custom = None
-    browser_connection_url = None
-    browser_steps = None
-    browser_steps_screenshot_path = None
-    content = None
-    error = None
-    fetcher_description = "No description"
-    headers = {}
-    instock_data = None
-    instock_data_js = ""
-    status_code = None
-    webdriver_js_execute_code = None
-    xpath_data = None
-    xpath_element_js = ""
-
-    # Will be needed in the future by the VisualSelector, always get this where possible.
-    screenshot = False
-    system_http_proxy = os.getenv('HTTP_PROXY')
-    system_https_proxy = os.getenv('HTTPS_PROXY')
-
-    # Time ONTOP of the system defined env minimum time
-    render_extract_delay = 0
-
-    def __init__(self):
-        from pkg_resources import resource_string
-        # The code that scrapes elements and makes a list of elements/size/position to click on in the VisualSelector
-        self.xpath_element_js = resource_string(__name__, "res/xpath_element_scraper.js").decode('utf-8')
-        self.instock_data_js = resource_string(__name__, "res/stock-not-in-stock.js").decode('utf-8')
-
-    @abstractmethod
-    def get_error(self):
-        return self.error
-
-    @abstractmethod
-    def run(self,
-            url,
-            timeout,
-            request_headers,
-            request_body,
-            request_method,
-            ignore_status_codes=False,
-            current_include_filters=None,
-            is_binary=False):
-        # Should set self.error, self.status_code and self.content
-        pass
-
-    @abstractmethod
-    def quit(self):
-        return
-
-    @abstractmethod
-    def get_last_status_code(self):
-        return self.status_code
-
-    @abstractmethod
-    def screenshot_step(self, step_n):
-        return None
-
-    @abstractmethod
-    # Return true/false if this checker is ready to run, in the case it needs todo some special config check etc
-    def is_ready(self):
-        return True
-
-    def get_all_headers(self):
-        """
-        Get all headers but ensure all keys are lowercase
-        :return:
-        """
-        return {k.lower(): v for k, v in self.headers.items()}
-
-    def browser_steps_get_valid_steps(self):
-        if self.browser_steps is not None and len(self.browser_steps):
-            valid_steps = filter(
-                lambda s: (s['operation'] and len(s['operation']) and s['operation'] != 'Choose one' and s['operation'] != 'Goto site'),
-                self.browser_steps)
-
-            return valid_steps
-
-        return None
-
-    def iterate_browser_steps(self):
-        from changedetectionio.blueprint.browser_steps.browser_steps import steppable_browser_interface
-        from playwright._impl._errors import TimeoutError, Error
-        from jinja2 import Environment
-        jinja2_env = Environment(extensions=['jinja2_time.TimeExtension'])
-
-        step_n = 0
-
-        if self.browser_steps is not None and len(self.browser_steps):
-            interface = steppable_browser_interface()
-            interface.page = self.page
-            valid_steps = self.browser_steps_get_valid_steps()
-
-            for step in valid_steps:
-                step_n += 1
-                logger.debug(f">> Iterating check - browser Step n {step_n} - {step['operation']}...")
-                self.screenshot_step("before-" + str(step_n))
-                self.save_step_html("before-" + str(step_n))
-                try:
-                    optional_value = step['optional_value']
-                    selector = step['selector']
-                    # Support for jinja2 template in step values, with date module added
-                    if '{%' in step['optional_value'] or '{{' in step['optional_value']:
-                        optional_value = str(jinja2_env.from_string(step['optional_value']).render())
-                    if '{%' in step['selector'] or '{{' in step['selector']:
-                        selector = str(jinja2_env.from_string(step['selector']).render())
-
-                    getattr(interface, "call_action")(action_name=step['operation'],
-                                                      selector=selector,
-                                                      optional_value=optional_value)
-                    self.screenshot_step(step_n)
-                    self.save_step_html(step_n)
-                except (Error, TimeoutError) as e:
-                    logger.debug(str(e))
-                    # Stop processing here
-                    raise BrowserStepsStepException(step_n=step_n, original_e=e)
-
-    # It's always good to reset these
-    def delete_browser_steps_screenshots(self):
-        import glob
-        if self.browser_steps_screenshot_path is not None:
-            dest = os.path.join(self.browser_steps_screenshot_path, 'step_*.jpeg')
-            files = glob.glob(dest)
-            for f in files:
-                if os.path.isfile(f):
-                    os.unlink(f)
-
-    def save_step_html(self, param):
-        pass

changedetectionio/content_fetchers/exceptions/__init__.py

@@ -1,97 +0,0 @@
-from loguru import logger
-
-
-class Non200ErrorCodeReceived(Exception):
-    def __init__(self, status_code, url, screenshot=None, xpath_data=None, page_html=None):
-        # Set this so we can use it in other parts of the app
-        self.status_code = status_code
-        self.url = url
-        self.screenshot = screenshot
-        self.xpath_data = xpath_data
-        self.page_text = None
-
-        if page_html:
-            from changedetectionio import html_tools
-            self.page_text = html_tools.html_to_text(page_html)
-        return
-
-
-class checksumFromPreviousCheckWasTheSame(Exception):
-    def __init__(self):
-        return
-
-
-class JSActionExceptions(Exception):
-    def __init__(self, status_code, url, screenshot, message=''):
-        self.status_code = status_code
-        self.url = url
-        self.screenshot = screenshot
-        self.message = message
-        return
-
-class BrowserConnectError(Exception):
-    msg = ''
-    def __init__(self, msg):
-        self.msg = msg
-        logger.error(f"Browser connection error {msg}")
-        return
-
-class BrowserFetchTimedOut(Exception):
-    msg = ''
-    def __init__(self, msg):
-        self.msg = msg
-        logger.error(f"Browser processing took too long - {msg}")
-        return
-
-class BrowserStepsStepException(Exception):
-    def __init__(self, step_n, original_e):
-        self.step_n = step_n
-        self.original_e = original_e
-        logger.debug(f"Browser Steps exception at step {self.step_n} {str(original_e)}")
-        return
-
-
-# @todo - make base Exception class that announces via logger()
-class PageUnloadable(Exception):
-    def __init__(self, status_code=None, url='', message='', screenshot=False):
-        # Set this so we can use it in other parts of the app
-        self.status_code = status_code
-        self.url = url
-        self.screenshot = screenshot
-        self.message = message
-        return
-
-class BrowserStepsInUnsupportedFetcher(Exception):
-    def __init__(self, url):
-        self.url = url
-        return
-
-class EmptyReply(Exception):
-    def __init__(self, status_code, url, screenshot=None):
-        # Set this so we can use it in other parts of the app
-        self.status_code = status_code
-        self.url = url
-        self.screenshot = screenshot
-        return
-
-
-class ScreenshotUnavailable(Exception):
-    def __init__(self, status_code, url, page_html=None):
-        # Set this so we can use it in other parts of the app
-        self.status_code = status_code
-        self.url = url
-        if page_html:
-            from html_tools import html_to_text
-            self.page_text = html_to_text(page_html)
-        return
-
-
-class ReplyWithContentButNoText(Exception):
-    def __init__(self, status_code, url, screenshot=None, has_filters=False, html_content=''):
-        # Set this so we can use it in other parts of the app
-        self.status_code = status_code
-        self.url = url
-        self.screenshot = screenshot
-        self.has_filters = has_filters
-        self.html_content = html_content
-        return

changedetectionio/content_fetchers/playwright.py

@@ -1,208 +0,0 @@
-import json
-import os
-from urllib.parse import urlparse
-
-from loguru import logger
-
-from changedetectionio.content_fetchers.base import Fetcher, manage_user_agent
-from changedetectionio.content_fetchers.exceptions import PageUnloadable, Non200ErrorCodeReceived, EmptyReply, ScreenshotUnavailable
-
-class fetcher(Fetcher):
-    fetcher_description = "Playwright {}/Javascript".format(
-        os.getenv("PLAYWRIGHT_BROWSER_TYPE", 'chromium').capitalize()
-    )
-    if os.getenv("PLAYWRIGHT_DRIVER_URL"):
-        fetcher_description += " via '{}'".format(os.getenv("PLAYWRIGHT_DRIVER_URL"))
-
-    browser_type = ''
-    command_executor = ''
-
-    # Configs for Proxy setup
-    # In the ENV vars, is prefixed with "playwright_proxy_", so it is for example "playwright_proxy_server"
-    playwright_proxy_settings_mappings = ['bypass', 'server', 'username', 'password']
-
-    proxy = None
-
-    def __init__(self, proxy_override=None, custom_browser_connection_url=None):
-        super().__init__()
-
-        self.browser_type = os.getenv("PLAYWRIGHT_BROWSER_TYPE", 'chromium').strip('"')
-
-        if custom_browser_connection_url:
-            self.browser_connection_is_custom = True
-            self.browser_connection_url = custom_browser_connection_url
-        else:
-            # Fallback to fetching from system
-            # .strip('"') is going to save someone a lot of time when they accidently wrap the env value
-            self.browser_connection_url = os.getenv("PLAYWRIGHT_DRIVER_URL", 'ws://playwright-chrome:3000').strip('"')
-
-        # If any proxy settings are enabled, then we should setup the proxy object
-        proxy_args = {}
-        for k in self.playwright_proxy_settings_mappings:
-            v = os.getenv('playwright_proxy_' + k, False)
-            if v:
-                proxy_args[k] = v.strip('"')
-
-        if proxy_args:
-            self.proxy = proxy_args
-
-        # allow per-watch proxy selection override
-        if proxy_override:
-            self.proxy = {'server': proxy_override}
-
-        if self.proxy:
-            # Playwright needs separate username and password values
-            parsed = urlparse(self.proxy.get('server'))
-            if parsed.username:
-                self.proxy['username'] = parsed.username
-                self.proxy['password'] = parsed.password
-
-    def screenshot_step(self, step_n=''):
-        screenshot = self.page.screenshot(type='jpeg', full_page=True, quality=int(os.getenv("SCREENSHOT_QUALITY", 72)))
-
-        if self.browser_steps_screenshot_path is not None:
-            destination = os.path.join(self.browser_steps_screenshot_path, 'step_{}.jpeg'.format(step_n))
-            logger.debug(f"Saving step screenshot to {destination}")
-            with open(destination, 'wb') as f:
-                f.write(screenshot)
-
-    def save_step_html(self, step_n):
-        content = self.page.content()
-        destination = os.path.join(self.browser_steps_screenshot_path, 'step_{}.html'.format(step_n))
-        logger.debug(f"Saving step HTML to {destination}")
-        with open(destination, 'w') as f:
-            f.write(content)
-
-    def run(self,
-            url,
-            timeout,
-            request_headers,
-            request_body,
-            request_method,
-            ignore_status_codes=False,
-            current_include_filters=None,
-            is_binary=False):
-
-        from playwright.sync_api import sync_playwright
-        import playwright._impl._errors
-        from changedetectionio.content_fetchers import visualselector_xpath_selectors
-        self.delete_browser_steps_screenshots()
-        response = None
-
-        with sync_playwright() as p:
-            browser_type = getattr(p, self.browser_type)
-
-            # Seemed to cause a connection Exception even tho I can see it connect
-            # self.browser = browser_type.connect(self.command_executor, timeout=timeout*1000)
-            # 60,000 connection timeout only
-            browser = browser_type.connect_over_cdp(self.browser_connection_url, timeout=60000)
-
-            # SOCKS5 with authentication is not supported (yet)
-            # https://github.com/microsoft/playwright/issues/10567
-
-            # Set user agent to prevent Cloudflare from blocking the browser
-            # Use the default one configured in the App.py model that's passed from fetch_site_status.py
-            context = browser.new_context(
-                accept_downloads=False,  # Should never be needed
-                bypass_csp=True,  # This is needed to enable JavaScript execution on GitHub and others
-                extra_http_headers=request_headers,
-                ignore_https_errors=True,
-                proxy=self.proxy,
-                service_workers=os.getenv('PLAYWRIGHT_SERVICE_WORKERS', 'allow'), # Should be `allow` or `block` - sites like YouTube can transmit large amounts of data via Service Workers
-                user_agent=manage_user_agent(headers=request_headers),
-            )
-
-            self.page = context.new_page()
-
-            # Listen for all console events and handle errors
-            self.page.on("console", lambda msg: print(f"Playwright console: Watch URL: {url} {msg.type}: {msg.text} {msg.args}"))
-
-            # Re-use as much code from browser steps as possible so its the same
-            from changedetectionio.blueprint.browser_steps.browser_steps import steppable_browser_interface
-            browsersteps_interface = steppable_browser_interface()
-            browsersteps_interface.page = self.page
-
-            response = browsersteps_interface.action_goto_url(value=url)
-            self.headers = response.all_headers()
-
-            if response is None:
-                context.close()
-                browser.close()
-                logger.debug("Content Fetcher > Response object was none")
-                raise EmptyReply(url=url, status_code=None)
-
-            try:
-                if self.webdriver_js_execute_code is not None and len(self.webdriver_js_execute_code):
-                    browsersteps_interface.action_execute_js(value=self.webdriver_js_execute_code, selector=None)
-            except playwright._impl._errors.TimeoutError as e:
-                context.close()
-                browser.close()
-                # This can be ok, we will try to grab what we could retrieve
-                pass
-            except Exception as e:
-                logger.debug(f"Content Fetcher > Other exception when executing custom JS code {str(e)}")
-                context.close()
-                browser.close()
-                raise PageUnloadable(url=url, status_code=None, message=str(e))
-
-            extra_wait = int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)) + self.render_extract_delay
-            self.page.wait_for_timeout(extra_wait * 1000)
-
-            try:
-                self.status_code = response.status
-            except Exception as e:
-                # https://github.com/dgtlmoon/changedetection.io/discussions/2122#discussioncomment-8241962
-                logger.critical(f"Response from the browser/Playwright did not have a status_code! Response follows.")
-                logger.critical(response)
-                context.close()
-                browser.close()
-                raise PageUnloadable(url=url, status_code=None, message=str(e))
-
-            if self.status_code != 200 and not ignore_status_codes:
-                screenshot = self.page.screenshot(type='jpeg', full_page=True,
-                                                  quality=int(os.getenv("SCREENSHOT_QUALITY", 72)))
-
-                raise Non200ErrorCodeReceived(url=url, status_code=self.status_code, screenshot=screenshot)
-
-            if len(self.page.content().strip()) == 0:
-                context.close()
-                browser.close()
-                logger.debug("Content Fetcher > Content was empty")
-                raise EmptyReply(url=url, status_code=response.status)
-
-            # Run Browser Steps here
-            if self.browser_steps_get_valid_steps():
-                self.iterate_browser_steps()
-
-            self.page.wait_for_timeout(extra_wait * 1000)
-
-            # So we can find an element on the page where its selector was entered manually (maybe not xPath etc)
-            if current_include_filters is not None:
-                self.page.evaluate("var include_filters={}".format(json.dumps(current_include_filters)))
-            else:
-                self.page.evaluate("var include_filters=''")
-
-            self.xpath_data = self.page.evaluate(
-                "async () => {" + self.xpath_element_js.replace('%ELEMENTS%', visualselector_xpath_selectors) + "}")
-            self.instock_data = self.page.evaluate("async () => {" + self.instock_data_js + "}")
-
-            self.content = self.page.content()
-            # Bug 3 in Playwright screenshot handling
-            # Some bug where it gives the wrong screenshot size, but making a request with the clip set first seems to solve it
-            # JPEG is better here because the screenshots can be very very large
-
-            # Screenshots also travel via the ws:// (websocket) meaning that the binary data is base64 encoded
-            # which will significantly increase the IO size between the server and client, it's recommended to use the lowest
-            # acceptable screenshot quality here
-            try:
-                # The actual screenshot - this always base64 and needs decoding! horrible! huge CPU usage
-                self.screenshot = self.page.screenshot(type='jpeg',
-                                                       full_page=True,
-                                                       quality=int(os.getenv("SCREENSHOT_QUALITY", 72)),
-                                                       )
-            except Exception as e:
-                # It's likely the screenshot was too long/big and something crashed
-                raise ScreenshotUnavailable(url=url, status_code=self.status_code)
-            finally:
-                context.close()
-                browser.close()

changedetectionio/content_fetchers/puppeteer.py

@@ -1,247 +0,0 @@
-import asyncio
-import json
-import os
-import websockets.exceptions
-from urllib.parse import urlparse
-
-from loguru import logger
-
-from changedetectionio.content_fetchers.base import Fetcher, manage_user_agent
-from changedetectionio.content_fetchers.exceptions import PageUnloadable, Non200ErrorCodeReceived, EmptyReply, BrowserFetchTimedOut, BrowserConnectError
-
-
-class fetcher(Fetcher):
-    fetcher_description = "Puppeteer/direct {}/Javascript".format(
-        os.getenv("PLAYWRIGHT_BROWSER_TYPE", 'chromium').capitalize()
-    )
-    if os.getenv("PLAYWRIGHT_DRIVER_URL"):
-        fetcher_description += " via '{}'".format(os.getenv("PLAYWRIGHT_DRIVER_URL"))
-
-    browser_type = ''
-    command_executor = ''
-
-    proxy = None
-
-    def __init__(self, proxy_override=None, custom_browser_connection_url=None):
-        super().__init__()
-
-        if custom_browser_connection_url:
-            self.browser_connection_is_custom = True
-            self.browser_connection_url = custom_browser_connection_url
-        else:
-            # Fallback to fetching from system
-            # .strip('"') is going to save someone a lot of time when they accidently wrap the env value
-            self.browser_connection_url = os.getenv("PLAYWRIGHT_DRIVER_URL", 'ws://playwright-chrome:3000').strip('"')
-
-        # allow per-watch proxy selection override
-        # @todo check global too?
-        if proxy_override:
-            # Playwright needs separate username and password values
-            parsed = urlparse(proxy_override)
-            if parsed:
-                self.proxy = {'username': parsed.username, 'password': parsed.password}
-                # Add the proxy server chrome start option, the username and password never gets added here
-                # (It always goes in via await self.page.authenticate(self.proxy))
-
-                # @todo filter some injection attack?
-                # check scheme when no scheme
-                proxy_url = parsed.scheme + "://" if parsed.scheme else 'http://'
-                r = "?" if not '?' in self.browser_connection_url else '&'
-                port = ":"+str(parsed.port) if parsed.port else ''
-                q = "?"+parsed.query if parsed.query else ''
-                proxy_url += f"{parsed.hostname}{port}{parsed.path}{q}"
-                self.browser_connection_url += f"{r}--proxy-server={proxy_url}"
-
-    # def screenshot_step(self, step_n=''):
-    #     screenshot = self.page.screenshot(type='jpeg', full_page=True, quality=85)
-    #
-    #     if self.browser_steps_screenshot_path is not None:
-    #         destination = os.path.join(self.browser_steps_screenshot_path, 'step_{}.jpeg'.format(step_n))
-    #         logger.debug(f"Saving step screenshot to {destination}")
-    #         with open(destination, 'wb') as f:
-    #             f.write(screenshot)
-    #
-    # def save_step_html(self, step_n):
-    #     content = self.page.content()
-    #     destination = os.path.join(self.browser_steps_screenshot_path, 'step_{}.html'.format(step_n))
-    #     logger.debug(f"Saving step HTML to {destination}")
-    #     with open(destination, 'w') as f:
-    #         f.write(content)
-
-    async def fetch_page(self,
-                         url,
-                         timeout,
-                         request_headers,
-                         request_body,
-                         request_method,
-                         ignore_status_codes,
-                         current_include_filters,
-                         is_binary
-                         ):
-
-        from changedetectionio.content_fetchers import visualselector_xpath_selectors
-        self.delete_browser_steps_screenshots()
-        extra_wait = int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)) + self.render_extract_delay
-
-        from pyppeteer import Pyppeteer
-        pyppeteer_instance = Pyppeteer()
-
-        # Connect directly using the specified browser_ws_endpoint
-        # @todo timeout
-        try:
-            browser = await pyppeteer_instance.connect(browserWSEndpoint=self.browser_connection_url,
-                                                       ignoreHTTPSErrors=True
-                                                       )
-        except websockets.exceptions.InvalidStatusCode as e:
-            raise BrowserConnectError(msg=f"Error while trying to connect the browser, Code {e.status_code} (check your access)")
-        except websockets.exceptions.InvalidURI:
-            raise BrowserConnectError(msg=f"Error connecting to the browser, check your browser connection address (should be ws:// or wss://")
-        except Exception as e:
-            raise BrowserConnectError(msg=f"Error connecting to the browser {str(e)}")
-        else:
-            self.page = await browser.newPage()
-
-        await self.page.setUserAgent(manage_user_agent(headers=request_headers, current_ua=await self.page.evaluate('navigator.userAgent')))
-
-        await self.page.setBypassCSP(True)
-        if request_headers:
-            await self.page.setExtraHTTPHeaders(request_headers)
-
-        # SOCKS5 with authentication is not supported (yet)
-        # https://github.com/microsoft/playwright/issues/10567
-        self.page.setDefaultNavigationTimeout(0)
-        await self.page.setCacheEnabled(True)
-        if self.proxy and self.proxy.get('username'):
-            # Setting Proxy-Authentication header is deprecated, and doing so can trigger header change errors from Puppeteer
-            # https://github.com/puppeteer/puppeteer/issues/676 ?
-            # https://help.brightdata.com/hc/en-us/articles/12632549957649-Proxy-Manager-How-to-Guides#h_01HAKWR4Q0AFS8RZTNYWRDFJC2
-            # https://cri.dev/posts/2020-03-30-How-to-solve-Puppeteer-Chrome-Error-ERR_INVALID_ARGUMENT/
-            await self.page.authenticate(self.proxy)
-
-        # Re-use as much code from browser steps as possible so its the same
-        # from changedetectionio.blueprint.browser_steps.browser_steps import steppable_browser_interface
-
-        # not yet used here, we fallback to playwright when browsersteps is required
-        #            browsersteps_interface = steppable_browser_interface()
-        #            browsersteps_interface.page = self.page
-
-        response = await self.page.goto(url, waitUntil="load")
-
-
-        if response is None:
-            await self.page.close()
-            await browser.close()
-            logger.warning("Content Fetcher > Response object was none")
-            raise EmptyReply(url=url, status_code=None)
-
-        self.headers = response.headers
-
-        try:
-            if self.webdriver_js_execute_code is not None and len(self.webdriver_js_execute_code):
-                await self.page.evaluate(self.webdriver_js_execute_code)
-        except Exception as e:
-            logger.warning("Got exception when running evaluate on custom JS code")
-            logger.error(str(e))
-            await self.page.close()
-            await browser.close()
-            # This can be ok, we will try to grab what we could retrieve
-            raise PageUnloadable(url=url, status_code=None, message=str(e))
-
-        try:
-            self.status_code = response.status
-        except Exception as e:
-            # https://github.com/dgtlmoon/changedetection.io/discussions/2122#discussioncomment-8241962
-            logger.critical(f"Response from the browser/Playwright did not have a status_code! Response follows.")
-            logger.critical(response)
-            await self.page.close()
-            await browser.close()
-            raise PageUnloadable(url=url, status_code=None, message=str(e))
-
-        if self.status_code != 200 and not ignore_status_codes:
-            screenshot = await self.page.screenshot(type_='jpeg',
-                                                    fullPage=True,
-                                                    quality=int(os.getenv("SCREENSHOT_QUALITY", 72)))
-
-            raise Non200ErrorCodeReceived(url=url, status_code=self.status_code, screenshot=screenshot)
-        content = await self.page.content
-        if len(content.strip()) == 0:
-            await self.page.close()
-            await browser.close()
-            logger.error("Content Fetcher > Content was empty")
-            raise EmptyReply(url=url, status_code=response.status)
-
-        # Run Browser Steps here
-        # @todo not yet supported, we switch to playwright in this case
-        #            if self.browser_steps_get_valid_steps():
-        #                self.iterate_browser_steps()
-
-        await asyncio.sleep(1 + extra_wait)
-
-        # So we can find an element on the page where its selector was entered manually (maybe not xPath etc)
-        # Setup the xPath/VisualSelector scraper
-        if current_include_filters is not None:
-            js = json.dumps(current_include_filters)
-            await self.page.evaluate(f"var include_filters={js}")
-        else:
-            await self.page.evaluate(f"var include_filters=''")
-
-        self.xpath_data = await self.page.evaluate(
-            "async () => {" + self.xpath_element_js.replace('%ELEMENTS%', visualselector_xpath_selectors) + "}")
-        self.instock_data = await self.page.evaluate("async () => {" + self.instock_data_js + "}")
-
-        self.content = await self.page.content
-        # Bug 3 in Playwright screenshot handling
-        # Some bug where it gives the wrong screenshot size, but making a request with the clip set first seems to solve it
-        # JPEG is better here because the screenshots can be very very large
-
-        # Screenshots also travel via the ws:// (websocket) meaning that the binary data is base64 encoded
-        # which will significantly increase the IO size between the server and client, it's recommended to use the lowest
-        # acceptable screenshot quality here
-        try:
-            self.screenshot = await self.page.screenshot(type_='jpeg',
-                                                         fullPage=True,
-                                                         quality=int(os.getenv("SCREENSHOT_QUALITY", 72)))
-        except Exception as e:
-            logger.error("Error fetching screenshot")
-            # // May fail on very large pages with 'WARNING: tile memory limits exceeded, some content may not draw'
-            # // @ todo after text extract, we can place some overlay text with red background to say 'croppped'
-            logger.error('ERROR: content-fetcher page was maybe too large for a screenshot, reverting to viewport only screenshot')
-            try:
-                self.screenshot = await self.page.screenshot(type_='jpeg',
-                                                             fullPage=False,
-                                                             quality=int(os.getenv("SCREENSHOT_QUALITY", 72)))
-            except Exception as e:
-                logger.error('ERROR: Failed to get viewport-only reduced screenshot :(')
-                pass
-        finally:
-            # It's good to log here in the case that the browser crashes on shutting down but we still get the data we need
-            logger.success(f"Fetching '{url}' complete, closing page")
-            await self.page.close()
-            logger.success(f"Fetching '{url}' complete, closing browser")
-            await browser.close()
-        logger.success(f"Fetching '{url}' complete, exiting puppeteer fetch.")
-
-    async def main(self, **kwargs):
-        await self.fetch_page(**kwargs)
-
-    def run(self, url, timeout, request_headers, request_body, request_method, ignore_status_codes=False,
-            current_include_filters=None, is_binary=False):
-
-        #@todo make update_worker async which could run any of these content_fetchers within memory and time constraints
-        max_time = os.getenv('PUPPETEER_MAX_PROCESSING_TIMEOUT_SECONDS', 180)
-
-        # This will work in 3.10 but not >= 3.11 because 3.11 wants tasks only
-        try:
-            asyncio.run(asyncio.wait_for(self.main(
-                url=url,
-                timeout=timeout,
-                request_headers=request_headers,
-                request_body=request_body,
-                request_method=request_method,
-                ignore_status_codes=ignore_status_codes,
-                current_include_filters=current_include_filters,
-                is_binary=is_binary
-            ), timeout=max_time))
-        except asyncio.TimeoutError:
-            raise(BrowserFetchTimedOut(msg=f"Browser connected but was unable to process the page in {max_time} seconds."))
-

changedetectionio/content_fetchers/requests.py

@@ -1,91 +0,0 @@
-import hashlib
-import os
-
-import chardet
-import requests
-
-from changedetectionio.content_fetchers.exceptions import BrowserStepsInUnsupportedFetcher, EmptyReply, Non200ErrorCodeReceived
-from changedetectionio.content_fetchers.base import Fetcher
-
-
-# "html_requests" is listed as the default fetcher in store.py!
-class fetcher(Fetcher):
-    fetcher_description = "Basic fast Plaintext/HTTP Client"
-
-    def __init__(self, proxy_override=None, custom_browser_connection_url=None):
-        super().__init__()
-        self.proxy_override = proxy_override
-        # browser_connection_url is none because its always 'launched locally'
-
-    def run(self,
-            url,
-            timeout,
-            request_headers,
-            request_body,
-            request_method,
-            ignore_status_codes=False,
-            current_include_filters=None,
-            is_binary=False):
-
-        if self.browser_steps_get_valid_steps():
-            raise BrowserStepsInUnsupportedFetcher(url=url)
-
-        # Make requests use a more modern looking user-agent
-        if not {k.lower(): v for k, v in request_headers.items()}.get('user-agent', None):
-            request_headers['User-Agent'] = os.getenv("DEFAULT_SETTINGS_HEADERS_USERAGENT",
-                                                      'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.66 Safari/537.36')
-
-        proxies = {}
-
-        # Allows override the proxy on a per-request basis
-
-        # https://requests.readthedocs.io/en/latest/user/advanced/#socks
-        # Should also work with `socks5://user:pass@host:port` type syntax.
-
-        if self.proxy_override:
-            proxies = {'http': self.proxy_override, 'https': self.proxy_override, 'ftp': self.proxy_override}
-        else:
-            if self.system_http_proxy:
-                proxies['http'] = self.system_http_proxy
-            if self.system_https_proxy:
-                proxies['https'] = self.system_https_proxy
-
-        r = requests.request(method=request_method,
-                             data=request_body,
-                             url=url,
-                             headers=request_headers,
-                             timeout=timeout,
-                             proxies=proxies,
-                             verify=False)
-
-        # If the response did not tell us what encoding format to expect, Then use chardet to override what `requests` thinks.
-        # For example - some sites don't tell us it's utf-8, but return utf-8 content
-        # This seems to not occur when using webdriver/selenium, it seems to detect the text encoding more reliably.
-        # https://github.com/psf/requests/issues/1604 good info about requests encoding detection
-        if not is_binary:
-            # Don't run this for PDF (and requests identified as binary) takes a _long_ time
-            if not r.headers.get('content-type') or not 'charset=' in r.headers.get('content-type'):
-                encoding = chardet.detect(r.content)['encoding']
-                if encoding:
-                    r.encoding = encoding
-
-        self.headers = r.headers
-
-        if not r.content or not len(r.content):
-            raise EmptyReply(url=url, status_code=r.status_code)
-
-        # @todo test this
-        # @todo maybe you really want to test zero-byte return pages?
-        if r.status_code != 200 and not ignore_status_codes:
-            # maybe check with content works?
-            raise Non200ErrorCodeReceived(url=url, status_code=r.status_code, page_html=r.text)
-
-        self.status_code = r.status_code
-        if is_binary:
-            # Binary files just return their checksum until we add something smarter
-            self.content = hashlib.md5(r.content).hexdigest()
-        else:
-            self.content = r.text
-
-
-        self.raw_content = r.content

changedetectionio/content_fetchers/webdriver_selenium.py

@@ -1,119 +0,0 @@
-import os
-import time
-
-from loguru import logger
-from changedetectionio.content_fetchers.base import Fetcher
-
-class fetcher(Fetcher):
-    if os.getenv("WEBDRIVER_URL"):
-        fetcher_description = "WebDriver Chrome/Javascript via '{}'".format(os.getenv("WEBDRIVER_URL"))
-    else:
-        fetcher_description = "WebDriver Chrome/Javascript"
-
-    # Configs for Proxy setup
-    # In the ENV vars, is prefixed with "webdriver_", so it is for example "webdriver_sslProxy"
-    selenium_proxy_settings_mappings = ['proxyType', 'ftpProxy', 'httpProxy', 'noProxy',
-                                        'proxyAutoconfigUrl', 'sslProxy', 'autodetect',
-                                        'socksProxy', 'socksVersion', 'socksUsername', 'socksPassword']
-    proxy = None
-
-    def __init__(self, proxy_override=None, custom_browser_connection_url=None):
-        super().__init__()
-        from selenium.webdriver.common.proxy import Proxy as SeleniumProxy
-
-        # .strip('"') is going to save someone a lot of time when they accidently wrap the env value
-        if not custom_browser_connection_url:
-            self.browser_connection_url = os.getenv("WEBDRIVER_URL", 'http://browser-chrome:4444/wd/hub').strip('"')
-        else:
-            self.browser_connection_is_custom = True
-            self.browser_connection_url = custom_browser_connection_url
-
-        # If any proxy settings are enabled, then we should setup the proxy object
-        proxy_args = {}
-        for k in self.selenium_proxy_settings_mappings:
-            v = os.getenv('webdriver_' + k, False)
-            if v:
-                proxy_args[k] = v.strip('"')
-
-        # Map back standard HTTP_ and HTTPS_PROXY to webDriver httpProxy/sslProxy
-        if not proxy_args.get('webdriver_httpProxy') and self.system_http_proxy:
-            proxy_args['httpProxy'] = self.system_http_proxy
-        if not proxy_args.get('webdriver_sslProxy') and self.system_https_proxy:
-            proxy_args['httpsProxy'] = self.system_https_proxy
-
-        # Allows override the proxy on a per-request basis
-        if proxy_override is not None:
-            proxy_args['httpProxy'] = proxy_override
-
-        if proxy_args:
-            self.proxy = SeleniumProxy(raw=proxy_args)
-
-    def run(self,
-            url,
-            timeout,
-            request_headers,
-            request_body,
-            request_method,
-            ignore_status_codes=False,
-            current_include_filters=None,
-            is_binary=False):
-
-        from selenium import webdriver
-        from selenium.webdriver.chrome.options import Options as ChromeOptions
-        from selenium.common.exceptions import WebDriverException
-        # request_body, request_method unused for now, until some magic in the future happens.
-
-        options = ChromeOptions()
-        if self.proxy:
-            options.proxy = self.proxy
-
-        self.driver = webdriver.Remote(
-            command_executor=self.browser_connection_url,
-            options=options)
-
-        try:
-            self.driver.get(url)
-        except WebDriverException as e:
-            # Be sure we close the session window
-            self.quit()
-            raise
-
-        self.driver.set_window_size(1280, 1024)
-        self.driver.implicitly_wait(int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)))
-
-        if self.webdriver_js_execute_code is not None:
-            self.driver.execute_script(self.webdriver_js_execute_code)
-            # Selenium doesn't automatically wait for actions as good as Playwright, so wait again
-            self.driver.implicitly_wait(int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)))
-
-        # @todo - how to check this? is it possible?
-        self.status_code = 200
-        # @todo somehow we should try to get this working for WebDriver
-        # raise EmptyReply(url=url, status_code=r.status_code)
-
-        # @todo - dom wait loaded?
-        time.sleep(int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)) + self.render_extract_delay)
-        self.content = self.driver.page_source
-        self.headers = {}
-
-        self.screenshot = self.driver.get_screenshot_as_png()
-
-    # Does the connection to the webdriver work? run a test connection.
-    def is_ready(self):
-        from selenium import webdriver
-        from selenium.webdriver.chrome.options import Options as ChromeOptions
-
-        self.driver = webdriver.Remote(
-            command_executor=self.command_executor,
-            options=ChromeOptions())
-
-        # driver.quit() seems to cause better exceptions
-        self.quit()
-        return True
-
-    def quit(self):
-        if self.driver:
-            try:
-                self.driver.quit()
-            except Exception as e:
-                logger.debug(f"Content Fetcher > Exception in chrome shutdown/quit {str(e)}")

changedetectionio/flask_app.py

@@ -1,19 +1,25 @@
 #!/usr/bin/python3
 
+from changedetectionio import queuedWatchMetaData
+from copy import deepcopy
+from distutils.util import strtobool
+from feedgen.feed import FeedGenerator
+from flask_compress import Compress as FlaskCompress
+from flask_login import current_user
+from flask_restful import abort, Api
+from flask_wtf import CSRFProtect
+from functools import wraps
+from threading import Event
 import datetime
+import flask_login
+from loguru import logger
 import os
+import pytz
 import queue
 import threading
 import time
-from copy import deepcopy
-from distutils.util import strtobool
-from functools import wraps
-from threading import Event
-
-import flask_login
-import pytz
 import timeago
-from feedgen.feed import FeedGenerator
+
 from flask import (
     Flask,
     abort,
@@ -26,15 +32,10 @@ from flask import (
     session,
     url_for,
 )
-from flask_compress import Compress as FlaskCompress
-from flask_login import current_user
+
 from flask_paginate import Pagination, get_page_parameter
-from flask_restful import abort, Api
-from flask_wtf import CSRFProtect
-from loguru import logger
 
 from changedetectionio import html_tools, __version__
-from changedetectionio import queuedWatchMetaData
 from changedetectionio.api import api_v1
 
 datastore = None
@@ -404,21 +405,17 @@ def changedetection_app(config=None, datastore_o=None):
         global datastore
         from changedetectionio import forms
 
-        active_tag_req = request.args.get('tag', '').lower().strip()
-        active_tag_uuid = active_tag = None
+        limit_tag = request.args.get('tag', '').lower().strip()
 
         # Be sure limit_tag is a uuid
-        if active_tag_req:
-            for uuid, tag in datastore.data['settings']['application'].get('tags', {}).items():
-                if active_tag_req == tag.get('title', '').lower().strip() or active_tag_req == uuid:
-                    active_tag = tag
-                    active_tag_uuid = uuid
-                    break
+        for uuid, tag in datastore.data['settings']['application'].get('tags', {}).items():
+            if limit_tag == tag.get('title', '').lower().strip():
+                limit_tag = uuid
 
 
         # Redirect for the old rss path which used the /?rss=true
         if request.args.get('rss'):
-            return redirect(url_for('rss', tag=active_tag_uuid))
+            return redirect(url_for('rss', tag=limit_tag))
 
         op = request.args.get('op')
         if op:
@@ -429,7 +426,7 @@ def changedetection_app(config=None, datastore_o=None):
                 datastore.data['watching'][uuid].toggle_mute()
 
             datastore.needs_write = True
-            return redirect(url_for('index', tag = active_tag_uuid))
+            return redirect(url_for('index', tag = limit_tag))
 
         # Sort by last_changed and add the uuid which is usually the key..
         sorted_watches = []
@@ -440,7 +437,7 @@ def changedetection_app(config=None, datastore_o=None):
             if with_errors and not watch.get('last_error'):
                 continue
 
-            if active_tag_uuid and not active_tag_uuid in watch['tags']:
+            if limit_tag and not limit_tag in watch['tags']:
                     continue
             if watch.get('last_error'):
                 errored_count += 1
@@ -459,12 +456,11 @@ def changedetection_app(config=None, datastore_o=None):
                                 total=total_count,
                                 per_page=datastore.data['settings']['application'].get('pager_size', 50), css_framework="semantic")
 
-        sorted_tags = sorted(datastore.data['settings']['application'].get('tags').items(), key=lambda x: x[1]['title'])
+
         output = render_template(
             "watch-overview.html",
                                  # Don't link to hosting when we're on the hosting environment
-                                 active_tag=active_tag,
-                                 active_tag_uuid=active_tag_uuid,
+                                 active_tag=limit_tag,
                                  app_rss_token=datastore.data['settings']['application']['rss_access_token'],
                                  datastore=datastore,
                                  errored_count=errored_count,
@@ -479,7 +475,7 @@ def changedetection_app(config=None, datastore_o=None):
                                  sort_attribute=request.args.get('sort') if request.args.get('sort') else request.cookies.get('sort'),
                                  sort_order=request.args.get('order') if request.args.get('order') else request.cookies.get('order'),
                                  system_default_fetcher=datastore.data['settings']['application'].get('fetch_backend'),
-                                 tags=sorted_tags,
+                                 tags=datastore.data['settings']['application'].get('tags'),
                                  watches=sorted_watches
                                  )
 
@@ -767,7 +763,7 @@ def changedetection_app(config=None, datastore_o=None):
     @app.route("/settings", methods=['GET', "POST"])
     @login_optionally_required
     def settings_page():
-        from changedetectionio import forms
+        from changedetectionio import content_fetcher, forms
 
         default = deepcopy(datastore.data['settings'])
         if datastore.proxy_list is not None:

changedetectionio/forms.py

@@ -27,7 +27,7 @@ from validators.url import url as url_validator
 # each select <option data-enabled="enabled-0-0"
 from changedetectionio.blueprint.browser_steps.browser_steps import browser_step_ui_config
 
-from changedetectionio import html_tools, content_fetchers
+from changedetectionio import content_fetcher, html_tools
 
 from changedetectionio.notification import (
     valid_notification_formats,
@@ -167,31 +167,33 @@ class ValidateContentFetcherIsReady(object):
         self.message = message
 
     def __call__(self, form, field):
+        import urllib3.exceptions
+        from changedetectionio import content_fetcher
         return
 
 # AttributeError: module 'changedetectionio.content_fetcher' has no attribute 'extra_browser_unlocked<>ASDF213r123r'
         # Better would be a radiohandler that keeps a reference to each class
-        # if field.data is not None and field.data != 'system':
-        #     klass = getattr(content_fetcher, field.data)
-        #     some_object = klass()
-        #     try:
-        #         ready = some_object.is_ready()
-        #
-        #     except urllib3.exceptions.MaxRetryError as e:
-        #         driver_url = some_object.command_executor
-        #         message = field.gettext('Content fetcher \'%s\' did not respond.' % (field.data))
-        #         message += '<br>' + field.gettext(
-        #             'Be sure that the selenium/webdriver runner is running and accessible via network from this container/host.')
-        #         message += '<br>' + field.gettext('Did you follow the instructions in the wiki?')
-        #         message += '<br><br>' + field.gettext('WebDriver Host: %s' % (driver_url))
-        #         message += '<br><a href="https://github.com/dgtlmoon/changedetection.io/wiki/Fetching-pages-with-WebDriver">Go here for more information</a>'
-        #         message += '<br>'+field.gettext('Content fetcher did not respond properly, unable to use it.\n %s' % (str(e)))
-        #
-        #         raise ValidationError(message)
-        #
-        #     except Exception as e:
-        #         message = field.gettext('Content fetcher \'%s\' did not respond properly, unable to use it.\n %s')
-        #         raise ValidationError(message % (field.data, e))
+        if field.data is not None and field.data != 'system':
+            klass = getattr(content_fetcher, field.data)
+            some_object = klass()
+            try:
+                ready = some_object.is_ready()
+
+            except urllib3.exceptions.MaxRetryError as e:
+                driver_url = some_object.command_executor
+                message = field.gettext('Content fetcher \'%s\' did not respond.' % (field.data))
+                message += '<br>' + field.gettext(
+                    'Be sure that the selenium/webdriver runner is running and accessible via network from this container/host.')
+                message += '<br>' + field.gettext('Did you follow the instructions in the wiki?')
+                message += '<br><br>' + field.gettext('WebDriver Host: %s' % (driver_url))
+                message += '<br><a href="https://github.com/dgtlmoon/changedetection.io/wiki/Fetching-pages-with-WebDriver">Go here for more information</a>'
+                message += '<br>'+field.gettext('Content fetcher did not respond properly, unable to use it.\n %s' % (str(e)))
+
+                raise ValidationError(message)
+
+            except Exception as e:
+                message = field.gettext('Content fetcher \'%s\' did not respond properly, unable to use it.\n %s')
+                raise ValidationError(message % (field.data, e))
 
 
 class ValidateNotificationBodyAndTitleWhenURLisSet(object):
@@ -419,7 +421,7 @@ class commonSettingsForm(Form):
     notification_title = StringField('Notification Title', default='ChangeDetection.io Notification - {{ watch_url }}', validators=[validators.Optional(), ValidateJinja2Template()])
     notification_body = TextAreaField('Notification Body', default='{{ watch_url }} had a change.', validators=[validators.Optional(), ValidateJinja2Template()])
     notification_format = SelectField('Notification format', choices=valid_notification_formats.keys())
-    fetch_backend = RadioField(u'Fetch Method', choices=content_fetchers.available_fetchers(), validators=[ValidateContentFetcherIsReady()])
+    fetch_backend = RadioField(u'Fetch Method', choices=content_fetcher.available_fetchers(), validators=[ValidateContentFetcherIsReady()])
     extract_title_as_title = BooleanField('Extract <title> from document and use as watch title', default=False)
     webdriver_delay = IntegerField('Wait seconds before extracting text', validators=[validators.Optional(), validators.NumberRange(min=1,
                                                                                                                                     message="Should contain one or more seconds")])
@@ -550,7 +552,7 @@ class globalSettingsApplicationForm(commonSettingsForm):
                            render_kw={"placeholder": os.getenv('BASE_URL', 'Not set')}
                            )
     empty_pages_are_a_change =  BooleanField('Treat empty pages as a change?', default=False)
-    fetch_backend = RadioField('Fetch Method', default="html_requests", choices=content_fetchers.available_fetchers(), validators=[ValidateContentFetcherIsReady()])
+    fetch_backend = RadioField('Fetch Method', default="html_requests", choices=content_fetcher.available_fetchers(), validators=[ValidateContentFetcherIsReady()])
     global_ignore_text = StringListField('Ignore Text', [ValidateListRegex()])
     global_subtractive_selectors = StringListField('Remove elements', [ValidateCSSJSONXPATHInput(allow_xpath=False, allow_json=False)])
     ignore_whitespace = BooleanField('Ignore whitespace')

changedetectionio/processors/__init__.py

@@ -2,6 +2,7 @@ from abc import abstractmethod
 import os
 import hashlib
 import re
+from changedetectionio import content_fetcher
 from copy import deepcopy
 from distutils.util import strtobool
 from loguru import logger
@@ -49,7 +50,7 @@ class difference_detection_processor():
             connection = list(
                 filter(lambda s: (s['browser_name'] == key), self.datastore.data['settings']['requests'].get('extra_browsers', [])))
             if connection:
-                prefer_fetch_backend = 'html_webdriver'
+                prefer_fetch_backend = 'base_html_playwright'
                 custom_browser_connection_url = connection[0].get('browser_connection_url')
 
         # PDF should be html_requests because playwright will serve it up (so far) in a embedded page
@@ -59,28 +60,17 @@ class difference_detection_processor():
            prefer_fetch_backend = "html_requests"
 
         # Grab the right kind of 'fetcher', (playwright, requests, etc)
-        from changedetectionio import content_fetchers
-        if hasattr(content_fetchers, prefer_fetch_backend):
-            # @todo TEMPORARY HACK - SWITCH BACK TO PLAYWRIGHT FOR BROWSERSTEPS
-            if prefer_fetch_backend == 'html_webdriver' and self.watch.has_browser_steps:
-                # This is never supported in selenium anyway
-                logger.warning("Using playwright fetcher override for possible puppeteer request in browsersteps, because puppetteer:browser steps is incomplete.")
-                from changedetectionio.content_fetchers.playwright import fetcher as playwright_fetcher
-                fetcher_obj = playwright_fetcher
-            else:
-                fetcher_obj = getattr(content_fetchers, prefer_fetch_backend)
+        if hasattr(content_fetcher, prefer_fetch_backend):
+            fetcher_obj = getattr(content_fetcher, prefer_fetch_backend)
         else:
-            # What it referenced doesnt exist, Just use a default
-            fetcher_obj = getattr(content_fetchers, "html_requests")
+            # If the klass doesnt exist, just use a default
+            fetcher_obj = getattr(content_fetcher, "html_requests")
+
 
         proxy_url = None
         if preferred_proxy_id:
-            # Custom browser endpoints should NOT have a proxy added
-            if not prefer_fetch_backend.startswith('extra_browser_'):
-                proxy_url = self.datastore.proxy_list.get(preferred_proxy_id).get('url')
-                logger.debug(f"Selected proxy key '{preferred_proxy_id}' as proxy URL '{proxy_url}' for {url}")
-            else:
-                logger.debug(f"Skipping adding proxy data when custom Browser endpoint is specified. ")
+            proxy_url = self.datastore.proxy_list.get(preferred_proxy_id).get('url')
+            logger.debug(f"Selected proxy key '{preferred_proxy_id}' as proxy URL '{proxy_url}' for {url}")
 
         # Now call the fetcher (playwright/requests/etc) with arguments that only a fetcher would need.
         # When browser_connection_url is None, it method should default to working out whats the best defaults (os env vars etc)

changedetectionio/processors/text_json_diff.py

@@ -8,9 +8,8 @@ import urllib3
 
 from . import difference_detection_processor
 from ..html_tools import PERL_STYLE_REGEX, cdata_in_document_to_text
-from changedetectionio import html_tools, content_fetchers
+from changedetectionio import content_fetcher, html_tools
 from changedetectionio.blueprint.price_data_follower import PRICE_DATA_TRACK_ACCEPT, PRICE_DATA_TRACK_REJECT
-import changedetectionio.content_fetchers
 from copy import deepcopy
 from loguru import logger
 
@@ -61,7 +60,7 @@ class perform_site_check(difference_detection_processor):
         update_obj['previous_md5_before_filters'] = hashlib.md5(self.fetcher.content.encode('utf-8')).hexdigest()
         if skip_when_checksum_same:
             if update_obj['previous_md5_before_filters'] == watch.get('previous_md5_before_filters'):
-                raise content_fetchers.exceptions.checksumFromPreviousCheckWasTheSame()
+                raise content_fetcher.checksumFromPreviousCheckWasTheSame()
 
         # Fetching complete, now filters
 
@@ -244,7 +243,7 @@ class perform_site_check(difference_detection_processor):
         # Treat pages with no renderable text content as a change? No by default
         empty_pages_are_a_change = self.datastore.data['settings']['application'].get('empty_pages_are_a_change', False)
         if not is_json and not empty_pages_are_a_change and len(stripped_text_from_html.strip()) == 0:
-            raise content_fetchers.exceptions.ReplyWithContentButNoText(url=url,
+            raise content_fetcher.ReplyWithContentButNoText(url=url,
                                                             status_code=self.fetcher.get_last_status_code(),
                                                             screenshot=screenshot,
                                                             has_filters=has_filter_rule,

changedetectionio/res/stock-not-in-stock.js

@@ -10,7 +10,6 @@ function isItemInStock() {
     const outOfStockTexts = [
         ' أخبرني عندما يتوفر',
         '0 in stock',
-        'actuellement indisponible',        
         'agotado',
         'article épuisé',
         'artikel zurzeit vergriffen',
@@ -18,7 +17,6 @@ function isItemInStock() {
         'ausverkauft', // sold out
         'available for back order',
         'back-order or out of stock',
-        'back in stock soon',
         'backordered',
         'benachrichtigt mich', // notify me
         'brak na stanie',
@@ -26,7 +24,6 @@ function isItemInStock() {
         'coming soon',
         'currently have any tickets for this',
         'currently unavailable',
-        'dieser artikel ist bald wieder verfügbar',
         'dostępne wkrótce',
         'en rupture de stock',
         'ist derzeit nicht auf lager',
@@ -60,13 +57,11 @@ function isItemInStock() {
         'tickets unavailable',
         'tijdelijk uitverkocht',
         'unavailable tickets',
-        'vorbestellung ist bald möglich',
         'we do not currently have an estimate of when this product will be back in stock.',
         'we don\'t know when or if this item will be back in stock.',
         'zur zeit nicht an lager',
         '品切れ',
         '已售完',
-        '已售',
         '품절'
     ];
 

changedetectionio/run_custom_browser_url_tests.sh

@@ -2,18 +2,16 @@
 
 # run some tests and look if the 'custom-browser-search-string=1' connect string appeared in the correct containers
 
-# @todo do it again but with the puppeteer one
-
 # enable debug
 set -x
 
 # A extra browser is configured, but we never chose to use it, so it should NOT show in the logs
 docker run --rm -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio;pytest tests/custom_browser_url/test_custom_browser_url.py::test_request_not_via_custom_browser_url'
-docker logs sockpuppetbrowser-custom-url &>log-custom.txt
-grep 'custom-browser-search-string=1' log-custom.txt
+docker logs sockpuppetbrowser-custom-url &>log.txt
+grep 'custom-browser-search-string=1' log.txt
 if [ $? -ne 1 ]
 then
-  echo "Saw a request in 'sockpuppetbrowser-custom-url' container with 'custom-browser-search-string=1' when I should not - log-custom.txt"
+  echo "Saw a request in 'sockpuppetbrowser-custom-url' container with 'custom-browser-search-string=1' when I should not"
   exit 1
 fi
 
@@ -27,11 +25,11 @@ fi
 
 # Special connect string should appear in the custom-url container, but not in the 'default' one
 docker run --rm -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio;pytest tests/custom_browser_url/test_custom_browser_url.py::test_request_via_custom_browser_url'
-docker logs sockpuppetbrowser-custom-url &>log-custom.txt
-grep 'custom-browser-search-string=1' log-custom.txt
+docker logs sockpuppetbrowser-custom-url &>log.txt
+grep 'custom-browser-search-string=1' log.txt
 if [ $? -ne 0 ]
 then
-  echo "Did not see request in 'sockpuppetbrowser-custom-url' container with 'custom-browser-search-string=1' when I should - log-custom.txt"
+  echo "Did not see request in 'sockpuppetbrowser-custom-url' container with 'custom-browser-search-string=1' when I should"
   exit 1
 fi
 

changedetectionio/run_proxy_tests.sh

@@ -10,7 +10,41 @@ set -x
 docker run --network changedet-network -d --name squid-one --hostname squid-one --rm -v `pwd`/tests/proxy_list/squid.conf:/etc/squid/conf.d/debian.conf ubuntu/squid:4.13-21.10_edge
 docker run --network changedet-network -d --name squid-two --hostname squid-two --rm -v `pwd`/tests/proxy_list/squid.conf:/etc/squid/conf.d/debian.conf ubuntu/squid:4.13-21.10_edge
 
-# Used for configuring a custom proxy URL via the UI - with username+password auth
+# SOCKS5 related - start simple Socks5 proxy server
+# SOCKSTEST=xyz should show in the logs of this service to confirm it fetched
+docker run --network changedet-network -d --hostname socks5proxy --name socks5proxy -p 1080:1080 -e PROXY_USER=proxy_user123 -e PROXY_PASSWORD=proxy_pass123 serjs/go-socks5-proxy
+docker run --network changedet-network -d --hostname socks5proxy-noauth -p 1081:1080 --name socks5proxy-noauth  serjs/go-socks5-proxy
+
+echo "---------------------------------- SOCKS5 -------------------"
+# SOCKS5 related - test from proxies.json
+docker run --network changedet-network \
+  -v `pwd`/tests/proxy_socks5/proxies.json-example:/app/changedetectionio/test-datastore/proxies.json \
+  --rm \
+  -e "SOCKSTEST=proxiesjson" \
+  test-changedetectionio \
+  bash -c 'cd changedetectionio && pytest tests/proxy_socks5/test_socks5_proxy_sources.py'
+
+# SOCKS5 related - by manually entering in UI
+docker run --network changedet-network \
+  --rm \
+  -e "SOCKSTEST=manual" \
+  test-changedetectionio \
+  bash -c 'cd changedetectionio && pytest tests/proxy_socks5/test_socks5_proxy.py'
+
+# SOCKS5 related - test from proxies.json via playwright - NOTE- PLAYWRIGHT DOESNT SUPPORT AUTHENTICATING PROXY
+docker run --network changedet-network \
+  -e "SOCKSTEST=manual-playwright" \
+  -v `pwd`/tests/proxy_socks5/proxies.json-example-noauth:/app/changedetectionio/test-datastore/proxies.json \
+  -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" \
+  --rm \
+  test-changedetectionio \
+  bash -c 'cd changedetectionio && pytest tests/proxy_socks5/test_socks5_proxy_sources.py'
+
+echo "socks5 server logs"
+docker logs socks5proxy
+echo "----------------------------------"
+
+# Used for configuring a custom proxy URL via the UI
 docker run --network changedet-network -d \
   --name squid-custom \
   --hostname squid-custom \
@@ -26,17 +60,15 @@ docker run --network changedet-network \
   test-changedetectionio \
   bash -c 'cd changedetectionio && pytest tests/proxy_list/test_multiple_proxy.py'
 
-set +e
-echo "- Looking for chosen.changedetection.io request in squid-one - it should NOT be here"
+
+## Should be a request in the default "first" squid
 docker logs squid-one 2>/dev/null|grep chosen.changedetection.io
-if [ $? -ne 1 ]
+if [ $? -ne 0 ]
 then
-  echo "Saw a request to chosen.changedetection.io in the squid logs (while checking preferred proxy - squid one) WHEN I SHOULD NOT"
+  echo "Did not see a request to chosen.changedetection.io in the squid logs (while checking preferred proxy - squid one)"
   exit 1
 fi
 
-set -e
-echo "- Looking for chosen.changedetection.io request in squid-two"
 # And one in the 'second' squid (user selects this as preferred)
 docker logs squid-two 2>/dev/null|grep chosen.changedetection.io
 if [ $? -ne 0 ]
@@ -45,6 +77,7 @@ then
   exit 1
 fi
 
+
 # Test the UI configurable proxies
 docker run --network changedet-network \
   test-changedetectionio \
@@ -52,7 +85,6 @@ docker run --network changedet-network \
 
 
 # Should see a request for one.changedetection.io in there
-echo "- Looking for .changedetection.io request in squid-custom"
 docker logs squid-custom 2>/dev/null|grep "TCP_TUNNEL.200.*changedetection.io"
 if [ $? -ne 0 ]
 then
@@ -69,7 +101,7 @@ docker run --network changedet-network \
 set +e
 # Check request was never seen in any container
 for c in $(echo "squid-one squid-two squid-custom"); do
-  echo ....Checking $c
+  echo Checking $c
   docker logs $c &> $c.txt
   grep noproxy $c.txt
   if [ $? -ne 1 ]

changedetectionio/run_socks_proxy_tests.sh

@@ -1,43 +0,0 @@
-#!/bin/bash
-
-# exit when any command fails
-set -e
-# enable debug
-set -x
-
-
-# SOCKS5 related - start simple Socks5 proxy server
-# SOCKSTEST=xyz should show in the logs of this service to confirm it fetched
-docker run --network changedet-network -d --hostname socks5proxy --rm  --name socks5proxy -p 1080:1080 -e PROXY_USER=proxy_user123 -e PROXY_PASSWORD=proxy_pass123 serjs/go-socks5-proxy
-docker run --network changedet-network -d --hostname socks5proxy-noauth --rm  -p 1081:1080 --name socks5proxy-noauth  serjs/go-socks5-proxy
-
-echo "---------------------------------- SOCKS5 -------------------"
-# SOCKS5 related - test from proxies.json
-docker run --network changedet-network \
-  -v `pwd`/tests/proxy_socks5/proxies.json-example:/app/changedetectionio/test-datastore/proxies.json \
-  --rm \
-  -e "SOCKSTEST=proxiesjson" \
-  test-changedetectionio \
-  bash -c 'cd changedetectionio && pytest tests/proxy_socks5/test_socks5_proxy_sources.py'
-
-# SOCKS5 related - by manually entering in UI
-docker run --network changedet-network \
-  --rm \
-  -e "SOCKSTEST=manual" \
-  test-changedetectionio \
-  bash -c 'cd changedetectionio && pytest tests/proxy_socks5/test_socks5_proxy.py'
-
-# SOCKS5 related - test from proxies.json via playwright - NOTE- PLAYWRIGHT DOESNT SUPPORT AUTHENTICATING PROXY
-docker run --network changedet-network \
-  -e "SOCKSTEST=manual-playwright" \
-  -v `pwd`/tests/proxy_socks5/proxies.json-example-noauth:/app/changedetectionio/test-datastore/proxies.json \
-  -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" \
-  --rm \
-  test-changedetectionio \
-  bash -c 'cd changedetectionio && pytest tests/proxy_socks5/test_socks5_proxy_sources.py'
-
-echo "socks5 server logs"
-docker logs socks5proxy
-echo "----------------------------------"
-
-docker kill socks5proxy socks5proxy-noauth

changedetectionio/static/js/browser-steps.js

@@ -160,12 +160,6 @@ $(document).ready(function () {
                     e.offsetX > item.left * y_scale && e.offsetX < item.left * y_scale + item.width * y_scale
 
                 ) {
-                    // Ignore really large ones, because we are scraping 'div' also from xpath_element_scraper but
-                    // that div or whatever could be some wrapper and would generally make you select the whole page
-                    if (item.width > 800 && item.height > 400) {
-                        return
-                    }
-
                     // There could be many elements here, record them all and then we'll find out which is the most 'useful'
                     // (input, textarea, button, A etc)
                     if (item.width < xpath_data['browser_width']) {

changedetectionio/templates/watch-overview.html

@@ -1,6 +1,6 @@
 {% extends 'base.html' %}
 {% block content %}
-{% from '_helpers.jinja' import render_simple_field, render_field, render_nolabel_field, sort_by_title %}
+{% from '_helpers.jinja' import render_simple_field, render_field, render_nolabel_field %}
 <script src="{{url_for('static_content', group='js', filename='jquery-3.6.0.min.js')}}"></script>
 <script src="{{url_for('static_content', group='js', filename='watch-overview.js')}}" defer></script>
 
@@ -13,7 +13,7 @@
             <div id="watch-add-wrapper-zone">
 
                     {{ render_nolabel_field(form.url, placeholder="https://...", required=true) }}
-                    {{ render_nolabel_field(form.tags, value=active_tag.title if active_tag else '', placeholder="watch label / tag") }}
+                    {{ render_nolabel_field(form.tags, value=tags[active_tag].title if active_tag else '', placeholder="watch label / tag") }}
                     {{ render_nolabel_field(form.watch_submit_button, title="Watch this URL!" ) }}
                     {{ render_nolabel_field(form.edit_and_watch_submit_button, title="Edit first then Watch") }}
             </div>
@@ -46,13 +46,11 @@
     {% if search_q %}<div id="search-result-info">Searching "<strong><i>{{search_q}}</i></strong>"</div>{% endif %}
     <div>
         <a href="{{url_for('index')}}" class="pure-button button-tag {{'active' if not active_tag }}">All</a>
-
-    <!-- tag list -->
-    {% for uuid, tag in tags %}
-        {% if tag != "" %}
-            <a href="{{url_for('index', tag=uuid) }}" class="pure-button button-tag {{'active' if active_tag_uuid == uuid }}">{{ tag.title }}</a>
-        {% endif %}
-    {% endfor %}
+        {% for uuid, tag in tags.items() %}
+            {% if tag != "" %}
+                <a href="{{url_for('index', tag=uuid) }}" class="pure-button button-tag {{'active' if active_tag == uuid }}">{{ tag.title }}</a>
+            {% endif %}
+        {% endfor %}
     </div>
 
     {% set sort_order = sort_order or 'asc' %}
@@ -199,8 +197,8 @@
             </li>
             {% endif %}
             <li>
-               <a href="{{ url_for('form_watch_checknow', tag=active_tag_uuid, with_errors=request.args.get('with_errors',0)) }}" class="pure-button button-tag ">Recheck
-                all {% if active_tag_uuid %} in "{{active_tag.title}}"{%endif%}</a>
+               <a href="{{ url_for('form_watch_checknow', tag=active_tag, with_errors=request.args.get('with_errors',0)) }}" class="pure-button button-tag ">Recheck
+                all {% if active_tag%} in "{{tags[active_tag].title}}"{%endif%}</a>
             </li>
             <li>
                 <a href="{{ url_for('rss', tag=active_tag , token=app_rss_token)}}"><img alt="RSS Feed" id="feed-icon" src="{{url_for('static_content', group='images', filename='Generic_Feed-icon.svg')}}" height="15"></a>

changedetectionio/tests/fetchers/test_custom_js_before_content.py

@@ -1,56 +0,0 @@
-import os
-from flask import url_for
-from ..util import live_server_setup, wait_for_all_checks, extract_UUID_from_client
-
-
-def test_execute_custom_js(client, live_server):
-
-    live_server_setup(live_server)
-    assert os.getenv('PLAYWRIGHT_DRIVER_URL'), "Needs PLAYWRIGHT_DRIVER_URL set for this test"
-
-    test_url = url_for('test_interactive_html_endpoint', _external=True)
-    test_url = test_url.replace('localhost.localdomain', 'cdio')
-    test_url = test_url.replace('localhost', 'cdio')
-
-    res = client.post(
-        url_for("form_quick_watch_add"),
-        data={"url": test_url, "tags": '', 'edit_and_watch_submit_button': 'Edit > Watch'},
-        follow_redirects=True
-    )
-
-    assert b"Watch added in Paused state, saving will unpause" in res.data
-
-    res = client.post(
-        url_for("edit_page", uuid="first", unpause_on_save=1),
-        data={
-            "url": test_url,
-            "tags": "",
-            'fetch_backend': "html_webdriver",
-            'webdriver_js_execute_code': 'document.querySelector("button[name=test-button]").click();',
-            'headers': "testheader: yes\buser-agent: MyCustomAgent",
-        },
-        follow_redirects=True
-    )
-    assert b"unpaused" in res.data
-    wait_for_all_checks(client)
-
-    uuid = extract_UUID_from_client(client)
-    assert live_server.app.config['DATASTORE'].data['watching'][uuid].history_n >= 1, "Watch history had atleast 1 (everything fetched OK)"
-
-    assert b"This text should be removed" not in res.data
-
-    # Check HTML conversion detected and workd
-    res = client.get(
-        url_for("preview_page", uuid=uuid),
-        follow_redirects=True
-    )
-    assert b"This text should be removed" not in res.data
-    assert b"I smell JavaScript because the button was pressed" in res.data
-
-    assert b"testheader: yes" in res.data
-    assert b"user-agent: mycustomagent" in res.data
-
-    client.get(
-        url_for("form_delete", uuid="all"),
-        follow_redirects=True
-    )

changedetectionio/tests/proxy_list/test_multiple_proxy.py

@@ -1,6 +1,6 @@
 #!/usr/bin/python3
 
-import os
+import time
 from flask import url_for
 from ..util import live_server_setup, wait_for_all_checks
 
@@ -9,20 +9,22 @@ def test_preferred_proxy(client, live_server):
     live_server_setup(live_server)
     url = "http://chosen.changedetection.io"
 
-
     res = client.post(
-        url_for("form_quick_watch_add"),
-        data={"url": url, "tags": '', 'edit_and_watch_submit_button': 'Edit > Watch'},
+        url_for("import_page"),
+        # Because a URL wont show in squid/proxy logs due it being SSLed
+        # Use plain HTTP or a specific domain-name here
+        data={"urls": url},
         follow_redirects=True
     )
-    assert b"Watch added in Paused state, saving will unpause" in res.data
+
+    assert b"1 Imported" in res.data
 
     wait_for_all_checks(client)
     res = client.post(
-        url_for("edit_page", uuid="first", unpause_on_save=1),
+        url_for("edit_page", uuid="first"),
         data={
                 "include_filters": "",
-                "fetch_backend": 'html_webdriver' if os.getenv('PLAYWRIGHT_DRIVER_URL') else 'html_requests',
+                "fetch_backend": "html_requests",
                 "headers": "",
                 "proxy": "proxy-two",
                 "tags": "",
@@ -30,6 +32,6 @@ def test_preferred_proxy(client, live_server):
               },
         follow_redirects=True
     )
-    assert b"unpaused" in res.data
+    assert b"Updated watch." in res.data
     wait_for_all_checks(client)
     # Now the request should appear in the second-squid logs

changedetectionio/tests/proxy_list/test_select_custom_proxy.py

@@ -3,7 +3,6 @@
 import time
 from flask import url_for
 from ..util import live_server_setup, wait_for_all_checks
-import os
 
 # just make a request, we will grep in the docker logs to see it actually got called
 def test_select_custom(client, live_server):
@@ -15,7 +14,7 @@ def test_select_custom(client, live_server):
         data={
             "requests-time_between_check-minutes": 180,
             "application-ignore_whitespace": "y",
-            "application-fetch_backend": 'html_webdriver' if os.getenv('PLAYWRIGHT_DRIVER_URL') else 'html_requests',
+            "application-fetch_backend": "html_requests",
             "requests-extra_proxies-0-proxy_name": "custom-test-proxy",
             # test:awesome is set in tests/proxy_list/squid-passwords.txt
             "requests-extra_proxies-0-proxy_url": "http://test:awesome@squid-custom:3128",

changedetectionio/tests/test_backend.py

@@ -29,7 +29,7 @@ def test_check_basic_change_detection_functionality(client, live_server):
 
     assert b"1 Imported" in res.data
 
-    wait_for_all_checks(client)
+    time.sleep(sleep_time_for_fetch_thread)
 
     # Do this a few times.. ensures we dont accidently set the status
     for n in range(3):

changedetectionio/tests/test_errorhandling.py

@@ -3,7 +3,7 @@
 import time
 
 from flask import url_for
-from .util import live_server_setup, wait_for_all_checks
+from . util import live_server_setup
 
 from ..html_tools import *
 
@@ -30,7 +30,7 @@ def _runner_test_http_errors(client, live_server, http_code, expected_text):
     assert b"1 Imported" in res.data
 
     # Give the thread time to pick it up
-    wait_for_all_checks(client)
+    time.sleep(2)
 
     res = client.get(url_for("index"))
     # no change
@@ -57,7 +57,7 @@ def _runner_test_http_errors(client, live_server, http_code, expected_text):
 def test_http_error_handler(client, live_server):
     _runner_test_http_errors(client, live_server, 403, 'Access denied')
     _runner_test_http_errors(client, live_server, 404, 'Page not found')
-    _runner_test_http_errors(client, live_server, 500, '(Internal server error) received')
+    _runner_test_http_errors(client, live_server, 500, '(Internal server Error) received')
     _runner_test_http_errors(client, live_server, 400, 'Error - Request returned a HTTP error code 400')
     res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
@@ -76,7 +76,7 @@ def test_DNS_errors(client, live_server):
     assert b"1 Imported" in res.data
 
     # Give the thread time to pick it up
-    wait_for_all_checks(client)
+    time.sleep(3)
 
     res = client.get(url_for("index"))
     found_name_resolution_error = b"Temporary failure in name resolution" in res.data or b"Name or service not known" in res.data
@@ -104,7 +104,7 @@ def test_low_level_errors_clear_correctly(client, live_server):
         follow_redirects=True
     )
     assert b"1 Imported" in res.data
-    wait_for_all_checks(client)
+    time.sleep(2)
 
     # We should see the DNS error
     res = client.get(url_for("index"))
@@ -121,7 +121,7 @@ def test_low_level_errors_clear_correctly(client, live_server):
     )
 
     # Now the error should be gone
-    wait_for_all_checks(client)
+    time.sleep(2)
     res = client.get(url_for("index"))
     found_name_resolution_error = b"Temporary failure in name resolution" in res.data or b"Name or service not known" in res.data
     assert not found_name_resolution_error

changedetectionio/tests/test_pdf.py

@@ -29,8 +29,7 @@ def test_fetch_pdf(client, live_server):
         follow_redirects=True
     )
 
-    # PDF header should not be there (it was converted to text)
-    assert b'PDF' not in res.data[:10]
+    assert b'PDF-1.5' not in res.data
     assert b'hello world' in res.data
 
     # So we know if the file changes in other ways

changedetectionio/tests/util.py

@@ -242,28 +242,5 @@ def live_server_setup(live_server):
             resp.headers['Content-Type'] = 'application/pdf'
             return resp
 
-    @live_server.app.route('/test-interactive-html-endpoint')
-    def test_interactive_html_endpoint():
-        header_text=""
-        for k,v in request.headers.items():
-            header_text += f"{k}: {v}<br>"
-
-        resp = make_response(f"""
-        <html>
-          <body>
-          Primitive JS check for <pre>changedetectionio/tests/visualselector/test_fetch_data.py</pre>
-            <p id="remove">This text should be removed</p>
-              <form onsubmit="event.preventDefault();">
-            <!-- obfuscated text so that we dont accidentally get a false positive due to conversion of the source :) --->
-                <button name="test-button" onclick="getElementById('remove').remove();getElementById('some-content').innerHTML = atob('SSBzbWVsbCBKYXZhU2NyaXB0IGJlY2F1c2UgdGhlIGJ1dHRvbiB3YXMgcHJlc3NlZCE=')">Click here</button>
-                <div id=some-content></div>
-                <pre>
-                {header_text.lower()}
-                </pre>
-              </body>
-         </html>""", 200)
-        resp.headers['Content-Type'] = 'text/html'
-        return resp
-
     live_server.start()
 

changedetectionio/tests/visualselector/test_fetch_data.py

@@ -1,5 +1,6 @@
 #!/usr/bin/python3
 
+import time
 import os
 from flask import url_for
 from ..util import live_server_setup, wait_for_all_checks, extract_UUID_from_client
@@ -7,19 +8,15 @@ from ..util import live_server_setup, wait_for_all_checks, extract_UUID_from_cli
 def test_setup(client, live_server):
     live_server_setup(live_server)
 
-
 # Add a site in paused mode, add an invalid filter, we should still have visual selector data ready
 def test_visual_selector_content_ready(client, live_server):
-
     import os
     import json
 
     assert os.getenv('PLAYWRIGHT_DRIVER_URL'), "Needs PLAYWRIGHT_DRIVER_URL set for this test"
 
     # Add our URL to the import page, because the docker container (playwright/selenium) wont be able to connect to our usual test url
-    test_url = url_for('test_interactive_html_endpoint', _external=True)
-    test_url = test_url.replace('localhost.localdomain', 'cdio')
-    test_url = test_url.replace('localhost', 'cdio')
+    test_url = "https://changedetection.io/ci-test/test-runjs.html"
 
     res = client.post(
         url_for("form_quick_watch_add"),
@@ -27,31 +24,28 @@ def test_visual_selector_content_ready(client, live_server):
         follow_redirects=True
     )
     assert b"Watch added in Paused state, saving will unpause" in res.data
-    uuid = extract_UUID_from_client(client)
+
     res = client.post(
-        url_for("edit_page", uuid=uuid, unpause_on_save=1),
+        url_for("edit_page", uuid="first", unpause_on_save=1),
         data={
-            "url": test_url,
-            "tags": "",
-            # For now, cookies doesnt work in headers because it must be a full cookiejar object
-            'headers': "testheader: yes\buser-agent: MyCustomAgent",
-            'fetch_backend': "html_webdriver",
+              "url": test_url,
+              "tags": "",
+              "headers": "",
+              'fetch_backend': "html_webdriver",
+              'webdriver_js_execute_code': 'document.querySelector("button[name=test-button]").click();'
         },
         follow_redirects=True
     )
     assert b"unpaused" in res.data
     wait_for_all_checks(client)
+    uuid = extract_UUID_from_client(client)
 
-
-    assert live_server.app.config['DATASTORE'].data['watching'][uuid].history_n >= 1, "Watch history had atleast 1 (everything fetched OK)"
-
+    # Check the JS execute code before extract worked
     res = client.get(
-        url_for("preview_page", uuid=uuid),
+        url_for("preview_page", uuid="first"),
         follow_redirects=True
     )
-    assert b"testheader: yes" in res.data
-    assert b"user-agent: mycustomagent" in res.data
-
+    assert b'I smell JavaScript' in res.data
 
     assert os.path.isfile(os.path.join('test-datastore', uuid, 'last-screenshot.png')), "last-screenshot.png should exist"
     assert os.path.isfile(os.path.join('test-datastore', uuid, 'elements.json')), "xpath elements.json data should exist"
@@ -81,33 +75,30 @@ def test_visual_selector_content_ready(client, live_server):
 
 def test_basic_browserstep(client, live_server):
 
-    #live_server_setup(live_server)
     assert os.getenv('PLAYWRIGHT_DRIVER_URL'), "Needs PLAYWRIGHT_DRIVER_URL set for this test"
+    #live_server_setup(live_server)
 
-    test_url = url_for('test_interactive_html_endpoint', _external=True)
-    test_url = test_url.replace('localhost.localdomain', 'cdio')
-    test_url = test_url.replace('localhost', 'cdio')
+    # Add our URL to the import page, because the docker container (playwright/selenium) wont be able to connect to our usual test url
+    test_url = "https://changedetection.io/ci-test/test-runjs.html"
 
     res = client.post(
         url_for("form_quick_watch_add"),
         data={"url": test_url, "tags": '', 'edit_and_watch_submit_button': 'Edit > Watch'},
         follow_redirects=True
     )
-
     assert b"Watch added in Paused state, saving will unpause" in res.data
 
     res = client.post(
         url_for("edit_page", uuid="first", unpause_on_save=1),
         data={
-            "url": test_url,
-            "tags": "",
-            'fetch_backend': "html_webdriver",
-            'browser_steps-0-operation': 'Goto site',
-            'browser_steps-1-operation': 'Click element',
-            'browser_steps-1-selector': 'button[name=test-button]',
-            'browser_steps-1-optional_value': '',
-            # For now, cookies doesnt work in headers because it must be a full cookiejar object
-            'headers': "testheader: yes\buser-agent: MyCustomAgent",
+              "url": test_url,
+              "tags": "",
+              "headers": "",
+              'fetch_backend': "html_webdriver",
+              'browser_steps-0-operation': 'Goto site',
+              'browser_steps-1-operation': 'Click element',
+              'browser_steps-1-selector': 'button[name=test-button]',
+              'browser_steps-1-optional_value': ''
         },
         follow_redirects=True
     )
@@ -115,9 +106,6 @@ def test_basic_browserstep(client, live_server):
     wait_for_all_checks(client)
 
     uuid = extract_UUID_from_client(client)
-    assert live_server.app.config['DATASTORE'].data['watching'][uuid].history_n >= 1, "Watch history had atleast 1 (everything fetched OK)"
-
-    assert b"This text should be removed" not in res.data
 
     # Check HTML conversion detected and workd
     res = client.get(
@@ -127,19 +115,13 @@ def test_basic_browserstep(client, live_server):
     assert b"This text should be removed" not in res.data
     assert b"I smell JavaScript because the button was pressed" in res.data
 
-    assert b"testheader: yes" in res.data
-    assert b"user-agent: mycustomagent" in res.data
-
-    four_o_four_url =  url_for('test_endpoint', status_code=404, _external=True)
-    four_o_four_url = four_o_four_url.replace('localhost.localdomain', 'cdio')
-    four_o_four_url = four_o_four_url.replace('localhost', 'cdio')
-
     # now test for 404 errors
     res = client.post(
         url_for("edit_page", uuid=uuid, unpause_on_save=1),
         data={
-              "url": four_o_four_url,
+              "url": "https://changedetection.io/404",
               "tags": "",
+              "headers": "",
               'fetch_backend': "html_webdriver",
               'browser_steps-0-operation': 'Goto site',
               'browser_steps-1-operation': 'Click element',

changedetectionio/update_worker.py

@@ -2,8 +2,8 @@ import os
 import threading
 import queue
 import time
-from . import content_fetchers
-from changedetectionio import html_tools
+
+from changedetectionio import content_fetcher, html_tools
 from .processors.text_json_diff import FilterNotFoundInResponse
 from .processors.restock_diff import UnableToExtractRestockData
 
@@ -290,7 +290,7 @@ class update_worker(threading.Thread):
                         logger.critical(f"File permission error updating file, watch: {uuid}")
                         logger.critical(str(e))
                         process_changedetection_results = False
-                    except content_fetchers.exceptions.ReplyWithContentButNoText as e:
+                    except content_fetcher.ReplyWithContentButNoText as e:
                         # Totally fine, it's by choice - just continue on, nothing more to care about
                         # Page had elements/content but no renderable text
                         # Backend (not filters) gave zero output
@@ -312,15 +312,13 @@ class update_worker(threading.Thread):
                             self.datastore.save_screenshot(watch_uuid=uuid, screenshot=e.screenshot)
                         process_changedetection_results = False
 
-                    except content_fetchers.exceptions.Non200ErrorCodeReceived as e:
+                    except content_fetcher.Non200ErrorCodeReceived as e:
                         if e.status_code == 403:
                             err_text = "Error - 403 (Access denied) received"
                         elif e.status_code == 404:
                             err_text = "Error - 404 (Page not found) received"
-                        elif e.status_code == 407:
-                            err_text = "Error - 407 (Proxy authentication required) received, did you need a username and password for the proxy?"
                         elif e.status_code == 500:
-                            err_text = "Error - 500 (Internal server error) received from the web site"
+                            err_text = "Error - 500 (Internal server Error) received"
                         else:
                             err_text = "Error - Request returned a HTTP error code {}".format(str(e.status_code))
 
@@ -358,24 +356,13 @@ class update_worker(threading.Thread):
 
                         process_changedetection_results = False
 
-                    except content_fetchers.exceptions.checksumFromPreviousCheckWasTheSame as e:
+                    except content_fetcher.checksumFromPreviousCheckWasTheSame as e:
                         # Yes fine, so nothing todo, don't continue to process.
                         process_changedetection_results = False
                         changed_detected = False
                         self.datastore.update_watch(uuid=uuid, update_obj={'last_error': False})
-                    except content_fetchers.exceptions.BrowserConnectError as e:
-                        self.datastore.update_watch(uuid=uuid,
-                                                    update_obj={'last_error': e.msg
-                                                                }
-                                                    )
-                        process_changedetection_results = False
-                    except content_fetchers.exceptions.BrowserFetchTimedOut as e:
-                        self.datastore.update_watch(uuid=uuid,
-                                                    update_obj={'last_error': e.msg
-                                                                }
-                                                    )
-                        process_changedetection_results = False
-                    except content_fetchers.exceptions.BrowserStepsStepException as e:
+
+                    except content_fetcher.BrowserStepsStepException as e:
 
                         if not self.datastore.data['watching'].get(uuid):
                             continue
@@ -417,25 +404,25 @@ class update_worker(threading.Thread):
 
                         process_changedetection_results = False
 
-                    except content_fetchers.exceptions.EmptyReply as e:
+                    except content_fetcher.EmptyReply as e:
                         # Some kind of custom to-str handler in the exception handler that does this?
                         err_text = "EmptyReply - try increasing 'Wait seconds before extracting text', Status Code {}".format(e.status_code)
                         self.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text,
                                                                            'last_check_status': e.status_code})
                         process_changedetection_results = False
-                    except content_fetchers.exceptions.ScreenshotUnavailable as e:
+                    except content_fetcher.ScreenshotUnavailable as e:
                         err_text = "Screenshot unavailable, page did not render fully in the expected time or page was too long - try increasing 'Wait seconds before extracting text'"
                         self.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text,
                                                                            'last_check_status': e.status_code})
                         process_changedetection_results = False
-                    except content_fetchers.exceptions.JSActionExceptions as e:
+                    except content_fetcher.JSActionExceptions as e:
                         err_text = "Error running JS Actions - Page request - "+e.message
                         if e.screenshot:
                             self.datastore.save_screenshot(watch_uuid=uuid, screenshot=e.screenshot, as_error=True)
                         self.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text,
                                                                            'last_check_status': e.status_code})
                         process_changedetection_results = False
-                    except content_fetchers.exceptions.PageUnloadable as e:
+                    except content_fetcher.PageUnloadable as e:
                         err_text = "Page request from server didnt respond correctly"
                         if e.message:
                             err_text = "{} - {}".format(err_text, e.message)
@@ -447,7 +434,7 @@ class update_worker(threading.Thread):
                                                                            'last_check_status': e.status_code,
                                                                            'has_ldjson_price_data': None})
                         process_changedetection_results = False
-                    except content_fetchers.exceptions.BrowserStepsInUnsupportedFetcher as e:
+                    except content_fetcher.BrowserStepsInUnsupportedFetcher as e:
                         err_text = "This watch has Browser Steps configured and so it cannot run with the 'Basic fast Plaintext/HTTP Client', either remove the Browser Steps or select a Chrome fetcher."
                         self.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text})
                         process_changedetection_results = False

requirements.txt

@@ -1,7 +1,4 @@
-# Used by Pyppeteer
-pyee
-
-eventlet==0.33.3 # related to dnspython fixes
+eventlet>=0.33.3 # related to dnspython fixes
 feedgen~=0.9
 flask-compress
 # 0.6.3 included compatibility fix for werkzeug 3.x (2.x had deprecation of url handlers)
@@ -22,15 +19,13 @@ validators~=0.21
 brotli~=1.0
 requests[socks]
 
-urllib3==1.26.18
+urllib3>1.26
 chardet>2.3.0
 
 wtforms~=3.0
 jsonpath-ng~=1.5.3
 
-# Pinned: module 'eventlet.green.select' has no attribute 'epoll'
-# https://github.com/eventlet/eventlet/issues/805#issuecomment-1640463482
-dnspython==2.3.0 # related to eventlet fixes
+dnspython~=2.4 # related to eventlet fixes
 
 # jq not available on Windows so must be installed manually
 
@@ -71,9 +66,6 @@ jq~=1.3; python_version >= "3.8" and sys_platform == "linux"
 pillow
 # playwright is installed at Dockerfile build time because it's not available on all platforms
 
-# experimental release
-pyppeteer-ng==2.0.0rc2
-
 # Include pytest, so if theres a support issue we can ask them to run these tests on their setup
 pytest ~=7.2
 pytest-flask ~=1.2