second attempt at plugins

Dockerfile

@@ -25,7 +25,7 @@ RUN pip install --target=/dependencies -r /requirements.txt
 # Playwright is an alternative to Selenium
 # Excluded this package from requirements.txt to prevent arm/v6 and arm/v7 builds from failing
 # https://github.com/dgtlmoon/changedetection.io/pull/1067 also musl/alpine (not supported)
-RUN pip install --target=/dependencies playwright~=1.40 \
+RUN pip install --target=/dependencies playwright~=1.39 \
     || echo "WARN: Failed to install Playwright. The application can still run, but the Playwright option will be disabled."
 
 # Final image stage

changedetectionio/__init__.py

@@ -16,6 +16,7 @@ import logging
 import os
 import pytz
 import queue
+import sys
 import threading
 import time
 import timeago
@@ -38,7 +39,7 @@ from flask_paginate import Pagination, get_page_parameter
 from changedetectionio import html_tools
 from changedetectionio.api import api_v1
 
-__version__ = '0.45.8.1'
+__version__ = '0.45.7.3'
 
 from changedetectionio.store import BASE_URL_NOT_SET_TEXT
 
@@ -80,6 +81,9 @@ csrf = CSRFProtect()
 csrf.init_app(app)
 notification_debug_log=[]
 
+from pathlib import Path
+sys.path.append(os.path.join(Path.home(), 'changedetectionio-plugins'))
+
 watch_api = Api(app, decorators=[csrf.exempt])
 
 def init_app_secret(datastore_path):

changedetectionio/api/api_v1.py

@@ -280,14 +280,11 @@ class CreateWatch(Resource):
             if tag_limit and not any(v.get('title').lower() == tag_limit for k, v in tags.items()):
                 continue
 
-            list[uuid] = {
-                'last_changed': watch.last_changed,
-                'last_checked': watch['last_checked'],
-                'last_error': watch['last_error'],
-                'title': watch['title'],
-                'url': watch['url'],
-                'viewed': watch.viewed
-            }
+            list[uuid] = {'url': watch['url'],
+                       'title': watch['title'],
+                       'last_checked': watch['last_checked'],
+                       'last_changed': watch.last_changed,
+                       'last_error': watch['last_error']}
 
         if request.args.get('recheck_all'):
             for uuid in self.datastore.data['watching'].keys():

changedetectionio/blueprint/browser_steps/__init__.py

@@ -97,7 +97,7 @@ def construct_blueprint(datastore: ChangeDetectionStore):
             proxy=proxy)
 
         # For test
-        #browsersteps_start_session['browserstepper'].action_goto_url(value="http://example.com?time="+str(time.time()))
+        #browsersteps_start_session['browserstepper'].action_goto_url(value="http://exbaseample.com?time="+str(time.time()))
 
         return browsersteps_start_session
 
@@ -152,7 +152,7 @@ def construct_blueprint(datastore: ChangeDetectionStore):
     @browser_steps_blueprint.route("/browsersteps_update", methods=['POST'])
     def browsersteps_ui_update():
         import base64
-        import playwright._impl._errors
+        import playwright._impl._api_types
         global browsersteps_sessions
         from changedetectionio.blueprint.browser_steps import browser_steps
 

changedetectionio/blueprint/browser_steps/browser_steps.py

@@ -110,7 +110,7 @@ class steppable_browser_interface():
         self.page.click(selector=selector, timeout=30 * 1000, delay=randint(200, 500))
 
     def action_click_element_if_exists(self, selector, value):
-        import playwright._impl._errors as _api_types
+        import playwright._impl._api_types as _api_types
         print("Clicking element if exists")
         if not len(selector.strip()):
             return
@@ -123,9 +123,6 @@ class steppable_browser_interface():
             return
 
     def action_click_x_y(self, selector, value):
-        if not re.match(r'^\s?\d+\s?,\s?\d+\s?$', value):
-            raise Exception("'Click X,Y' step should be in the format of '100 , 90'")
-
         x, y = value.strip().split(',')
         x = int(float(x.strip()))
         y = int(float(y.strip()))

changedetectionio/blueprint/check_proxies/__init__.py

@@ -41,7 +41,7 @@ def construct_blueprint(datastore: ChangeDetectionStore):
         now = time.time()
         try:
             update_handler = text_json_diff.perform_site_check(datastore=datastore, watch_uuid=uuid)
-            update_handler.call_browser()
+            update_handler.fetch_content()
         # title, size is len contents not len xfer
         except content_fetcher.Non200ErrorCodeReceived as e:
             if e.status_code == 404:

changedetectionio/content_fetcher.py

@@ -172,7 +172,7 @@ class Fetcher():
 
     def iterate_browser_steps(self):
         from changedetectionio.blueprint.browser_steps.browser_steps import steppable_browser_interface
-        from playwright._impl._errors import TimeoutError
+        from playwright._impl._api_types import TimeoutError
         from jinja2 import Environment
         jinja2_env = Environment(extensions=['jinja2_time.TimeExtension'])
 
@@ -436,7 +436,7 @@ class base_html_playwright(Fetcher):
                     is_binary)
 
         from playwright.sync_api import sync_playwright
-        import playwright._impl._errors
+        import playwright._impl._api_types
 
         self.delete_browser_steps_screenshots()
         response = None
@@ -489,7 +489,7 @@ class base_html_playwright(Fetcher):
             try:
                 if self.webdriver_js_execute_code is not None and len(self.webdriver_js_execute_code):
                     browsersteps_interface.action_execute_js(value=self.webdriver_js_execute_code, selector=None)
-            except playwright._impl._errors.TimeoutError as e:
+            except playwright._impl._api_types.TimeoutError as e:
                 context.close()
                 browser.close()
                 # This can be ok, we will try to grab what we could retrieve

changedetectionio/model/Watch.py

@@ -113,8 +113,7 @@ class model(dict):
 
     @property
     def viewed(self):
-        # Don't return viewed when last_viewed is 0 and newest_key is 0
-        if int(self['last_viewed']) and int(self['last_viewed']) >= int(self.newest_history_key) :
+        if int(self['last_viewed']) >= int(self.newest_history_key) :
             return True
 
         return False

changedetectionio/notification.py

@@ -46,9 +46,6 @@ from apprise.decorators import notify
 @notify(on="puts")
 def apprise_custom_api_call_wrapper(body, title, notify_type, *args, **kwargs):
     import requests
-    from apprise.utils import parse_url as apprise_parse_url
-    from apprise.URLBase import URLBase
-
     url = kwargs['meta'].get('url')
 
     if url.startswith('post'):
@@ -71,45 +68,16 @@ def apprise_custom_api_call_wrapper(body, title, notify_type, *args, **kwargs):
     url = url.replace('delete://', 'http://')
     url = url.replace('deletes://', 'https://')
 
-    headers = {}
-    params = {}
-    auth = None
-
-    # Convert /foobar?+some-header=hello to proper header dictionary
-    results = apprise_parse_url(url)
-    if results:
-        # Add our headers that the user can potentially over-ride if they wish
-        # to to our returned result set and tidy entries by unquoting them
-        headers = {URLBase.unquote(x): URLBase.unquote(y)
-                   for x, y in results['qsd+'].items()}
-
-        # https://github.com/caronc/apprise/wiki/Notify_Custom_JSON#get-parameter-manipulation
-        # In Apprise, it relies on prefixing each request arg with "-", because it uses say &method=update as a flag for apprise
-        # but here we are making straight requests, so we need todo convert this against apprise's logic
-        for k, v in results['qsd'].items():
-            if not k.strip('+-') in results['qsd+'].keys():
-                params[URLBase.unquote(k)] = URLBase.unquote(v)
-
-        # Determine Authentication
-        auth = ''
-        if results.get('user') and results.get('password'):
-            auth = (URLBase.unquote(results.get('user')), URLBase.unquote(results.get('user')))
-        elif results.get('user'):
-            auth = (URLBase.unquote(results.get('user')))
-
     # Try to auto-guess if it's JSON
+    headers = {}
     try:
         json.loads(body)
-        headers['Content-Type'] = 'application/json; charset=utf-8'
+        headers = {'Content-Type': 'application/json; charset=utf-8'}
     except ValueError as e:
         pass
 
-    r(results.get('url'),
-      auth=auth,
-      data=body,
-      headers=headers,
-      params=params
-      )
+
+    r(url, headers=headers, data=body)
 
 
 def process_notification(n_object, datastore):

changedetectionio/processors/__init__.py

@@ -4,10 +4,8 @@ import hashlib
 import re
 from changedetectionio import content_fetcher
 from copy import deepcopy
-from distutils.util import strtobool
-
-class difference_detection_processor():
 
+class difference_detection_processor_interface():
     browser_steps = None
     datastore = None
     fetcher = None
@@ -15,57 +13,36 @@ class difference_detection_processor():
     watch = None
     xpath_data = None
 
-    def __init__(self, *args, datastore, watch_uuid, **kwargs):
-        super().__init__(*args, **kwargs)
+
+    @abstractmethod
+    def run_changedetection(self, uuid, skip_when_checksum_same=True):
+        update_obj = {'last_notification_error': False, 'last_error': False}
+        some_data = 'xxxxx'
+        update_obj["previous_md5"] = hashlib.md5(some_data.encode('utf-8')).hexdigest()
+        changed_detected = False
+        return changed_detected, update_obj, ''.encode('utf-8')
+
+
+class text_content_difference_detection_processor(difference_detection_processor_interface):
+
+    def __init__(self, *args, datastore, watch_uuid, prefer_fetch_backend, **kwargs):
         self.datastore = datastore
         self.watch = deepcopy(self.datastore.data['watching'].get(watch_uuid))
+        self.prefer_fetch_backend = prefer_fetch_backend
+        super().__init__(*args, **kwargs)
 
-    def call_browser(self):
-
-        # Protect against file:// access
-        if re.search(r'^file://', self.watch.get('url', '').strip(), re.IGNORECASE):
-            if not strtobool(os.getenv('ALLOW_FILE_URI', 'false')):
-                raise Exception(
-                    "file:// type access is denied for security reasons."
-                )
-
-        url = self.watch.link
-
-        # Requests, playwright, other browser via wss:// etc, fetch_extra_something
-        prefer_fetch_backend = self.watch.get('fetch_backend', 'system')
-
-        # Proxy ID "key"
-        preferred_proxy_id = self.datastore.get_preferred_proxy_for_watch(uuid=self.watch.get('uuid'))
-
-        # Pluggable content self.fetcher
-        if not prefer_fetch_backend or prefer_fetch_backend == 'system':
-            prefer_fetch_backend = self.datastore.data['settings']['application'].get('fetch_backend')
-
-        # In the case that the preferred fetcher was a browser config with custom connection URL..
-        # @todo - on save watch, if its extra_browser_ then it should be obvious it will use playwright (like if its requests now..)
-        browser_connection_url = None
-        if prefer_fetch_backend.startswith('extra_browser_'):
-            (t, key) = prefer_fetch_backend.split('extra_browser_')
-            connection = list(
-                filter(lambda s: (s['browser_name'] == key), self.datastore.data['settings']['requests'].get('extra_browsers', [])))
-            if connection:
-                prefer_fetch_backend = 'base_html_playwright'
-                browser_connection_url = connection[0].get('browser_connection_url')
-
-        # PDF should be html_requests because playwright will serve it up (so far) in a embedded page
-        # @todo https://github.com/dgtlmoon/changedetection.io/issues/2019
-        # @todo needs test to or a fix
-        if self.watch.is_pdf:
-           prefer_fetch_backend = "html_requests"
-
+        ########################################
+        # Attach the correct fetcher and proxy #
+        ########################################
         # Grab the right kind of 'fetcher', (playwright, requests, etc)
-        if hasattr(content_fetcher, prefer_fetch_backend):
-            fetcher_obj = getattr(content_fetcher, prefer_fetch_backend)
+        if hasattr(content_fetcher, self.prefer_fetch_backend):
+            fetcher_obj = getattr(content_fetcher, self.prefer_fetch_backend)
         else:
             # If the klass doesnt exist, just use a default
             fetcher_obj = getattr(content_fetcher, "html_requests")
 
-
+        # Proxy ID "key"
+        preferred_proxy_id = self.datastore.get_preferred_proxy_for_watch(uuid=self.watch.get('uuid'))
         proxy_url = None
         if preferred_proxy_id:
             proxy_url = self.datastore.proxy_list.get(preferred_proxy_id).get('url')
@@ -74,9 +51,23 @@ class difference_detection_processor():
         # Now call the fetcher (playwright/requests/etc) with arguments that only a fetcher would need.
         # When browser_connection_url is None, it method should default to working out whats the best defaults (os env vars etc)
         self.fetcher = fetcher_obj(proxy_override=proxy_url,
-                                   browser_connection_url=browser_connection_url
+                                   browser_connection_url=None # Default, let each fetcher work it out
                                    )
 
+    def fetch_content(self):
+
+        url = self.watch.link
+
+        # In the case that the preferred fetcher was a browser config with custom connection URL..
+        # @todo - on save watch, if its extra_browser_ then it should be obvious it will use playwright (like if its requests now..)
+        if self.prefer_fetch_backend.startswith('extra_browser_'):
+            (t, key) = self.prefer_fetch_backend.split('extra_browser_')
+            connection = list(
+                filter(lambda s: (s['browser_name'] == key), self.datastore.data['settings']['requests'].get('extra_browsers', [])))
+            if connection:
+                prefer_fetch_backend = 'base_html_playwright'
+                browser_connection_url = connection[0].get('browser_connection_url')
+
         if self.watch.has_browser_steps:
             self.fetcher.browser_steps = self.watch.get('browser_steps', [])
             self.fetcher.browser_steps_screenshot_path = os.path.join(self.datastore.datastore_path, self.watch.get('uuid'))
@@ -120,14 +111,6 @@ class difference_detection_processor():
 
         # After init, call run_changedetection() which will do the actual change-detection
 
-    @abstractmethod
-    def run_changedetection(self, uuid, skip_when_checksum_same=True):
-        update_obj = {'last_notification_error': False, 'last_error': False}
-        some_data = 'xxxxx'
-        update_obj["previous_md5"] = hashlib.md5(some_data.encode('utf-8')).hexdigest()
-        changed_detected = False
-        return changed_detected, update_obj, ''.encode('utf-8')
-
 
 def available_processors():
     from . import restock_diff, text_json_diff

changedetectionio/processors/restock_diff.py

@@ -1,8 +1,9 @@
 
 import hashlib
 import urllib3
-from . import difference_detection_processor
+#from . import browser_content_difference_detection_processor
 from copy import deepcopy
+from . import text_content_difference_detection_processor
 
 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 
@@ -15,7 +16,7 @@ class UnableToExtractRestockData(Exception):
         self.status_code = status_code
         return
 
-class perform_site_check(difference_detection_processor):
+class perform_site_check(text_content_difference_detection_processor):
     screenshot = None
     xpath_data = None
 

changedetectionio/processors/text_json_diff.py

@@ -10,8 +10,8 @@ import urllib3
 from changedetectionio import content_fetcher, html_tools
 from changedetectionio.blueprint.price_data_follower import PRICE_DATA_TRACK_ACCEPT, PRICE_DATA_TRACK_REJECT
 from copy import deepcopy
-from . import difference_detection_processor
 from ..html_tools import PERL_STYLE_REGEX, cdata_in_document_to_text
+from . import text_content_difference_detection_processor
 
 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 
@@ -31,7 +31,7 @@ class PDFToHTMLToolNotFound(ValueError):
 
 # Some common stuff here that can be moved to a base class
 # (set_proxy_from_list)
-class perform_site_check(difference_detection_processor):
+class perform_site_check(text_content_difference_detection_processor):
 
     def run_changedetection(self, uuid, skip_when_checksum_same=True):
         changed_detected = False

changedetectionio/res/stock-not-in-stock.js

@@ -1,7 +1,6 @@
 function isItemInStock() {
   // @todo Pass these in so the same list can be used in non-JS fetchers
   const outOfStockTexts = [
-    ' أخبرني عندما يتوفر',
     '0 in stock',
     'agotado',
     'artikel zurzeit vergriffen',
@@ -17,12 +16,9 @@ function isItemInStock() {
     'currently have any tickets for this',
     'currently unavailable',
     'dostępne wkrótce',
-    'dostępne wkrótce',
     'en rupture de stock',
     'ist derzeit nicht auf lager',
-    'ist derzeit nicht auf lager',
     'item is no longer available',
-    'let me know when it\'s available',
     'message if back in stock',
     'nachricht bei',
     'nicht auf lager',
@@ -46,9 +42,7 @@ function isItemInStock() {
     'unavailable tickets',
     'we do not currently have an estimate of when this product will be back in stock.',
     'zur zeit nicht an lager',
-    '品切れ',
     '已售完',
-    '품절'
   ];
 
 

changedetectionio/static/images/bell-off.svg

@@ -1,4 +1,4 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<svg width="15" height="16.363636" viewBox="0 0 15 16.363636" xmlns="http://www.w3.org/2000/svg" >
+<svg width="15" height="16.363636" viewBox="0 0 15 16.363636" xmlns="http://www.w3.org/2000/svg" xmlns:svg="http://www.w3.org/2000/svg">
   <path d="m 14.318182,11.762045 v 1.1925 H 5.4102273 L 11.849318,7.1140909 C 12.234545,9.1561364 12.54,11.181818 14.318182,11.762045 Z m -6.7984093,4.601591 c 1.0759091,0 2.0256823,-0.955909 2.0256823,-2.045454 H 5.4545455 c 0,1.089545 0.9879545,2.045454 2.0652272,2.045454 z M 15,2.8622727 0.9177273,15.636136 0,14.627045 l 1.8443182,-1.6725 h -1.1625 v -1.1925 C 4.0070455,10.677273 2.1784091,4.5388636 5.3611364,2.6897727 5.8009091,2.4347727 6.0709091,1.9609091 6.0702273,1.4488636 v -0.00205 C 6.0702273,0.64772727 6.7104545,0 7.5,0 8.2895455,0 8.9297727,0.64772727 8.9297727,1.4468182 v 0.00205 C 8.9290909,1.9602319 9.199773,2.4354591 9.638864,2.6897773 10.364318,3.111141 10.827273,3.7568228 11.1525,4.5129591 L 14.085682,1.8531818 Z M 6.8181818,1.3636364 C 6.8181818,1.74 7.1236364,2.0454545 7.5,2.0454545 7.8763636,2.0454545 8.1818182,1.74 8.1818182,1.3636364 8.1818182,0.98795455 7.8763636,0.68181818 7.5,0.68181818 c -0.3763636,0 -0.6818182,0.30613637 -0.6818182,0.68181822 z" id="path2" style="fill:#f8321b;stroke-width:0.681818;fill-opacity:1"/>
 </svg>

changedetectionio/static/images/brightdata.svg

@@ -10,7 +10,7 @@
    xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
    xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
    xmlns="http://www.w3.org/2000/svg"
-   >
+   xmlns:svg="http://www.w3.org/2000/svg">
   <defs
      id="defs16" />
   <sodipodi:namedview

changedetectionio/static/images/copy.svg

@@ -12,7 +12,7 @@
    xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
    xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
    xmlns="http://www.w3.org/2000/svg"
-   ><defs
+   xmlns:svg="http://www.w3.org/2000/svg"><defs
      id="defs11" /><sodipodi:namedview
      id="namedview9"
      pagecolor="#ffffff"

changedetectionio/static/images/email.svg

@@ -10,7 +10,7 @@
    viewBox="0 0 7.1975545 4.7993639"
    xml:space="preserve"
    xmlns="http://www.w3.org/2000/svg"
-   ><defs
+   xmlns:svg="http://www.w3.org/2000/svg"><defs
    id="defs19" />
 <g
    id="g14"

changedetectionio/static/images/notice.svg

@@ -9,7 +9,7 @@
    id="svg5"
    xmlns:xlink="http://www.w3.org/1999/xlink"
    xmlns="http://www.w3.org/2000/svg"
-   >
+   xmlns:svg="http://www.w3.org/2000/svg">
   <defs
      id="defs2" />
   <g

changedetectionio/static/images/oxylabs.svg

@@ -10,7 +10,7 @@
    xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
    xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
    xmlns="http://www.w3.org/2000/svg"
-   >
+   xmlns:svg="http://www.w3.org/2000/svg">
   <defs
      id="defs12" />
   <sodipodi:namedview

changedetectionio/static/images/pause.svg

@@ -3,6 +3,7 @@
    xmlns:dc="http://purl.org/dc/elements/1.1/"
    xmlns:cc="http://creativecommons.org/ns#"
    xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+   xmlns:svg="http://www.w3.org/2000/svg"
    xmlns="http://www.w3.org/2000/svg"
    version="1.1"
    id="Capa_1"

changedetectionio/static/images/play.svg

@@ -13,6 +13,7 @@
    xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
    xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
    xmlns="http://www.w3.org/2000/svg"
+   xmlns:svg="http://www.w3.org/2000/svg"
    xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
    xmlns:cc="http://creativecommons.org/ns#"
    xmlns:dc="http://purl.org/dc/elements/1.1/"><sodipodi:namedview

changedetectionio/static/images/spread-white.svg

@@ -6,7 +6,7 @@
    version="1.1"
    id="svg6"
    xmlns="http://www.w3.org/2000/svg"
-   >
+   xmlns:svg="http://www.w3.org/2000/svg">
   <defs
      id="defs10" />
   <path

changedetectionio/static/images/spread.svg

@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<svg width="18" height="19.92" viewBox="0 0 18 19.92" xmlns="http://www.w3.org/2000/svg" >
+<svg width="18" height="19.92" viewBox="0 0 18 19.92" xmlns="http://www.w3.org/2000/svg" xmlns:svg="http://www.w3.org/2000/svg">
   <path d="M -3,-2 H 21 V 22 H -3 Z" fill="none" id="path2"/>
   <path d="m 15,14.08 c -0.76,0 -1.44,0.3 -1.96,0.77 L 5.91,10.7 C 5.96,10.47 6,10.24 6,10 6,9.76 5.96,9.53 5.91,9.3 L 12.96,5.19 C 13.5,5.69 14.21,6 15,6 16.66,6 18,4.66 18,3 18,1.34 16.66,0 15,0 c -1.66,0 -3,1.34 -3,3 0,0.24 0.04,0.47 0.09,0.7 L 5.04,7.81 C 4.5,7.31 3.79,7 3,7 1.34,7 0,8.34 0,10 c 0,1.66 1.34,3 3,3 0.79,0 1.5,-0.31 2.04,-0.81 l 7.12,4.16 c -0.05,0.21 -0.08,0.43 -0.08,0.65 0,1.61 1.31,2.92 2.92,2.92 1.61,0 2.92,-1.31 2.92,-2.92 0,-1.61 -1.31,-2.92 -2.92,-2.92 z" id="path4" style="fill:#0078e7;fill-opacity:1"/>
 </svg>

changedetectionio/static/js/global-settings.js

@@ -1,4 +1,19 @@
 $(document).ready(function () {
+    function toggle() {
+        if ($('input[name="application-fetch_backend"]:checked').val() != 'html_requests') {
+            $('#requests-override-options').hide();
+            $('#webdriver-override-options').show();
+        } else {
+            $('#requests-override-options').show();
+            $('#webdriver-override-options').hide();
+        }
+    }
+
+    $('input[name="application-fetch_backend"]').click(function (e) {
+        toggle();
+    });
+    toggle();
+
     $("#api-key").hover(
         function () {
             $("#api-key-copy").html('copy').fadeIn();

changedetectionio/static/js/vis.js

@@ -1,29 +0,0 @@
-$(document).ready(function () {
-
-    // Lazy Hide/Show elements mechanism
-    $('[data-visible-for]').hide();
-    function show_related_elem(e) {
-        var n = $(e).attr('name') + "=" + $(e).val();
-        if (n === 'fetch_backend=system') {
-            n = "fetch_backend=" + default_system_fetch_backend;
-        }
-        $(`[data-visible-for~="${n}"]`).show();
-    }
-    $(':radio').on('keyup keypress blur change click', function (e) {
-        $(`[data-visible-for]`).hide();
-        $('.advanced-options').hide();
-        show_related_elem(this);
-    });
-
-    $(':radio:checked').each(function (e) {
-       show_related_elem(this);
-    })
-
-
-    // Show advanced
-    $('.show-advanced').click(function (e) {
-        $(this).closest('.tab-pane-inner').find('.advanced-options').each(function (e) {
-            $(this).toggle();
-        })
-    });
-});

changedetectionio/static/js/watch-settings.js

@@ -1,4 +1,40 @@
 $(document).ready(function () {
+    function toggle() {
+        if ($('input[name="fetch_backend"]:checked').val() == 'html_webdriver') {
+            if (playwright_enabled) {
+                // playwright supports headers, so hide everything else
+                // See #664
+                $('#requests-override-options #request-method').hide();
+                $('#requests-override-options #request-body').hide();
+
+                // @todo connect this one up
+                $('#ignore-status-codes-option').hide();
+            } else {
+                // selenium/webdriver doesnt support anything afaik, hide it all
+                $('#requests-override-options').hide();
+            }
+
+            $('#webdriver-override-options').show();
+
+        } else if ($('input[name="fetch_backend"]:checked').val() == 'system') {
+            $('#requests-override-options #request-method').hide();
+            $('#requests-override-options #request-body').hide();
+            $('#ignore-status-codes-option').hide();
+            $('#requests-override-options').hide();
+            $('#webdriver-override-options').hide();
+        } else {
+
+            $('#requests-override-options').show();
+            $('#requests-override-options *:hidden').show();
+            $('#webdriver-override-options').hide();
+        }
+    }
+
+    $('input[name="fetch_backend"]').click(function (e) {
+        toggle();
+    });
+    toggle();
+
     $('#notification-setting-reset-to-default').click(function (e) {
         $('#notification_title').val('');
         $('#notification_body').val('');

changedetectionio/static/styles/scss/styles.scss

@@ -402,24 +402,8 @@ label {
   }
 
   #watch-add-wrapper-zone {
-
-    @media only screen and (min-width: 760px) {
-      display: flex;
-      gap: 0.3rem;
-      flex-direction: row;
-    }
-    /* URL field grows always, other stay static in width */
-    > span {
-      flex-grow: 0;
-
-      input {
-        width: 100%;
-        padding-right: 1em;
-      }
-
-      &:first-child {
-        flex-grow: 1;
-      }
+    >div {
+      display: inline-block;
     }
 
     @media only screen and (max-width: 760px) {
@@ -960,8 +944,10 @@ ul {
 
 @import "parts/_visualselector";
 
-#webdriver_delay {
+#webdriver-override-options {
+  input[type="number"] {
     width: 5em;
+  }
 }
 
 #api-key {

changedetectionio/static/styles/styles.css

@@ -683,23 +683,11 @@ label:hover {
   #new-watch-form legend {
     color: var(--color-text-legend);
     font-weight: bold; }
-  #new-watch-form #watch-add-wrapper-zone {
-    /* URL field grows always, other stay static in width */ }
-    @media only screen and (min-width: 760px) {
-      #new-watch-form #watch-add-wrapper-zone {
-        display: flex;
-        gap: 0.3rem;
-        flex-direction: row; } }
-    #new-watch-form #watch-add-wrapper-zone > span {
-      flex-grow: 0; }
-      #new-watch-form #watch-add-wrapper-zone > span input {
-        width: 100%;
-        padding-right: 1em; }
-      #new-watch-form #watch-add-wrapper-zone > span:first-child {
-        flex-grow: 1; }
-    @media only screen and (max-width: 760px) {
-      #new-watch-form #watch-add-wrapper-zone #url {
-        width: 100%; } }
+  #new-watch-form #watch-add-wrapper-zone > div {
+    display: inline-block; }
+  @media only screen and (max-width: 760px) {
+    #new-watch-form #watch-add-wrapper-zone #url {
+      width: 100%; } }
 
 #diff-col {
   padding-left: 40px; }
@@ -1077,7 +1065,7 @@ ul {
 #selector-current-xpath {
   font-size: 80%; }
 
-#webdriver_delay {
+#webdriver-override-options input[type="number"] {
   width: 5em; }
 
 #api-key:hover {

changedetectionio/templates/_common_fields.jinja

@@ -13,10 +13,10 @@
                             <div class="pure-form-message-inline">
                               <ul>
                                 <li>Use <a target=_new href="https://github.com/caronc/apprise">AppRise URLs</a> for notification to just about any service! <i><a target=_new href="https://github.com/dgtlmoon/changedetection.io/wiki/Notification-configuration-notes">Please read the notification services wiki here for important configuration notes</a></i>.</li>
-                                <li><code><a target=_new href="https://github.com/caronc/apprise/wiki/Notify_discord">discord://</a></code> (or <code>https://discord.com/api/webhooks...</code>)) only supports a maximum <strong>2,000 characters</strong> of notification text, including the title.</li>
+                                <li><code><a target=_new href="https://github.com/caronc/apprise/wiki/Notify_discord">discord://</a></code> (or <code>https://discord.com/api/webhooks...</code>)) </code> only supports a maximum <strong>2,000 characters</strong> of notification text, including the title.</li>
                                 <li><code><a target=_new href="https://github.com/caronc/apprise/wiki/Notify_telegram">tgram://</a></code> bots can't send messages to other bots, so you should specify chat ID of non-bot user.</li>
                                 <li><code><a target=_new href="https://github.com/caronc/apprise/wiki/Notify_telegram">tgram://</a></code> only supports very limited HTML and can fail when extra tags are sent, <a href="https://core.telegram.org/bots/api#html-style">read more here</a> (or use plaintext/markdown format)</li>
-                                <li><code>gets://</code>, <code>posts://</code>, <code>puts://</code>, <code>deletes://</code> for direct API calls (or omit the "<code>s</code>" for non-SSL ie <code>get://</code>) <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Notification-configuration-notes#postposts">more help here</a></li>
+                                <li><code>gets://</code>, <code>posts://</code>, <code>puts://</code>, <code>deletes://</code> for direct API calls (or omit the "<code>s</code>" for non-SSL ie <code>get://</code>)</li>
                                   <li>Accepts the <code>{{ '{{token}}' }}</code> placeholders listed below</li>
                               </ul>
                             </div>

changedetectionio/templates/_helpers.jinja

@@ -39,24 +39,6 @@
 {% endmacro %}
 
 
-{% macro render_nolabel_field(field) %}
-    <span>
-    {{ field(**kwargs)|safe }}
-        {% if field.errors %}
-            <span class="error">
-      {% if field.errors %}
-          <ul class=errors>
-        {% for error in field.errors %}
-            <li>{{ error }}</li>
-        {% endfor %}
-        </ul>
-      {% endif %}
-      </span>
-        {% endif %}
-    </span>
-{% endmacro %}
-
-
 {% macro render_button(field) %}
   {{ field(**kwargs)|safe }}
 {% endmacro %}

changedetectionio/templates/base.html

@@ -116,7 +116,7 @@
                    viewBox="0 0 16.9 16.1"
                    id="svg-heart"
                    xmlns="http://www.w3.org/2000/svg"
-                   >
+                   xmlns:svg="http://www.w3.org/2000/svg">
                   <path id="heartpath" d="M 5.338316,0.50302766 C 0.71136983,0.50647126 -3.9576371,7.2707777 8.5004254,15.503028 23.833425,5.3700277 13.220206,-2.5384409 8.6762066,1.6475589 c -0.060791,0.054322 -0.11943,0.1110064 -0.1757812,0.1699219 -0.057,-0.059 -0.1157813,-0.116875 -0.1757812,-0.171875 C 7.4724566,0.86129334 6.4060729,0.50223298 5.338316,0.50302766 Z"
                      style="fill:var(--color-background);fill-opacity:1;stroke:#ff0000;stroke-opacity:1" />
                 </svg>
@@ -170,6 +170,7 @@
                     And tell your friends and colleagues :)
                 </li>
                 </ul>
+                </p>
                 <p>
                     The more popular changedetection.io is, the more time we can dedicate to adding amazing features!
                 </p>

changedetectionio/templates/edit.html

@@ -3,7 +3,6 @@
 {% from '_helpers.jinja' import render_field, render_checkbox_field, render_button %}
 {% from '_common_fields.jinja' import render_common_settings_form %}
 <script src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
-<script src="{{url_for('static_content', group='js', filename='vis.js')}}" defer></script>
 <script>
     const browser_steps_available_screenshots=JSON.parse('{{ watch.get_browsersteps_available_screenshots|tojson }}');
     const browser_steps_config=JSON.parse('{{ browser_steps_config|tojson }}');
@@ -20,7 +19,7 @@
     const proxy_recheck_status_url="{{url_for('check_proxies.get_recheck_status', uuid=uuid)}}";
     const screenshot_url="{{url_for('static_content', group='screenshot', filename=uuid)}}";
     const watch_visual_selector_data_url="{{url_for('static_content', group='visual_selector_data', filename=uuid)}}";
-    const default_system_fetch_backend="{{ settings_application['fetch_backend'] }}";
+
 </script>
 
 <script src="{{url_for('static_content', group='js', filename='watch-settings.js')}}" defer></script>
@@ -125,9 +124,10 @@
                         </span>
                     </div>
                 {% endif %}
-
-                <!-- webdriver always -->
-                <fieldset data-visible-for="fetch_backend=html_webdriver"  style="display: none;">
+                    <div  class="pure-control-group inline-radio">
+                        {{ render_checkbox_field(form.ignore_status_codes) }}
+                    </div>
+                <fieldset id="webdriver-override-options">
                     <div class="pure-control-group">
                         {{ render_field(form.webdriver_delay) }}
                         <div class="pure-form-message-inline">
@@ -140,40 +140,23 @@
                         </div>
                     </div>
                     <div class="pure-control-group">
-                        <a class="pure-button button-secondary button-xsmall show-advanced">Show advanced options</a>
-                    </div>
-                    <div class="advanced-options"  style="display: none;">
                         {{ render_field(form.webdriver_js_execute_code) }}
                         <div class="pure-form-message-inline">
-                            Run this code before performing change detection, handy for filling in fields and other
-                            actions <a
-                                href="https://github.com/dgtlmoon/changedetection.io/wiki/Run-JavaScript-before-change-detection">More
-                            help and examples here</a>
+                            Run this code before performing change detection, handy for filling in fields and other actions <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Run-JavaScript-before-change-detection">More help and examples here</a>
                         </div>
                     </div>
                 </fieldset>
-                <!-- html requests always -->
-                <fieldset data-visible-for="fetch_backend=html_requests">
-                    <div class="pure-control-group">
-                        <a class="pure-button button-secondary button-xsmall show-advanced">Show advanced options</a>
-                    </div>
-                    <div class="advanced-options"  style="display: none;">
-                        <div class="pure-control-group" id="request-method">
-                            {{ render_field(form.method) }}
-                        </div>
-                        <div id="request-body">
-                                            {{ render_field(form.body, rows=5, placeholder="Example
-{
-   \"name\":\"John\",
-   \"age\":30,
-   \"car\":null
-}") }}
+                <fieldset class="pure-group" id="requests-override-options">
+                    {% if not playwright_enabled %}
+                        <div class="pure-form-message-inline">
+                            <strong>Request override is currently only used by the <i>Basic fast Plaintext/HTTP Client</i> method.</strong>
                         </div>
+                    {% endif %}
+                    <div class="pure-control-group" id="request-method">
+                        {{ render_field(form.method) }}
                     </div>
-                </fieldset>
-            <!-- hmm -->
-                <div class="pure-control-group advanced-options"  style="display: none;">
-                    {{ render_field(form.headers, rows=5, placeholder="Example
+                    <div class="pure-control-group" id="request-headers">
+{{ render_field(form.headers, rows=5, placeholder="Example
 Cookie: foobar
 User-Agent: wonderbra 1.0") }}
 
@@ -186,12 +169,17 @@ User-Agent: wonderbra 1.0") }}
                             <br>
                             (Not supported by Selenium browser)
                         </div>
+
                     </div>
-            <fieldset data-visible-for="fetch_backend=html_requests fetch_backend=html_webdriver" >
-                    <div class="pure-control-group inline-radio advanced-options"  style="display: none;">
-                    {{ render_checkbox_field(form.ignore_status_codes) }}
+                    <div class="pure-control-group" id="request-body">
+                                        {{ render_field(form.body, rows=5, placeholder="Example
+{
+   \"name\":\"John\",
+   \"age\":30,
+   \"car\":null
+}") }}
                     </div>
-            </fieldset>
+                </fieldset>
             </div>
             {% if playwright_enabled %}
             <div class="tab-pane-inner" id="browser-steps">

changedetectionio/templates/settings.html

@@ -11,7 +11,7 @@
 </script>
 <script src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
 <script src="{{url_for('static_content', group='js', filename='notifications.js')}}" defer></script>
-<script src="{{url_for('static_content', group='js', filename='vis.js')}}" defer></script>
+
 <script src="{{url_for('static_content', group='js', filename='global-settings.js')}}" defer></script>
 <div class="edit-form">
     <div class="tabs collapsable">
@@ -111,7 +111,7 @@
                     <br>
                     Tip: <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Proxy-configuration#brightdata-proxy-support">Connect using Bright Data and Oxylabs Proxies, find out more here.</a>
                 </div>
-                <fieldset class="pure-group" id="webdriver-override-options" data-visible-for="application-fetch_backend=html_webdriver">
+                <fieldset class="pure-group" id="webdriver-override-options">
                     <div class="pure-form-message-inline">
                         <strong>If you're having trouble waiting for the page to be fully rendered (text missing etc), try increasing the 'wait' time here.</strong>
                         <br>
@@ -236,11 +236,8 @@ nav
                 <span class="pure-form-message-inline">SOCKS5 proxies with authentication are only supported with 'plain requests' fetcher, for other fetchers you should whitelist the IP access instead</span>
                 </div>
                 <div class="pure-control-group" id="extra-browsers-setting">
-                    <p>
-                    <span class="pure-form-message-inline"><i>Extra Browsers</i> can be attached to further defeat CAPTCHA's on websites that are particularly hard to scrape.</span><br>
-                    <span class="pure-form-message-inline">Simply paste the connection address into the box, <a href="https://changedetection.io/tutorial/using-bright-datas-scraping-browser-pass-captchas-and-other-protection-when-monitoring">More instructions and examples here</a> </span>
-                    </p>
-                    {{ render_field(form.requests.form.extra_browsers) }}
+                    <span class="pure-form-message-inline"><i>Extra Browsers</i> allow changedetection.io to communicate with a different web-browser.</span><br>
+                  {{ render_field(form.requests.form.extra_browsers) }}
                 </div>
             </div>
             <div id="actions">

changedetectionio/templates/svgs/search-icon.svg

@@ -1 +1 @@
-<svg version="1.1" id="Layer_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 0 122.879 119.799" enable-background="new 0 0 122.879 119.799" xml:space="preserve"><g><path d="M49.988,0h0.016v0.007C63.803,0.011,76.298,5.608,85.34,14.652c9.027,9.031,14.619,21.515,14.628,35.303h0.007v0.033v0.04 h-0.007c-0.005,5.557-0.917,10.905-2.594,15.892c-0.281,0.837-0.575,1.641-0.877,2.409v0.007c-1.446,3.66-3.315,7.12-5.547,10.307 l29.082,26.139l0.018,0.016l0.157,0.146l0.011,0.011c1.642,1.563,2.536,3.656,2.649,5.78c0.11,2.1-0.543,4.248-1.979,5.971 l-0.011,0.016l-0.175,0.203l-0.035,0.035l-0.146,0.16l-0.016,0.021c-1.565,1.642-3.654,2.534-5.78,2.646 c-2.097,0.111-4.247-0.54-5.971-1.978l-0.015-0.011l-0.204-0.175l-0.029-0.024L78.761,90.865c-0.88,0.62-1.778,1.209-2.687,1.765 c-1.233,0.755-2.51,1.466-3.813,2.115c-6.699,3.342-14.269,5.222-22.272,5.222v0.007h-0.016v-0.007 c-13.799-0.004-26.296-5.601-35.338-14.645C5.605,76.291,0.016,63.805,0.007,50.021H0v-0.033v-0.016h0.007 c0.004-13.799,5.601-26.296,14.645-35.338C23.683,5.608,36.167,0.016,49.955,0.007V0H49.988L49.988,0z M50.004,11.21v0.007h-0.016 h-0.033V11.21c-10.686,0.007-20.372,4.35-27.384,11.359C15.56,29.578,11.213,39.274,11.21,49.973h0.007v0.016v0.033H11.21 c0.007,10.686,4.347,20.367,11.359,27.381c7.009,7.012,16.705,11.359,27.403,11.361v-0.007h0.016h0.033v0.007 c10.686-0.007,20.368-4.348,27.382-11.359c7.011-7.009,11.358-16.702,11.36-27.4h-0.006v-0.016v-0.033h0.006 c-0.006-10.686-4.35-20.372-11.358-27.384C70.396,15.56,60.703,11.213,50.004,11.21L50.004,11.21z"/></g></svg>
+<?xml version="1.0" encoding="utf-8"?><svg version="1.1" id="Layer_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 0 122.879 119.799" enable-background="new 0 0 122.879 119.799" xml:space="preserve"><g><path d="M49.988,0h0.016v0.007C63.803,0.011,76.298,5.608,85.34,14.652c9.027,9.031,14.619,21.515,14.628,35.303h0.007v0.033v0.04 h-0.007c-0.005,5.557-0.917,10.905-2.594,15.892c-0.281,0.837-0.575,1.641-0.877,2.409v0.007c-1.446,3.66-3.315,7.12-5.547,10.307 l29.082,26.139l0.018,0.016l0.157,0.146l0.011,0.011c1.642,1.563,2.536,3.656,2.649,5.78c0.11,2.1-0.543,4.248-1.979,5.971 l-0.011,0.016l-0.175,0.203l-0.035,0.035l-0.146,0.16l-0.016,0.021c-1.565,1.642-3.654,2.534-5.78,2.646 c-2.097,0.111-4.247-0.54-5.971-1.978l-0.015-0.011l-0.204-0.175l-0.029-0.024L78.761,90.865c-0.88,0.62-1.778,1.209-2.687,1.765 c-1.233,0.755-2.51,1.466-3.813,2.115c-6.699,3.342-14.269,5.222-22.272,5.222v0.007h-0.016v-0.007 c-13.799-0.004-26.296-5.601-35.338-14.645C5.605,76.291,0.016,63.805,0.007,50.021H0v-0.033v-0.016h0.007 c0.004-13.799,5.601-26.296,14.645-35.338C23.683,5.608,36.167,0.016,49.955,0.007V0H49.988L49.988,0z M50.004,11.21v0.007h-0.016 h-0.033V11.21c-10.686,0.007-20.372,4.35-27.384,11.359C15.56,29.578,11.213,39.274,11.21,49.973h0.007v0.016v0.033H11.21 c0.007,10.686,4.347,20.367,11.359,27.381c7.009,7.012,16.705,11.359,27.403,11.361v-0.007h0.016h0.033v0.007 c10.686-0.007,20.368-4.348,27.382-11.359c7.011-7.009,11.358-16.702,11.36-27.4h-0.006v-0.016v-0.033h0.006 c-0.006-10.686-4.35-20.372-11.358-27.384C70.396,15.56,60.703,11.213,50.004,11.21L50.004,11.21z"/></g></svg>

changedetectionio/templates/watch-overview.html

@@ -1,6 +1,6 @@
 {% extends 'base.html' %}
 {% block content %}
-{% from '_helpers.jinja' import render_simple_field, render_field, render_nolabel_field %}
+{% from '_helpers.jinja' import render_simple_field, render_field %}
 <script src="{{url_for('static_content', group='js', filename='jquery-3.6.0.min.js')}}"></script>
 <script src="{{url_for('static_content', group='js', filename='watch-overview.js')}}" defer></script>
 
@@ -11,14 +11,17 @@
         <fieldset>
             <legend>Add a new change detection watch</legend>
             <div id="watch-add-wrapper-zone">
-
-                    {{ render_nolabel_field(form.url, placeholder="https://...", required=true) }}
-                    {{ render_nolabel_field(form.tags, value=tags[active_tag].title if active_tag else '', placeholder="watch label / tag") }}
-                    {{ render_nolabel_field(form.watch_submit_button, title="Watch this URL!" ) }}
-                    {{ render_nolabel_field(form.edit_and_watch_submit_button, title="Edit first then Watch") }}
+                <div>
+                    {{ render_simple_field(form.url, placeholder="https://...", required=true) }}
+                    {{ render_simple_field(form.tags, value=tags[active_tag].title if active_tag else '', placeholder="watch label / tag") }}
+                </div>
+                <div>
+                    {{ render_simple_field(form.watch_submit_button, title="Watch this URL!" ) }}
+                    {{ render_simple_field(form.edit_and_watch_submit_button, title="Edit first then Watch") }}
+                </div>
             </div>
             <div id="quick-watch-processor-type">
-                {{ render_simple_field(form.processor) }}
+                {{ render_simple_field(form.processor, title="Edit first then Watch") }}
             </div>
 
         </fieldset>

changedetectionio/tests/conftest.py

@@ -13,17 +13,22 @@ global app
 
 
 def cleanup(datastore_path):
-    import glob
     # Unlink test output files
-
-    for g in ["*.txt", "*.json", "*.pdf"]:
-        files = glob.glob(os.path.join(datastore_path, g))
-        for f in files:
-            if 'proxies.json' in f:
-                # Usually mounted by docker container during test time
-                continue
-            if os.path.isfile(f):
-                os.unlink(f)
+    files = [
+        'count.txt',
+        'endpoint-content.txt'
+        'headers.txt',
+        'headers-testtag.txt',
+        'notification.txt',
+        'secret.txt',
+        'url-watches.json',
+        'output.txt',
+    ]
+    for file in files:
+        try:
+            os.unlink("{}/{}".format(datastore_path, file))
+        except FileNotFoundError:
+            pass
 
 @pytest.fixture(scope='session')
 def app(request):

changedetectionio/tests/test_api.py

@@ -96,9 +96,7 @@ def test_api_simple(client, live_server):
     )
     assert watch_uuid in res.json.keys()
     before_recheck_info = res.json[watch_uuid]
-
     assert before_recheck_info['last_checked'] != 0
-
     #705 `last_changed` should be zero on the first check
     assert before_recheck_info['last_changed'] == 0
     assert before_recheck_info['title'] == 'My test URL'
@@ -159,18 +157,6 @@ def test_api_simple(client, live_server):
     # @todo how to handle None/default global values?
     assert watch['history_n'] == 2, "Found replacement history section, which is in its own API"
 
-    assert watch.get('viewed') == False
-    # Loading the most recent snapshot should force viewed to become true
-    client.get(url_for("diff_history_page", uuid="first"), follow_redirects=True)
-
-    # Fetch the whole watch again, viewed should be true
-    res = client.get(
-        url_for("watch", uuid=watch_uuid),
-        headers={'x-api-key': api_key}
-    )
-    watch = res.json
-    assert watch.get('viewed') == True
-
     # basic systeminfo check
     res = client.get(
         url_for("systeminfo"),

changedetectionio/tests/test_notification.py

@@ -281,8 +281,7 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server):
 
     # CUSTOM JSON BODY CHECK for POST://
     set_original_response()
-    # https://github.com/caronc/apprise/wiki/Notify_Custom_JSON#header-manipulation
-    test_notification_url = url_for('test_notification_endpoint', _external=True).replace('http://', 'post://')+"?xxx={{ watch_url }}&+custom-header=123"
+    test_notification_url = url_for('test_notification_endpoint', _external=True).replace('http://', 'post://')+"?xxx={{ watch_url }}"
 
     res = client.post(
         url_for("settings_page"),
@@ -298,7 +297,10 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server):
         follow_redirects=True
     )
     assert b'Settings updated' in res.data
-
+    client.get(
+        url_for("form_delete", uuid="all"),
+        follow_redirects=True
+    )
     # Add a watch and trigger a HTTP POST
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
@@ -313,9 +315,7 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server):
     set_modified_response()
 
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    wait_for_all_checks(client)
-
-    time.sleep(2) # plus extra delay for notifications to fire
+    time.sleep(2)
 
     with open("test-datastore/notification.txt", 'r') as f:
         x = f.read()
@@ -328,13 +328,6 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server):
     with open("test-datastore/notification-url.txt", 'r') as f:
         notification_url = f.read()
         assert 'xxx=http' in notification_url
-        # apprise style headers should be stripped
-        assert 'custom-header' not in notification_url
-
-    with open("test-datastore/notification-headers.txt", 'r') as f:
-        notification_headers = f.read()
-        assert 'custom-header: 123' in notification_headers.lower()
-
 
     # Should always be automatically detected as JSON content type even when we set it as 'Text' (default)
     assert os.path.isfile("test-datastore/notification-content-type.txt")
@@ -342,8 +335,3 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server):
         assert 'application/json' in f.read()
 
     os.unlink("test-datastore/notification-url.txt")
-
-    client.get(
-        url_for("form_delete", uuid="all"),
-        follow_redirects=True
-    )

changedetectionio/tests/test_pdf.py

@@ -2,8 +2,9 @@
 
 import time
 from flask import url_for
-from .util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks
+from .util import set_original_response, set_modified_response, live_server_setup
 
+sleep_time_for_fetch_thread = 3
 
 # `subtractive_selectors` should still work in `source:` type requests
 def test_fetch_pdf(client, live_server):
@@ -21,9 +22,7 @@ def test_fetch_pdf(client, live_server):
 
     assert b"1 Imported" in res.data
 
-
-    wait_for_all_checks(client)
-
+    time.sleep(sleep_time_for_fetch_thread)
     res = client.get(
         url_for("preview_page", uuid="first"),
         follow_redirects=True
@@ -34,42 +33,8 @@ def test_fetch_pdf(client, live_server):
 
     # So we know if the file changes in other ways
     import hashlib
-    original_md5 = hashlib.md5(open("test-datastore/endpoint-test.pdf", 'rb').read()).hexdigest().upper()
+    md5 = hashlib.md5(open("test-datastore/endpoint-test.pdf", 'rb').read()).hexdigest().upper()
     # We should have one
-    assert len(original_md5) >0
+    assert len(md5) >0
     # And it's going to be in the document
-    assert b'Document checksum - '+bytes(str(original_md5).encode('utf-8')) in res.data
-
-
-    shutil.copy("tests/test2.pdf", "test-datastore/endpoint-test.pdf")
-    changed_md5 = hashlib.md5(open("test-datastore/endpoint-test.pdf", 'rb').read()).hexdigest().upper()
-    res = client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    assert b'1 watches queued for rechecking.' in res.data
-
-    wait_for_all_checks(client)
-
-    # Now something should be ready, indicated by having a 'unviewed' class
-    res = client.get(url_for("index"))
-    assert b'unviewed' in res.data
-
-    # The original checksum should be not be here anymore (cdio adds it to the bottom of the text)
-
-    res = client.get(
-        url_for("preview_page", uuid="first"),
-        follow_redirects=True
-    )
-
-    assert original_md5.encode('utf-8') not in res.data
-    assert changed_md5.encode('utf-8') in res.data
-
-
-    res = client.get(
-        url_for("diff_history_page", uuid="first"),
-        follow_redirects=True
-    )
-
-    assert original_md5.encode('utf-8') in res.data
-    assert changed_md5.encode('utf-8') in res.data
-
-    assert b'here is a change' in res.data
-    
+    assert b'Document checksum - '+bytes(str(md5).encode('utf-8')) in res.data

changedetectionio/tests/util.py

@@ -205,9 +205,6 @@ def live_server_setup(live_server):
         with open("test-datastore/notification-url.txt", "w") as f:
             f.write(request.url)
 
-        with open("test-datastore/notification-headers.txt", "w") as f:
-            f.write(str(request.headers))
-
         if request.content_type:
             with open("test-datastore/notification-content-type.txt", "w") as f:
                 f.write(request.content_type)

changedetectionio/update_worker.py

@@ -1,9 +1,13 @@
+import importlib
 import os
+import re
 import threading
 import queue
 import time
+from distutils.util import strtobool
 
 from changedetectionio import content_fetcher, html_tools
+
 from .processors.text_json_diff import FilterNotFoundInResponse
 from .processors.restock_diff import UnableToExtractRestockData
 
@@ -15,6 +19,7 @@ from .processors.restock_diff import UnableToExtractRestockData
 import logging
 import sys
 
+
 class update_worker(threading.Thread):
     current_uuid = None
 
@@ -24,6 +29,7 @@ class update_worker(threading.Thread):
         self.app = app
         self.notification_q = notification_q
         self.datastore = datastore
+
         super().__init__(*args, **kwargs)
 
     def queue_notification_for_watch(self, n_object, watch):
@@ -209,7 +215,7 @@ class update_worker(threading.Thread):
         from .processors import text_json_diff, restock_diff
 
         while not self.app.config.exit.is_set():
-            update_handler = None
+            change_processor = None
 
             try:
                 queued_item_data = self.q.get(block=False)
@@ -230,35 +236,46 @@ class update_worker(threading.Thread):
                     now = time.time()
 
                     try:
-                        # Processor is what we are using for detecting the "Change"
+                        # Protect against file:// access
+                        if re.search(r'^file://', self.datastore.data['watching'][uuid].get('url', '').strip(), re.IGNORECASE):
+                            if not strtobool(os.getenv('ALLOW_FILE_URI', 'false')):
+                                raise Exception(
+                                    "file:// type access is denied for security reasons."
+                                )
+
+                        prefer_fetch_backend = self.datastore.data['watching'][uuid].get('fetch_backend', 'system')
+                        if not prefer_fetch_backend or prefer_fetch_backend == 'system':
+                            prefer_fetch_backend = self.datastore.data['settings']['application'].get('fetch_backend')
+
                         processor = self.datastore.data['watching'][uuid].get('processor', 'text_json_diff')
-                        # if system...
 
-                        # Abort processing when the content was the same as the last fetch
-                        skip_when_same_checksum = queued_item_data.item.get('skip_when_checksum_same')
+                        processor = 'cdio_whois_diff'
 
-
-                        # @todo some way to switch by name
-                        # Init a new 'difference_detection_processor'
-
-                        if processor == 'restock_diff':
-                            update_handler = restock_diff.perform_site_check(datastore=self.datastore,
-                                                                             watch_uuid=uuid
-                                                                             )
+                        if processor in ['text_json_diff', 'restock_diff']:
+                            base_processor_module = f"changedetectionio.processors.{processor}"
                         else:
-                            # Used as a default and also by some tests
-                            update_handler = text_json_diff.perform_site_check(datastore=self.datastore,
-                                                                               watch_uuid=uuid
-                                                                               )
+                            # Each plugin is one processor exactly
+                            base_processor_module = f"{processor}.processor"
+
+# its correct that processor dictates which fethcer it uses i think
+
+                        # these should inherit the right fetcher too
+                        module = importlib.import_module(base_processor_module)
+                        change_processor = getattr(module, 'perform_site_check')
+                        change_processor = change_processor(datastore=self.datastore,
+                                                            watch_uuid=uuid,
+                                                            prefer_fetch_backend=prefer_fetch_backend
+                                                            )
 
                         # Clear last errors (move to preflight func?)
                         self.datastore.data['watching'][uuid]['browser_steps_last_error_step'] = None
 
-                        update_handler.call_browser()
-
-                        changed_detected, update_obj, contents = update_handler.run_changedetection(uuid,
-                                                                                    skip_when_checksum_same=skip_when_same_checksum,
-                                                                                    )
+                        skip_when_same_checksum = queued_item_data.item.get('skip_when_checksum_same')
+                        # Each processor extends base class of the kind of fetcher it needs to run anyway
+                        change_processor.fetch_content()
+                        changed_detected, update_obj, contents = change_processor.run_changedetection(uuid,
+                                                                                                      skip_when_checksum_same=skip_when_same_checksum
+                                                                                                      )
 
                         # Re #342
                         # In Python 3, all strings are sequences of Unicode characters. There is a bytes type that holds raw bytes.
@@ -465,10 +482,10 @@ class update_worker(threading.Thread):
                                                                            })
 
                         # Always save the screenshot if it's available
-                        if update_handler.screenshot:
-                            self.datastore.save_screenshot(watch_uuid=uuid, screenshot=update_handler.screenshot)
-                        if update_handler.xpath_data:
-                            self.datastore.save_xpath_data(watch_uuid=uuid, data=update_handler.xpath_data)
+                        if change_processor.screenshot:
+                            self.datastore.save_screenshot(watch_uuid=uuid, screenshot=change_processor.screenshot)
+                        if change_processor.xpath_data:
+                            self.datastore.save_xpath_data(watch_uuid=uuid, data=change_processor.xpath_data)
 
 
                 self.current_uuid = None  # Done