Revert "UI Edit/Stats - Add levenshtein distance info, explains how "different" the last two snapshot are (#3109)"

This reverts commit cfb4decf67.

.github/workflows/test-only.yml

@@ -28,7 +28,6 @@ jobs:
     uses: ./.github/workflows/test-stack-reusable-workflow.yml
     with:
       python-version: '3.11'
-      skip-pypuppeteer: true
 
   test-application-3-12:
     needs: lint-code

.github/workflows/test-stack-reusable-workflow.yml

@@ -7,7 +7,7 @@ on:
         description: 'Python version to use'
         required: true
         type: string
-        default: '3.10'
+        default: '3.11'
       skip-pypuppeteer:
         description: 'Skip PyPuppeteer (not supported in 3.11/3.12)'
         required: false

Dockerfile

@@ -1,8 +1,5 @@
 # pip dependencies install stage
 
-# @NOTE! I would love to move to 3.11 but it breaks the async handler in changedetectionio/content_fetchers/puppeteer.py
-#        If you know how to fix it, please do! and test it for both 3.10 and 3.11
-
 ARG PYTHON_VERSION=3.11
 
 FROM python:${PYTHON_VERSION}-slim-bookworm AS builder
@@ -71,7 +68,7 @@ COPY changedetection.py /app/changedetection.py
 # Github Action test purpose(test-only.yml).
 # On production, it is effectively LOGGER_LEVEL=''.
 ARG LOGGER_LEVEL=''
-ENV LOGGER_LEVEL "$LOGGER_LEVEL"
+ENV LOGGER_LEVEL="$LOGGER_LEVEL"
 
 WORKDIR /app
 CMD ["python", "./changedetection.py", "-d", "/datastore"]

MANIFEST.in

@@ -1,9 +1,9 @@
 recursive-include changedetectionio/api *
-recursive-include changedetectionio/apprise_plugin *
 recursive-include changedetectionio/blueprint *
 recursive-include changedetectionio/content_fetchers *
 recursive-include changedetectionio/conditions *
 recursive-include changedetectionio/model *
+recursive-include changedetectionio/notification *
 recursive-include changedetectionio/processors *
 recursive-include changedetectionio/static *
 recursive-include changedetectionio/templates *

README.md

@@ -89,7 +89,7 @@ _Need an actual Chrome runner with Javascript support? We support fetching via W
 #### Key Features
 
 - Lots of trigger filters, such as "Trigger on text", "Remove text by selector", "Ignore text", "Extract text", also using regular-expressions!
-- Target elements with xPath(1.0) and CSS Selectors, Easily monitor complex JSON with JSONPath or jq
+- Target elements with xPath 1 and xPath 2, CSS Selectors, Easily monitor complex JSON with JSONPath or jq
 - Switch between fast non-JS and Chrome JS based "fetchers"
 - Track changes in PDF files (Monitor text changed in the PDF, Also monitor PDF filesize and checksums)
 - Easily specify how often a site should be checked
@@ -105,6 +105,12 @@ We [recommend and use Bright Data](https://brightdata.grsm.io/n0r16zf7eivq) glob
 
 Please :star: star :star: this project and help it grow! https://github.com/dgtlmoon/changedetection.io/
 
+### Conditional web page changes
+
+Easily [configure conditional actions](https://changedetection.io/tutorial/conditional-actions-web-page-changes), for example, only trigger when a price is above or below a preset amount, or [when a web page includes (or does not include) a keyword](https://changedetection.io/tutorial/how-monitor-keywords-any-website)
+
+<img src="./docs/web-page-change-conditions.png" style="max-width:80%;" alt="Conditional web page changes"  title="Conditional web page changes"  />
+
 ### Schedule web page watches in any timezone, limit by day of week and time.
 
 Easily set a re-check schedule, for example you could limit the web page change detection to only operate during business hours.

changedetectionio/__init__.py

@@ -2,7 +2,7 @@
 
 # Read more https://github.com/dgtlmoon/changedetection.io/wiki
 
-__version__ = '0.49.7'
+__version__ = '0.49.13'
 
 from changedetectionio.strtobool import strtobool
 from json.decoder import JSONDecodeError
@@ -11,6 +11,7 @@ os.environ['EVENTLET_NO_GREENDNS'] = 'yes'
 import eventlet
 import eventlet.wsgi
 import getopt
+import platform
 import signal
 import socket
 import sys
@@ -19,7 +20,6 @@ from changedetectionio import store
 from changedetectionio.flask_app import changedetection_app
 from loguru import logger
 
-
 # Only global so we can access it in the signal handler
 app = None
 datastore = None
@@ -29,8 +29,6 @@ def get_version():
 
 # Parent wrapper or OS sends us a SIGTERM/SIGINT, do everything required for a clean shutdown
 def sigshutdown_handler(_signo, _stack_frame):
-    global app
-    global datastore
     name = signal.Signals(_signo).name
     logger.critical(f'Shutdown: Got Signal - {name} ({_signo}), Saving DB to disk and calling shutdown')
     datastore.sync_to_json()
@@ -147,6 +145,19 @@ def main():
 
     signal.signal(signal.SIGTERM, sigshutdown_handler)
     signal.signal(signal.SIGINT, sigshutdown_handler)
+    
+    # Custom signal handler for memory cleanup
+    def sigusr_clean_handler(_signo, _stack_frame):
+        from changedetectionio.gc_cleanup import memory_cleanup
+        logger.info('SIGUSR1 received: Running memory cleanup')
+        return memory_cleanup(app)
+
+    # Register the SIGUSR1 signal handler
+    # Only register the signal handler if running on Linux
+    if platform.system() == "Linux":
+        signal.signal(signal.SIGUSR1, sigusr_clean_handler)
+    else:
+        logger.info("SIGUSR1 handler only registered on Linux, skipped.")
 
     # Go into cleanup mode
     if do_cleanup:

changedetectionio/api/Notifications.py

@@ -0,0 +1,145 @@
+from flask_expects_json import expects_json
+from flask_restful import Resource
+from . import auth
+from flask_restful import abort, Resource
+from flask import request
+from . import auth
+from . import schema_create_notification_urls, schema_delete_notification_urls
+
+class Notifications(Resource):
+    def __init__(self, **kwargs):
+        # datastore is a black box dependency
+        self.datastore = kwargs['datastore']
+
+    @auth.check_token
+    def get(self):
+        """
+        @api {get} /api/v1/notifications Return Notification URL List
+        @apiDescription Return the Notification URL List from the configuration
+        @apiExample {curl} Example usage:
+            curl http://localhost:5000/api/v1/notifications -H"x-api-key:813031b16330fe25e3780cf0325daa45"
+            HTTP/1.0 200
+            {
+                'notification_urls': ["notification-urls-list"]
+            }
+        @apiName Get
+        @apiGroup Notifications
+        """
+
+        notification_urls = self.datastore.data.get('settings', {}).get('application', {}).get('notification_urls', [])        
+
+        return {
+                'notification_urls': notification_urls,
+               }, 200
+    
+    @auth.check_token
+    @expects_json(schema_create_notification_urls)
+    def post(self):
+        """
+        @api {post} /api/v1/notifications Create Notification URLs
+        @apiDescription Add one or more notification URLs from the configuration
+        @apiExample {curl} Example usage:
+            curl http://localhost:5000/api/v1/notifications/batch -H"x-api-key:813031b16330fe25e3780cf0325daa45" -H "Content-Type: application/json" -d '{"notification_urls": ["url1", "url2"]}'
+        @apiName CreateBatch
+        @apiGroup Notifications
+        @apiSuccess (201) {Object[]} notification_urls List of added notification URLs
+        @apiError (400) {String} Invalid input
+        """
+
+        json_data = request.get_json()
+        notification_urls = json_data.get("notification_urls", [])
+
+        from wtforms import ValidationError
+        try:
+            validate_notification_urls(notification_urls)
+        except ValidationError as e:
+            return str(e), 400
+
+        added_urls = []
+
+        for url in notification_urls:
+            clean_url = url.strip()
+            added_url = self.datastore.add_notification_url(clean_url)
+            if added_url:
+                added_urls.append(added_url)
+
+        if not added_urls:
+            return "No valid notification URLs were added", 400
+
+        return {'notification_urls': added_urls}, 201
+    
+    @auth.check_token
+    @expects_json(schema_create_notification_urls)
+    def put(self):
+        """
+        @api {put} /api/v1/notifications Replace Notification URLs
+        @apiDescription Replace all notification URLs with the provided list (can be empty)
+        @apiExample {curl} Example usage:
+            curl -X PUT http://localhost:5000/api/v1/notifications -H"x-api-key:813031b16330fe25e3780cf0325daa45" -H "Content-Type: application/json" -d '{"notification_urls": ["url1", "url2"]}'
+        @apiName Replace
+        @apiGroup Notifications
+        @apiSuccess (200) {Object[]} notification_urls List of current notification URLs
+        @apiError (400) {String} Invalid input
+        """
+        json_data = request.get_json()
+        notification_urls = json_data.get("notification_urls", [])
+
+        from wtforms import ValidationError
+        try:
+            validate_notification_urls(notification_urls)
+        except ValidationError as e:
+            return str(e), 400
+        
+        if not isinstance(notification_urls, list):
+            return "Invalid input format", 400
+
+        clean_urls = [url.strip() for url in notification_urls if isinstance(url, str)]
+        self.datastore.data['settings']['application']['notification_urls'] = clean_urls
+        self.datastore.needs_write = True
+
+        return {'notification_urls': clean_urls}, 200
+        
+    @auth.check_token
+    @expects_json(schema_delete_notification_urls)
+    def delete(self):
+        """
+        @api {delete} /api/v1/notifications Delete Notification URLs
+        @apiDescription Deletes one or more notification URLs from the configuration
+        @apiExample {curl} Example usage:
+            curl http://localhost:5000/api/v1/notifications -X DELETE -H"x-api-key:813031b16330fe25e3780cf0325daa45" -H "Content-Type: application/json" -d '{"notification_urls": ["url1", "url2"]}'
+        @apiParam {String[]} notification_urls The notification URLs to delete.
+        @apiName Delete
+        @apiGroup Notifications
+        @apiSuccess (204) {String} OK Deleted
+        @apiError (400) {String} No matching notification URLs found.
+        """
+
+        json_data = request.get_json()
+        urls_to_delete = json_data.get("notification_urls", [])
+        if not isinstance(urls_to_delete, list):
+            abort(400, message="Expected a list of notification URLs.")
+
+        notification_urls = self.datastore.data['settings']['application'].get('notification_urls', [])
+        deleted = []
+
+        for url in urls_to_delete:
+            clean_url = url.strip()
+            if clean_url in notification_urls:
+                notification_urls.remove(clean_url)
+                deleted.append(clean_url)
+
+        if not deleted:
+            abort(400, message="No matching notification URLs found.")
+
+        self.datastore.data['settings']['application']['notification_urls'] = notification_urls
+        self.datastore.needs_write = True
+
+        return 'OK', 204
+    
+def validate_notification_urls(notification_urls):
+    from changedetectionio.forms import ValidateAppRiseServers
+    validator = ValidateAppRiseServers()
+    class DummyForm: pass
+    dummy_form = DummyForm()
+    field = type("Field", (object,), {"data": notification_urls, "gettext": lambda self, x: x})()
+    validator(dummy_form, field)

changedetectionio/api/Search.py

@@ -0,0 +1,51 @@
+from flask_restful import Resource, abort
+from flask import request
+from . import auth
+
+class Search(Resource):
+    def __init__(self, **kwargs):
+        # datastore is a black box dependency
+        self.datastore = kwargs['datastore']
+
+    @auth.check_token
+    def get(self):
+        """
+        @api {get} /api/v1/search Search for watches
+        @apiDescription Search watches by URL or title text
+        @apiExample {curl} Example usage:
+            curl "http://localhost:5000/api/v1/search?q=https://example.com/page1" -H"x-api-key:813031b16330fe25e3780cf0325daa45"
+            curl "http://localhost:5000/api/v1/search?q=https://example.com/page1?tag=Favourites" -H"x-api-key:813031b16330fe25e3780cf0325daa45"
+            curl "http://localhost:5000/api/v1/search?q=https://example.com?partial=true" -H"x-api-key:813031b16330fe25e3780cf0325daa45"
+        @apiName Search
+        @apiGroup Watch Management
+        @apiQuery {String} q Search query to match against watch URLs and titles
+        @apiQuery {String} [tag] Optional name of tag to limit results (name not UUID)
+        @apiQuery {String} [partial] Allow partial matching of URL query
+        @apiSuccess (200) {Object} JSON Object containing matched watches
+        """
+        query = request.args.get('q', '').strip()
+        tag_limit = request.args.get('tag', '').strip()
+        from changedetectionio.strtobool import strtobool
+        partial = bool(strtobool(request.args.get('partial', '0'))) if 'partial' in request.args else False
+
+        # Require a search query
+        if not query:
+            abort(400, message="Search query 'q' parameter is required")
+
+        # Use the search function from the datastore
+        matching_uuids = self.datastore.search_watches_for_url(query=query, tag_limit=tag_limit, partial=partial)
+
+        # Build the response with watch details
+        results = {}
+        for uuid in matching_uuids:
+            watch = self.datastore.data['watching'].get(uuid)
+            results[uuid] = {
+                'last_changed': watch.last_changed,
+                'last_checked': watch['last_checked'],
+                'last_error': watch['last_error'],
+                'title': watch['title'],
+                'url': watch['url'],
+                'viewed': watch.viewed
+            }
+
+        return results, 200

changedetectionio/api/__init__.py

@@ -19,8 +19,15 @@ schema_create_tag['required'] = ['title']
 schema_update_tag = copy.deepcopy(schema_tag)
 schema_update_tag['additionalProperties'] = False
 
+schema_notification_urls = copy.deepcopy(schema)
+schema_create_notification_urls = copy.deepcopy(schema_notification_urls)
+schema_create_notification_urls['required'] = ['notification_urls']
+schema_delete_notification_urls = copy.deepcopy(schema_notification_urls)
+schema_delete_notification_urls['required'] = ['notification_urls']
+
 # Import all API resources
 from .Watch import Watch, WatchHistory, WatchSingleHistory, CreateWatch
 from .Tags import Tags, Tag
 from .Import import Import
 from .SystemInfo import SystemInfo
+from .Notifications import Notifications

changedetectionio/api/api_schema.py

@@ -1,5 +1,7 @@
 # Responsible for building the storage dict into a set of rules ("JSON Schema") acceptable via the API
 # Probably other ways to solve this when the backend switches to some ORM
+from changedetectionio.notification import valid_notification_formats
+
 
 def build_time_between_check_json_schema():
     # Setup time between check schema
@@ -98,8 +100,6 @@ def build_watch_json_schema(d):
         }
     }
 
-    from changedetectionio.notification import valid_notification_formats
-
     schema['properties']['notification_format'] = {'type': 'string',
                                                    'enum': list(valid_notification_formats.keys())
                                                    }

changedetectionio/apprise_asset.py

@@ -1,12 +0,0 @@
-from changedetectionio import apprise_plugin
-import apprise
-
-# Create our AppriseAsset and populate it with some of our new values:
-# https://github.com/caronc/apprise/wiki/Development_API#the-apprise-asset-object
-asset = apprise.AppriseAsset(
-   image_url_logo='https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/changedetectionio/static/images/avatar-256x256.png'
-)
-
-asset.app_id = "changedetection.io"
-asset.app_desc = "ChangeDetection.io best and simplest website monitoring and change detection"
-asset.app_url = "https://changedetection.io"

changedetectionio/apprise_plugin/__init__.py

@@ -1,98 +0,0 @@
-# include the decorator
-from apprise.decorators import notify
-from loguru import logger
-from requests.structures import CaseInsensitiveDict
-
-
-@notify(on="delete")
-@notify(on="deletes")
-@notify(on="get")
-@notify(on="gets")
-@notify(on="post")
-@notify(on="posts")
-@notify(on="put")
-@notify(on="puts")
-def apprise_custom_api_call_wrapper(body, title, notify_type, *args, **kwargs):
-    import requests
-    import json
-    import re
-
-    from urllib.parse import unquote_plus
-    from apprise.utils.parse import parse_url as apprise_parse_url
-
-    url = kwargs['meta'].get('url')
-    schema = kwargs['meta'].get('schema').lower().strip()
-
-    # Choose POST, GET etc from requests
-    method =  re.sub(rf's$', '', schema)
-    requests_method = getattr(requests, method)
-
-    params = CaseInsensitiveDict({}) # Added to requests
-    auth = None
-    has_error = False
-
-    # Convert /foobar?+some-header=hello to proper header dictionary
-    results = apprise_parse_url(url)
-
-    # Add our headers that the user can potentially over-ride if they wish
-    # to to our returned result set and tidy entries by unquoting them
-    headers = CaseInsensitiveDict({unquote_plus(x): unquote_plus(y)
-               for x, y in results['qsd+'].items()})
-
-    # https://github.com/caronc/apprise/wiki/Notify_Custom_JSON#get-parameter-manipulation
-    # In Apprise, it relies on prefixing each request arg with "-", because it uses say &method=update as a flag for apprise
-    # but here we are making straight requests, so we need todo convert this against apprise's logic
-    for k, v in results['qsd'].items():
-        if not k.strip('+-') in results['qsd+'].keys():
-            params[unquote_plus(k)] = unquote_plus(v)
-
-    # Determine Authentication
-    auth = ''
-    if results.get('user') and results.get('password'):
-        auth = (unquote_plus(results.get('user')), unquote_plus(results.get('user')))
-    elif results.get('user'):
-        auth = (unquote_plus(results.get('user')))
-
-    # If it smells like it could be JSON and no content-type was already set, offer a default content type.
-    if body and '{' in body[:100] and not headers.get('Content-Type'):
-        json_header = 'application/json; charset=utf-8'
-        try:
-            # Try if it's JSON
-            json.loads(body)
-            headers['Content-Type'] = json_header
-        except ValueError as e:
-            logger.warning(f"Could not automatically add '{json_header}' header to the notification because the document failed to parse as JSON: {e}")
-            pass
-
-    # POSTS -> HTTPS etc
-    if schema.lower().endswith('s'):
-        url = re.sub(rf'^{schema}', 'https', results.get('url'))
-    else:
-        url = re.sub(rf'^{schema}', 'http', results.get('url'))
-
-    status_str = ''
-    try:
-        r = requests_method(url,
-          auth=auth,
-          data=body.encode('utf-8') if type(body) is str else body,
-          headers=headers,
-          params=params
-        )
-
-        if not (200 <= r.status_code < 300):
-            status_str = f"Error sending '{method.upper()}' request to {url} - Status: {r.status_code}: '{r.reason}'"
-            logger.error(status_str)
-            has_error = True
-        else:
-            logger.info(f"Sent '{method.upper()}' request to {url}")
-            has_error = False
-
-    except requests.RequestException as e:
-        status_str = f"Error sending '{method.upper()}' request to {url} - {str(e)}"
-        logger.error(status_str)
-        has_error = True
-
-    if has_error:
-        raise TypeError(status_str)
-
-    return True

changedetectionio/auth_decorator.py

@@ -20,10 +20,7 @@ def login_optionally_required(func):
         has_password_enabled = datastore.data['settings']['application'].get('password') or os.getenv("SALTED_PASS", False)
 
         # Permitted
-        if request.endpoint and 'static_content' in request.endpoint and request.view_args and request.view_args.get('group') == 'styles':
-            return func(*args, **kwargs)
-        # Permitted
-        elif request.endpoint and 'diff_history_page' in request.endpoint and datastore.data['settings']['application'].get('shared_diff_access'):
+        if request.endpoint and 'diff_history_page' in request.endpoint and datastore.data['settings']['application'].get('shared_diff_access'):
             return func(*args, **kwargs)
         elif request.method in flask_login.config.EXEMPT_METHODS:
             return func(*args, **kwargs)

changedetectionio/blueprint/backups/__init__.py

@@ -138,7 +138,7 @@ def construct_blueprint(datastore: ChangeDetectionStore):
         return send_from_directory(os.path.abspath(datastore.datastore_path), filename, as_attachment=True)
 
     @login_optionally_required
-    @backups_blueprint.route("/", methods=['GET'])
+    @backups_blueprint.route("", methods=['GET'])
     def index():
         backups = find_backups()
         output = render_template("overview.html",

changedetectionio/blueprint/browser_steps/__init__.py

@@ -23,7 +23,6 @@ from loguru import logger
 browsersteps_sessions = {}
 io_interface_context = None
 import json
-import base64
 import hashlib
 from flask import Response
 
@@ -34,10 +33,8 @@ def construct_blueprint(datastore: ChangeDetectionStore):
         from . import nonContext
         from . import browser_steps
         import time
-        global browsersteps_sessions
         global io_interface_context
 
-
         # We keep the playwright session open for many minutes
         keepalive_seconds = int(os.getenv('BROWSERSTEPS_MINUTES_KEEPALIVE', 10)) * 60
 
@@ -104,8 +101,6 @@ def construct_blueprint(datastore: ChangeDetectionStore):
         # A new session was requested, return sessionID
 
         import uuid
-        global browsersteps_sessions
-
         browsersteps_session_id = str(uuid.uuid4())
         watch_uuid = request.args.get('uuid')
 
@@ -149,7 +144,6 @@ def construct_blueprint(datastore: ChangeDetectionStore):
     def browsersteps_ui_update():
         import base64
         import playwright._impl._errors
-        global browsersteps_sessions
         from changedetectionio.blueprint.browser_steps import browser_steps
 
         remaining =0

changedetectionio/blueprint/browser_steps/browser_steps.py

@@ -4,7 +4,7 @@ import re
 from random import randint
 from loguru import logger
 
-from changedetectionio.content_fetchers.helpers import capture_stitched_together_full_page, SCREENSHOT_SIZE_STITCH_THRESHOLD
+from changedetectionio.content_fetchers import SCREENSHOT_MAX_HEIGHT_DEFAULT
 from changedetectionio.content_fetchers.base import manage_user_agent
 from changedetectionio.safe_jinja import render as jinja_render
 
@@ -293,19 +293,16 @@ class browsersteps_live_ui(steppable_browser_interface):
     def get_current_state(self):
         """Return the screenshot and interactive elements mapping, generally always called after action_()"""
         import importlib.resources
+        import json
+        # because we for now only run browser steps in playwright mode (not puppeteer mode)
+        from changedetectionio.content_fetchers.playwright import capture_full_page
+
         xpath_element_js = importlib.resources.files("changedetectionio.content_fetchers.res").joinpath('xpath_element_scraper.js').read_text()
 
         now = time.time()
         self.page.wait_for_timeout(1 * 1000)
 
-
-        full_height = self.page.evaluate("document.documentElement.scrollHeight")
-
-        if full_height >= SCREENSHOT_SIZE_STITCH_THRESHOLD:
-            logger.warning(f"Page full Height: {full_height}px longer than {SCREENSHOT_SIZE_STITCH_THRESHOLD}px, using 'stitched screenshot method'.")
-            screenshot = capture_stitched_together_full_page(self.page)
-        else:
-            screenshot = self.page.screenshot(type='jpeg', full_page=True, quality=40)
+        screenshot = capture_full_page(page=self.page)
 
         logger.debug(f"Time to get screenshot from browser {time.time() - now:.2f}s")
 
@@ -313,13 +310,21 @@ class browsersteps_live_ui(steppable_browser_interface):
         self.page.evaluate("var include_filters=''")
         # Go find the interactive elements
         # @todo in the future, something smarter that can scan for elements with .click/focus etc event handlers?
-        elements = 'a,button,input,select,textarea,i,th,td,p,li,h1,h2,h3,h4,div,span'
-        xpath_element_js = xpath_element_js.replace('%ELEMENTS%', elements)
 
-        xpath_data = self.page.evaluate("async () => {" + xpath_element_js + "}")
+        self.page.request_gc()
+
+        scan_elements = 'a,button,input,select,textarea,i,th,td,p,li,h1,h2,h3,h4,div,span'
+
+        MAX_TOTAL_HEIGHT = int(os.getenv("SCREENSHOT_MAX_HEIGHT", SCREENSHOT_MAX_HEIGHT_DEFAULT))
+        xpath_data = json.loads(self.page.evaluate(xpath_element_js, {
+            "visualselector_xpath_selectors": scan_elements,
+            "max_height": MAX_TOTAL_HEIGHT
+        }))
+        self.page.request_gc()
+
         # So the JS will find the smallest one first
         xpath_data['size_pos'] = sorted(xpath_data['size_pos'], key=lambda k: k['width'] * k['height'], reverse=True)
-        logger.debug(f"Time to scrape xpath element data in browser {time.time()-now:.2f}s")
+        logger.debug(f"Time to scrape xPath element data in browser {time.time()-now:.2f}s")
 
         # playwright._impl._api_types.Error: Browser closed.
         # @todo show some countdown timer?

changedetectionio/blueprint/imports/__init__.py

@@ -27,7 +27,7 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMe
                     update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid}))
 
                 if len(importer_handler.remaining_data) == 0:
-                    return redirect(url_for('index'))
+                    return redirect(url_for('watchlist.index'))
                 else:
                     remaining_urls = importer_handler.remaining_data
 

changedetectionio/blueprint/price_data_follower/__init__.py

@@ -20,13 +20,13 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q: PriorityQueue
         datastore.data['watching'][uuid]['processor'] = 'restock_diff'
         datastore.data['watching'][uuid].clear_watch()
         update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid}))
-        return redirect(url_for("index"))
+        return redirect(url_for("watchlist.index"))
 
     @login_required
     @price_data_follower_blueprint.route("/<string:uuid>/reject", methods=['GET'])
     def reject(uuid):
         datastore.data['watching'][uuid]['track_ldjson_price_data'] = PRICE_DATA_TRACK_REJECT
-        return redirect(url_for("index"))
+        return redirect(url_for("watchlist.index"))
 
 
     return price_data_follower_blueprint

changedetectionio/blueprint/rss/__init__.py

@@ -1,103 +1 @@
-import time
-import datetime
-import pytz
-from flask import Blueprint, make_response, request, url_for
-from loguru import logger
-from feedgen.feed import FeedGenerator
-
-from changedetectionio.store import ChangeDetectionStore
-from changedetectionio.safe_jinja import render as jinja_render
-
-def construct_blueprint(datastore: ChangeDetectionStore):
-    rss_blueprint = Blueprint('rss', __name__)
-    
-    # Import the login decorator if needed
-    # from changedetectionio.auth_decorator import login_optionally_required
-
-    @rss_blueprint.route("/", methods=['GET'])
-    def feed():
-        now = time.time()
-        # Always requires token set
-        app_rss_token = datastore.data['settings']['application'].get('rss_access_token')
-        rss_url_token = request.args.get('token')
-        if rss_url_token != app_rss_token:
-            return "Access denied, bad token", 403
-
-        from changedetectionio import diff
-        limit_tag = request.args.get('tag', '').lower().strip()
-        # Be sure limit_tag is a uuid
-        for uuid, tag in datastore.data['settings']['application'].get('tags', {}).items():
-            if limit_tag == tag.get('title', '').lower().strip():
-                limit_tag = uuid
-
-        # Sort by last_changed and add the uuid which is usually the key..
-        sorted_watches = []
-
-        # @todo needs a .itemsWithTag() or something - then we can use that in Jinaj2 and throw this away
-        for uuid, watch in datastore.data['watching'].items():
-            # @todo tag notification_muted skip also (improve Watch model)
-            if datastore.data['settings']['application'].get('rss_hide_muted_watches') and watch.get('notification_muted'):
-                continue
-            if limit_tag and not limit_tag in watch['tags']:
-                continue
-            watch['uuid'] = uuid
-            sorted_watches.append(watch)
-
-        sorted_watches.sort(key=lambda x: x.last_changed, reverse=False)
-
-        fg = FeedGenerator()
-        fg.title('changedetection.io')
-        fg.description('Feed description')
-        fg.link(href='https://changedetection.io')
-
-        for watch in sorted_watches:
-
-            dates = list(watch.history.keys())
-            # Re #521 - Don't bother processing this one if theres less than 2 snapshots, means we never had a change detected.
-            if len(dates) < 2:
-                continue
-
-            if not watch.viewed:
-                # Re #239 - GUID needs to be individual for each event
-                # @todo In the future make this a configurable link back (see work on BASE_URL https://github.com/dgtlmoon/changedetection.io/pull/228)
-                guid = "{}/{}".format(watch['uuid'], watch.last_changed)
-                fe = fg.add_entry()
-
-                # Include a link to the diff page, they will have to login here to see if password protection is enabled.
-                # Description is the page you watch, link takes you to the diff JS UI page
-                # Dict val base_url will get overriden with the env var if it is set.
-                ext_base_url = datastore.data['settings']['application'].get('active_base_url')
-
-                # Because we are called via whatever web server, flask should figure out the right path (
-                diff_link = {'href': url_for('ui.ui_views.diff_history_page', uuid=watch['uuid'], _external=True)}
-
-                fe.link(link=diff_link)
-
-                # @todo watch should be a getter - watch.get('title') (internally if URL else..)
-
-                watch_title = watch.get('title') if watch.get('title') else watch.get('url')
-                fe.title(title=watch_title)
-
-                html_diff = diff.render_diff(previous_version_file_contents=watch.get_history_snapshot(dates[-2]),
-                                             newest_version_file_contents=watch.get_history_snapshot(dates[-1]),
-                                             include_equal=False,
-                                             line_feed_sep="<br>")
-
-                # @todo Make this configurable and also consider html-colored markup
-                # @todo User could decide if <link> goes to the diff page, or to the watch link
-                rss_template = "<html><body>\n<h4><a href=\"{{watch_url}}\">{{watch_title}}</a></h4>\n<p>{{html_diff}}</p>\n</body></html>\n"
-                content = jinja_render(template_str=rss_template, watch_title=watch_title, html_diff=html_diff, watch_url=watch.link)
-
-                fe.content(content=content, type='CDATA')
-
-                fe.guid(guid, permalink=False)
-                dt = datetime.datetime.fromtimestamp(int(watch.newest_history_key))
-                dt = dt.replace(tzinfo=pytz.UTC)
-                fe.pubDate(dt)
-
-        response = make_response(fg.rss_str())
-        response.headers.set('Content-Type', 'application/rss+xml;charset=utf-8')
-        logger.trace(f"RSS generated in {time.time() - now:.3f}s")
-        return response
-
-    return rss_blueprint
+RSS_FORMAT_TYPES = [('plaintext', 'Plain text'), ('html', 'HTML Color')]

changedetectionio/blueprint/rss/blueprint.py

@@ -0,0 +1,147 @@
+
+from changedetectionio.safe_jinja import render as jinja_render
+from changedetectionio.store import ChangeDetectionStore
+from feedgen.feed import FeedGenerator
+from flask import Blueprint, make_response, request, url_for, redirect
+from loguru import logger
+import datetime
+import pytz
+import re
+import time
+
+
+BAD_CHARS_REGEX=r'[\x00-\x08\x0B\x0C\x0E-\x1F]'
+
+# Anything that is not text/UTF-8 should be stripped before it breaks feedgen (such as binary data etc)
+def scan_invalid_chars_in_rss(content):
+    for match in re.finditer(BAD_CHARS_REGEX, content):
+        i = match.start()
+        bad_char = content[i]
+        hex_value = f"0x{ord(bad_char):02x}"
+        # Grab context
+        start = max(0, i - 20)
+        end = min(len(content), i + 21)
+        context = content[start:end].replace('\n', '\\n').replace('\r', '\\r')
+        logger.warning(f"Invalid char {hex_value} at pos {i}: ...{context}...")
+        # First match is enough
+        return True
+
+    return False
+
+
+def clean_entry_content(content):
+    cleaned = re.sub(BAD_CHARS_REGEX, '', content)
+    return cleaned
+
+def construct_blueprint(datastore: ChangeDetectionStore):
+    rss_blueprint = Blueprint('rss', __name__)
+
+    # Some RSS reader situations ended up with rss/ (forward slash after RSS) due
+    # to some earlier blueprint rerouting work, it should goto feed.
+    @rss_blueprint.route("/", methods=['GET'])
+    def extraslash():
+        return redirect(url_for('rss.feed'))
+
+    # Import the login decorator if needed
+    # from changedetectionio.auth_decorator import login_optionally_required
+    @rss_blueprint.route("", methods=['GET'])
+    def feed():
+        now = time.time()
+        # Always requires token set
+        app_rss_token = datastore.data['settings']['application'].get('rss_access_token')
+        rss_url_token = request.args.get('token')
+        if rss_url_token != app_rss_token:
+            return "Access denied, bad token", 403
+
+        from changedetectionio import diff
+        limit_tag = request.args.get('tag', '').lower().strip()
+        # Be sure limit_tag is a uuid
+        for uuid, tag in datastore.data['settings']['application'].get('tags', {}).items():
+            if limit_tag == tag.get('title', '').lower().strip():
+                limit_tag = uuid
+
+        # Sort by last_changed and add the uuid which is usually the key..
+        sorted_watches = []
+
+        # @todo needs a .itemsWithTag() or something - then we can use that in Jinaj2 and throw this away
+        for uuid, watch in datastore.data['watching'].items():
+            # @todo tag notification_muted skip also (improve Watch model)
+            if datastore.data['settings']['application'].get('rss_hide_muted_watches') and watch.get('notification_muted'):
+                continue
+            if limit_tag and not limit_tag in watch['tags']:
+                continue
+            watch['uuid'] = uuid
+            sorted_watches.append(watch)
+
+        sorted_watches.sort(key=lambda x: x.last_changed, reverse=False)
+
+        fg = FeedGenerator()
+        fg.title('changedetection.io')
+        fg.description('Feed description')
+        fg.link(href='https://changedetection.io')
+
+        html_colour_enable = False
+        if datastore.data['settings']['application'].get('rss_content_format') == 'html':
+            html_colour_enable = True
+
+        for watch in sorted_watches:
+
+            dates = list(watch.history.keys())
+            # Re #521 - Don't bother processing this one if theres less than 2 snapshots, means we never had a change detected.
+            if len(dates) < 2:
+                continue
+
+            if not watch.viewed:
+                # Re #239 - GUID needs to be individual for each event
+                # @todo In the future make this a configurable link back (see work on BASE_URL https://github.com/dgtlmoon/changedetection.io/pull/228)
+                guid = "{}/{}".format(watch['uuid'], watch.last_changed)
+                fe = fg.add_entry()
+
+                # Include a link to the diff page, they will have to login here to see if password protection is enabled.
+                # Description is the page you watch, link takes you to the diff JS UI page
+                # Dict val base_url will get overriden with the env var if it is set.
+                ext_base_url = datastore.data['settings']['application'].get('active_base_url')
+                # @todo fix
+
+                # Because we are called via whatever web server, flask should figure out the right path (
+                diff_link = {'href': url_for('ui.ui_views.diff_history_page', uuid=watch['uuid'], _external=True)}
+
+                fe.link(link=diff_link)
+
+                # @todo watch should be a getter - watch.get('title') (internally if URL else..)
+
+                watch_title = watch.get('title') if watch.get('title') else watch.get('url')
+                fe.title(title=watch_title)
+                try:
+
+                    html_diff = diff.render_diff(previous_version_file_contents=watch.get_history_snapshot(dates[-2]),
+                                                 newest_version_file_contents=watch.get_history_snapshot(dates[-1]),
+                                                 include_equal=False,
+                                                 line_feed_sep="<br>",
+                                                 html_colour=html_colour_enable
+                                                 )
+                except FileNotFoundError as e:
+                    html_diff = f"History snapshot file for watch {watch.get('uuid')}@{watch.last_changed} - '{watch.get('title')} not found."
+
+                # @todo Make this configurable and also consider html-colored markup
+                # @todo User could decide if <link> goes to the diff page, or to the watch link
+                rss_template = "<html><body>\n<h4><a href=\"{{watch_url}}\">{{watch_title}}</a></h4>\n<p>{{html_diff}}</p>\n</body></html>\n"
+
+                content = jinja_render(template_str=rss_template, watch_title=watch_title, html_diff=html_diff, watch_url=watch.link)
+
+                # Out of range chars could also break feedgen
+                if scan_invalid_chars_in_rss(content):
+                    content = clean_entry_content(content)
+
+                fe.content(content=content, type='CDATA')
+                fe.guid(guid, permalink=False)
+                dt = datetime.datetime.fromtimestamp(int(watch.newest_history_key))
+                dt = dt.replace(tzinfo=pytz.UTC)
+                fe.pubDate(dt)
+
+        response = make_response(fg.rss_str())
+        response.headers.set('Content-Type', 'application/rss+xml;charset=utf-8')
+        logger.trace(f"RSS generated in {time.time() - now:.3f}s")
+        return response
+
+    return rss_blueprint

changedetectionio/blueprint/settings/__init__.py

@@ -13,7 +13,7 @@ from changedetectionio.auth_decorator import login_optionally_required
 def construct_blueprint(datastore: ChangeDetectionStore):
     settings_blueprint = Blueprint('settings', __name__, template_folder="templates")
 
-    @settings_blueprint.route("/", methods=['GET', "POST"])
+    @settings_blueprint.route("", methods=['GET', "POST"])
     @login_optionally_required
     def settings_page():
         from changedetectionio import forms
@@ -74,7 +74,7 @@ def construct_blueprint(datastore: ChangeDetectionStore):
                     datastore.needs_write_urgent = True
                     flash("Password protection enabled.", 'notice')
                     flask_login.logout_user()
-                    return redirect(url_for('index'))
+                    return redirect(url_for('watchlist.index'))
 
                 datastore.needs_write_urgent = True
                 flash("Settings updated.")

changedetectionio/blueprint/settings/templates/settings.html

@@ -22,6 +22,7 @@
             <li class="tab"><a href="#notifications">Notifications</a></li>
             <li class="tab"><a href="#fetching">Fetching</a></li>
             <li class="tab"><a href="#filters">Global Filters</a></li>
+            <li class="tab"><a href="#ui-options">UI Options</a></li>
             <li class="tab"><a href="#api">API</a></li>
             <li class="tab"><a href="#timedate">Time &amp Date</a></li>
             <li class="tab"><a href="#proxies">CAPTCHA &amp; Proxies</a></li>
@@ -78,7 +79,10 @@
                         {{ render_field(form.application.form.pager_size) }}
                         <span class="pure-form-message-inline">Number of items per page in the watch overview list, 0 to disable.</span>
                     </div>
-
+                    <div class="pure-control-group">
+                        {{ render_field(form.application.form.rss_content_format) }}
+                        <span class="pure-form-message-inline">Love RSS? Does your reader support HTML? Set it here</span>
+                    </div>
                     <div class="pure-control-group">
                         {{ render_checkbox_field(form.application.form.extract_title_as_title) }}
                         <span class="pure-form-message-inline">Note: This will automatically apply to all existing watches.</span>
@@ -214,7 +218,7 @@ nav
                         <a id="chrome-extension-link"
                            title="Try our new Chrome Extension!"
                            href="https://chromewebstore.google.com/detail/changedetectionio-website/kefcfmgmlhmankjmnbijimhofdjekbop">
-                            <img alt="Chrome store icon" src="{{ url_for('static_content', group='images', filename='Google-Chrome-icon.png') }}" alt="Chrome">
+                            <img alt="Chrome store icon" src="{{ url_for('static_content', group='images', filename='google-chrome-icon.png') }}" alt="Chrome">
                             Chrome Webstore
                         </a>
                     </p>
@@ -237,6 +241,12 @@ nav
                     </p>
                 </div>
             </div>
+            <div class="tab-pane-inner" id="ui-options">
+                <div class="pure-control-group">
+                    {{ render_checkbox_field(form.application.form.ui.form.open_diff_in_new_tab, class="open_diff_in_new_tab") }}
+                    <span class="pure-form-message-inline">Enable this setting to open the diff page in a new tab. If disabled, the diff page will open in the current tab.</span>
+                </div>
+            </div>
             <div class="tab-pane-inner" id="proxies">
                 <div id="recommended-proxy">
                     <div>
@@ -299,7 +309,7 @@ nav
             <div id="actions">
                 <div class="pure-control-group">
                     {{ render_button(form.save_button) }}
-                    <a href="{{url_for('index')}}" class="pure-button button-small button-cancel">Back</a>
+                    <a href="{{url_for('watchlist.index')}}" class="pure-button button-small button-cancel">Back</a>
                     <a href="{{url_for('ui.clear_all_history')}}" class="pure-button button-small button-error">Clear Snapshot History</a>
                 </div>
             </div>

changedetectionio/blueprint/tags/__init__.py

@@ -104,6 +104,9 @@ def construct_blueprint(datastore: ChangeDetectionStore):
             uuid = list(datastore.data['settings']['application']['tags'].keys()).pop()
 
         default = datastore.data['settings']['application']['tags'].get(uuid)
+        if not default:
+            flash("Tag not found", "error")
+            return redirect(url_for('watchlist.index'))
 
         form = group_restock_settings_form(
                                        formdata=request.form if request.method == 'POST' else None,

changedetectionio/blueprint/tags/templates/edit-tag.html

@@ -13,6 +13,7 @@
     /*const email_notification_prefix=JSON.parse('{{ emailprefix|tojson }}');*/
 /*{% endif %}*/
 
+{% set has_tag_filters_extra='' %}
 
 </script>
 
@@ -46,59 +47,12 @@
             </div>
 
             <div class="tab-pane-inner" id="filters-and-triggers">
-                    <div class="pure-control-group">
-                        {% set field = render_field(form.include_filters,
-                            rows=5,
-                            placeholder="#example
-xpath://body/div/span[contains(@class, 'example-class')]",
-                            class="m-d")
-                        %}
-                        {{ field }}
-                        {% if '/text()' in  field %}
-                          <span class="pure-form-message-inline"><strong>Note!: //text() function does not work where the &lt;element&gt; contains &lt;![CDATA[]]&gt;</strong></span><br>
-                        {% endif %}
-                        <span class="pure-form-message-inline">One CSS, xPath, JSON Path/JQ selector per line, <i>any</i> rules that matches will be used.<br>
-                    <div data-target="#advanced-help-selectors" class="toggle-show pure-button button-tag button-xsmall">Show advanced help and tips</div>
-                    <ul id="advanced-help-selectors">
-                        <li>CSS - Limit text to this CSS rule, only text matching this CSS rule is included.</li>
-                        <li>JSON - Limit text to this JSON rule, using either <a href="https://pypi.org/project/jsonpath-ng/" target="new">JSONPath</a> or <a href="https://stedolan.github.io/jq/" target="new">jq</a> (if installed).
-                            <ul>
-                                <li>JSONPath: Prefix with <code>json:</code>, use <code>json:$</code> to force re-formatting if required,  <a href="https://jsonpath.com/" target="new">test your JSONPath here</a>.</li>
-                                {% if jq_support %}
-                                <li>jq: Prefix with <code>jq:</code> and <a href="https://jqplay.org/" target="new">test your jq here</a>. Using <a href="https://stedolan.github.io/jq/" target="new">jq</a> allows for complex filtering and processing of JSON data with built-in functions, regex, filtering, and more. See examples and documentation <a href="https://stedolan.github.io/jq/manual/" target="new">here</a>. Prefix <code>jqraw:</code> outputs the results as text instead of a JSON list.</li>
-                                {% else %}
-                                <li>jq support not installed</li>
-                                {% endif %}
-                            </ul>
-                        </li>
-                        <li>XPath - Limit text to this XPath rule, simply start with a forward-slash. To specify XPath to be used explicitly or the XPath rule starts with an XPath function: Prefix with <code>xpath:</code>
-                            <ul>
-                                <li>Example:  <code>//*[contains(@class, 'sametext')]</code> or <code>xpath:count(//*[contains(@class, 'sametext')])</code>, <a
-                                href="http://xpather.com/" target="new">test your XPath here</a></li>
-                                <li>Example: Get all titles from an RSS feed <code>//title/text()</code></li>
-                                <li>To use XPath1.0: Prefix with <code>xpath1:</code></li>
-                            </ul>
-                            </li>
-                    </ul>
-                    Please be sure that you thoroughly understand how to write CSS, JSONPath, XPath{% if jq_support %}, or jq selector{%endif%} rules before filing an issue on GitHub! <a
-                                href="https://github.com/dgtlmoon/changedetection.io/wiki/CSS-Selector-help">here for more CSS selector help</a>.<br>
-                </span>
-                    </div>
-                <fieldset class="pure-control-group">
-                    {{ render_field(form.subtractive_selectors, rows=5, placeholder="header
-footer
-nav
-.stockticker
-//*[contains(text(), 'Advertisement')]") }}
-                    <span class="pure-form-message-inline">
-                        <ul>
-                          <li> Remove HTML element(s) by CSS and XPath selectors before text conversion. </li>
-                          <li> Don't paste HTML here, use only CSS and XPath selectors </li>
-                          <li> Add multiple elements, CSS or XPath selectors per line to ignore multiple parts of the HTML. </li>
-                        </ul>
-                      </span>
-                </fieldset>
-
+                <p>These settings are <strong><i>added</i></strong> to any existing watch configurations.</p>
+                {% include "edit/include_subtract.html" %}
+                <div class="text-filtering border-fieldset">
+                    <h3>Text filtering</h3>
+                    {% include "edit/text-options.html" %}
+                </div>
             </div>
 
         {# rendered sub Template #}
@@ -112,7 +66,7 @@ nav
                     <div  class="pure-control-group inline-radio">
                       {{ render_checkbox_field(form.notification_muted) }}
                     </div>
-                    {% if is_html_webdriver %}
+                    {% if 1 %}
                     <div class="pure-control-group inline-radio">
                       {{ render_checkbox_field(form.notification_screenshot) }}
                         <span class="pure-form-message-inline">

changedetectionio/blueprint/tags/templates/groups-overview.html

@@ -47,7 +47,7 @@
                     <a class="link-mute state-{{'on' if tag.notification_muted else 'off'}}" href="{{url_for('tags.mute', uuid=tag.uuid)}}"><img src="{{url_for('static_content', group='images', filename='bell-off.svg')}}" alt="Mute notifications" title="Mute notifications" class="icon icon-mute" ></a>
                 </td>
                 <td>{{ "{:,}".format(tag_count[uuid]) if uuid in tag_count else 0 }}</td>
-                <td class="title-col inline"> <a href="{{url_for('index', tag=uuid) }}">{{ tag.title }}</a></td>
+                <td class="title-col inline"> <a href="{{url_for('watchlist.index', tag=uuid) }}">{{ tag.title }}</a></td>
                 <td>
                     <a class="pure-button pure-button-primary" href="{{ url_for('tags.form_tag_edit', uuid=uuid) }}">Edit</a>&nbsp;
                     <a class="pure-button pure-button-primary" href="{{ url_for('tags.delete', uuid=uuid) }}" title="Deletes and removes tag">Delete</a>

changedetectionio/blueprint/ui/__init__.py

@@ -36,7 +36,7 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, running_updat
         else:
             flash("Cleared snapshot history for watch {}".format(uuid))
 
-        return redirect(url_for('index'))
+        return redirect(url_for('watchlist.index'))
 
     @ui_blueprint.route("/clear_history", methods=['GET', 'POST'])
     @login_optionally_required
@@ -52,7 +52,7 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, running_updat
             else:
                 flash('Incorrect confirmation text.', 'error')
 
-            return redirect(url_for('index'))
+            return redirect(url_for('watchlist.index'))
 
         output = render_template("clear_all_history.html")
         return output
@@ -68,7 +68,7 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, running_updat
                 continue
             datastore.set_last_viewed(watch_uuid, int(time.time()))
 
-        return redirect(url_for('index'))
+        return redirect(url_for('watchlist.index'))
 
     @ui_blueprint.route("/delete", methods=['GET'])
     @login_optionally_required
@@ -77,7 +77,7 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, running_updat
 
         if uuid != 'all' and not uuid in datastore.data['watching'].keys():
             flash('The watch by UUID {} does not exist.'.format(uuid), 'error')
-            return redirect(url_for('index'))
+            return redirect(url_for('watchlist.index'))
 
         # More for testing, possible to return the first/only
         if uuid == 'first':
@@ -85,7 +85,7 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, running_updat
         datastore.delete(uuid)
         flash('Deleted.')
 
-        return redirect(url_for('index'))
+        return redirect(url_for('watchlist.index'))
 
     @ui_blueprint.route("/clone", methods=['GET'])
     @login_optionally_required
@@ -96,12 +96,13 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, running_updat
             uuid = list(datastore.data['watching'].keys()).pop()
 
         new_uuid = datastore.clone(uuid)
-        if new_uuid:
-            if not datastore.data['watching'].get(uuid).get('paused'):
-                update_q.put(queuedWatchMetaData.PrioritizedItem(priority=5, item={'uuid': new_uuid}))
-            flash('Cloned.')
 
-        return redirect(url_for('index'))
+        if not datastore.data['watching'].get(uuid).get('paused'):
+            update_q.put(queuedWatchMetaData.PrioritizedItem(priority=5, item={'uuid': new_uuid}))
+
+        flash('Cloned, you are editing the new watch.')
+
+        return redirect(url_for("ui.ui_edit.edit_page", uuid=new_uuid))
 
     @ui_blueprint.route("/checknow", methods=['GET'])
     @login_optionally_required
@@ -124,7 +125,10 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, running_updat
 
         else:
             # Recheck all, including muted
-            for watch_uuid, watch in datastore.data['watching'].items():
+            # Get most overdue first
+            for k in sorted(datastore.data['watching'].items(), key=lambda item: item[1].get('last_checked', 0)):
+                watch_uuid = k[0]
+                watch = k[1]
                 if not watch['paused']:
                     if watch_uuid not in running_uuids:
                         if with_errors and not watch.get('last_error'):
@@ -139,11 +143,11 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, running_updat
         if i == 1:
             flash("Queued 1 watch for rechecking.")
         if i > 1:
-            flash("Queued {} watches for rechecking.".format(i))
+            flash(f"Queued {i} watches for rechecking.")
         if i == 0:
             flash("No watches available to recheck.")
 
-        return redirect(url_for('index'))
+        return redirect(url_for('watchlist.index'))
 
     @ui_blueprint.route("/form/checkbox-operations", methods=['POST'])
     @login_optionally_required
@@ -244,7 +248,7 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, running_updat
 
             flash(f"{len(uuids)} watches were tagged")
 
-        return redirect(url_for('index'))
+        return redirect(url_for('watchlist.index'))
 
 
     @ui_blueprint.route("/share-url/<string:uuid>", methods=['GET'])
@@ -296,6 +300,6 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, running_updat
             logger.error(f"Error sharing -{str(e)}")
             flash(f"Could not share, something went wrong while communicating with the share server - {str(e)}", 'error')
 
-        return redirect(url_for('index'))
+        return redirect(url_for('watchlist.index'))
 
     return ui_blueprint

changedetectionio/blueprint/ui/edit.py

@@ -32,14 +32,14 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMe
         # More for testing, possible to return the first/only
         if not datastore.data['watching'].keys():
             flash("No watches to edit", "error")
-            return redirect(url_for('index'))
+            return redirect(url_for('watchlist.index'))
 
         if uuid == 'first':
             uuid = list(datastore.data['watching'].keys()).pop()
 
         if not uuid in datastore.data['watching']:
             flash("No watch with the UUID %s found." % (uuid), "error")
-            return redirect(url_for('index'))
+            return redirect(url_for('watchlist.index'))
 
         switch_processor = request.args.get('switch_processor')
         if switch_processor:
@@ -66,7 +66,7 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMe
         processor_classes = next((tpl for tpl in processors.find_processors() if tpl[1] == processor_name), None)
         if not processor_classes:
             flash(f"Cannot load the edit form for processor/plugin '{processor_classes[1]}', plugin missing?", 'error')
-            return redirect(url_for('index'))
+            return redirect(url_for('watchlist.index'))
 
         parent_module = processors.get_parent_module(processor_classes[0])
 
@@ -207,15 +207,12 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMe
             if request.args.get("next") and request.args.get("next") == 'diff':
                 return redirect(url_for('ui.ui_views.diff_history_page', uuid=uuid))
 
-            return redirect(url_for('index', tag=request.args.get("tag",'')))
+            return redirect(url_for('watchlist.index', tag=request.args.get("tag",'')))
 
         else:
             if request.method == 'POST' and not form.validate():
                 flash("An error occurred, please see below.", "error")
 
-            visualselector_data_is_ready = datastore.visualselector_data_is_ready(uuid)
-
-
             # JQ is difficult to install on windows and must be manually added (outside requirements.txt)
             jq_support = True
             try:
@@ -225,11 +222,12 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMe
 
             watch = datastore.data['watching'].get(uuid)
 
+            # if system or watch is configured to need a chrome type browser
             system_uses_webdriver = datastore.data['settings']['application']['fetch_backend'] == 'html_webdriver'
-
-            watch_uses_webdriver = False
+            watch_needs_selenium_or_playwright = False
             if (watch.get('fetch_backend') == 'system' and system_uses_webdriver) or watch.get('fetch_backend') == 'html_webdriver' or watch.get('fetch_backend', '').startswith('extra_browser_'):
-                watch_uses_webdriver = True
+                watch_needs_selenium_or_playwright = True
+
 
             from zoneinfo import available_timezones
 
@@ -247,14 +245,17 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMe
                 'has_default_notification_urls': True if len(datastore.data['settings']['application']['notification_urls']) else False,
                 'has_extra_headers_file': len(datastore.get_all_headers_in_textfile_for_watch(uuid=uuid)) > 0,
                 'has_special_tag_options': _watch_has_tag_options_set(watch=watch),
-                'watch_uses_webdriver': watch_uses_webdriver,
                 'jq_support': jq_support,
                 'playwright_enabled': os.getenv('PLAYWRIGHT_DRIVER_URL', False),
                 'settings_application': datastore.data['settings']['application'],
+                'system_has_playwright_configured': os.getenv('PLAYWRIGHT_DRIVER_URL'),
+                'system_has_webdriver_configured': os.getenv('WEBDRIVER_URL'),
+                'visual_selector_data_ready': datastore.visualselector_data_is_ready(watch_uuid=uuid),
                 'timezone_default_config': datastore.data['settings']['application'].get('timezone'),
                 'using_global_webdriver_wait': not default['webdriver_delay'],
                 'uuid': uuid,
-                'watch': watch
+                'watch': watch,
+                'watch_needs_selenium_or_playwright': watch_needs_selenium_or_playwright,
             }
 
             included_content = None

changedetectionio/blueprint/ui/notification.py

@@ -17,11 +17,13 @@ def construct_blueprint(datastore: ChangeDetectionStore):
 
         # Watch_uuid could be unset in the case it`s used in tag editor, global settings
         import apprise
-        from changedetectionio.apprise_asset import asset
-        apobj = apprise.Apprise(asset=asset)
+        from changedetectionio.notification.handler import process_notification
+        from changedetectionio.notification.apprise_plugin.assets import apprise_asset
+
+        from changedetectionio.notification.apprise_plugin.custom_handlers import apprise_http_custom_handler
+
+        apobj = apprise.Apprise(asset=apprise_asset)
 
-        # so that the custom endpoints are registered
-        from changedetectionio.apprise_plugin import apprise_custom_api_call_wrapper
         is_global_settings_form = request.args.get('mode', '') == 'global-settings'
         is_group_settings_form = request.args.get('mode', '') == 'group-settings'
 
@@ -90,7 +92,6 @@ def construct_blueprint(datastore: ChangeDetectionStore):
 
             n_object['as_async'] = False
             n_object.update(watch.extra_notification_token_values())
-            from changedetectionio.notification import process_notification
             sent_obj = process_notification(n_object, datastore)
 
         except Exception as e:

changedetectionio/blueprint/ui/templates/clear_all_history.html

@@ -37,7 +37,7 @@
         </div>
         <br />
         <div class="pure-control-group">
-          <a href="{{url_for('index')}}" class="pure-button button-cancel"
+          <a href="{{url_for('watchlist.index')}}" class="pure-button button-cancel"
             >Cancel</a
           >
         </div>

changedetectionio/blueprint/ui/views.py

@@ -26,7 +26,7 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMe
             watch = datastore.data['watching'][uuid]
         except KeyError:
             flash("No history found for the specified link, bad link?", "error")
-            return redirect(url_for('index'))
+            return redirect(url_for('watchlist.index'))
 
         system_uses_webdriver = datastore.data['settings']['application']['fetch_backend'] == 'html_webdriver'
         extra_stylesheets = [url_for('static_content', group='styles', filename='diff.css')]
@@ -91,7 +91,7 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMe
             watch = datastore.data['watching'][uuid]
         except KeyError:
             flash("No history found for the specified link, bad link?", "error")
-            return redirect(url_for('index'))
+            return redirect(url_for('watchlist.index'))
 
         # For submission of requesting an extract
         extract_form = forms.extractDataForm(request.form)
@@ -119,7 +119,7 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMe
 
         if len(dates) < 2:
             flash("Not enough saved change detection snapshots to produce a report.", "error")
-            return redirect(url_for('index'))
+            return redirect(url_for('watchlist.index'))
 
         # Save the current newest history as the most recently viewed
         datastore.set_last_viewed(uuid, time.time())
@@ -196,7 +196,7 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMe
         if not form.validate():
             for widget, l in form.errors.items():
                 flash(','.join(l), 'error')
-            return redirect(url_for('index'))
+            return redirect(url_for('watchlist.index'))
 
         url = request.form.get('url').strip()
         if datastore.url_exists(url):
@@ -215,6 +215,6 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMe
                 update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': new_uuid}))
                 flash("Watch added.")
 
-        return redirect(url_for('index', tag=request.args.get('tag','')))
+        return redirect(url_for('watchlist.index', tag=request.args.get('tag','')))
 
     return views_blueprint

changedetectionio/blueprint/watchlist/__init__.py

@@ -0,0 +1,111 @@
+import os
+import time
+
+from flask import Blueprint, request, make_response, render_template, redirect, url_for, flash, session
+from flask_login import current_user
+from flask_paginate import Pagination, get_page_parameter
+
+from changedetectionio import forms
+from changedetectionio.store import ChangeDetectionStore
+from changedetectionio.auth_decorator import login_optionally_required
+
+def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMetaData):
+    watchlist_blueprint = Blueprint('watchlist', __name__, template_folder="templates")
+    
+    @watchlist_blueprint.route("/", methods=['GET'])
+    @login_optionally_required
+    def index():
+        active_tag_req = request.args.get('tag', '').lower().strip()
+        active_tag_uuid = active_tag = None
+
+        # Be sure limit_tag is a uuid
+        if active_tag_req:
+            for uuid, tag in datastore.data['settings']['application'].get('tags', {}).items():
+                if active_tag_req == tag.get('title', '').lower().strip() or active_tag_req == uuid:
+                    active_tag = tag
+                    active_tag_uuid = uuid
+                    break
+
+        # Redirect for the old rss path which used the /?rss=true
+        if request.args.get('rss'):
+            return redirect(url_for('rss.feed', tag=active_tag_uuid))
+
+        op = request.args.get('op')
+        if op:
+            uuid = request.args.get('uuid')
+            if op == 'pause':
+                datastore.data['watching'][uuid].toggle_pause()
+            elif op == 'mute':
+                datastore.data['watching'][uuid].toggle_mute()
+
+            datastore.needs_write = True
+            return redirect(url_for('watchlist.index', tag = active_tag_uuid))
+
+        # Sort by last_changed and add the uuid which is usually the key..
+        sorted_watches = []
+        with_errors = request.args.get('with_errors') == "1"
+        errored_count = 0
+        search_q = request.args.get('q').strip().lower() if request.args.get('q') else False
+        for uuid, watch in datastore.data['watching'].items():
+            if with_errors and not watch.get('last_error'):
+                continue
+
+            if active_tag_uuid and not active_tag_uuid in watch['tags']:
+                    continue
+            if watch.get('last_error'):
+                errored_count += 1
+
+            if search_q:
+                if (watch.get('title') and search_q in watch.get('title').lower()) or search_q in watch.get('url', '').lower():
+                    sorted_watches.append(watch)
+                elif watch.get('last_error') and search_q in watch.get('last_error').lower():
+                    sorted_watches.append(watch)
+            else:
+                sorted_watches.append(watch)
+
+        form = forms.quickWatchForm(request.form)
+        page = request.args.get(get_page_parameter(), type=int, default=1)
+        total_count = len(sorted_watches)
+
+        pagination = Pagination(page=page,
+                                total=total_count,
+                                per_page=datastore.data['settings']['application'].get('pager_size', 50), css_framework="semantic")
+
+        sorted_tags = sorted(datastore.data['settings']['application'].get('tags').items(), key=lambda x: x[1]['title'])
+        output = render_template(
+            "watch-overview.html",
+                                 active_tag=active_tag,
+                                 active_tag_uuid=active_tag_uuid,
+                                 app_rss_token=datastore.data['settings']['application'].get('rss_access_token'),
+                                 datastore=datastore,
+                                 errored_count=errored_count,
+                                 form=form,
+                                 guid=datastore.data['app_guid'],
+                                 has_proxies=datastore.proxy_list,
+                                 has_unviewed=datastore.has_unviewed,
+                                 hosted_sticky=os.getenv("SALTED_PASS", False) == False,
+                                 now_time_server=time.time(),
+                                 pagination=pagination,
+                                 queued_uuids=[q_uuid.item['uuid'] for q_uuid in update_q.queue],
+                                 search_q=request.args.get('q', '').strip(),
+                                 sort_attribute=request.args.get('sort') if request.args.get('sort') else request.cookies.get('sort'),
+                                 sort_order=request.args.get('order') if request.args.get('order') else request.cookies.get('order'),
+                                 system_default_fetcher=datastore.data['settings']['application'].get('fetch_backend'),
+                                 tags=sorted_tags,
+                                 watches=sorted_watches
+                                 )
+
+        if session.get('share-link'):
+            del(session['share-link'])
+
+        resp = make_response(output)
+
+        # The template can run on cookie or url query info
+        if request.args.get('sort'):
+            resp.set_cookie('sort', request.args.get('sort'))
+        if request.args.get('order'):
+            resp.set_cookie('order', request.args.get('order'))
+
+        return resp
+        
+    return watchlist_blueprint

changedetectionio/blueprint/watchlist/templates/watch-overview.html

@@ -3,7 +3,16 @@
 {% from '_helpers.html' import render_simple_field, render_field, render_nolabel_field, sort_by_title %}
 <script src="{{url_for('static_content', group='js', filename='jquery-3.6.0.min.js')}}"></script>
 <script src="{{url_for('static_content', group='js', filename='watch-overview.js')}}" defer></script>
+<script>let nowtimeserver={{ now_time_server }};</script>
 
+<style>
+.checking-now .last-checked {
+    background-image: linear-gradient(to bottom, transparent 0%, rgba(0,0,0,0.05) 40%, rgba(0,0,0,0.1) 100%);
+    background-size: 0 100%;
+    background-repeat: no-repeat;
+    transition: background-size 0.9s ease
+}
+</style>
 <div class="box">
 
     <form class="pure-form" action="{{ url_for('ui.ui_views.form_quick_watch_add', tag=active_tag_uuid) }}" method="POST" id="new-watch-form">
@@ -46,12 +55,12 @@
     {% endif %}
     {% if search_q %}<div id="search-result-info">Searching "<strong><i>{{search_q}}</i></strong>"</div>{% endif %}
     <div>
-        <a href="{{url_for('index')}}" class="pure-button button-tag {{'active' if not active_tag_uuid }}">All</a>
+        <a href="{{url_for('watchlist.index')}}" class="pure-button button-tag {{'active' if not active_tag_uuid }}">All</a>
 
     <!-- tag list -->
     {% for uuid, tag in tags %}
         {% if tag != "" %}
-            <a href="{{url_for('index', tag=uuid) }}" class="pure-button button-tag {{'active' if active_tag_uuid == uuid }}">{{ tag.title }}</a>
+            <a href="{{url_for('watchlist.index', tag=uuid) }}" class="pure-button button-tag {{'active' if active_tag_uuid == uuid }}">{{ tag.title }}</a>
         {% endif %}
     {% endfor %}
     </div>
@@ -72,14 +81,14 @@
             <tr>
                 {% set link_order = "desc" if sort_order  == 'asc' else "asc" %}
                 {% set arrow_span = "" %}
-                <th><input style="vertical-align: middle" type="checkbox" id="check-all" > <a class="{{ 'active '+link_order if sort_attribute == 'date_created' else 'inactive' }}"  href="{{url_for('index', sort='date_created', order=link_order, tag=active_tag_uuid)}}"># <span class='arrow {{link_order}}'></span></a></th>
+                <th><input style="vertical-align: middle" type="checkbox" id="check-all" > <a class="{{ 'active '+link_order if sort_attribute == 'date_created' else 'inactive' }}"  href="{{url_for('watchlist.index', sort='date_created', order=link_order, tag=active_tag_uuid)}}"># <span class='arrow {{link_order}}'></span></a></th>
                 <th class="empty-cell"></th>
-                <th><a class="{{ 'active '+link_order if sort_attribute == 'label' else 'inactive' }}" href="{{url_for('index', sort='label', order=link_order, tag=active_tag_uuid)}}">Website <span class='arrow {{link_order}}'></span></a></th>
+                <th><a class="{{ 'active '+link_order if sort_attribute == 'label' else 'inactive' }}" href="{{url_for('watchlist.index', sort='label', order=link_order, tag=active_tag_uuid)}}">Website <span class='arrow {{link_order}}'></span></a></th>
              {% if any_has_restock_price_processor %}
                 <th>Restock &amp; Price</th>
              {% endif %}
-                <th><a class="{{ 'active '+link_order if sort_attribute == 'last_checked' else 'inactive' }}" href="{{url_for('index', sort='last_checked', order=link_order, tag=active_tag_uuid)}}"><span class="hide-on-mobile">Last</span> Checked <span class='arrow {{link_order}}'></span></a></th>
-                <th><a class="{{ 'active '+link_order if sort_attribute == 'last_changed' else 'inactive' }}" href="{{url_for('index', sort='last_changed', order=link_order, tag=active_tag_uuid)}}"><span class="hide-on-mobile">Last</span> Changed <span class='arrow {{link_order}}'></span></a></th>
+                <th><a class="{{ 'active '+link_order if sort_attribute == 'last_checked' else 'inactive' }}" href="{{url_for('watchlist.index', sort='last_checked', order=link_order, tag=active_tag_uuid)}}"><span class="hide-on-mobile">Last</span> Checked <span class='arrow {{link_order}}'></span></a></th>
+                <th><a class="{{ 'active '+link_order if sort_attribute == 'last_changed' else 'inactive' }}" href="{{url_for('watchlist.index', sort='last_changed', order=link_order, tag=active_tag_uuid)}}"><span class="hide-on-mobile">Last</span> Changed <span class='arrow {{link_order}}'></span></a></th>
                 <th class="empty-cell"></th>
             </tr>
             </thead>
@@ -91,8 +100,8 @@
             {% endif %}
             {% for watch in (watches|sort(attribute=sort_attribute, reverse=sort_order == 'asc'))|pagination_slice(skip=pagination.skip) %}
 
-                {% set is_unviewed =  watch.newest_history_key| int > watch.last_viewed and watch.history_n>=2 %}
-
+                {% set is_unviewed = watch.newest_history_key| int > watch.last_viewed and watch.history_n>=2 %}
+                {% set checking_now = is_checking_now(watch) %}
             <tr id="{{ watch.uuid }}"
                 class="{{ loop.cycle('pure-table-odd', 'pure-table-even') }} processor-{{ watch['processor'] }}
                 {% if watch.last_error is defined and watch.last_error != False %}error{% endif %}
@@ -100,16 +109,18 @@
                 {% if watch.paused is defined and watch.paused != False %}paused{% endif %}
                 {% if is_unviewed %}unviewed{% endif %}
                 {% if watch.has_restock_info %} has-restock-info {% if watch['restock']['in_stock'] %}in-stock{% else %}not-in-stock{% endif %} {% else %}no-restock-info{% endif %}
-                {% if watch.uuid in queued_uuids %}queued{% endif %}">
+                {% if watch.uuid in queued_uuids %}queued{% endif %}
+                {% if checking_now %}checking-now{% endif %}
+                ">
                 <td class="inline checkbox-uuid" ><input name="uuids"  type="checkbox" value="{{ watch.uuid}} " > <span>{{ loop.index+pagination.skip }}</span></td>
                 <td class="inline watch-controls">
                     {% if not watch.paused %}
-                    <a class="state-off" href="{{url_for('index', op='pause', uuid=watch.uuid, tag=active_tag_uuid)}}"><img src="{{url_for('static_content', group='images', filename='pause.svg')}}" alt="Pause checks" title="Pause checks" class="icon icon-pause" ></a>
+                    <a class="state-off" href="{{url_for('watchlist.index', op='pause', uuid=watch.uuid, tag=active_tag_uuid)}}"><img src="{{url_for('static_content', group='images', filename='pause.svg')}}" alt="Pause checks" title="Pause checks" class="icon icon-pause" ></a>
                     {% else %}
-                    <a class="state-on" href="{{url_for('index', op='pause', uuid=watch.uuid, tag=active_tag_uuid)}}"><img src="{{url_for('static_content', group='images', filename='play.svg')}}" alt="UnPause checks" title="UnPause checks" class="icon icon-unpause" ></a>
+                    <a class="state-on" href="{{url_for('watchlist.index', op='pause', uuid=watch.uuid, tag=active_tag_uuid)}}"><img src="{{url_for('static_content', group='images', filename='play.svg')}}" alt="UnPause checks" title="UnPause checks" class="icon icon-unpause" ></a>
                     {% endif %}
                     {% set mute_label = 'UnMute notification' if watch.notification_muted else 'Mute notification' %}
-                    <a class="link-mute state-{{'on' if watch.notification_muted else 'off'}}" href="{{url_for('index', op='mute', uuid=watch.uuid, tag=active_tag_uuid)}}"><img src="{{url_for('static_content', group='images', filename='bell-off.svg')}}" alt="{{ mute_label }}" title="{{ mute_label }}" class="icon icon-mute" ></a>
+                    <a class="link-mute state-{{'on' if watch.notification_muted else 'off'}}" href="{{url_for('watchlist.index', op='mute', uuid=watch.uuid, tag=active_tag_uuid)}}"><img src="{{url_for('static_content', group='images', filename='bell-off.svg')}}" alt="{{ mute_label }}" title="{{ mute_label }}" class="icon icon-mute" ></a>
                 </td>
                 <td class="title-col inline">{{watch.title if watch.title is not none and watch.title|length > 0 else watch.url}}
                     <a class="external" target="_blank" rel="noopener" href="{{ watch.link.replace('source:','') }}"></a>
@@ -119,7 +130,7 @@
                          or (  watch.get_fetch_backend == "system" and system_default_fetcher == 'html_webdriver'  )
                          or "extra_browser_" in watch.get_fetch_backend
                     %}
-                    <img class="status-icon" src="{{url_for('static_content', group='images', filename='Google-Chrome-icon.png')}}" alt="Using a Chrome browser" title="Using a Chrome browser" >
+                    <img class="status-icon" src="{{url_for('static_content', group='images', filename='google-chrome-icon.png')}}" alt="Using a Chrome browser" title="Using a Chrome browser" >
                     {% endif %}
 
                     {%if watch.is_pdf  %}<img class="status-icon" src="{{url_for('static_content', group='images', filename='pdf-icon.svg')}}" title="Converting PDF to text" >{% endif %}
@@ -178,7 +189,14 @@
                     {% endif %}
                 </td>
 {% endif %}
-                <td class="last-checked" data-timestamp="{{ watch.last_checked }}">{{watch|format_last_checked_time|safe}}</td>
+            {#last_checked becomes fetch-start-time#}
+                <td class="last-checked" data-timestamp="{{ watch.last_checked }}" {% if checking_now %} data-fetchduration={{ watch.fetch_time }} data-eta_complete="{{ watch.last_checked+watch.fetch_time }}" {% endif %} >
+                    {% if checking_now %}
+                        <span class="spinner"></span><span> Checking now</span>
+                    {% else %}
+                        {{watch|format_last_checked_time|safe}}</td>
+                    {% endif %}
+
                 <td class="last-changed" data-timestamp="{{ watch.last_changed }}">{% if watch.history_n >=2 and watch.last_changed >0 %}
                     {{watch.last_changed|format_timestamp_timeago}}
                     {% else %}
@@ -191,15 +209,18 @@
                     <a href="{{ url_for('ui.ui_edit.edit_page', uuid=watch.uuid, tag=active_tag_uuid)}}#general" class="pure-button pure-button-primary">Edit</a>
                     {% if watch.history_n >= 2 %}
 
+                        {% set open_diff_in_new_tab = datastore.data['settings']['application']['ui'].get('open_diff_in_new_tab') %}
+                        {% set target_attr = ' target="' ~ watch.uuid ~ '"' if open_diff_in_new_tab else '' %}
+
                         {%  if is_unviewed %}
-                           <a href="{{ url_for('ui.ui_views.diff_history_page', uuid=watch.uuid, from_version=watch.get_from_version_based_on_last_viewed) }}" target="{{watch.uuid}}" class="pure-button pure-button-primary diff-link">History</a>
+                           <a href="{{ url_for('ui.ui_views.diff_history_page', uuid=watch.uuid, from_version=watch.get_from_version_based_on_last_viewed) }}" {{target_attr}} class="pure-button pure-button-primary diff-link">History</a>
                         {% else %}
-                           <a href="{{ url_for('ui.ui_views.diff_history_page', uuid=watch.uuid)}}" target="{{watch.uuid}}" class="pure-button pure-button-primary diff-link">History</a>
+                           <a href="{{ url_for('ui.ui_views.diff_history_page', uuid=watch.uuid)}}" {{target_attr}} class="pure-button pure-button-primary diff-link">History</a>
                         {% endif %}
 
                     {% else %}
                         {% if watch.history_n == 1 or (watch.history_n ==0 and watch.error_text_ctime )%}
-                            <a href="{{ url_for('ui.ui_views.preview_page', uuid=watch.uuid)}}" target="{{watch.uuid}}" class="pure-button pure-button-primary">Preview</a>
+                            <a href="{{ url_for('ui.ui_views.preview_page', uuid=watch.uuid)}}" {{target_attr}} class="pure-button pure-button-primary">Preview</a>
                         {% endif %}
                     {% endif %}
                 </td>
@@ -210,7 +231,7 @@
         <ul id="post-list-buttons">
             {% if errored_count %}
             <li>
-                <a href="{{url_for('index', with_errors=1, tag=request.args.get('tag')) }}" class="pure-button button-tag button-error ">With errors ({{ errored_count }})</a>
+                <a href="{{url_for('watchlist.index', with_errors=1, tag=request.args.get('tag')) }}" class="pure-button button-tag button-error ">With errors ({{ errored_count }})</a>
             </li>
             {% endif %}
             {% if has_unviewed %}
@@ -223,7 +244,7 @@
                 all {% if active_tag_uuid %} in "{{active_tag.title}}"{%endif%}</a>
             </li>
             <li>
-                <a href="{{ url_for('rss.feed', tag=active_tag_uuid, token=app_rss_token)}}"><img alt="RSS Feed" id="feed-icon" src="{{url_for('static_content', group='images', filename='Generic_Feed-icon.svg')}}" height="15"></a>
+                <a href="{{ url_for('rss.feed', tag=active_tag_uuid, token=app_rss_token)}}"><img alt="RSS Feed" id="feed-icon" src="{{url_for('static_content', group='images', filename='generic_feed-icon.svg')}}" height="15"></a>
             </li>
         </ul>
         {{ pagination.links }}

changedetectionio/conditions/__init__.py

@@ -8,7 +8,7 @@ from . import default_plugin
 
 # List of all supported JSON Logic operators
 operator_choices = [
-    (None, "Choose one"),
+    (None, "Choose one - Operator"),
     (">", "Greater Than"),
     ("<", "Less Than"),
     (">=", "Greater Than or Equal To"),
@@ -21,7 +21,7 @@ operator_choices = [
 
 # Fields available in the rules
 field_choices = [
-    (None, "Choose one"),
+    (None, "Choose one - Field"),
 ]
 
 # The data we will feed the JSON Rules to see if it passes the test/conditions or not
@@ -94,11 +94,11 @@ def execute_ruleset_against_all_plugins(current_watch_uuid: str, application_dat
     EXECUTE_DATA = {}
     result = True
     
-    ruleset_settings = application_datastruct['watching'].get(current_watch_uuid)
+    watch = application_datastruct['watching'].get(current_watch_uuid)
 
-    if ruleset_settings.get("conditions"):
-        logic_operator = "and" if ruleset_settings.get("conditions_match_logic", "ALL") == "ALL" else "or"
-        complete_rules = filter_complete_rules(ruleset_settings['conditions'])
+    if watch and watch.get("conditions"):
+        logic_operator = "and" if watch.get("conditions_match_logic", "ALL") == "ALL" else "or"
+        complete_rules = filter_complete_rules(watch['conditions'])
         if complete_rules:
             # Give all plugins a chance to update the data dict again (that we will test the conditions against)
             for plugin in plugin_manager.get_plugins():
@@ -116,8 +116,7 @@ def execute_ruleset_against_all_plugins(current_watch_uuid: str, application_dat
             if not jsonLogic(logic=ruleset, data=EXECUTE_DATA, operations=CUSTOM_OPERATIONS):
                 result = False
 
-    return result
-
+    return {'executed_data': EXECUTE_DATA, 'result': result}
 
 # Load plugins dynamically
 for plugin in plugin_manager.get_plugins():

changedetectionio/conditions/blueprint.py

@@ -67,7 +67,8 @@ def construct_blueprint(datastore):
 
             return jsonify({
                 'status': 'success',
-                'result': result,
+                'result': result.get('result'),
+                'data': result.get('executed_data'),
                 'message': 'Condition passes' if result else 'Condition does not pass'
             })
 

changedetectionio/conditions/form.py

@@ -19,7 +19,7 @@ class ConditionFormRow(Form):
         validators=[validators.Optional()]
     )
 
-    value = StringField("Value", validators=[validators.Optional()])
+    value = StringField("Value", validators=[validators.Optional()], render_kw={"placeholder": "A value"})
 
     def validate(self, extra_validators=None):
         # First, run the default validators

changedetectionio/content_fetchers/__init__.py

@@ -7,11 +7,29 @@ import os
 # Visual Selector scraper - 'Button' is there because some sites have <button>OUT OF STOCK</button>.
 visualselector_xpath_selectors = 'div,span,form,table,tbody,tr,td,a,p,ul,li,h1,h2,h3,h4,header,footer,section,article,aside,details,main,nav,section,summary,button'
 
+SCREENSHOT_MAX_HEIGHT_DEFAULT = 20000
+SCREENSHOT_DEFAULT_QUALITY = 40
+
+# Maximum total height for the final image (When in stitch mode).
+# We limit this to 16000px due to the huge amount of RAM that was being used
+# Example: 16000 × 1400 × 3 = 67,200,000 bytes ≈ 64.1 MB (not including buffers in PIL etc)
+SCREENSHOT_MAX_TOTAL_HEIGHT = int(os.getenv("SCREENSHOT_MAX_HEIGHT", SCREENSHOT_MAX_HEIGHT_DEFAULT))
+
+# The size at which we will switch to stitching method, when below this (and
+# MAX_TOTAL_HEIGHT which can be set by a user) we will use the default
+# screenshot method.
+SCREENSHOT_SIZE_STITCH_THRESHOLD = 8000
 
 # available_fetchers() will scan this implementation looking for anything starting with html_
 # this information is used in the form selections
 from changedetectionio.content_fetchers.requests import fetcher as html_requests
 
+
+import importlib.resources
+XPATH_ELEMENT_JS = importlib.resources.files("changedetectionio.content_fetchers.res").joinpath('xpath_element_scraper.js').read_text(encoding='utf-8')
+INSTOCK_DATA_JS = importlib.resources.files("changedetectionio.content_fetchers.res").joinpath('stock-not-in-stock.js').read_text(encoding='utf-8')
+
+
 def available_fetchers():
     # See the if statement at the bottom of this file for how we switch between playwright and webdriver
     import inspect

changedetectionio/content_fetchers/base.py

@@ -63,11 +63,6 @@ class Fetcher():
     # Time ONTOP of the system defined env minimum time
     render_extract_delay = 0
 
-    def __init__(self):
-        import importlib.resources
-        self.xpath_element_js = importlib.resources.files("changedetectionio.content_fetchers.res").joinpath('xpath_element_scraper.js').read_text(encoding='utf-8')
-        self.instock_data_js = importlib.resources.files("changedetectionio.content_fetchers.res").joinpath('stock-not-in-stock.js').read_text(encoding='utf-8')
-
     @abstractmethod
     def get_error(self):
         return self.error
@@ -87,7 +82,7 @@ class Fetcher():
         pass
 
     @abstractmethod
-    def quit(self):
+    def quit(self, watch=None):
         return
 
     @abstractmethod
@@ -143,6 +138,7 @@ class Fetcher():
                 logger.debug(f">> Iterating check - browser Step n {step_n} - {step['operation']}...")
                 self.screenshot_step("before-" + str(step_n))
                 self.save_step_html("before-" + str(step_n))
+
                 try:
                     optional_value = step['optional_value']
                     selector = step['selector']

changedetectionio/content_fetchers/helpers.py

@@ -1,104 +0,0 @@
-
-# Pages with a vertical height longer than this will use the 'stitch together' method.
-
-# - Many GPUs have a max texture size of 16384x16384px (or lower on older devices).
-# - If a page is taller than ~8000–10000px, it risks exceeding GPU memory limits.
-# - This is especially important on headless Chromium, where Playwright may fail to allocate a massive full-page buffer.
-
-
-# The size at which we will switch to stitching method
-SCREENSHOT_SIZE_STITCH_THRESHOLD=8000
-
-from loguru import logger
-
-def capture_stitched_together_full_page(page):
-    import io
-    import os
-    import time
-    from PIL import Image, ImageDraw, ImageFont
-
-    MAX_TOTAL_HEIGHT = SCREENSHOT_SIZE_STITCH_THRESHOLD*4  # Maximum total height for the final image (When in stitch mode)
-    MAX_CHUNK_HEIGHT = 4000  # Height per screenshot chunk
-    WARNING_TEXT_HEIGHT = 20  # Height of the warning text overlay
-
-    # Save the original viewport size
-    original_viewport = page.viewport_size
-    now = time.time()
-
-    try:
-        viewport = page.viewport_size
-        page_height = page.evaluate("document.documentElement.scrollHeight")
-
-        # Limit the total capture height
-        capture_height = min(page_height, MAX_TOTAL_HEIGHT)
-
-        images = []
-        total_captured_height = 0
-
-        for offset in range(0, capture_height, MAX_CHUNK_HEIGHT):
-            # Ensure we do not exceed the total height limit
-            chunk_height = min(MAX_CHUNK_HEIGHT, MAX_TOTAL_HEIGHT - total_captured_height)
-
-            # Adjust viewport size for this chunk
-            page.set_viewport_size({"width": viewport["width"], "height": chunk_height})
-
-            # Scroll to the correct position
-            page.evaluate(f"window.scrollTo(0, {offset})")
-
-            # Capture screenshot chunk
-            screenshot_bytes = page.screenshot(type='jpeg', quality=int(os.getenv("SCREENSHOT_QUALITY", 30)))
-            images.append(Image.open(io.BytesIO(screenshot_bytes)))
-
-            total_captured_height += chunk_height
-
-            # Stop if we reached the maximum total height
-            if total_captured_height >= MAX_TOTAL_HEIGHT:
-                break
-
-        # Create the final stitched image
-        stitched_image = Image.new('RGB', (viewport["width"], total_captured_height))
-        y_offset = 0
-
-        # Stitch the screenshot chunks together
-        for img in images:
-            stitched_image.paste(img, (0, y_offset))
-            y_offset += img.height
-
-        logger.debug(f"Screenshot stitched together in {time.time()-now:.2f}s")
-
-        # Overlay warning text if the screenshot was trimmed
-        if page_height > MAX_TOTAL_HEIGHT:
-            draw = ImageDraw.Draw(stitched_image)
-            warning_text = f"WARNING: Screenshot was {page_height}px but trimmed to {MAX_TOTAL_HEIGHT}px because it was too long"
-
-            # Load font (default system font if Arial is unavailable)
-            try:
-                font = ImageFont.truetype("arial.ttf", WARNING_TEXT_HEIGHT)  # Arial (Windows/Mac)
-            except IOError:
-                font = ImageFont.load_default()  # Default font if Arial not found
-
-            # Get text bounding box (correct method for newer Pillow versions)
-            text_bbox = draw.textbbox((0, 0), warning_text, font=font)
-            text_width = text_bbox[2] - text_bbox[0]  # Calculate text width
-            text_height = text_bbox[3] - text_bbox[1]  # Calculate text height
-
-            # Define background rectangle (top of the image)
-            draw.rectangle([(0, 0), (viewport["width"], WARNING_TEXT_HEIGHT)], fill="white")
-
-            # Center text horizontally within the warning area
-            text_x = (viewport["width"] - text_width) // 2
-            text_y = (WARNING_TEXT_HEIGHT - text_height) // 2
-
-            # Draw the warning text in red
-            draw.text((text_x, text_y), warning_text, fill="red", font=font)
-
-        # Save or return the final image
-        output = io.BytesIO()
-        stitched_image.save(output, format="JPEG", quality=int(os.getenv("SCREENSHOT_QUALITY", 30)))
-        screenshot = output.getvalue()
-
-    finally:
-        # Restore the original viewport size
-        page.set_viewport_size(original_viewport)
-
-    return screenshot

changedetectionio/content_fetchers/playwright.py

@@ -4,10 +4,76 @@ from urllib.parse import urlparse
 
 from loguru import logger
 
-from changedetectionio.content_fetchers.helpers import capture_stitched_together_full_page, SCREENSHOT_SIZE_STITCH_THRESHOLD
+from changedetectionio.content_fetchers import SCREENSHOT_MAX_HEIGHT_DEFAULT, visualselector_xpath_selectors, \
+    SCREENSHOT_SIZE_STITCH_THRESHOLD, SCREENSHOT_MAX_TOTAL_HEIGHT, XPATH_ELEMENT_JS, INSTOCK_DATA_JS
 from changedetectionio.content_fetchers.base import Fetcher, manage_user_agent
 from changedetectionio.content_fetchers.exceptions import PageUnloadable, Non200ErrorCodeReceived, EmptyReply, ScreenshotUnavailable
 
+def capture_full_page(page):
+    import os
+    import time
+    from multiprocessing import Process, Pipe
+
+    start = time.time()
+
+    page_height = page.evaluate("document.documentElement.scrollHeight")
+    page_width = page.evaluate("document.documentElement.scrollWidth")
+    original_viewport = page.viewport_size
+
+    logger.debug(f"Playwright viewport size {page.viewport_size} page height {page_height} page width {page_width}")
+
+    # Use an approach similar to puppeteer: set a larger viewport and take screenshots in chunks
+    step_size = SCREENSHOT_SIZE_STITCH_THRESHOLD # Size that won't cause GPU to overflow
+    screenshot_chunks = []
+    y = 0
+
+    if page_height > page.viewport_size['height']:
+        if page_height < step_size:
+            step_size = page_height # Incase page is bigger than default viewport but smaller than proposed step size
+        logger.debug(f"Setting bigger viewport to step through large page width W{page.viewport_size['width']}xH{step_size} because page_height > viewport_size")
+        # Set viewport to a larger size to capture more content at once
+        page.set_viewport_size({'width': page.viewport_size['width'], 'height': step_size})
+
+    # Capture screenshots in chunks up to the max total height
+    while y < min(page_height, SCREENSHOT_MAX_TOTAL_HEIGHT):
+        page.request_gc()
+        page.evaluate(f"window.scrollTo(0, {y})")
+        page.request_gc()
+        screenshot_chunks.append(page.screenshot(
+            type="jpeg",
+            full_page=False,
+            quality=int(os.getenv("SCREENSHOT_QUALITY", 72))
+        ))
+        y += step_size
+        page.request_gc()
+
+    # Restore original viewport size
+    page.set_viewport_size({'width': original_viewport['width'], 'height': original_viewport['height']})
+
+    # If we have multiple chunks, stitch them together
+    if len(screenshot_chunks) > 1:
+        from changedetectionio.content_fetchers.screenshot_handler import stitch_images_worker
+        logger.debug(f"Screenshot stitching {len(screenshot_chunks)} chunks together")
+        parent_conn, child_conn = Pipe()
+        p = Process(target=stitch_images_worker, args=(child_conn, screenshot_chunks, page_height, SCREENSHOT_MAX_TOTAL_HEIGHT))
+        p.start()
+        screenshot = parent_conn.recv_bytes()
+        p.join()
+        logger.debug(
+            f"Screenshot (chunked/stitched) - Page height: {page_height} Capture height: {SCREENSHOT_MAX_TOTAL_HEIGHT} - Stitched together in {time.time() - start:.2f}s")
+        # Explicit cleanup
+        del screenshot_chunks
+        del p
+        del parent_conn, child_conn
+        screenshot_chunks = None
+        return screenshot
+
+    logger.debug(
+        f"Screenshot Page height: {page_height} Capture height: {SCREENSHOT_MAX_TOTAL_HEIGHT} - Stitched together in {time.time() - start:.2f}s")
+
+    return screenshot_chunks[0]
+
+
 class fetcher(Fetcher):
     fetcher_description = "Playwright {}/Javascript".format(
         os.getenv("PLAYWRIGHT_BROWSER_TYPE", 'chromium').capitalize()
@@ -60,7 +126,8 @@ class fetcher(Fetcher):
 
     def screenshot_step(self, step_n=''):
         super().screenshot_step(step_n=step_n)
-        screenshot = self.page.screenshot(type='jpeg', full_page=True, quality=int(os.getenv("SCREENSHOT_QUALITY", 72)))
+        screenshot = capture_full_page(page=self.page)
+
 
         if self.browser_steps_screenshot_path is not None:
             destination = os.path.join(self.browser_steps_screenshot_path, 'step_{}.jpeg'.format(step_n))
@@ -89,7 +156,6 @@ class fetcher(Fetcher):
 
         from playwright.sync_api import sync_playwright
         import playwright._impl._errors
-        from changedetectionio.content_fetchers import visualselector_xpath_selectors
         import time
         self.delete_browser_steps_screenshots()
         response = None
@@ -164,9 +230,7 @@ class fetcher(Fetcher):
                 raise PageUnloadable(url=url, status_code=None, message=str(e))
 
             if self.status_code != 200 and not ignore_status_codes:
-                screenshot = self.page.screenshot(type='jpeg', full_page=True,
-                                                  quality=int(os.getenv("SCREENSHOT_QUALITY", 72)))
-
+                screenshot = capture_full_page(self.page)
                 raise Non200ErrorCodeReceived(url=url, status_code=self.status_code, screenshot=screenshot)
 
             if not empty_pages_are_a_change and len(self.page.content().strip()) == 0:
@@ -187,13 +251,23 @@ class fetcher(Fetcher):
                 self.page.evaluate("var include_filters={}".format(json.dumps(current_include_filters)))
             else:
                 self.page.evaluate("var include_filters=''")
+            self.page.request_gc()
 
-            self.xpath_data = self.page.evaluate(
-                "async () => {" + self.xpath_element_js.replace('%ELEMENTS%', visualselector_xpath_selectors) + "}")
-            self.instock_data = self.page.evaluate("async () => {" + self.instock_data_js + "}")
+            # request_gc before and after evaluate to free up memory
+            # @todo browsersteps etc
+            MAX_TOTAL_HEIGHT = int(os.getenv("SCREENSHOT_MAX_HEIGHT", SCREENSHOT_MAX_HEIGHT_DEFAULT))
+            self.xpath_data = self.page.evaluate(XPATH_ELEMENT_JS, {
+                "visualselector_xpath_selectors": visualselector_xpath_selectors,
+                "max_height": MAX_TOTAL_HEIGHT
+            })
+            self.page.request_gc()
+
+            self.instock_data = self.page.evaluate(INSTOCK_DATA_JS)
+            self.page.request_gc()
 
             self.content = self.page.content()
-            logger.debug(f"Time to scrape xpath element data in browser {time.time() - now:.2f}s")
+            self.page.request_gc()
+            logger.debug(f"Scrape xPath element data in browser done in {time.time() - now:.2f}s")
 
             # Bug 3 in Playwright screenshot handling
             # Some bug where it gives the wrong screenshot size, but making a request with the clip set first seems to solve it
@@ -204,18 +278,41 @@ class fetcher(Fetcher):
             # acceptable screenshot quality here
             try:
                 # The actual screenshot - this always base64 and needs decoding! horrible! huge CPU usage
-                full_height = self.page.evaluate("document.documentElement.scrollHeight")
-
-                if full_height >= SCREENSHOT_SIZE_STITCH_THRESHOLD:
-                    logger.warning(
-                        f"Page full Height: {full_height}px longer than {SCREENSHOT_SIZE_STITCH_THRESHOLD}px, using 'stitched screenshot method'.")
-                    self.screenshot = capture_stitched_together_full_page(self.page)
-                else:
-                    self.screenshot = self.page.screenshot(type='jpeg', full_page=True, quality=int(os.getenv("SCREENSHOT_QUALITY", 30)))
+                self.screenshot = capture_full_page(page=self.page)
 
             except Exception as e:
                 # It's likely the screenshot was too long/big and something crashed
                 raise ScreenshotUnavailable(url=url, status_code=self.status_code)
             finally:
-                context.close()
-                browser.close()
+                # Request garbage collection one more time before closing
+                try:
+                    self.page.request_gc()
+                except:
+                    pass
+                
+                # Clean up resources properly
+                try:
+                    self.page.request_gc()
+                except:
+                    pass
+
+                try:
+                    self.page.close()
+                except:
+                    pass
+                self.page = None
+
+                try:
+                    context.close()
+                except:
+                    pass
+                context = None
+
+                try:
+                    browser.close()
+                except:
+                    pass
+                browser = None
+
+
+

changedetectionio/content_fetchers/puppeteer.py

@@ -6,8 +6,77 @@ from urllib.parse import urlparse
 
 from loguru import logger
 
+from changedetectionio.content_fetchers import SCREENSHOT_MAX_HEIGHT_DEFAULT, visualselector_xpath_selectors, \
+    SCREENSHOT_SIZE_STITCH_THRESHOLD, SCREENSHOT_DEFAULT_QUALITY, XPATH_ELEMENT_JS, INSTOCK_DATA_JS, \
+    SCREENSHOT_MAX_TOTAL_HEIGHT
 from changedetectionio.content_fetchers.base import Fetcher, manage_user_agent
-from changedetectionio.content_fetchers.exceptions import PageUnloadable, Non200ErrorCodeReceived, EmptyReply, BrowserFetchTimedOut, BrowserConnectError
+from changedetectionio.content_fetchers.exceptions import PageUnloadable, Non200ErrorCodeReceived, EmptyReply, BrowserFetchTimedOut, \
+    BrowserConnectError
+
+
+# Bug 3 in Playwright screenshot handling
+# Some bug where it gives the wrong screenshot size, but making a request with the clip set first seems to solve it
+
+# Screenshots also travel via the ws:// (websocket) meaning that the binary data is base64 encoded
+# which will significantly increase the IO size between the server and client, it's recommended to use the lowest
+# acceptable screenshot quality here
+async def capture_full_page(page):
+    import os
+    import time
+    from multiprocessing import Process, Pipe
+
+    start = time.time()
+
+    page_height = await page.evaluate("document.documentElement.scrollHeight")
+    page_width = await page.evaluate("document.documentElement.scrollWidth")
+    original_viewport = page.viewport
+
+    logger.debug(f"Puppeteer viewport size {page.viewport} page height {page_height} page width {page_width}")
+
+    # Bug 3 in Playwright screenshot handling
+    # Some bug where it gives the wrong screenshot size, but making a request with the clip set first seems to solve it
+    # JPEG is better here because the screenshots can be very very large
+
+    # Screenshots also travel via the ws:// (websocket) meaning that the binary data is base64 encoded
+    # which will significantly increase the IO size between the server and client, it's recommended to use the lowest
+    # acceptable screenshot quality here
+
+
+    step_size = SCREENSHOT_SIZE_STITCH_THRESHOLD # Something that will not cause the GPU to overflow when taking the screenshot
+    screenshot_chunks = []
+    y = 0
+    if page_height > page.viewport['height']:
+        if page_height < step_size:
+            step_size = page_height # Incase page is bigger than default viewport but smaller than proposed step size
+        await page.setViewport({'width': page.viewport['width'], 'height': step_size})
+
+    while y < min(page_height, SCREENSHOT_MAX_TOTAL_HEIGHT):
+        await page.evaluate(f"window.scrollTo(0, {y})")
+        screenshot_chunks.append(await page.screenshot(type_='jpeg',
+                                                       fullPage=False,
+                                                       quality=int(os.getenv("SCREENSHOT_QUALITY", 72))))
+        y += step_size
+
+    await page.setViewport({'width': original_viewport['width'], 'height': original_viewport['height']})
+
+    if len(screenshot_chunks) > 1:
+        from changedetectionio.content_fetchers.screenshot_handler import stitch_images_worker
+        logger.debug(f"Screenshot stitching {len(screenshot_chunks)} chunks together")
+        parent_conn, child_conn = Pipe()
+        p = Process(target=stitch_images_worker, args=(child_conn, screenshot_chunks, page_height, SCREENSHOT_MAX_TOTAL_HEIGHT))
+        p.start()
+        screenshot = parent_conn.recv_bytes()
+        p.join()
+        logger.debug(
+            f"Screenshot (chunked/stitched) - Page height: {page_height} Capture height: {SCREENSHOT_MAX_TOTAL_HEIGHT} - Stitched together in {time.time() - start:.2f}s")
+
+        screenshot_chunks = None
+        return screenshot
+
+    logger.debug(
+        f"Screenshot Page height: {page_height} Capture height: {SCREENSHOT_MAX_TOTAL_HEIGHT} - Stitched together in {time.time() - start:.2f}s")
+    return screenshot_chunks[0]
+
 
 class fetcher(Fetcher):
     fetcher_description = "Puppeteer/direct {}/Javascript".format(
@@ -79,7 +148,6 @@ class fetcher(Fetcher):
                          empty_pages_are_a_change
                          ):
 
-        from changedetectionio.content_fetchers import visualselector_xpath_selectors
         self.delete_browser_steps_screenshots()
         extra_wait = int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)) + self.render_extract_delay
 
@@ -181,11 +249,10 @@ class fetcher(Fetcher):
             raise PageUnloadable(url=url, status_code=None, message=str(e))
 
         if self.status_code != 200 and not ignore_status_codes:
-            screenshot = await self.page.screenshot(type_='jpeg',
-                                                    fullPage=True,
-                                                    quality=int(os.getenv("SCREENSHOT_QUALITY", 72)))
+            screenshot = await capture_full_page(page=self.page)
 
             raise Non200ErrorCodeReceived(url=url, status_code=self.status_code, screenshot=screenshot)
+
         content = await self.page.content
 
         if not empty_pages_are_a_change and len(content.strip()) == 0:
@@ -203,46 +270,31 @@ class fetcher(Fetcher):
 
         # So we can find an element on the page where its selector was entered manually (maybe not xPath etc)
         # Setup the xPath/VisualSelector scraper
-        if current_include_filters is not None:
+        if current_include_filters:
             js = json.dumps(current_include_filters)
             await self.page.evaluate(f"var include_filters={js}")
         else:
             await self.page.evaluate(f"var include_filters=''")
 
-        self.xpath_data = await self.page.evaluate(
-            "async () => {" + self.xpath_element_js.replace('%ELEMENTS%', visualselector_xpath_selectors) + "}")
-        self.instock_data = await self.page.evaluate("async () => {" + self.instock_data_js + "}")
+        MAX_TOTAL_HEIGHT = int(os.getenv("SCREENSHOT_MAX_HEIGHT", SCREENSHOT_MAX_HEIGHT_DEFAULT))
+        self.xpath_data = await self.page.evaluate(XPATH_ELEMENT_JS, {
+            "visualselector_xpath_selectors": visualselector_xpath_selectors,
+            "max_height": MAX_TOTAL_HEIGHT
+        })
+        if not self.xpath_data:
+            raise Exception(f"Content Fetcher > xPath scraper failed. Please report this URL so we can fix it :)")
+
+        self.instock_data = await self.page.evaluate(INSTOCK_DATA_JS)
 
         self.content = await self.page.content
-        # Bug 3 in Playwright screenshot handling
-        # Some bug where it gives the wrong screenshot size, but making a request with the clip set first seems to solve it
-        # JPEG is better here because the screenshots can be very very large
 
-        # Screenshots also travel via the ws:// (websocket) meaning that the binary data is base64 encoded
-        # which will significantly increase the IO size between the server and client, it's recommended to use the lowest
-        # acceptable screenshot quality here
-        try:
-            self.screenshot = await self.page.screenshot(type_='jpeg',
-                                                         fullPage=True,
-                                                         quality=int(os.getenv("SCREENSHOT_QUALITY", 72)))
-        except Exception as e:
-            logger.error("Error fetching screenshot")
-            # // May fail on very large pages with 'WARNING: tile memory limits exceeded, some content may not draw'
-            # // @ todo after text extract, we can place some overlay text with red background to say 'croppped'
-            logger.error('ERROR: content-fetcher page was maybe too large for a screenshot, reverting to viewport only screenshot')
-            try:
-                self.screenshot = await self.page.screenshot(type_='jpeg',
-                                                             fullPage=False,
-                                                             quality=int(os.getenv("SCREENSHOT_QUALITY", 72)))
-            except Exception as e:
-                logger.error('ERROR: Failed to get viewport-only reduced screenshot :(')
-                pass
-        finally:
-            # It's good to log here in the case that the browser crashes on shutting down but we still get the data we need
-            logger.success(f"Fetching '{url}' complete, closing page")
-            await self.page.close()
-            logger.success(f"Fetching '{url}' complete, closing browser")
-            await browser.close()
+        self.screenshot = await capture_full_page(page=self.page)
+
+        # It's good to log here in the case that the browser crashes on shutting down but we still get the data we need
+        logger.success(f"Fetching '{url}' complete, closing page")
+        await self.page.close()
+        logger.success(f"Fetching '{url}' complete, closing browser")
+        await browser.close()
         logger.success(f"Fetching '{url}' complete, exiting puppeteer fetch.")
 
     async def main(self, **kwargs):

changedetectionio/content_fetchers/requests.py

@@ -96,3 +96,17 @@ class fetcher(Fetcher):
 
 
         self.raw_content = r.content
+
+    def quit(self, watch=None):
+
+        # In case they switched to `requests` fetcher from something else
+        # Then the screenshot could be old, in any case, it's not used here.
+        # REMOVE_REQUESTS_OLD_SCREENSHOTS - Mainly used for testing
+        if strtobool(os.getenv("REMOVE_REQUESTS_OLD_SCREENSHOTS", 'true')):
+            screenshot = watch.get_screenshot()
+            if screenshot:
+                try:
+                    os.unlink(screenshot)
+                except Exception as e:
+                    logger.warning(f"Failed to unlink screenshot: {screenshot} - {e}")
+

changedetectionio/content_fetchers/res/puppeteer_fetch.js

@@ -1,190 +0,0 @@
-module.exports = async ({page, context}) => {
-
-    var {
-        url,
-        execute_js,
-        user_agent,
-        extra_wait_ms,
-        req_headers,
-        include_filters,
-        xpath_element_js,
-        screenshot_quality,
-        proxy_username,
-        proxy_password,
-        disk_cache_dir,
-        no_cache_list,
-        block_url_list,
-    } = context;
-
-    await page.setBypassCSP(true)
-    await page.setExtraHTTPHeaders(req_headers);
-
-    if (user_agent) {
-        await page.setUserAgent(user_agent);
-    }
-    // https://ourcodeworld.com/articles/read/1106/how-to-solve-puppeteer-timeouterror-navigation-timeout-of-30000-ms-exceeded
-
-    await page.setDefaultNavigationTimeout(0);
-
-    if (proxy_username) {
-        // Setting Proxy-Authentication header is deprecated, and doing so can trigger header change errors from Puppeteer
-        // https://github.com/puppeteer/puppeteer/issues/676 ?
-        // https://help.brightdata.com/hc/en-us/articles/12632549957649-Proxy-Manager-How-to-Guides#h_01HAKWR4Q0AFS8RZTNYWRDFJC2
-        // https://cri.dev/posts/2020-03-30-How-to-solve-Puppeteer-Chrome-Error-ERR_INVALID_ARGUMENT/
-        await page.authenticate({
-            username: proxy_username,
-            password: proxy_password
-        });
-    }
-
-    await page.setViewport({
-        width: 1024,
-        height: 768,
-        deviceScaleFactor: 1,
-    });
-
-    await page.setRequestInterception(true);
-    if (disk_cache_dir) {
-        console.log(">>>>>>>>>>>>>>> LOCAL DISK CACHE ENABLED <<<<<<<<<<<<<<<<<<<<<");
-    }
-    const fs = require('fs');
-    const crypto = require('crypto');
-
-    function file_is_expired(file_path) {
-        if (!fs.existsSync(file_path)) {
-            return true;
-        }
-        var stats = fs.statSync(file_path);
-        const now_date = new Date();
-        const expire_seconds = 300;
-        if ((now_date / 1000) - (stats.mtime.getTime() / 1000) > expire_seconds) {
-            console.log("CACHE EXPIRED: " + file_path);
-            return true;
-        }
-        return false;
-
-    }
-
-    page.on('request', async (request) => {
-        // General blocking of requests that waste traffic
-        if (block_url_list.some(substring => request.url().toLowerCase().includes(substring))) return request.abort();
-
-        if (disk_cache_dir) {
-            const url = request.url();
-            const key = crypto.createHash('md5').update(url).digest("hex");
-            const dir_path = disk_cache_dir + key.slice(0, 1) + '/' + key.slice(1, 2) + '/' + key.slice(2, 3) + '/';
-
-            // https://stackoverflow.com/questions/4482686/check-synchronously-if-file-directory-exists-in-node-js
-
-            if (fs.existsSync(dir_path + key)) {
-                console.log("* CACHE HIT , using - " + dir_path + key + " - " + url);
-                const cached_data = fs.readFileSync(dir_path + key);
-                // @todo headers can come from dir_path+key+".meta" json file
-                request.respond({
-                    status: 200,
-                    //contentType: 'text/html', //@todo
-                    body: cached_data
-                });
-                return;
-            }
-        }
-        request.continue();
-    });
-
-
-    if (disk_cache_dir) {
-        page.on('response', async (response) => {
-            const url = response.url();
-            // Basic filtering for sane responses
-            if (response.request().method() != 'GET' || response.request().resourceType() == 'xhr' || response.request().resourceType() == 'document' || response.status() != 200) {
-                console.log("Skipping (not useful) - Status:" + response.status() + " Method:" + response.request().method() + " ResourceType:" + response.request().resourceType() + " " + url);
-                return;
-            }
-            if (no_cache_list.some(substring => url.toLowerCase().includes(substring))) {
-                console.log("Skipping (no_cache_list) - " + url);
-                return;
-            }
-            if (url.toLowerCase().includes('data:')) {
-                console.log("Skipping (embedded-data) - " + url);
-                return;
-            }
-            response.buffer().then(buffer => {
-                if (buffer.length > 100) {
-                    console.log("Cache - Saving " + response.request().method() + " - " + url + " - " + response.request().resourceType());
-
-                    const key = crypto.createHash('md5').update(url).digest("hex");
-                    const dir_path = disk_cache_dir + key.slice(0, 1) + '/' + key.slice(1, 2) + '/' + key.slice(2, 3) + '/';
-
-                    if (!fs.existsSync(dir_path)) {
-                        fs.mkdirSync(dir_path, {recursive: true})
-                    }
-
-                    if (fs.existsSync(dir_path + key)) {
-                        if (file_is_expired(dir_path + key)) {
-                            fs.writeFileSync(dir_path + key, buffer);
-                        }
-                    } else {
-                        fs.writeFileSync(dir_path + key, buffer);
-                    }
-                }
-            });
-        });
-    }
-
-    const r = await page.goto(url, {
-        waitUntil: 'load'
-    });
-
-    await page.waitForTimeout(1000);
-    await page.waitForTimeout(extra_wait_ms);
-
-    if (execute_js) {
-        await page.evaluate(execute_js);
-        await page.waitForTimeout(200);
-    }
-
-    var xpath_data;
-    var instock_data;
-    try {
-        // Not sure the best way here, in the future this should be a new package added to npm then run in evaluatedCode
-        // (Once the old playwright is removed)
-        xpath_data = await page.evaluate((include_filters) => {%xpath_scrape_code%}, include_filters);
-        instock_data = await page.evaluate(() => {%instock_scrape_code%});
-    } catch (e) {
-        console.log(e);
-    }
-
-    // Protocol error (Page.captureScreenshot): Cannot take screenshot with 0 width can come from a proxy auth failure
-    // Wrap it here (for now)
-
-    var b64s = false;
-    try {
-        b64s = await page.screenshot({encoding: "base64", fullPage: true, quality: screenshot_quality, type: 'jpeg'});
-    } catch (e) {
-        console.log(e);
-    }
-
-    // May fail on very large pages with 'WARNING: tile memory limits exceeded, some content may not draw'
-    if (!b64s) {
-        // @todo after text extract, we can place some overlay text with red background to say 'croppped'
-        console.error('ERROR: content-fetcher page was maybe too large for a screenshot, reverting to viewport only screenshot');
-        try {
-            b64s = await page.screenshot({encoding: "base64", quality: screenshot_quality, type: 'jpeg'});
-        } catch (e) {
-            console.log(e);
-        }
-    }
-
-    var html = await page.content();
-    return {
-        data: {
-            'content': html,
-            'headers': r.headers(),
-            'instock_data': instock_data,
-            'screenshot': b64s,
-            'status_code': r.status(),
-            'xpath_data': xpath_data
-        },
-        type: 'application/json',
-    };
-};

changedetectionio/content_fetchers/res/stock-not-in-stock.js

@@ -1,223 +1,220 @@
-// Restock Detector
-// (c) Leigh Morresi dgtlmoon@gmail.com
-//
-// Assumes the product is in stock to begin with, unless the following appears above the fold ;
-// - outOfStockTexts appears above the fold (out of stock)
-// - negateOutOfStockRegex (really is in stock)
+async () => {
 
-function isItemInStock() {
-    // @todo Pass these in so the same list can be used in non-JS fetchers
-    const outOfStockTexts = [
-        ' أخبرني عندما يتوفر',
-        '0 in stock',
-        'actuellement indisponible',
-        'agotado',
-        'article épuisé',
-        'artikel zurzeit vergriffen',
-        'as soon as stock is available',
-        'ausverkauft', // sold out
-        'available for back order',
-        'awaiting stock',
-        'back in stock soon',
-        'back-order or out of stock',
-        'backordered',
-        'benachrichtigt mich', // notify me
-        'brak na stanie',
-        'brak w magazynie',
-        'coming soon',
-        'currently have any tickets for this',
-        'currently unavailable',
-        'dieser artikel ist bald wieder verfügbar',
-        'dostępne wkrótce',
-        'en rupture',
-        'en rupture de stock',
-        'épuisé',
-        'esgotado',
-        'indisponible',
-        'indisponível',
-        'isn\'t in stock right now',
-        'isnt in stock right now',
-        'isn’t in stock right now',
-        'item is no longer available',
-        'let me know when it\'s available',
-        'mail me when available',
-        'message if back in stock',
-        'mevcut değil',
-        'nachricht bei',
-        'nicht auf lager',
-        'nicht lagernd',
-        'nicht lieferbar',
-        'nicht verfügbar',
-        'nicht vorrätig',
-        'nicht zur verfügung',
-        'nie znaleziono produktów',
-        'niet beschikbaar',
-        'niet leverbaar',
-        'niet op voorraad',
-        'no disponible',
-        'non disponibile',
-        'non disponible',
-        'no longer in stock',
-        'no tickets available',
-        'not available',
-        'not currently available',
-        'not in stock',
-        'notify me when available',
-        'notify me',
-        'notify when available',
-        'não disponível',
-        'não estamos a aceitar encomendas',
-        'out of stock',
-        'out-of-stock',
-        'plus disponible',
-        'prodotto esaurito',
-        'produkt niedostępny',
-        'rupture',
-        'sold out',
-        'sold-out',
-        'stokta yok',
-        'temporarily out of stock',
-        'temporarily unavailable',
-        'there were no search results for',
-        'this item is currently unavailable',
-        'tickets unavailable',
-        'tijdelijk uitverkocht',
-        'tükendi',
-        'unavailable nearby',
-        'unavailable tickets',
-        'vergriffen',
-        'vorbestellen',
-        'vorbestellung ist bald möglich',
-        'we don\'t currently have any',
-        'we couldn\'t find any products that match',
-        'we do not currently have an estimate of when this product will be back in stock.',
-        'we don\'t know when or if this item will be back in stock.',
-        'we were not able to find a match',
-        'when this arrives in stock',
-        'zur zeit nicht an lager',
-        '品切れ',
-        '已售',
-        '已售完',
-        '품절'
-    ];
+    function isItemInStock() {
+        // @todo Pass these in so the same list can be used in non-JS fetchers
+        const outOfStockTexts = [
+            ' أخبرني عندما يتوفر',
+            '0 in stock',
+            'actuellement indisponible',
+            'agotado',
+            'article épuisé',
+            'artikel zurzeit vergriffen',
+            'as soon as stock is available',
+            'ausverkauft', // sold out
+            'available for back order',
+            'awaiting stock',
+            'back in stock soon',
+            'back-order or out of stock',
+            'backordered',
+            'benachrichtigt mich', // notify me
+            'brak na stanie',
+            'brak w magazynie',
+            'coming soon',
+            'currently have any tickets for this',
+            'currently unavailable',
+            'dieser artikel ist bald wieder verfügbar',
+            'dostępne wkrótce',
+            'en rupture',
+            'en rupture de stock',
+            'épuisé',
+            'esgotado',
+            'indisponible',
+            'indisponível',
+            'isn\'t in stock right now',
+            'isnt in stock right now',
+            'isn’t in stock right now',
+            'item is no longer available',
+            'let me know when it\'s available',
+            'mail me when available',
+            'message if back in stock',
+            'mevcut değil',
+            'nachricht bei',
+            'nicht auf lager',
+            'nicht lagernd',
+            'nicht lieferbar',
+            'nicht verfügbar',
+            'nicht vorrätig',
+            'nicht zur verfügung',
+            'nie znaleziono produktów',
+            'niet beschikbaar',
+            'niet leverbaar',
+            'niet op voorraad',
+            'no disponible',
+            'non disponibile',
+            'non disponible',
+            'no longer in stock',
+            'no tickets available',
+            'not available',
+            'not currently available',
+            'not in stock',
+            'notify me when available',
+            'notify me',
+            'notify when available',
+            'não disponível',
+            'não estamos a aceitar encomendas',
+            'out of stock',
+            'out-of-stock',
+            'plus disponible',
+            'prodotto esaurito',
+            'produkt niedostępny',
+            'rupture',
+            'sold out',
+            'sold-out',
+            'stok habis',
+            'stok kosong',
+            'stok varian ini habis',
+            'stokta yok',
+            'temporarily out of stock',
+            'temporarily unavailable',
+            'there were no search results for',
+            'this item is currently unavailable',
+            'tickets unavailable',
+            'tidak dijual',
+            'tidak tersedia',
+            'tijdelijk uitverkocht',
+            'tiket tidak tersedia',
+            'tükendi',
+            'unavailable nearby',
+            'unavailable tickets',
+            'vergriffen',
+            'vorbestellen',
+            'vorbestellung ist bald möglich',
+            'we don\'t currently have any',
+            'we couldn\'t find any products that match',
+            'we do not currently have an estimate of when this product will be back in stock.',
+            'we don\'t know when or if this item will be back in stock.',
+            'we were not able to find a match',
+            'when this arrives in stock',
+            'zur zeit nicht an lager',
+            '品切れ',
+            '已售',
+            '已售完',
+            '품절'
+        ];
 
 
-    const vh = Math.max(document.documentElement.clientHeight || 0, window.innerHeight || 0);
+        const vh = Math.max(document.documentElement.clientHeight || 0, window.innerHeight || 0);
 
-    function getElementBaseText(element) {
-        // .textContent can include text from children which may give the wrong results
-        // scan only immediate TEXT_NODEs, which will be a child of the element
-        var text = "";
-        for (var i = 0; i < element.childNodes.length; ++i)
-            if (element.childNodes[i].nodeType === Node.TEXT_NODE)
-                text += element.childNodes[i].textContent;
-        return text.toLowerCase().trim();
-    }
+        function getElementBaseText(element) {
+            // .textContent can include text from children which may give the wrong results
+            // scan only immediate TEXT_NODEs, which will be a child of the element
+            var text = "";
+            for (var i = 0; i < element.childNodes.length; ++i)
+                if (element.childNodes[i].nodeType === Node.TEXT_NODE)
+                    text += element.childNodes[i].textContent;
+            return text.toLowerCase().trim();
+        }
 
-    const negateOutOfStockRegex = new RegExp('^([0-9] in stock|add to cart|in stock)', 'ig');
+        const negateOutOfStockRegex = new RegExp('^([0-9] in stock|add to cart|in stock)', 'ig');
 
-    // The out-of-stock or in-stock-text is generally always above-the-fold
-    // and often below-the-fold is a list of related products that may or may not contain trigger text
-    // so it's good to filter to just the 'above the fold' elements
-    // and it should be atleast 100px from the top to ignore items in the toolbar, sometimes menu items like "Coming soon" exist
+        // The out-of-stock or in-stock-text is generally always above-the-fold
+        // and often below-the-fold is a list of related products that may or may not contain trigger text
+        // so it's good to filter to just the 'above the fold' elements
+        // and it should be atleast 100px from the top to ignore items in the toolbar, sometimes menu items like "Coming soon" exist
 
 
 // @todo - if it's SVG or IMG, go into image diff mode
-// %ELEMENTS% replaced at injection time because different interfaces use it with different settings
 
-    console.log("Scanning %ELEMENTS%");
+        function collectVisibleElements(parent, visibleElements) {
+            if (!parent) return; // Base case: if parent is null or undefined, return
 
-    function collectVisibleElements(parent, visibleElements) {
-        if (!parent) return; // Base case: if parent is null or undefined, return
+            // Add the parent itself to the visible elements array if it's of the specified types
+            visibleElements.push(parent);
 
-        // Add the parent itself to the visible elements array if it's of the specified types
-        visibleElements.push(parent);
-
-        // Iterate over the parent's children
-        const children = parent.children;
-        for (let i = 0; i < children.length; i++) {
-            const child = children[i];
-            if (
-                child.nodeType === Node.ELEMENT_NODE &&
-                window.getComputedStyle(child).display !== 'none' &&
-                window.getComputedStyle(child).visibility !== 'hidden' &&
-                child.offsetWidth >= 0 &&
-                child.offsetHeight >= 0 &&
-                window.getComputedStyle(child).contentVisibility !== 'hidden'
-            ) {
-                // If the child is an element and is visible, recursively collect visible elements
-                collectVisibleElements(child, visibleElements);
+            // Iterate over the parent's children
+            const children = parent.children;
+            for (let i = 0; i < children.length; i++) {
+                const child = children[i];
+                if (
+                    child.nodeType === Node.ELEMENT_NODE &&
+                    window.getComputedStyle(child).display !== 'none' &&
+                    window.getComputedStyle(child).visibility !== 'hidden' &&
+                    child.offsetWidth >= 0 &&
+                    child.offsetHeight >= 0 &&
+                    window.getComputedStyle(child).contentVisibility !== 'hidden'
+                ) {
+                    // If the child is an element and is visible, recursively collect visible elements
+                    collectVisibleElements(child, visibleElements);
+                }
             }
         }
-    }
 
-    const elementsToScan = [];
-    collectVisibleElements(document.body, elementsToScan);
+        const elementsToScan = [];
+        collectVisibleElements(document.body, elementsToScan);
 
-    var elementText = "";
+        var elementText = "";
 
-    // REGEXS THAT REALLY MEAN IT'S IN STOCK
-    for (let i = elementsToScan.length - 1; i >= 0; i--) {
-        const element = elementsToScan[i];
+        // REGEXS THAT REALLY MEAN IT'S IN STOCK
+        for (let i = elementsToScan.length - 1; i >= 0; i--) {
+            const element = elementsToScan[i];
 
-        // outside the 'fold' or some weird text in the heading area
-        // .getBoundingClientRect() was causing a crash in chrome 119, can only be run on contentVisibility != hidden
-        if (element.getBoundingClientRect().top + window.scrollY >= vh || element.getBoundingClientRect().top + window.scrollY <= 100) {
-            continue
+            // outside the 'fold' or some weird text in the heading area
+            // .getBoundingClientRect() was causing a crash in chrome 119, can only be run on contentVisibility != hidden
+            if (element.getBoundingClientRect().top + window.scrollY >= vh || element.getBoundingClientRect().top + window.scrollY <= 100) {
+                continue
+            }
+
+            elementText = "";
+            try {
+                if (element.tagName.toLowerCase() === "input") {
+                    elementText = element.value.toLowerCase().trim();
+                } else {
+                    elementText = getElementBaseText(element);
+                }
+            } catch (e) {
+                console.warn('stock-not-in-stock.js scraper - handling element for gettext failed', e);
+            }
+
+            if (elementText.length) {
+                // try which ones could mean its in stock
+                if (negateOutOfStockRegex.test(elementText) && !elementText.includes('(0 products)')) {
+                    console.log(`Negating/overriding 'Out of Stock' back to "Possibly in stock" found "${elementText}"`)
+                    return 'Possibly in stock';
+                }
+            }
         }
 
-        elementText = "";
-        try {
+        // OTHER STUFF THAT COULD BE THAT IT'S OUT OF STOCK
+        for (let i = elementsToScan.length - 1; i >= 0; i--) {
+            const element = elementsToScan[i];
+            // outside the 'fold' or some weird text in the heading area
+            // .getBoundingClientRect() was causing a crash in chrome 119, can only be run on contentVisibility != hidden
+            // Note: theres also an automated test that places the 'out of stock' text fairly low down
+            if (element.getBoundingClientRect().top + window.scrollY >= vh + 250 || element.getBoundingClientRect().top + window.scrollY <= 100) {
+                continue
+            }
+            elementText = "";
             if (element.tagName.toLowerCase() === "input") {
                 elementText = element.value.toLowerCase().trim();
             } else {
                 elementText = getElementBaseText(element);
             }
-        } catch (e) {
-            console.warn('stock-not-in-stock.js scraper - handling element for gettext failed', e);
-        }
 
-        if (elementText.length) {
-            // try which ones could mean its in stock
-            if (negateOutOfStockRegex.test(elementText) && !elementText.includes('(0 products)')) {
-                console.log(`Negating/overriding 'Out of Stock' back to "Possibly in stock" found "${elementText}"`)
-                return 'Possibly in stock';
-            }
-        }
-    }
-
-    // OTHER STUFF THAT COULD BE THAT IT'S OUT OF STOCK
-    for (let i = elementsToScan.length - 1; i >= 0; i--) {
-        const element = elementsToScan[i];
-        // outside the 'fold' or some weird text in the heading area
-        // .getBoundingClientRect() was causing a crash in chrome 119, can only be run on contentVisibility != hidden
-        // Note: theres also an automated test that places the 'out of stock' text fairly low down
-        if (element.getBoundingClientRect().top + window.scrollY >= vh + 250 || element.getBoundingClientRect().top + window.scrollY <= 100) {
-            continue
-        }
-        elementText = "";
-        if (element.tagName.toLowerCase() === "input") {
-            elementText = element.value.toLowerCase().trim();
-        } else {
-            elementText = getElementBaseText(element);
-        }
-
-        if (elementText.length) {
-            // and these mean its out of stock
-            for (const outOfStockText of outOfStockTexts) {
-                if (elementText.includes(outOfStockText)) {
-                    console.log(`Selected 'Out of Stock' - found text "${outOfStockText}" - "${elementText}" - offset top ${element.getBoundingClientRect().top}, page height is ${vh}`)
-                    return outOfStockText; // item is out of stock
+            if (elementText.length) {
+                // and these mean its out of stock
+                for (const outOfStockText of outOfStockTexts) {
+                    if (elementText.includes(outOfStockText)) {
+                        console.log(`Selected 'Out of Stock' - found text "${outOfStockText}" - "${elementText}" - offset top ${element.getBoundingClientRect().top}, page height is ${vh}`)
+                        return outOfStockText; // item is out of stock
+                    }
                 }
             }
         }
+
+        console.log(`Returning 'Possibly in stock' - cant' find any useful matching text`)
+        return 'Possibly in stock'; // possibly in stock, cant decide otherwise.
     }
 
-    console.log(`Returning 'Possibly in stock' - cant' find any useful matching text`)
-    return 'Possibly in stock'; // possibly in stock, cant decide otherwise.
-}
-
 // returns the element text that makes it think it's out of stock
-return isItemInStock().trim()
-
-
+    return isItemInStock().trim()
+}

changedetectionio/content_fetchers/res/xpath_element_scraper.js

@@ -1,285 +1,285 @@
-// Copyright (C) 2021 Leigh Morresi (dgtlmoon@gmail.com)
-// All rights reserved.
+async (options) => {
 
-// @file Scrape the page looking for elements of concern (%ELEMENTS%)
-// http://matatk.agrip.org.uk/tests/position-and-width/
-// https://stackoverflow.com/questions/26813480/when-is-element-getboundingclientrect-guaranteed-to-be-updated-accurate
-//
-// Some pages like https://www.londonstockexchange.com/stock/NCCL/ncondezi-energy-limited/analysis
-// will automatically force a scroll somewhere, so include the position offset
-// Lets hope the position doesnt change while we iterate the bbox's, but this is better than nothing
-var scroll_y = 0;
-try {
-    scroll_y = +document.documentElement.scrollTop || document.body.scrollTop
-} catch (e) {
-    console.log(e);
-}
+    let visualselector_xpath_selectors = options.visualselector_xpath_selectors
+    let max_height = options.max_height
 
-
-// Include the getXpath script directly, easier than fetching
-function getxpath(e) {
-    var n = e;
-    if (n && n.id) return '//*[@id="' + n.id + '"]';
-    for (var o = []; n && Node.ELEMENT_NODE === n.nodeType;) {
-        for (var i = 0, r = !1, d = n.previousSibling; d;) d.nodeType !== Node.DOCUMENT_TYPE_NODE && d.nodeName === n.nodeName && i++, d = d.previousSibling;
-        for (d = n.nextSibling; d;) {
-            if (d.nodeName === n.nodeName) {
-                r = !0;
-                break
-            }
-            d = d.nextSibling
-        }
-        o.push((n.prefix ? n.prefix + ":" : "") + n.localName + (i || r ? "[" + (i + 1) + "]" : "")), n = n.parentNode
-    }
-    return o.length ? "/" + o.reverse().join("/") : ""
-}
-
-const findUpTag = (el) => {
-    let r = el
-    chained_css = [];
-    depth = 0;
-
-    //  Strategy 1: If it's an input, with name, and there's only one, prefer that
-    if (el.name !== undefined && el.name.length) {
-        var proposed = el.tagName + "[name=\"" + CSS.escape(el.name) + "\"]";
-        var proposed_element = window.document.querySelectorAll(proposed);
-        if (proposed_element.length) {
-            if (proposed_element.length === 1) {
-                return proposed;
-            } else {
-                // Some sites change ID but name= stays the same, we can hit it if we know the index
-                // Find all the elements that match and work out the input[n]
-                var n = Array.from(proposed_element).indexOf(el);
-                // Return a Playwright selector for nthinput[name=zipcode]
-                return proposed + " >> nth=" + n;
-            }
-        }
-    }
-
-    // Strategy 2: Keep going up until we hit an ID tag, imagine it's like  #list-widget div h4
-    while (r.parentNode) {
-        if (depth === 5) {
-            break;
-        }
-        if ('' !== r.id) {
-            chained_css.unshift("#" + CSS.escape(r.id));
-            final_selector = chained_css.join(' > ');
-            // Be sure theres only one, some sites have multiples of the same ID tag :-(
-            if (window.document.querySelectorAll(final_selector).length === 1) {
-                return final_selector;
-            }
-            return null;
-        } else {
-            chained_css.unshift(r.tagName.toLowerCase());
-        }
-        r = r.parentNode;
-        depth += 1;
-    }
-    return null;
-}
-
-
-// @todo - if it's SVG or IMG, go into image diff mode
-// %ELEMENTS% replaced at injection time because different interfaces use it with different settings
-
-var size_pos = [];
-// after page fetch, inject this JS
-// build a map of all elements and their positions (maybe that only include text?)
-var bbox;
-console.log("Scanning %ELEMENTS%");
-
-function collectVisibleElements(parent, visibleElements) {
-    if (!parent) return; // Base case: if parent is null or undefined, return
-
-
-    // Add the parent itself to the visible elements array if it's of the specified types
-    const tagName = parent.tagName.toLowerCase();
-    if ("%ELEMENTS%".split(',').includes(tagName)) {
-        visibleElements.push(parent);
-    }
-
-    // Iterate over the parent's children
-    const children = parent.children;
-    for (let i = 0; i < children.length; i++) {
-        const child = children[i];
-        const computedStyle = window.getComputedStyle(child);
-
-        if (
-            child.nodeType === Node.ELEMENT_NODE &&
-            computedStyle.display !== 'none' &&
-            computedStyle.visibility !== 'hidden' &&
-            child.offsetWidth >= 0 &&
-            child.offsetHeight >= 0 &&
-            computedStyle.contentVisibility !== 'hidden'
-        ) {
-            // If the child is an element and is visible, recursively collect visible elements
-            collectVisibleElements(child, visibleElements);
-        }
-    }
-}
-
-// Create an array to hold the visible elements
-const visibleElementsArray = [];
-
-// Call collectVisibleElements with the starting parent element
-collectVisibleElements(document.body, visibleElementsArray);
-
-
-visibleElementsArray.forEach(function (element) {
-
-    bbox = element.getBoundingClientRect();
-
-    // Skip really small ones, and where width or height ==0
-    if (bbox['width'] * bbox['height'] < 10) {
-        return
-    }
-
-    // Don't include elements that are offset from canvas
-    if (bbox['top'] + scroll_y < 0 || bbox['left'] < 0) {
-        return
-    }
-
-    // @todo the getXpath kind of sucks, it doesnt know when there is for example just one ID sometimes
-    // it should not traverse when we know we can anchor off just an ID one level up etc..
-    // maybe, get current class or id, keep traversing up looking for only class or id until there is just one match
-
-    // 1st primitive - if it has class, try joining it all and select, if theres only one.. well thats us.
-    xpath_result = false;
+    var scroll_y = 0;
     try {
-        var d = findUpTag(element);
-        if (d) {
-            xpath_result = d;
-        }
+        scroll_y = +document.documentElement.scrollTop || document.body.scrollTop
     } catch (e) {
         console.log(e);
     }
-    // You could swap it and default to getXpath and then try the smarter one
-    // default back to the less intelligent one
-    if (!xpath_result) {
-        try {
-            // I've seen on FB and eBay that this doesnt work
-            // ReferenceError: getXPath is not defined at eval (eval at evaluate (:152:29), <anonymous>:67:20) at UtilityScript.evaluate (<anonymous>:159:18) at UtilityScript.<anonymous> (<anonymous>:1:44)
-            xpath_result = getxpath(element);
-        } catch (e) {
-            console.log(e);
-            return
+
+// Include the getXpath script directly, easier than fetching
+    function getxpath(e) {
+        var n = e;
+        if (n && n.id) return '//*[@id="' + n.id + '"]';
+        for (var o = []; n && Node.ELEMENT_NODE === n.nodeType;) {
+            for (var i = 0, r = !1, d = n.previousSibling; d;) d.nodeType !== Node.DOCUMENT_TYPE_NODE && d.nodeName === n.nodeName && i++, d = d.previousSibling;
+            for (d = n.nextSibling; d;) {
+                if (d.nodeName === n.nodeName) {
+                    r = !0;
+                    break
+                }
+                d = d.nextSibling
+            }
+            o.push((n.prefix ? n.prefix + ":" : "") + n.localName + (i || r ? "[" + (i + 1) + "]" : "")), n = n.parentNode
+        }
+        return o.length ? "/" + o.reverse().join("/") : ""
+    }
+
+    const findUpTag = (el) => {
+        let r = el
+        chained_css = [];
+        depth = 0;
+
+        //  Strategy 1: If it's an input, with name, and there's only one, prefer that
+        if (el.name !== undefined && el.name.length) {
+            var proposed = el.tagName + "[name=\"" + CSS.escape(el.name) + "\"]";
+            var proposed_element = window.document.querySelectorAll(proposed);
+            if (proposed_element.length) {
+                if (proposed_element.length === 1) {
+                    return proposed;
+                } else {
+                    // Some sites change ID but name= stays the same, we can hit it if we know the index
+                    // Find all the elements that match and work out the input[n]
+                    var n = Array.from(proposed_element).indexOf(el);
+                    // Return a Playwright selector for nthinput[name=zipcode]
+                    return proposed + " >> nth=" + n;
+                }
+            }
+        }
+
+        // Strategy 2: Keep going up until we hit an ID tag, imagine it's like  #list-widget div h4
+        while (r.parentNode) {
+            if (depth === 5) {
+                break;
+            }
+            if ('' !== r.id) {
+                chained_css.unshift("#" + CSS.escape(r.id));
+                final_selector = chained_css.join(' > ');
+                // Be sure theres only one, some sites have multiples of the same ID tag :-(
+                if (window.document.querySelectorAll(final_selector).length === 1) {
+                    return final_selector;
+                }
+                return null;
+            } else {
+                chained_css.unshift(r.tagName.toLowerCase());
+            }
+            r = r.parentNode;
+            depth += 1;
+        }
+        return null;
+    }
+
+
+// @todo - if it's SVG or IMG, go into image diff mode
+
+    var size_pos = [];
+// after page fetch, inject this JS
+// build a map of all elements and their positions (maybe that only include text?)
+    var bbox;
+    console.log(`Scanning for "${visualselector_xpath_selectors}"`);
+
+    function collectVisibleElements(parent, visibleElements) {
+        if (!parent) return; // Base case: if parent is null or undefined, return
+
+
+        // Add the parent itself to the visible elements array if it's of the specified types
+        const tagName = parent.tagName.toLowerCase();
+        if (visualselector_xpath_selectors.split(',').includes(tagName)) {
+            visibleElements.push(parent);
+        }
+
+        // Iterate over the parent's children
+        const children = parent.children;
+        for (let i = 0; i < children.length; i++) {
+            const child = children[i];
+            const computedStyle = window.getComputedStyle(child);
+
+            if (
+                child.nodeType === Node.ELEMENT_NODE &&
+                computedStyle.display !== 'none' &&
+                computedStyle.visibility !== 'hidden' &&
+                child.offsetWidth >= 0 &&
+                child.offsetHeight >= 0 &&
+                computedStyle.contentVisibility !== 'hidden'
+            ) {
+                // If the child is an element and is visible, recursively collect visible elements
+                collectVisibleElements(child, visibleElements);
+            }
         }
     }
 
-    let label = "not-interesting" // A placeholder, the actual labels for training are done by hand for now
+// Create an array to hold the visible elements
+    const visibleElementsArray = [];
 
-    let text = element.textContent.trim().slice(0, 30).trim();
-    while (/\n{2,}|\t{2,}/.test(text)) {
-        text = text.replace(/\n{2,}/g, '\n').replace(/\t{2,}/g, '\t')
-    }
+// Call collectVisibleElements with the starting parent element
+    collectVisibleElements(document.body, visibleElementsArray);
 
-    // Try to identify any possible currency amounts "Sale: 4000" or "Sale now 3000 Kc", can help with the training.
-    const hasDigitCurrency = (/\d/.test(text.slice(0, 6)) || /\d/.test(text.slice(-6)) ) &&  /([€£$¥₩₹]|USD|AUD|EUR|Kč|kr|SEK|,–)/.test(text) ;
-    const computedStyle = window.getComputedStyle(element);
 
-    size_pos.push({
-        xpath: xpath_result,
-        width: Math.round(bbox['width']),
-        height: Math.round(bbox['height']),
-        left: Math.floor(bbox['left']),
-        top: Math.floor(bbox['top']) + scroll_y,
-        // tagName used by Browser Steps
-        tagName: (element.tagName) ? element.tagName.toLowerCase() : '',
-        // tagtype used by Browser Steps
-        tagtype: (element.tagName.toLowerCase() === 'input' && element.type) ? element.type.toLowerCase() : '',
-        isClickable: computedStyle.cursor === "pointer",
-        // Used by the keras trainer
-        fontSize: computedStyle.getPropertyValue('font-size'),
-        fontWeight: computedStyle.getPropertyValue('font-weight'),
-        hasDigitCurrency: hasDigitCurrency,
-        label: label,
+    visibleElementsArray.forEach(function (element) {
+
+        bbox = element.getBoundingClientRect();
+
+        // Skip really small ones, and where width or height ==0
+        if (bbox['width'] * bbox['height'] < 10) {
+            return
+        }
+
+        // Don't include elements that are offset from canvas
+        if (bbox['top'] + scroll_y < 0 || bbox['left'] < 0) {
+            return
+        }
+
+        // @todo the getXpath kind of sucks, it doesnt know when there is for example just one ID sometimes
+        // it should not traverse when we know we can anchor off just an ID one level up etc..
+        // maybe, get current class or id, keep traversing up looking for only class or id until there is just one match
+
+        // 1st primitive - if it has class, try joining it all and select, if theres only one.. well thats us.
+        xpath_result = false;
+        try {
+            var d = findUpTag(element);
+            if (d) {
+                xpath_result = d;
+            }
+        } catch (e) {
+            console.log(e);
+        }
+        // You could swap it and default to getXpath and then try the smarter one
+        // default back to the less intelligent one
+        if (!xpath_result) {
+            try {
+                // I've seen on FB and eBay that this doesnt work
+                // ReferenceError: getXPath is not defined at eval (eval at evaluate (:152:29), <anonymous>:67:20) at UtilityScript.evaluate (<anonymous>:159:18) at UtilityScript.<anonymous> (<anonymous>:1:44)
+                xpath_result = getxpath(element);
+            } catch (e) {
+                console.log(e);
+                return
+            }
+        }
+
+        let label = "not-interesting" // A placeholder, the actual labels for training are done by hand for now
+
+        let text = element.textContent.trim().slice(0, 30).trim();
+        while (/\n{2,}|\t{2,}/.test(text)) {
+            text = text.replace(/\n{2,}/g, '\n').replace(/\t{2,}/g, '\t')
+        }
+
+        // Try to identify any possible currency amounts "Sale: 4000" or "Sale now 3000 Kc", can help with the training.
+        const hasDigitCurrency = (/\d/.test(text.slice(0, 6)) || /\d/.test(text.slice(-6))) && /([€£$¥₩₹]|USD|AUD|EUR|Kč|kr|SEK|,–)/.test(text);
+        const computedStyle = window.getComputedStyle(element);
+
+        if (Math.floor(bbox['top']) + scroll_y > max_height) {
+            return
+        }
+
+        size_pos.push({
+            xpath: xpath_result,
+            width: Math.round(bbox['width']),
+            height: Math.round(bbox['height']),
+            left: Math.floor(bbox['left']),
+            top: Math.floor(bbox['top']) + scroll_y,
+            // tagName used by Browser Steps
+            tagName: (element.tagName) ? element.tagName.toLowerCase() : '',
+            // tagtype used by Browser Steps
+            tagtype: (element.tagName.toLowerCase() === 'input' && element.type) ? element.type.toLowerCase() : '',
+            isClickable: computedStyle.cursor === "pointer",
+            // Used by the keras trainer
+            fontSize: computedStyle.getPropertyValue('font-size'),
+            fontWeight: computedStyle.getPropertyValue('font-weight'),
+            hasDigitCurrency: hasDigitCurrency,
+            label: label,
+        });
+
     });
 
-});
-
 
 // Inject the current one set in the include_filters, which may be a CSS rule
 // used for displaying the current one in VisualSelector, where its not one we generated.
-if (include_filters.length) {
-    let results;
-    // Foreach filter, go and find it on the page and add it to the results so we can visualise it again
-    for (const f of include_filters) {
-        bbox = false;
-        q = false;
+    if (include_filters.length) {
+        let results;
+        // Foreach filter, go and find it on the page and add it to the results so we can visualise it again
+        for (const f of include_filters) {
+            bbox = false;
+            q = false;
 
-        if (!f.length) {
-            console.log("xpath_element_scraper: Empty filter, skipping");
-            continue;
-        }
-
-        try {
-            // is it xpath?
-            if (f.startsWith('/') || f.startsWith('xpath')) {
-                var qry_f = f.replace(/xpath(:|\d:)/, '')
-                console.log("[xpath] Scanning for included filter " + qry_f)
-                let xpathResult = document.evaluate(qry_f, document, null, XPathResult.ORDERED_NODE_SNAPSHOT_TYPE, null);
-                results = [];
-                for (let i = 0; i < xpathResult.snapshotLength; i++) {
-                    results.push(xpathResult.snapshotItem(i));
-                }
-            } else {
-                console.log("[css] Scanning for included filter " + f)
-                console.log("[css] Scanning for included filter " + f);
-                results = document.querySelectorAll(f);
+            if (!f.length) {
+                console.log("xpath_element_scraper: Empty filter, skipping");
+                continue;
             }
-        } catch (e) {
-            // Maybe catch DOMException and alert?
-            console.log("xpath_element_scraper: Exception selecting element from filter " + f);
-            console.log(e);
-        }
 
-        if (results != null && results.length) {
-
-            // Iterate over the results
-            results.forEach(node => {
-                // Try to resolve //something/text() back to its /something so we can atleast get the bounding box
-                try {
-                    if (typeof node.nodeName == 'string' && node.nodeName === '#text') {
-                        node = node.parentElement
+            try {
+                // is it xpath?
+                if (f.startsWith('/') || f.startsWith('xpath')) {
+                    var qry_f = f.replace(/xpath(:|\d:)/, '')
+                    console.log("[xpath] Scanning for included filter " + qry_f)
+                    let xpathResult = document.evaluate(qry_f, document, null, XPathResult.ORDERED_NODE_SNAPSHOT_TYPE, null);
+                    results = [];
+                    for (let i = 0; i < xpathResult.snapshotLength; i++) {
+                        results.push(xpathResult.snapshotItem(i));
                     }
-                } catch (e) {
-                    console.log(e)
-                    console.log("xpath_element_scraper: #text resolver")
-                }
-
-                // #1231 - IN the case XPath attribute filter is applied, we will have to traverse up and find the element.
-                if (typeof node.getBoundingClientRect == 'function') {
-                    bbox = node.getBoundingClientRect();
-                    console.log("xpath_element_scraper: Got filter element, scroll from top was " + scroll_y)
                 } else {
+                    console.log("[css] Scanning for included filter " + f)
+                    console.log("[css] Scanning for included filter " + f);
+                    results = document.querySelectorAll(f);
+                }
+            } catch (e) {
+                // Maybe catch DOMException and alert?
+                console.log("xpath_element_scraper: Exception selecting element from filter " + f);
+                console.log(e);
+            }
+
+            if (results != null && results.length) {
+
+                // Iterate over the results
+                results.forEach(node => {
+                    // Try to resolve //something/text() back to its /something so we can atleast get the bounding box
                     try {
-                        // Try and see we can find its ownerElement
-                        bbox = node.ownerElement.getBoundingClientRect();
-                        console.log("xpath_element_scraper: Got filter by ownerElement element, scroll from top was " + scroll_y)
+                        if (typeof node.nodeName == 'string' && node.nodeName === '#text') {
+                            node = node.parentElement
+                        }
                     } catch (e) {
                         console.log(e)
-                        console.log("xpath_element_scraper: error looking up q.ownerElement")
+                        console.log("xpath_element_scraper: #text resolver")
                     }
-                }
 
-                if (bbox && bbox['width'] > 0 && bbox['height'] > 0) {
-                    size_pos.push({
-                        xpath: f,
-                        width: parseInt(bbox['width']),
-                        height: parseInt(bbox['height']),
-                        left: parseInt(bbox['left']),
-                        top: parseInt(bbox['top']) + scroll_y,
-                        highlight_as_custom_filter: true
-                    });
-                }
-            });
+                    // #1231 - IN the case XPath attribute filter is applied, we will have to traverse up and find the element.
+                    if (typeof node.getBoundingClientRect == 'function') {
+                        bbox = node.getBoundingClientRect();
+                        console.log("xpath_element_scraper: Got filter element, scroll from top was " + scroll_y)
+                    } else {
+                        try {
+                            // Try and see we can find its ownerElement
+                            bbox = node.ownerElement.getBoundingClientRect();
+                            console.log("xpath_element_scraper: Got filter by ownerElement element, scroll from top was " + scroll_y)
+                        } catch (e) {
+                            console.log(e)
+                            console.log("xpath_element_scraper: error looking up q.ownerElement")
+                        }
+                    }
+
+                    if (bbox && bbox['width'] > 0 && bbox['height'] > 0) {
+                        size_pos.push({
+                            xpath: f,
+                            width: parseInt(bbox['width']),
+                            height: parseInt(bbox['height']),
+                            left: parseInt(bbox['left']),
+                            top: parseInt(bbox['top']) + scroll_y,
+                            highlight_as_custom_filter: true
+                        });
+                    }
+                });
+            }
         }
     }
-}
 
 // Sort the elements so we find the smallest one first, in other words, we find the smallest one matching in that area
 // so that we dont select the wrapping element by mistake and be unable to select what we want
-size_pos.sort((a, b) => (a.width * a.height > b.width * b.height) ? 1 : -1)
+    size_pos.sort((a, b) => (a.width * a.height > b.width * b.height) ? 1 : -1)
+
+// browser_width required for proper scaling in the frontend
+    // Return as a string to save playwright for juggling thousands of objects
+    return JSON.stringify({'size_pos': size_pos, 'browser_width': window.innerWidth});
+}
 
-// Window.width required for proper scaling in the frontend
-return {'size_pos': size_pos, 'browser_width': window.innerWidth};

changedetectionio/content_fetchers/screenshot_handler.py

@@ -0,0 +1,73 @@
+# Pages with a vertical height longer than this will use the 'stitch together' method.
+
+# - Many GPUs have a max texture size of 16384x16384px (or lower on older devices).
+# - If a page is taller than ~8000–10000px, it risks exceeding GPU memory limits.
+# - This is especially important on headless Chromium, where Playwright may fail to allocate a massive full-page buffer.
+
+from loguru import logger
+
+from changedetectionio.content_fetchers import SCREENSHOT_MAX_HEIGHT_DEFAULT, SCREENSHOT_DEFAULT_QUALITY
+
+
+def stitch_images_worker(pipe_conn, chunks_bytes, original_page_height, capture_height):
+    import os
+    import io
+    from PIL import Image, ImageDraw, ImageFont
+
+    try:
+
+        # Load images from byte chunks
+        images = [Image.open(io.BytesIO(b)) for b in chunks_bytes]
+        total_height = sum(im.height for im in images)
+        max_width = max(im.width for im in images)
+
+        # Create stitched image
+        stitched = Image.new('RGB', (max_width, total_height))
+        y_offset = 0
+        for im in images:
+            stitched.paste(im, (0, y_offset))
+            y_offset += im.height
+
+        # Draw caption on top (overlaid, not extending canvas)
+        draw = ImageDraw.Draw(stitched)
+
+
+        caption_text = f"WARNING: Screenshot was {original_page_height}px but trimmed to {capture_height}px because it was too long"
+        padding = 10
+        font_size = 35
+        font_color = (255, 0, 0)
+        background_color = (255, 255, 255)
+
+
+        # Try to load a proper font
+        try:
+            font = ImageFont.truetype("arial.ttf", font_size)
+        except IOError:
+            font = ImageFont.load_default()
+
+        bbox = draw.textbbox((0, 0), caption_text, font=font)
+        text_width = bbox[2] - bbox[0]
+        text_height = bbox[3] - bbox[1]
+
+        # Draw white rectangle background behind text
+        rect_top = 0
+        rect_bottom = text_height + 2 * padding
+        draw.rectangle([(0, rect_top), (max_width, rect_bottom)], fill=background_color)
+
+        # Draw text centered horizontally, 10px padding from top of the rectangle
+        text_x = (max_width - text_width) // 2
+        text_y = padding
+        draw.text((text_x, text_y), caption_text, font=font, fill=font_color)
+
+        # Encode and send image
+        output = io.BytesIO()
+        stitched.save(output, format="JPEG", quality=int(os.getenv("SCREENSHOT_QUALITY", SCREENSHOT_DEFAULT_QUALITY)))
+        pipe_conn.send_bytes(output.getvalue())
+
+        stitched.close()
+    except Exception as e:
+        pipe_conn.send(f"error:{e}")
+    finally:
+        pipe_conn.close()
+
+

changedetectionio/content_fetchers/webdriver_selenium.py

@@ -65,6 +65,17 @@ class fetcher(Fetcher):
         # request_body, request_method unused for now, until some magic in the future happens.
 
         options = ChromeOptions()
+
+        # Load Chrome options from env
+        CHROME_OPTIONS = [
+            line.strip()
+            for line in os.getenv("CHROME_OPTIONS", "").strip().splitlines()
+            if line.strip()
+        ]
+
+        for opt in CHROME_OPTIONS:
+            options.add_argument(opt)
+
         if self.proxy:
             options.proxy = self.proxy
 
@@ -79,7 +90,9 @@ class fetcher(Fetcher):
             self.quit()
             raise
 
-        self.driver.set_window_size(1280, 1024)
+        if not "--window-size" in os.getenv("CHROME_OPTIONS", ""):
+            self.driver.set_window_size(1280, 1024)
+
         self.driver.implicitly_wait(int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)))
 
         if self.webdriver_js_execute_code is not None:
@@ -87,6 +100,7 @@ class fetcher(Fetcher):
             # Selenium doesn't automatically wait for actions as good as Playwright, so wait again
             self.driver.implicitly_wait(int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)))
 
+
         # @todo - how to check this? is it possible?
         self.status_code = 200
         # @todo somehow we should try to get this working for WebDriver
@@ -112,9 +126,9 @@ class fetcher(Fetcher):
         self.quit()
         return True
 
-    def quit(self):
+    def quit(self, watch=None):
         if self.driver:
             try:
                 self.driver.quit()
             except Exception as e:
-                logger.debug(f"Content Fetcher > Exception in chrome shutdown/quit {str(e)}")
+                logger.debug(f"Content Fetcher > Exception in chrome shutdown/quit {str(e)}")

changedetectionio/flask_app.py

@@ -33,7 +33,8 @@ from loguru import logger
 
 from changedetectionio import __version__
 from changedetectionio import queuedWatchMetaData
-from changedetectionio.api import Watch, WatchHistory, WatchSingleHistory, CreateWatch, Import, SystemInfo, Tag, Tags
+from changedetectionio.api import Watch, WatchHistory, WatchSingleHistory, CreateWatch, Import, SystemInfo, Tag, Tags, Notifications
+from changedetectionio.api.Search import Search
 from .time_handler import is_within_schedule
 
 datastore = None
@@ -122,14 +123,18 @@ def _jinja2_filter_format_number_locale(value: float) -> str:
 
     return formatted_value
 
+@app.template_global('is_checking_now')
+def _watch_is_checking_now(watch_obj, format="%Y-%m-%d %H:%M:%S"):
+    # Worker thread tells us which UUID it is currently processing.
+    for t in running_update_threads:
+        if t.current_uuid == watch_obj['uuid']:
+            return True
+
+
 # We use the whole watch object from the store/JSON so we can see if there's some related status in terms of a thread
 # running or something similar.
 @app.template_filter('format_last_checked_time')
 def _jinja2_filter_datetime(watch_obj, format="%Y-%m-%d %H:%M:%S"):
-    # Worker thread tells us which UUID it is currently processing.
-    for t in running_update_threads:
-        if t.current_uuid == watch_obj['uuid']:
-            return '<span class="spinner"></span><span> Checking now</span>'
 
     if watch_obj['last_checked'] == 0:
         return 'Not yet'
@@ -228,7 +233,8 @@ def changedetection_app(config=None, datastore_o=None):
 
         if has_password_enabled and not flask_login.current_user.is_authenticated:
             # Permitted
-            if request.endpoint and 'static_content' in request.endpoint and request.view_args and request.view_args.get('group') == 'styles':
+            if request.endpoint and request.endpoint == 'static_content' and request.view_args:
+                # Handled by static_content handler
                 return None
             # Permitted
             elif request.endpoint and 'login' in request.endpoint:
@@ -275,8 +281,12 @@ def changedetection_app(config=None, datastore_o=None):
 
     watch_api.add_resource(Tag, '/api/v1/tag', '/api/v1/tag/<string:uuid>',
                            resource_class_kwargs={'datastore': datastore})
+                           
+    watch_api.add_resource(Search, '/api/v1/search',
+                           resource_class_kwargs={'datastore': datastore})
 
-
+    watch_api.add_resource(Notifications, '/api/v1/notifications',
+                           resource_class_kwargs={'datastore': datastore})
 
     @login_manager.user_loader
     def user_loader(email):
@@ -287,12 +297,12 @@ def changedetection_app(config=None, datastore_o=None):
     @login_manager.unauthorized_handler
     def unauthorized_handler():
         flash("You must be logged in, please log in.", 'error')
-        return redirect(url_for('login', next=url_for('index')))
+        return redirect(url_for('login', next=url_for('watchlist.index')))
 
     @app.route('/logout')
     def logout():
         flask_login.logout_user()
-        return redirect(url_for('index'))
+        return redirect(url_for('watchlist.index'))
 
     # https://github.com/pallets/flask/blob/93dd1709d05a1cf0e886df6223377bdab3b077fb/examples/tutorial/flaskr/__init__.py#L39
     # You can divide up the stuff like this
@@ -302,7 +312,7 @@ def changedetection_app(config=None, datastore_o=None):
         if request.method == 'GET':
             if flask_login.current_user.is_authenticated:
                 flash("Already logged in")
-                return redirect(url_for("index"))
+                return redirect(url_for("watchlist.index"))
 
             output = render_template("login.html")
             return output
@@ -319,13 +329,13 @@ def changedetection_app(config=None, datastore_o=None):
             # It's more reliable and safe to ignore the 'next' redirect
             # When we used...
             # next = request.args.get('next')
-            # return redirect(next or url_for('index'))
+            # return redirect(next or url_for('watchlist.index'))
             # We would sometimes get login loop errors on sites hosted in sub-paths
 
             # note for the future:
             #            if not is_safe_url(next):
             #                return flask.abort(400)
-            return redirect(url_for('index'))
+            return redirect(url_for('watchlist.index'))
 
         else:
             flash('Incorrect password', 'error')
@@ -338,118 +348,20 @@ def changedetection_app(config=None, datastore_o=None):
         if os.getenv('USE_X_SETTINGS') and 'X-Forwarded-Prefix' in request.headers:
             app.config['REMEMBER_COOKIE_PATH'] = request.headers['X-Forwarded-Prefix']
             app.config['SESSION_COOKIE_PATH'] = request.headers['X-Forwarded-Prefix']
-
         return None
 
-
-    @app.route("/", methods=['GET'])
-    @login_optionally_required
-    def index():
-        global datastore
-        from changedetectionio import forms
-
-        active_tag_req = request.args.get('tag', '').lower().strip()
-        active_tag_uuid = active_tag = None
-
-        # Be sure limit_tag is a uuid
-        if active_tag_req:
-            for uuid, tag in datastore.data['settings']['application'].get('tags', {}).items():
-                if active_tag_req == tag.get('title', '').lower().strip() or active_tag_req == uuid:
-                    active_tag = tag
-                    active_tag_uuid = uuid
-                    break
-
-
-        # Redirect for the old rss path which used the /?rss=true
-        if request.args.get('rss'):
-            return redirect(url_for('rss.feed', tag=active_tag_uuid))
-
-        op = request.args.get('op')
-        if op:
-            uuid = request.args.get('uuid')
-            if op == 'pause':
-                datastore.data['watching'][uuid].toggle_pause()
-            elif op == 'mute':
-                datastore.data['watching'][uuid].toggle_mute()
-
-            datastore.needs_write = True
-            return redirect(url_for('index', tag = active_tag_uuid))
-
-        # Sort by last_changed and add the uuid which is usually the key..
-        sorted_watches = []
-        with_errors = request.args.get('with_errors') == "1"
-        errored_count = 0
-        search_q = request.args.get('q').strip().lower() if request.args.get('q') else False
-        for uuid, watch in datastore.data['watching'].items():
-            if with_errors and not watch.get('last_error'):
-                continue
-
-            if active_tag_uuid and not active_tag_uuid in watch['tags']:
-                    continue
-            if watch.get('last_error'):
-                errored_count += 1
-
-            if search_q:
-                if (watch.get('title') and search_q in watch.get('title').lower()) or search_q in watch.get('url', '').lower():
-                    sorted_watches.append(watch)
-                elif watch.get('last_error') and search_q in watch.get('last_error').lower():
-                    sorted_watches.append(watch)
-            else:
-                sorted_watches.append(watch)
-
-        form = forms.quickWatchForm(request.form)
-        page = request.args.get(get_page_parameter(), type=int, default=1)
-        total_count = len(sorted_watches)
-
-        pagination = Pagination(page=page,
-                                total=total_count,
-                                per_page=datastore.data['settings']['application'].get('pager_size', 50), css_framework="semantic")
-
-        sorted_tags = sorted(datastore.data['settings']['application'].get('tags').items(), key=lambda x: x[1]['title'])
-        output = render_template(
-            "watch-overview.html",
-                                 # Don't link to hosting when we're on the hosting environment
-                                 active_tag=active_tag,
-                                 active_tag_uuid=active_tag_uuid,
-                                 app_rss_token=datastore.data['settings']['application'].get('rss_access_token'),
-                                 datastore=datastore,
-                                 errored_count=errored_count,
-                                 form=form,
-                                 guid=datastore.data['app_guid'],
-                                 has_proxies=datastore.proxy_list,
-                                 has_unviewed=datastore.has_unviewed,
-                                 hosted_sticky=os.getenv("SALTED_PASS", False) == False,
-                                 pagination=pagination,
-                                 queued_uuids=[q_uuid.item['uuid'] for q_uuid in update_q.queue],
-                                 search_q=request.args.get('q','').strip(),
-                                 sort_attribute=request.args.get('sort') if request.args.get('sort') else request.cookies.get('sort'),
-                                 sort_order=request.args.get('order') if request.args.get('order') else request.cookies.get('order'),
-                                 system_default_fetcher=datastore.data['settings']['application'].get('fetch_backend'),
-                                 tags=sorted_tags,
-                                 watches=sorted_watches
-                                 )
-
-        if session.get('share-link'):
-            del(session['share-link'])
-
-        resp = make_response(output)
-
-        # The template can run on cookie or url query info
-        if request.args.get('sort'):
-            resp.set_cookie('sort', request.args.get('sort'))
-        if request.args.get('order'):
-            resp.set_cookie('order', request.args.get('order'))
-
-        return resp
-
     @app.route("/static/<string:group>/<string:filename>", methods=['GET'])
     def static_content(group, filename):
         from flask import make_response
+        import re
+        group = re.sub(r'[^\w.-]+', '', group.lower())
+        filename = re.sub(r'[^\w.-]+', '', filename.lower())
 
         if group == 'screenshot':
             # Could be sensitive, follow password requirements
             if datastore.data['settings']['application']['password'] and not flask_login.current_user.is_authenticated:
-                abort(403)
+                if not datastore.data['settings']['application'].get('shared_diff_access'):
+                    abort(403)
 
             screenshot_filename = "last-screenshot.png" if not request.args.get('error_screenshot') else "last-error-screenshot.png"
 
@@ -483,7 +395,7 @@ def changedetection_app(config=None, datastore_o=None):
                     response.headers['Content-Type'] = 'application/json'
                     response.headers['Content-Encoding'] = 'deflate'
                 else:
-                    logger.error(f'Request elements.deflate at "{watch_directory}" but was notfound.')
+                    logger.error(f'Request elements.deflate at "{watch_directory}" but was not found.')
                     abort(404)
 
                 if response:
@@ -498,7 +410,7 @@ def changedetection_app(config=None, datastore_o=None):
 
         # These files should be in our subdirectory
         try:
-            return send_from_directory("static/{}".format(group), path=filename)
+            return send_from_directory(f"static/{group}", path=filename)
         except FileNotFoundError:
             abort(404)
 
@@ -527,12 +439,25 @@ def changedetection_app(config=None, datastore_o=None):
     import changedetectionio.conditions.blueprint as conditions
     app.register_blueprint(conditions.construct_blueprint(datastore), url_prefix='/conditions')
 
-    import changedetectionio.blueprint.rss as rss
+    import changedetectionio.blueprint.rss.blueprint as rss
     app.register_blueprint(rss.construct_blueprint(datastore), url_prefix='/rss')
-    
+
+    # watchlist UI buttons etc
     import changedetectionio.blueprint.ui as ui
     app.register_blueprint(ui.construct_blueprint(datastore, update_q, running_update_threads, queuedWatchMetaData))
 
+    import changedetectionio.blueprint.watchlist as watchlist
+    app.register_blueprint(watchlist.construct_blueprint(datastore=datastore, update_q=update_q, queuedWatchMetaData=queuedWatchMetaData), url_prefix='')
+    
+    # Memory cleanup endpoint
+    @app.route('/gc-cleanup', methods=['GET'])
+    @login_optionally_required
+    def gc_cleanup():
+        from changedetectionio.gc_cleanup import memory_cleanup
+        from flask import jsonify
+
+        result = memory_cleanup(app)
+        return jsonify({"status": "success", "message": "Memory cleanup completed", "result": result})
 
     # @todo handle ctrl break
     ticker_thread = threading.Thread(target=ticker_thread_check_time_launch_checks).start()
@@ -590,7 +515,8 @@ def notification_runner():
             sent_obj = None
 
             try:
-                from changedetectionio import notification
+                from changedetectionio.notification.handler import process_notification
+
                 # Fallback to system config if not set
                 if not n_object.get('notification_body') and datastore.data['settings']['application'].get('notification_body'):
                     n_object['notification_body'] = datastore.data['settings']['application'].get('notification_body')
@@ -600,8 +526,8 @@ def notification_runner():
 
                 if not n_object.get('notification_format') and datastore.data['settings']['application'].get('notification_format'):
                     n_object['notification_format'] = datastore.data['settings']['application'].get('notification_format')
-
-                sent_obj = notification.process_notification(n_object, datastore)
+                if n_object.get('notification_urls', {}):
+                    sent_obj = process_notification(n_object, datastore)
 
             except Exception as e:
                 logger.error(f"Watch URL: {n_object['watch_url']}  Error {str(e)}")

changedetectionio/forms.py

@@ -3,6 +3,7 @@ import re
 from loguru import logger
 from wtforms.widgets.core import TimeInput
 
+from changedetectionio.blueprint.rss import RSS_FORMAT_TYPES
 from changedetectionio.conditions.form import ConditionFormRow
 from changedetectionio.strtobool import strtobool
 
@@ -305,10 +306,10 @@ class ValidateAppRiseServers(object):
 
     def __call__(self, form, field):
         import apprise
-        apobj = apprise.Apprise()
+        from .notification.apprise_plugin.assets import apprise_asset
+        from .notification.apprise_plugin.custom_handlers import apprise_http_custom_handler  # noqa: F401
 
-        # so that the custom endpoints are registered
-        from .apprise_asset import asset
+        apobj = apprise.Apprise(asset=apprise_asset)
 
         for server_url in field.data:
             url = server_url.strip()
@@ -585,7 +586,7 @@ class processor_text_json_diff_form(commonSettingsForm):
     filter_text_replaced = BooleanField('Replaced/changed lines', default=True)
     filter_text_removed = BooleanField('Removed lines', default=True)
 
-    trigger_text = StringListField('Trigger/wait for text', [validators.Optional(), ValidateListRegex()])
+    trigger_text = StringListField('Keyword triggers - Trigger/wait for text', [validators.Optional(), ValidateListRegex()])
     if os.getenv("PLAYWRIGHT_DRIVER_URL"):
         browser_steps = FieldList(FormField(SingleBrowserStep), min_entries=10)
     text_should_not_be_present = StringListField('Block change-detection while text matches', [validators.Optional(), ValidateListRegex()])
@@ -720,6 +721,8 @@ class globalSettingsRequestForm(Form):
                     self.extra_proxies.errors.append('Both a name, and a Proxy URL is required.')
                     return False
 
+class globalSettingsApplicationUIForm(Form):
+    open_diff_in_new_tab = BooleanField('Open diff page in a new tab', default=True, validators=[validators.Optional()])
 
 # datastore.data['settings']['application']..
 class globalSettingsApplicationForm(commonSettingsForm):
@@ -739,6 +742,9 @@ class globalSettingsApplicationForm(commonSettingsForm):
                               render_kw={"style": "width: 5em;"},
                               validators=[validators.NumberRange(min=0,
                                                                  message="Should be atleast zero (disabled)")])
+
+    rss_content_format = SelectField('RSS Content format', choices=RSS_FORMAT_TYPES)
+
     removepassword_button = SubmitField('Remove password', render_kw={"class": "pure-button pure-button-primary"})
     render_anchor_tag_content = BooleanField('Render anchor tag content', default=False)
     shared_diff_access = BooleanField('Allow access to view diff page when password is enabled', default=False, validators=[validators.Optional()])
@@ -748,6 +754,7 @@ class globalSettingsApplicationForm(commonSettingsForm):
                                                                   render_kw={"style": "width: 5em;"},
                                                                   validators=[validators.NumberRange(min=0,
                                                                                                      message="Should contain zero or more attempts")])
+    ui = FormField(globalSettingsApplicationUIForm)
 
 
 class globalSettingsForm(Form):

changedetectionio/gc_cleanup.py

@@ -0,0 +1,162 @@
+#!/usr/bin/env python3
+
+import ctypes
+import gc
+import re
+import psutil
+import sys
+import threading
+import importlib
+from loguru import logger
+
+def memory_cleanup(app=None):
+    """
+    Perform comprehensive memory cleanup operations and log memory usage
+    at each step with nicely formatted numbers.
+    
+    Args:
+        app: Optional Flask app instance for clearing Flask-specific caches
+        
+    Returns:
+        str: Status message
+    """
+    # Get current process
+    process = psutil.Process()
+    
+    # Log initial memory usage with nicely formatted numbers
+    current_memory = process.memory_info().rss / 1024 / 1024
+    logger.debug(f"Memory cleanup started - Current memory usage: {current_memory:,.2f} MB")
+
+    # 1. Standard garbage collection - force full collection on all generations
+    gc.collect(0)  # Collect youngest generation
+    gc.collect(1)  # Collect middle generation
+    gc.collect(2)  # Collect oldest generation
+
+    # Run full collection again to ensure maximum cleanup
+    gc.collect()
+    current_memory = process.memory_info().rss / 1024 / 1024
+    logger.debug(f"After full gc.collect() - Memory usage: {current_memory:,.2f} MB")
+    
+
+    # 3. Call libc's malloc_trim to release memory back to the OS
+    libc = ctypes.CDLL("libc.so.6")
+    libc.malloc_trim(0)
+    current_memory = process.memory_info().rss / 1024 / 1024
+    logger.debug(f"After malloc_trim(0) - Memory usage: {current_memory:,.2f} MB")
+    
+    # 4. Clear Python's regex cache
+    re.purge()
+    current_memory = process.memory_info().rss / 1024 / 1024
+    logger.debug(f"After re.purge() - Memory usage: {current_memory:,.2f} MB")
+
+    # 5. Reset thread-local storage
+    # Create a new thread local object to encourage cleanup of old ones
+    threading.local()
+    current_memory = process.memory_info().rss / 1024 / 1024
+    logger.debug(f"After threading.local() - Memory usage: {current_memory:,.2f} MB")
+
+    # 6. Clear sys.intern cache if Python version supports it
+    try:
+        sys.intern.clear()
+        current_memory = process.memory_info().rss / 1024 / 1024
+        logger.debug(f"After sys.intern.clear() - Memory usage: {current_memory:,.2f} MB")
+    except (AttributeError, TypeError):
+        logger.debug("sys.intern.clear() not supported in this Python version")
+    
+    # 7. Clear XML/lxml caches if available
+    try:
+        # Check if lxml.etree is in use
+        lxml_etree = sys.modules.get('lxml.etree')
+        if lxml_etree:
+            # Clear module-level caches
+            if hasattr(lxml_etree, 'clear_error_log'):
+                lxml_etree.clear_error_log()
+            
+            # Check for _ErrorLog and _RotatingErrorLog objects and clear them
+            for obj in gc.get_objects():
+                if hasattr(obj, '__class__') and hasattr(obj.__class__, '__name__'):
+                    class_name = obj.__class__.__name__
+                    if class_name in ('_ErrorLog', '_RotatingErrorLog', '_DomainErrorLog') and hasattr(obj, 'clear'):
+                        try:
+                            obj.clear()
+                        except (AttributeError, TypeError):
+                            pass
+                    
+                    # Clear Element objects which can hold references to documents
+                    elif class_name in ('_Element', 'ElementBase') and hasattr(obj, 'clear'):
+                        try:
+                            obj.clear()
+                        except (AttributeError, TypeError):
+                            pass
+            
+            current_memory = process.memory_info().rss / 1024 / 1024
+            logger.debug(f"After lxml.etree cleanup - Memory usage: {current_memory:,.2f} MB")
+
+        # Check if lxml.html is in use
+        lxml_html = sys.modules.get('lxml.html')
+        if lxml_html:
+            # Clear HTML-specific element types
+            for obj in gc.get_objects():
+                if hasattr(obj, '__class__') and hasattr(obj.__class__, '__name__'):
+                    class_name = obj.__class__.__name__
+                    if class_name in ('HtmlElement', 'FormElement', 'InputElement',
+                                    'SelectElement', 'TextareaElement', 'CheckboxGroup',
+                                    'RadioGroup', 'MultipleSelectOptions', 'FieldsDict') and hasattr(obj, 'clear'):
+                        try:
+                            obj.clear()
+                        except (AttributeError, TypeError):
+                            pass
+
+            current_memory = process.memory_info().rss / 1024 / 1024
+            logger.debug(f"After lxml.html cleanup - Memory usage: {current_memory:,.2f} MB")
+    except (ImportError, AttributeError):
+        logger.debug("lxml cleanup not applicable")
+    
+    # 8. Clear JSON parser caches if applicable
+    try:
+        # Check if json module is being used and try to clear its cache
+        json_module = sys.modules.get('json')
+        if json_module and hasattr(json_module, '_default_encoder'):
+            json_module._default_encoder.markers.clear()
+            current_memory = process.memory_info().rss / 1024 / 1024
+            logger.debug(f"After JSON parser cleanup - Memory usage: {current_memory:,.2f} MB")
+    except (AttributeError, KeyError):
+        logger.debug("JSON cleanup not applicable")
+    
+    # 9. Force Python's memory allocator to release unused memory
+    try:
+        if hasattr(sys, 'pypy_version_info'):
+            # PyPy has different memory management
+            gc.collect()
+        else:
+            # CPython - try to release unused memory
+            ctypes.pythonapi.PyGC_Collect()
+            current_memory = process.memory_info().rss / 1024 / 1024
+            logger.debug(f"After PyGC_Collect - Memory usage: {current_memory:,.2f} MB")
+    except (AttributeError, TypeError):
+        logger.debug("PyGC_Collect not supported")
+    
+    # 10. Clear Flask-specific caches if applicable
+    if app:
+        try:
+            # Clear Flask caches if they exist
+            for key in list(app.config.get('_cache', {}).keys()):
+                app.config['_cache'].pop(key, None)
+            
+            # Clear Jinja2 template cache if available
+            if hasattr(app, 'jinja_env') and hasattr(app.jinja_env, 'cache'):
+                app.jinja_env.cache.clear()
+            
+            current_memory = process.memory_info().rss / 1024 / 1024
+            logger.debug(f"After Flask cache clear - Memory usage: {current_memory:,.2f} MB")
+        except (AttributeError, KeyError):
+            logger.debug("No Flask cache to clear")
+    
+    # Final garbage collection pass
+    gc.collect()
+    libc.malloc_trim(0)
+    
+    # Log final memory usage
+    final_memory = process.memory_info().rss / 1024 / 1024
+    logger.info(f"Memory cleanup completed - Final memory usage: {final_memory:,.2f} MB")
+    return "cleaned"

changedetectionio/html_tools.py

@@ -366,22 +366,41 @@ def extract_json_as_string(content, json_filter, ensure_is_ldjson_info_type=None
 # wordlist - list of regex's (str) or words (str)
 # Preserves all linefeeds and other whitespacing, its not the job of this to remove that
 def strip_ignore_text(content, wordlist, mode="content"):
-    i = 0
-    output = []
     ignore_text = []
     ignore_regex = []
-    ignored_line_numbers = []
+    ignore_regex_multiline = []
+    ignored_lines = []
 
     for k in wordlist:
         # Is it a regex?
         res = re.search(PERL_STYLE_REGEX, k, re.IGNORECASE)
         if res:
-            ignore_regex.append(re.compile(perl_style_slash_enclosed_regex_to_options(k)))
+            res = re.compile(perl_style_slash_enclosed_regex_to_options(k))
+            if res.flags & re.DOTALL or res.flags & re.MULTILINE:
+                ignore_regex_multiline.append(res)
+            else:
+                ignore_regex.append(res)
         else:
             ignore_text.append(k.strip())
 
-    for line in content.splitlines(keepends=True):
-        i += 1
+    for r in ignore_regex_multiline:
+        for match in r.finditer(content):
+            content_lines = content[:match.end()].splitlines(keepends=True)
+            match_lines = content[match.start():match.end()].splitlines(keepends=True)
+
+            end_line = len(content_lines)
+            start_line = end_line - len(match_lines)
+
+            if end_line - start_line <= 1:
+                # Match is empty or in the middle of the line
+                ignored_lines.append(start_line)
+            else:
+                for i in range(start_line, end_line):
+                    ignored_lines.append(i)
+
+    line_index = 0
+    lines = content.splitlines(keepends=True)
+    for line in lines:
         # Always ignore blank lines in this mode. (when this function gets called)
         got_match = False
         for l in ignore_text:
@@ -393,17 +412,19 @@ def strip_ignore_text(content, wordlist, mode="content"):
                 if r.search(line):
                     got_match = True
 
-        if not got_match:
-            # Not ignored, and should preserve "keepends"
-            output.append(line)
-        else:
-            ignored_line_numbers.append(i)
+        if got_match:
+            ignored_lines.append(line_index)
+
+        line_index += 1
+
+    ignored_lines = set([i for i in ignored_lines if i >= 0 and i < len(lines)])
 
     # Used for finding out what to highlight
     if mode == "line numbers":
-        return ignored_line_numbers
+        return [i + 1 for i in ignored_lines]
 
-    return ''.join(output)
+    output_lines = set(range(len(lines))) - ignored_lines
+    return ''.join([lines[i] for i in output_lines])
 
 def cdata_in_document_to_text(html_content: str, render_anchor_tag_content=False) -> str:
     from xml.sax.saxutils import escape as xml_escape
@@ -414,7 +435,9 @@ def cdata_in_document_to_text(html_content: str, render_anchor_tag_content=False
 
     return re.sub(pattern, repl, html_content)
 
-def html_to_text(html_content: str, render_anchor_tag_content=False, is_rss=False) -> str:
+
+def html_to_text_sub_worker(conn, html_content: str, render_anchor_tag_content=False, is_rss=False):
+
     from inscriptis import get_text
     from inscriptis.model.config import ParserConfig
 
@@ -449,15 +472,27 @@ def html_to_text(html_content: str, render_anchor_tag_content=False, is_rss=Fals
         html_content = re.sub(r'</title>', r'</h1>', html_content)
 
     text_content = get_text(html_content, config=parser_config)
+    conn.send(text_content)
+    conn.close()
 
-    return text_content
+# NOTE!! ANYTHING LIBXML, HTML5LIB ETC WILL CAUSE SOME SMALL MEMORY LEAK IN THE LOCAL "LIB" IMPLEMENTATION OUTSIDE PYTHON
+def html_to_text(html_content: str, render_anchor_tag_content=False, is_rss=False):
+    from multiprocessing import Process, Pipe
 
+    parent_conn, child_conn = Pipe()
+    p = Process(target=html_to_text_sub_worker, args=(child_conn, html_content, render_anchor_tag_content, is_rss))
+    p.start()
+    text = parent_conn.recv()
+    p.join()
+    return text
 
 # Does LD+JSON exist with a @type=='product' and a .price set anywhere?
 def has_ldjson_product_info(content):
     try:
-        lc = content.lower()
-        if 'application/ld+json' in lc and lc.count('"price"') == 1 and '"pricecurrency"' in lc:
+        # Better than .lower() which can use a lot of ram
+        if (re.search(r'application/ld\+json', content, re.IGNORECASE) and
+            re.search(r'"price"', content, re.IGNORECASE) and
+            re.search(r'"pricecurrency"', content, re.IGNORECASE)):
             return True
 
 #       On some pages this is really terribly expensive when they dont really need it

changedetectionio/model/App.py

@@ -1,4 +1,7 @@
 from os import getenv
+
+from changedetectionio.blueprint.rss import RSS_FORMAT_TYPES
+
 from changedetectionio.notification import (
     default_notification_body,
     default_notification_format,
@@ -9,6 +12,8 @@ from changedetectionio.notification import (
 _FILTER_FAILURE_THRESHOLD_ATTEMPTS_DEFAULT = 6
 DEFAULT_SETTINGS_HEADERS_USERAGENT='Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.66 Safari/537.36'
 
+
+
 class model(dict):
     base_config = {
             'note': "Hello! If you change this file manually, please be sure to restart your changedetection.io instance!",
@@ -48,12 +53,16 @@ class model(dict):
                     'password': False,
                     'render_anchor_tag_content': False,
                     'rss_access_token': None,
+                    'rss_content_format': RSS_FORMAT_TYPES[0][0],
                     'rss_hide_muted_watches': True,
                     'schema_version' : 0,
                     'shared_diff_access': False,
                     'webdriver_delay': None , # Extra delay in seconds before extracting text
                     'tags': {}, #@todo use Tag.model initialisers
                     'timezone': None, # Default IANA timezone name
+                    'ui': {
+                        'open_diff_in_new_tab': True,
+                    },
                 }
             }
         }

changedetectionio/model/Watch.py

@@ -553,7 +553,10 @@ class model(watch_base):
         self.ensure_data_dir_exists()
 
         with open(target_path, 'wb') as f:
-            f.write(zlib.compress(json.dumps(data).encode()))
+            if not isinstance(data, str):
+                f.write(zlib.compress(json.dumps(data).encode()))
+            else:
+                f.write(zlib.compress(data.encode()))
             f.close()
 
     # Save as PNG, PNG is larger but better for doing visual diff in the future
@@ -575,7 +578,7 @@ class model(watch_base):
         import brotli
         filepath = os.path.join(self.watch_data_dir, 'last-fetched.br')
 
-        if not os.path.isfile(filepath):
+        if not os.path.isfile(filepath) or os.path.getsize(filepath) == 0:
             # If a previous attempt doesnt yet exist, just snarf the previous snapshot instead
             dates = list(self.history.keys())
             if len(dates):

changedetectionio/model/__init__.py

@@ -2,7 +2,7 @@ import os
 import uuid
 
 from changedetectionio import strtobool
-from changedetectionio.notification import default_notification_format_for_watch
+default_notification_format_for_watch = 'System default'
 
 class watch_base(dict):
 

changedetectionio/notification/__init__.py

@@ -0,0 +1,35 @@
+from changedetectionio.model import default_notification_format_for_watch
+
+ult_notification_format_for_watch = 'System default'
+default_notification_format = 'HTML Color'
+default_notification_body = '{{watch_url}} had a change.\n---\n{{diff}}\n---\n'
+default_notification_title = 'ChangeDetection.io Notification - {{watch_url}}'
+
+# The values (markdown etc) are from apprise NotifyFormat,
+# But to avoid importing the whole heavy module just use the same strings here.
+valid_notification_formats = {
+    'Text': 'text',
+    'Markdown': 'markdown',
+    'HTML': 'html',
+    'HTML Color': 'htmlcolor',
+    # Used only for editing a watch (not for global)
+    default_notification_format_for_watch: default_notification_format_for_watch
+}
+
+
+valid_tokens = {
+    'base_url': '',
+    'current_snapshot': '',
+    'diff': '',
+    'diff_added': '',
+    'diff_full': '',
+    'diff_patch': '',
+    'diff_removed': '',
+    'diff_url': '',
+    'preview_url': '',
+    'triggered_text': '',
+    'watch_tag': '',
+    'watch_title': '',
+    'watch_url': '',
+    'watch_uuid': '',
+}

changedetectionio/notification/apprise_plugin/assets.py

@@ -0,0 +1,16 @@
+from apprise import AppriseAsset
+
+# Refer to:
+# https://github.com/caronc/apprise/wiki/Development_API#the-apprise-asset-object
+
+APPRISE_APP_ID = "changedetection.io"
+APPRISE_APP_DESC = "ChangeDetection.io best and simplest website monitoring and change detection"
+APPRISE_APP_URL = "https://changedetection.io"
+APPRISE_AVATAR_URL = "https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/changedetectionio/static/images/avatar-256x256.png"
+
+apprise_asset = AppriseAsset(
+    app_id=APPRISE_APP_ID,
+    app_desc=APPRISE_APP_DESC,
+    app_url=APPRISE_APP_URL,
+    image_url_logo=APPRISE_AVATAR_URL,
+)

changedetectionio/notification/apprise_plugin/custom_handlers.py

@@ -0,0 +1,112 @@
+import json
+import re
+from urllib.parse import unquote_plus
+
+import requests
+from apprise.decorators import notify
+from apprise.utils.parse import parse_url as apprise_parse_url
+from loguru import logger
+from requests.structures import CaseInsensitiveDict
+
+SUPPORTED_HTTP_METHODS = {"get", "post", "put", "delete", "patch", "head"}
+
+
+def notify_supported_methods(func):
+    for method in SUPPORTED_HTTP_METHODS:
+        func = notify(on=method)(func)
+        # Add support for https, for each supported http method
+        func = notify(on=f"{method}s")(func)
+    return func
+
+
+def _get_auth(parsed_url: dict) -> str | tuple[str, str]:
+    user: str | None = parsed_url.get("user")
+    password: str | None = parsed_url.get("password")
+
+    if user is not None and password is not None:
+        return (unquote_plus(user), unquote_plus(password))
+
+    if user is not None:
+        return unquote_plus(user)
+
+    return ""
+
+
+def _get_headers(parsed_url: dict, body: str) -> CaseInsensitiveDict:
+    headers = CaseInsensitiveDict(
+        {unquote_plus(k).title(): unquote_plus(v) for k, v in parsed_url["qsd+"].items()}
+    )
+
+    # If Content-Type is not specified, guess if the body is a valid JSON
+    if headers.get("Content-Type") is None:
+        try:
+            json.loads(body)
+            headers["Content-Type"] = "application/json; charset=utf-8"
+        except Exception:
+            pass
+
+    return headers
+
+
+def _get_params(parsed_url: dict) -> CaseInsensitiveDict:
+    # https://github.com/caronc/apprise/wiki/Notify_Custom_JSON#get-parameter-manipulation
+    # In Apprise, it relies on prefixing each request arg with "-", because it uses say &method=update as a flag for apprise
+    # but here we are making straight requests, so we need todo convert this against apprise's logic
+    params = CaseInsensitiveDict(
+        {
+            unquote_plus(k): unquote_plus(v)
+            for k, v in parsed_url["qsd"].items()
+            if k.strip("-") not in parsed_url["qsd-"]
+            and k.strip("+") not in parsed_url["qsd+"]
+        }
+    )
+
+    return params
+
+
+@notify_supported_methods
+def apprise_http_custom_handler(
+    body: str,
+    title: str,
+    notify_type: str,
+    meta: dict,
+    *args,
+    **kwargs,
+) -> bool:
+    url: str = meta.get("url")
+    schema: str = meta.get("schema")
+    method: str = re.sub(r"s$", "", schema).upper()
+
+    # Convert /foobar?+some-header=hello to proper header dictionary
+    parsed_url: dict[str, str | dict | None] | None = apprise_parse_url(url)
+    if parsed_url is None:
+        return False
+
+    auth = _get_auth(parsed_url=parsed_url)
+    headers = _get_headers(parsed_url=parsed_url, body=body)
+    params = _get_params(parsed_url=parsed_url)
+
+    url = re.sub(rf"^{schema}", "https" if schema.endswith("s") else "http", parsed_url.get("url"))
+
+    try:
+        response = requests.request(
+            method=method,
+            url=url,
+            auth=auth,
+            headers=headers,
+            params=params,
+            data=body.encode("utf-8") if isinstance(body, str) else body,
+        )
+
+        response.raise_for_status()
+
+        logger.info(f"Successfully sent custom notification to {url}")
+        return True
+
+    except requests.RequestException as e:
+        logger.error(f"Remote host error while sending custom notification to {url}: {e}")
+        return False
+
+    except Exception as e:
+        logger.error(f"Unexpected error occurred while sending custom notification to {url}: {e}")
+        return False

changedetectionio/notification/handler.py

@@ -1,48 +1,17 @@
 
 import time
-from apprise import NotifyFormat
 import apprise
 from loguru import logger
 
-
-valid_tokens = {
-    'base_url': '',
-    'current_snapshot': '',
-    'diff': '',
-    'diff_added': '',
-    'diff_full': '',
-    'diff_patch': '',
-    'diff_removed': '',
-    'diff_url': '',
-    'preview_url': '',
-    'triggered_text': '',
-    'watch_tag': '',
-    'watch_title': '',
-    'watch_url': '',
-    'watch_uuid': '',
-}
-
-default_notification_format_for_watch = 'System default'
-default_notification_format = 'HTML Color'
-default_notification_body = '{{watch_url}} had a change.\n---\n{{diff}}\n---\n'
-default_notification_title = 'ChangeDetection.io Notification - {{watch_url}}'
-
-valid_notification_formats = {
-    'Text': NotifyFormat.TEXT,
-    'Markdown': NotifyFormat.MARKDOWN,
-    'HTML': NotifyFormat.HTML,
-    'HTML Color': 'htmlcolor',
-    # Used only for editing a watch (not for global)
-    default_notification_format_for_watch: default_notification_format_for_watch
-}
-
+from .apprise_plugin.assets import apprise_asset, APPRISE_AVATAR_URL
 
 
 def process_notification(n_object, datastore):
-    # so that the custom endpoints are registered
-    from changedetectionio.apprise_plugin import apprise_custom_api_call_wrapper
+    from changedetectionio.safe_jinja import render as jinja_render
+    from . import default_notification_format_for_watch, default_notification_format, valid_notification_formats
+    # be sure its registered
+    from .apprise_plugin.custom_handlers import apprise_http_custom_handler
 
-    from .safe_jinja import render as jinja_render
     now = time.time()
     if n_object.get('notification_timestamp'):
         logger.trace(f"Time since queued {now-n_object['notification_timestamp']:.3f}s")
@@ -59,19 +28,18 @@ def process_notification(n_object, datastore):
         # Initially text or whatever
         n_format = datastore.data['settings']['application'].get('notification_format', valid_notification_formats[default_notification_format])
 
-    logger.trace(f"Complete notification body including Jinja and placeholders calculated in  {time.time() - now:.3f}s")
+    logger.trace(f"Complete notification body including Jinja and placeholders calculated in  {time.time() - now:.2f}s")
 
     # https://github.com/caronc/apprise/wiki/Development_LogCapture
     # Anything higher than or equal to WARNING (which covers things like Connection errors)
     # raise it as an exception
 
     sent_objs = []
-    from .apprise_asset import asset
 
     if 'as_async' in n_object:
-        asset.async_mode = n_object.get('as_async')
+        apprise_asset.async_mode = n_object.get('as_async')
 
-    apobj = apprise.Apprise(debug=True, asset=asset)
+    apobj = apprise.Apprise(debug=True, asset=apprise_asset)
 
     if not n_object.get('notification_urls'):
         return None
@@ -112,7 +80,7 @@ def process_notification(n_object, datastore):
                     and not url.startswith('get') \
                     and not url.startswith('delete') \
                     and not url.startswith('put'):
-                url += k + 'avatar_url=https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/changedetectionio/static/images/avatar-256x256.png'
+                url += k + f"avatar_url={APPRISE_AVATAR_URL}"
 
             if url.startswith('tgram://'):
                 # Telegram only supports a limit subset of HTML, remove the '<br>' we place in.
@@ -177,6 +145,7 @@ def process_notification(n_object, datastore):
 # ( Where we prepare the tokens in the notification to be replaced with actual values )
 def create_notification_parameters(n_object, datastore):
     from copy import deepcopy
+    from . import valid_tokens
 
     # in the case we send a test notification from the main settings, there is no UUID.
     uuid = n_object['uuid'] if 'uuid' in n_object else ''

changedetectionio/processors/__init__.py

@@ -159,7 +159,7 @@ class difference_detection_processor():
                          )
 
         #@todo .quit here could go on close object, so we can run JS if change-detected
-        self.fetcher.quit()
+        self.fetcher.quit(watch=self.watch)
 
         # After init, call run_changedetection() which will do the actual change-detection
 

changedetectionio/processors/text_json_diff/processor.py

@@ -252,6 +252,7 @@ class perform_site_check(difference_detection_processor):
 
         # 615 Extract text by regex
         extract_text = watch.get('extract_text', [])
+        extract_text += self.datastore.get_tag_overrides_for_watch(uuid=watch.get('uuid'), attr='extract_text')
         if len(extract_text) > 0:
             regex_matched_output = []
             for s_re in extract_text:
@@ -296,6 +297,8 @@ class perform_site_check(difference_detection_processor):
 ### CALCULATE MD5
         # If there's text to ignore
         text_to_ignore = watch.get('ignore_text', []) + self.datastore.data['settings']['application'].get('global_ignore_text', [])
+        text_to_ignore += self.datastore.get_tag_overrides_for_watch(uuid=watch.get('uuid'), attr='ignore_text')
+
         text_for_checksuming = stripped_text_from_html
         if text_to_ignore:
             text_for_checksuming = html_tools.strip_ignore_text(stripped_text_from_html, text_to_ignore)
@@ -308,8 +311,8 @@ class perform_site_check(difference_detection_processor):
 
         ############ Blocking rules, after checksum #################
         blocked = False
-
         trigger_text = watch.get('trigger_text', [])
+        trigger_text += self.datastore.get_tag_overrides_for_watch(uuid=watch.get('uuid'), attr='trigger_text')
         if len(trigger_text):
             # Assume blocked
             blocked = True
@@ -324,6 +327,7 @@ class perform_site_check(difference_detection_processor):
                 blocked = False
 
         text_should_not_be_present = watch.get('text_should_not_be_present', [])
+        text_should_not_be_present += self.datastore.get_tag_overrides_for_watch(uuid=watch.get('uuid'), attr='text_should_not_be_present')
         if len(text_should_not_be_present):
             # If anything matched, then we should block a change from happening
             result = html_tools.strip_ignore_text(content=str(stripped_text_from_html),
@@ -334,12 +338,14 @@ class perform_site_check(difference_detection_processor):
 
         # And check if 'conditions' will let this pass through
         if watch.get('conditions') and watch.get('conditions_match_logic'):
-            if not execute_ruleset_against_all_plugins(current_watch_uuid=watch.get('uuid'),
-                                                application_datastruct=self.datastore.data,
-                                                ephemeral_data={
-                                                    'text': stripped_text_from_html
-                                                }
-                                                ):
+            conditions_result = execute_ruleset_against_all_plugins(current_watch_uuid=watch.get('uuid'),
+                                                                    application_datastruct=self.datastore.data,
+                                                                    ephemeral_data={
+                                                                        'text': stripped_text_from_html
+                                                                    }
+                                                                    )
+
+            if not conditions_result.get('result'):
                 # Conditions say "Condition not met" so we block it.
                 blocked = True
 

changedetectionio/run_basic_tests.sh

@@ -14,7 +14,8 @@ SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 find tests/test_*py -type f|while read test_name
 do
   echo "TEST RUNNING $test_name"
-  pytest $test_name
+  # REMOVE_REQUESTS_OLD_SCREENSHOTS disabled so that we can write a screenshot and send it in test_notifications.py without a real browser
+  REMOVE_REQUESTS_OLD_SCREENSHOTS=false pytest $test_name
 done
 
 echo "RUNNING WITH BASE_URL SET"
@@ -22,7 +23,7 @@ echo "RUNNING WITH BASE_URL SET"
 # Now re-run some tests with BASE_URL enabled
 # Re #65 - Ability to include a link back to the installation, in the notification.
 export BASE_URL="https://really-unique-domain.io"
-pytest tests/test_notification.py
+REMOVE_REQUESTS_OLD_SCREENSHOTS=false pytest tests/test_notification.py
 
 
 # Re-run with HIDE_REFERER set - could affect login
@@ -32,7 +33,7 @@ pytest tests/test_access_control.py
 # Re-run a few tests that will trigger brotli based storage
 export SNAPSHOT_BROTLI_COMPRESSION_THRESHOLD=5
 pytest tests/test_access_control.py
-pytest tests/test_notification.py
+REMOVE_REQUESTS_OLD_SCREENSHOTS=false pytest tests/test_notification.py
 pytest tests/test_backend.py
 pytest tests/test_rss.py
 pytest tests/test_unique_lines.py

changedetectionio/static/js/conditions.js

@@ -8,7 +8,7 @@ $(document).ready(function () {
         $(".addRuleRow").on("click", function(e) {
             e.preventDefault();
             
-            let currentRow = $(this).closest("tr");
+            let currentRow = $(this).closest(".fieldlist-row");
             
             // Clone without events
             let newRow = currentRow.clone(false);
@@ -29,8 +29,8 @@ $(document).ready(function () {
             e.preventDefault();
             
             // Only remove if there's more than one row
-            if ($("#rulesTable tbody tr").length > 1) {
-                $(this).closest("tr").remove();
+            if ($("#rulesTable .fieldlist-row").length > 1) {
+                $(this).closest(".fieldlist-row").remove();
                 reindexRules();
             }
         });
@@ -39,7 +39,7 @@ $(document).ready(function () {
         $(".verifyRuleRow").on("click", function(e) {
             e.preventDefault();
             
-            let row = $(this).closest("tr");
+            let row = $(this).closest(".fieldlist-row");
             let field = row.find("select[name$='field']").val();
             let operator = row.find("select[name$='operator']").val();
             let value = row.find("input[name$='value']").val();
@@ -52,7 +52,7 @@ $(document).ready(function () {
 
             
             // Create a rule object
-            const rule = {
+            let rule = {
                 field: field,
                 operator: operator,
                 value: value
@@ -96,6 +96,10 @@ $(document).ready(function () {
                 contentType: false, // Let the browser set the correct content type
                 success: function (response) {
                     if (response.status === "success") {
+                        if(rule['field'] !== "page_filtered_text") {
+                            // A little debug helper for the user
+                            $('#verify-state-text').text(`${rule['field']} was value "${response.data[rule['field']]}"`)
+                        }
                         if (response.result) {
                             alert("✅ Condition PASSES verification against current snapshot!");
                         } else {
@@ -124,7 +128,7 @@ $(document).ready(function () {
         $(".addRuleRow, .removeRuleRow, .verifyRuleRow").off("click");
         
         // Reindex all form elements
-        $("#rulesTable tbody tr").each(function(index) {
+        $("#rulesTable .fieldlist-row").each(function(index) {
             $(this).find("select, input").each(function() {
                 let oldName = $(this).attr("name");
                 let oldId = $(this).attr("id");

changedetectionio/static/js/watch-overview.js

@@ -48,6 +48,8 @@ $(function () {
         $('input[type=checkbox]').not(this).prop('checked', this.checked);
     });
 
+    const time_check_step_size_seconds=1;
+
     // checkboxes - show/hide buttons
     $("input[type=checkbox]").click(function (e) {
         if ($('input[type=checkbox]:checked').length) {
@@ -57,5 +59,30 @@ $(function () {
         }
     });
 
+    setInterval(function () {
+        // Background ETA completion for 'checking now'
+        $(".watch-table .checking-now .last-checked").each(function () {
+            const eta_complete = parseFloat($(this).data('eta_complete'));
+            const fetch_duration = parseInt($(this).data('fetchduration'));
+
+            if (eta_complete + 2 > nowtimeserver && fetch_duration > 3) {
+                const remaining_seconds = Math.abs(eta_complete) - nowtimeserver - 1;
+
+                let r = (1.0 - (remaining_seconds / fetch_duration)) * 100;
+                if (r < 10) {
+                    r = 10;
+                }
+                if (r >= 90) {
+                    r = 100;
+                }
+                $(this).css('background-size', `${r}% 100%`);
+                //$(this).text(`${r}% remain ${remaining_seconds}`);
+            } else {
+                $(this).css('background-size', `100% 100%`);
+            }
+        });
+
+        nowtimeserver = nowtimeserver + time_check_step_size_seconds;
+    }, time_check_step_size_seconds * 1000);
 });
 

changedetectionio/static/styles/scss/parts/_conditions_table.scss

@@ -0,0 +1,135 @@
+/* Styles for the flexbox-based table replacement for conditions */
+.fieldlist_formfields {
+  width: 100%;
+  background-color: var(--color-background, #fff);
+  border-radius: 4px;
+  border: 1px solid var(--color-border-table-cell, #cbcbcb);
+  
+  /* Header row */
+  .fieldlist-header {
+    display: flex;
+    background-color: var(--color-background-table-thead, #e0e0e0);
+    font-weight: bold;
+    border-bottom: 1px solid var(--color-border-table-cell, #cbcbcb);
+  }
+  
+  .fieldlist-header-cell {
+    flex: 1;
+    padding: 0.5em 1em;
+    text-align: left;
+    
+    &:last-child {
+      flex: 0 0 120px; /* Fixed width for actions column */
+    }
+  }
+  
+  /* Body rows */
+  .fieldlist-body {
+    display: flex;
+    flex-direction: column;
+  }
+  
+  .fieldlist-row {
+    display: flex;
+    border-bottom: 1px solid var(--color-border-table-cell, #cbcbcb);
+    
+    &:last-child {
+      border-bottom: none;
+    }
+    
+    &:nth-child(2n-1) {
+      background-color: var(--color-table-stripe, #f2f2f2);
+    }
+    
+    &.error-row {
+      background-color: var(--color-error-input, #ffdddd);
+    }
+  }
+  
+  .fieldlist-cell {
+    flex: 1;
+    padding: 0.5em 1em;
+    display: flex;
+    flex-direction: column;
+    justify-content: center;
+    
+    /* Make inputs take up full width of their cell */
+    input, select {
+      width: 100%;
+    }
+    
+    &.fieldlist-actions {
+      flex: 0 0 120px; /* Fixed width for actions column */
+      display: flex;
+      flex-direction: row;
+      align-items: center;
+      gap: 4px;
+    }
+  }
+  
+  /* Error styling */
+  ul.errors {
+    margin-top: 0.5em;
+    margin-bottom: 0;
+    padding: 0.5em;
+    background-color: var(--color-error-background-snapshot-age, #ffdddd);
+    border-radius: 4px;
+    list-style-position: inside;
+  }
+  
+  /* Responsive styles */
+  @media only screen and (max-width: 760px) {
+    .fieldlist-header, .fieldlist-row {
+      flex-direction: column;
+    }
+    
+    .fieldlist-header-cell {
+      display: none;
+    }
+    
+    .fieldlist-row {
+      padding: 0.5em 0;
+      border-bottom: 2px solid var(--color-border-table-cell, #cbcbcb);
+    }
+    
+    .fieldlist-cell {
+      padding: 0.25em 0.5em;
+      
+      &.fieldlist-actions {
+        flex: 1;
+        justify-content: flex-start;
+        padding-top: 0.5em;
+      }
+    }
+    
+    /* Add some spacing between fields on mobile */
+    .fieldlist-cell:not(:last-child) {
+      margin-bottom: 0.5em;
+    }
+    
+    /* Label each cell on mobile view */
+    .fieldlist-cell::before {
+      content: attr(data-label);
+      font-weight: bold;
+      margin-bottom: 0.25em;
+    }
+  }
+}
+
+/* Button styling */
+.fieldlist_formfields {
+  .addRuleRow, .removeRuleRow, .verifyRuleRow {
+    cursor: pointer;
+    border: none;
+    padding: 4px 8px;
+    border-radius: 3px;
+    font-weight: bold;
+    background-color: #aaa;
+    color: var(--color-foreground-text, #fff);
+
+    &:hover {
+      background-color: #999;
+    }
+  }
+
+}

changedetectionio/static/styles/scss/styles.scss

@@ -14,6 +14,7 @@
 @import "parts/_love";
 @import "parts/preview_text_filter";
 @import "parts/_edit";
+@import "parts/_conditions_table";
 
 body {
   color: var(--color-text);

changedetectionio/static/styles/styles.css

@@ -530,6 +530,99 @@ ul#conditions_match_logic {
   ul#conditions_match_logic li {
     padding-right: 1em; }
 
+/* Styles for the flexbox-based table replacement for conditions */
+.fieldlist_formfields {
+  width: 100%;
+  background-color: var(--color-background, #fff);
+  border-radius: 4px;
+  border: 1px solid var(--color-border-table-cell, #cbcbcb);
+  /* Header row */
+  /* Body rows */
+  /* Error styling */
+  /* Responsive styles */ }
+  .fieldlist_formfields .fieldlist-header {
+    display: flex;
+    background-color: var(--color-background-table-thead, #e0e0e0);
+    font-weight: bold;
+    border-bottom: 1px solid var(--color-border-table-cell, #cbcbcb); }
+  .fieldlist_formfields .fieldlist-header-cell {
+    flex: 1;
+    padding: 0.5em 1em;
+    text-align: left; }
+    .fieldlist_formfields .fieldlist-header-cell:last-child {
+      flex: 0 0 120px;
+      /* Fixed width for actions column */ }
+  .fieldlist_formfields .fieldlist-body {
+    display: flex;
+    flex-direction: column; }
+  .fieldlist_formfields .fieldlist-row {
+    display: flex;
+    border-bottom: 1px solid var(--color-border-table-cell, #cbcbcb); }
+    .fieldlist_formfields .fieldlist-row:last-child {
+      border-bottom: none; }
+    .fieldlist_formfields .fieldlist-row:nth-child(2n-1) {
+      background-color: var(--color-table-stripe, #f2f2f2); }
+    .fieldlist_formfields .fieldlist-row.error-row {
+      background-color: var(--color-error-input, #ffdddd); }
+  .fieldlist_formfields .fieldlist-cell {
+    flex: 1;
+    padding: 0.5em 1em;
+    display: flex;
+    flex-direction: column;
+    justify-content: center;
+    /* Make inputs take up full width of their cell */ }
+    .fieldlist_formfields .fieldlist-cell input, .fieldlist_formfields .fieldlist-cell select {
+      width: 100%; }
+    .fieldlist_formfields .fieldlist-cell.fieldlist-actions {
+      flex: 0 0 120px;
+      /* Fixed width for actions column */
+      display: flex;
+      flex-direction: row;
+      align-items: center;
+      gap: 4px; }
+  .fieldlist_formfields ul.errors {
+    margin-top: 0.5em;
+    margin-bottom: 0;
+    padding: 0.5em;
+    background-color: var(--color-error-background-snapshot-age, #ffdddd);
+    border-radius: 4px;
+    list-style-position: inside; }
+  @media only screen and (max-width: 760px) {
+    .fieldlist_formfields {
+      /* Add some spacing between fields on mobile */
+      /* Label each cell on mobile view */ }
+      .fieldlist_formfields .fieldlist-header, .fieldlist_formfields .fieldlist-row {
+        flex-direction: column; }
+      .fieldlist_formfields .fieldlist-header-cell {
+        display: none; }
+      .fieldlist_formfields .fieldlist-row {
+        padding: 0.5em 0;
+        border-bottom: 2px solid var(--color-border-table-cell, #cbcbcb); }
+      .fieldlist_formfields .fieldlist-cell {
+        padding: 0.25em 0.5em; }
+        .fieldlist_formfields .fieldlist-cell.fieldlist-actions {
+          flex: 1;
+          justify-content: flex-start;
+          padding-top: 0.5em; }
+      .fieldlist_formfields .fieldlist-cell:not(:last-child) {
+        margin-bottom: 0.5em; }
+      .fieldlist_formfields .fieldlist-cell::before {
+        content: attr(data-label);
+        font-weight: bold;
+        margin-bottom: 0.25em; } }
+
+/* Button styling */
+.fieldlist_formfields .addRuleRow, .fieldlist_formfields .removeRuleRow, .fieldlist_formfields .verifyRuleRow {
+  cursor: pointer;
+  border: none;
+  padding: 4px 8px;
+  border-radius: 3px;
+  font-weight: bold;
+  background-color: #aaa;
+  color: var(--color-foreground-text, #fff); }
+  .fieldlist_formfields .addRuleRow:hover, .fieldlist_formfields .removeRuleRow:hover, .fieldlist_formfields .verifyRuleRow:hover {
+    background-color: #999; }
+
 body {
   color: var(--color-text);
   background: var(--color-background-page);

changedetectionio/store.py

@@ -251,8 +251,14 @@ class ChangeDetectionStore:
     # Clone a watch by UUID
     def clone(self, uuid):
         url = self.data['watching'][uuid].get('url')
-        extras = self.data['watching'][uuid]
+        extras = deepcopy(self.data['watching'][uuid])
         new_uuid = self.add_watch(url=url, extras=extras)
+        watch = self.data['watching'][new_uuid]
+
+        if self.data['settings']['application'].get('extract_title_as_title') or watch['extract_title_as_title']:
+            # Because it will be recalculated on the next fetch
+            self.data['watching'][new_uuid]['title'] = None
+
         return new_uuid
 
     def url_exists(self, url):
@@ -363,7 +369,6 @@ class ChangeDetectionStore:
         new_watch.ensure_data_dir_exists()
         self.__data['watching'][new_uuid] = new_watch
 
-
         if write_to_disk_now:
             self.sync_to_json()
 
@@ -631,6 +636,41 @@ class ChangeDetectionStore:
             if watch.get('processor') == processor_name:
                 return True
         return False
+        
+    def search_watches_for_url(self, query, tag_limit=None, partial=False):
+        """Search watches by URL, title, or error messages
+        
+        Args:
+            query (str): Search term to match against watch URLs, titles, and error messages
+            tag_limit (str, optional): Optional tag name to limit search results
+            partial: (bool, optional): sub-string matching
+
+        Returns:
+            list: List of UUIDs of watches that match the search criteria
+        """
+        matching_uuids = []
+        query = query.lower().strip()
+        tag = self.tag_exists_by_name(tag_limit) if tag_limit else False
+
+        for uuid, watch in self.data['watching'].items():
+            # Filter by tag if requested
+            if tag_limit:
+                if not tag.get('uuid') in watch.get('tags', []):
+                    continue
+
+            # Search in URL, title, or error messages
+            if partial:
+                if ((watch.get('title') and query in watch.get('title').lower()) or
+                    query in watch.get('url', '').lower() or
+                    (watch.get('last_error') and query in watch.get('last_error').lower())):
+                    matching_uuids.append(uuid)
+            else:
+                if ((watch.get('title') and query == watch.get('title').lower()) or
+                        query == watch.get('url', '').lower() or
+                        (watch.get('last_error') and query == watch.get('last_error').lower())):
+                    matching_uuids.append(uuid)
+
+        return matching_uuids
 
     def get_unique_notification_tokens_available(self):
         # Ask each type of watch if they have any extra notification token to add to the validation
@@ -924,3 +964,25 @@ class ChangeDetectionStore:
                         f_d.write(zlib.compress(f_j.read()))
                         os.unlink(json_path)
 
+    def add_notification_url(self, notification_url):
+        
+        logger.debug(f">>> Adding new notification_url - '{notification_url}'")
+
+        notification_urls = self.data['settings']['application'].get('notification_urls', [])
+
+        if notification_url in notification_urls:
+            return notification_url
+
+        with self.lock:
+            notification_urls = self.__data['settings']['application'].get('notification_urls', [])
+
+            if notification_url in notification_urls:
+                return notification_url
+
+            # Append and update the datastore
+            notification_urls.append(notification_url)
+            self.__data['settings']['application']['notification_urls'] = notification_urls
+            self.needs_write = True
+
+        return notification_url
+

changedetectionio/templates/_helpers.html

@@ -61,21 +61,20 @@
   {{ field(**kwargs)|safe }}
 {% endmacro %}
 
-{% macro render_fieldlist_of_formfields_as_table(fieldlist, table_id="rulesTable") %}
-  <table class="fieldlist_formfields pure-table" id="{{ table_id }}">
-    <thead>
-      <tr>
-        {% for subfield in fieldlist[0] %}
-          <th>{{ subfield.label }}</th>
-        {% endfor %}
-        <th>Actions</th>
-      </tr>
-    </thead>
-    <tbody>
+{% macro render_conditions_fieldlist_of_formfields_as_table(fieldlist, table_id="rulesTable") %}
+  <div class="fieldlist_formfields" id="{{ table_id }}">
+    <div class="fieldlist-header">
+      {% for subfield in fieldlist[0] %}
+        <div class="fieldlist-header-cell">{{ subfield.label }}</div>
+      {% endfor %}
+      <div class="fieldlist-header-cell">Actions</div>
+    </div>
+    <div class="fieldlist-body">
       {% for form_row in fieldlist %}
-        <tr {% if form_row.errors %} class="error-row" {% endif %}>
+        <div class="fieldlist-row {% if form_row.errors %}error-row{% endif %}">
           {% for subfield in form_row %}
-            <td>
+            <div class="fieldlist-cell">
+
               {{ subfield()|safe }}
               {% if subfield.errors %}
                 <ul class="errors">
@@ -84,30 +83,28 @@
                   {% endfor %}
                 </ul>
               {% endif %}
-            </td>
+            </div>
           {% endfor %}
-          <td>
-            <button type="button" class="addRuleRow">+</button>
-            <button type="button" class="removeRuleRow">-</button>
+          <div class="fieldlist-cell fieldlist-actions">
+            <button type="button" class="addRuleRow" title="Add a row/rule after">+</button>
+            <button type="button" class="removeRuleRow" title="Remove this row/rule">-</button>
             <button type="button" class="verifyRuleRow" title="Verify this rule against current snapshot">✓</button>
-          </td>
-        </tr>
+          </div>
+        </div>
       {% endfor %}
-    </tbody>
-  </table>
+    </div>
+  </div>
 {% endmacro %}
 
 
 {% macro playwright_warning() %}
-    <p><strong>Error - Playwright support for Chrome based fetching is not enabled.</strong> Alternatively try our <a href="https://changedetection.io">very affordable subscription based service which has all this setup for you</a>.</p>
+    <p><strong>Error - This watch needs Chrome (with playwright/sockpuppetbrowser), but Chrome based fetching is not enabled.</strong> Alternatively try our <a href="https://changedetection.io">very affordable subscription based service which has all this setup for you</a>.</p>
     <p>You may need to <a href="https://github.com/dgtlmoon/changedetection.io/blob/09ebc6ec6338545bdd694dc6eee57f2e9d2b8075/docker-compose.yml#L31">Enable playwright environment variable</a> and uncomment the <strong>sockpuppetbrowser</strong> in the <a href="https://github.com/dgtlmoon/changedetection.io/blob/master/docker-compose.yml">docker-compose.yml</a> file.</p>
     <br>
-    <p>(Also Selenium/WebDriver can not extract full page screenshots reliably so Playwright is recommended here)</p>
-
 {% endmacro %}
 
-{% macro only_webdriver_type_watches_warning() %}
-    <p><strong>Sorry, this functionality only works with Playwright/Chrome enabled watches.<br>You need to <a href="#request">Set the fetch method to Playwright/Chrome mode and resave</a> and have the Playwright connection enabled.</strong></p><br>
+{% macro only_playwright_type_watches_warning() %}
+    <p><strong>Sorry, this functionality only works with Playwright/Chrome enabled watches.<br>You need to <a href="#request">Set the fetch method to Playwright/Chrome mode and resave</a> and have the SockpuppetBrowser/Playwright or Selenium enabled.</strong></p><br>
 {% endmacro %}
 
 {% macro render_time_schedule_form(form, available_timezones, timezone_default_config) %}

changedetectionio/templates/base.html

@@ -42,7 +42,7 @@
           <a class="pure-menu-heading" href="https://changedetection.io" rel="noopener">
             <strong>Change</strong>Detection.io</a>
         {% else %}
-          <a class="pure-menu-heading" href="{{url_for('index')}}">
+          <a class="pure-menu-heading" href="{{url_for('watchlist.index')}}">
             <strong>Change</strong>Detection.io</a>
         {% endif %}
         {% if current_diff_url %}
@@ -157,15 +157,13 @@
                 <h4>Try our Chrome extension</h4>
                 <p>
                     <a id="chrome-extension-link"
-                       title="Try our new Chrome Extension!"
+                       title="Chrome Extension - Web Page Change Detection with changedetection.io!"
                        href="https://chromewebstore.google.com/detail/changedetectionio-website/kefcfmgmlhmankjmnbijimhofdjekbop">
-                        <img alt="Chrome store icon" src="{{url_for('static_content', group='images', filename='Google-Chrome-icon.png')}}">
+                        <img alt="Chrome store icon" src="{{url_for('static_content', group='images', filename='google-chrome-icon.png')}}">
                         Chrome Webstore
                     </a>
                 </p>
-
                 Easily add the current web-page from your browser directly into your changedetection.io tool, more great features coming soon!
-
                 <h4>Changedetection.io needs your support!</h4>
                 <p>
                     You can help us by supporting changedetection.io on these platforms;
@@ -173,17 +171,20 @@
                 <p>
                 <ul>
                     <li>
-                        <a href="https://alternativeto.net/software/changedetection-io/about/">Rate us at
+                        <a href="https://alternativeto.net/software/changedetection-io/about/" title="Web page change detection at alternativeto.net">Rate us at
                         AlternativeTo.net</a>
                     </li>
                 <li>
-                    <a href="https://github.com/dgtlmoon/changedetection.io">Star us on GitHub</a>
+                    <a href="https://github.com/dgtlmoon/changedetection.io" title="Web page change detection on GitHub">Star us on GitHub</a>
                 </li>
                 <li>
-                    <a href="https://twitter.com/change_det_io">Follow us at Twitter/X</a>
+                    <a rel="nofollow" href="https://twitter.com/change_det_io" title="Web page change detection on Twitter">Follow us at Twitter/X</a>
                 </li>
                 <li>
-                    <a href="https://www.linkedin.com/company/changedetection-io">Check us out on LinkedIn</a>
+                    <a rel="nofollow" href="https://www.g2.com/products/changedetection-io/reviews" title="Web page change detection reviews at G2">G2 Software reviews</a>
+                </li>
+                <li>
+                    <a rel="nofollow" href="https://www.linkedin.com/company/changedetection-io" title="Visit web page change detection at LinkedIn">Check us out on LinkedIn</a>
                 </li>
                 <li>
                     And tell your friends and colleagues :)

changedetectionio/templates/edit.html

@@ -1,6 +1,6 @@
 {% extends 'base.html' %}
 {% block content %}
-{% from '_helpers.html' import render_field, render_checkbox_field, render_button, render_time_schedule_form, playwright_warning, only_webdriver_type_watches_warning, render_fieldlist_of_formfields_as_table %}
+{% from '_helpers.html' import render_field, render_checkbox_field, render_button, render_time_schedule_form, playwright_warning, only_playwright_type_watches_warning, render_conditions_fieldlist_of_formfields_as_table %}
 {% from '_common_fields.html' import render_common_settings_form %}
 <script src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
 <script src="{{url_for('static_content', group='js', filename='vis.js')}}" defer></script>
@@ -204,7 +204,9 @@ Math: {{ 1 + 1 }}") }}
             </div>
 
             <div class="tab-pane-inner" id="browser-steps">
-            {% if playwright_enabled and watch_uses_webdriver %}
+            {% if watch_needs_selenium_or_playwright %}
+                {# Only works with playwright #}
+                {% if system_has_playwright_configured %}
                 <img class="beta-logo" src="{{url_for('static_content', group='images', filename='beta-logo.png')}}" alt="New beta functionality">
                 <fieldset>
                     <div class="pure-control-group">
@@ -223,7 +225,6 @@ Math: {{ 1 + 1 }}") }}
                         <div class="flex-wrapper" >
 
                             <div id="browser-steps-ui" class="noselect">
-
                                 <div class="noselect"  id="browsersteps-selector-wrapper" style="width: 100%">
                                     <span class="loader" >
                                         <span id="browsersteps-click-start">
@@ -245,15 +246,16 @@ Math: {{ 1 + 1 }}") }}
                     </div>
                 </fieldset>
                 {% else %}
-                    <span class="pure-form-message-inline">
-                        {% if not watch_uses_webdriver %}
-                            {{ only_webdriver_type_watches_warning() }}
-                        {% endif %}
-                        {%  if not playwright_enabled %}
-                            {{ playwright_warning() }}
-                        {% endif %}
-                    </span>
+                    {# it's configured to use selenium or chrome but system says its not configured #}
+                    {{ playwright_warning() }}
+                    {% if system_has_webdriver_configured %}
+                        <strong>Selenium/Webdriver cant be used here because it wont fetch screenshots reliably.</strong>
+                    {% endif %}
                 {% endif %}
+            {% else %}
+                {# "This functionality needs chrome.." #}
+                {{ only_playwright_type_watches_warning() }}
+            {% endif %}
             </div>
 
 
@@ -262,7 +264,7 @@ Math: {{ 1 + 1 }}") }}
                     <div  class="pure-control-group inline-radio">
                       {{ render_checkbox_field(form.notification_muted) }}
                     </div>
-                    {% if watch_uses_webdriver %}
+                    {% if watch_needs_selenium_or_playwright %}
                     <div class="pure-control-group inline-radio">
                       {{ render_checkbox_field(form.notification_screenshot) }}
                         <span class="pure-form-message-inline">
@@ -289,25 +291,13 @@ Math: {{ 1 + 1 }}") }}
                     <script>
                         const verify_condition_rule_url="{{url_for('conditions.verify_condition_single_rule', watch_uuid=uuid)}}";
                     </script>
-                <style>
-                    .verifyRuleRow {
-                        background-color: #4caf50;
-                        color: white;
-                        border: none;
-                        cursor: pointer;
-                        font-weight: bold;
-                    }
-                    .verifyRuleRow:hover {
-                        background-color: #45a049;
-                    }
-                </style>
                 <div class="pure-control-group">
                     {{ render_field(form.conditions_match_logic) }}
-                    {{ render_fieldlist_of_formfields_as_table(form.conditions) }}
+                    {{ render_conditions_fieldlist_of_formfields_as_table(form.conditions) }}
                     <div class="pure-form-message-inline">
-                        <br>
-                        Use the verify (✓) button to test if a condition passes against the current snapshot.<br><br>
-                        Did you know that <strong>conditions</strong> can be extended with your own custom plugin? tutorials coming soon!<br>
+
+                        <p id="verify-state-text">Use the verify (✓) button to test if a condition passes against the current snapshot.</p>
+                       Read a quick tutorial about <a href="https://changedetection.io/tutorial/conditional-actions-web-page-changes">using conditional web page changes here</a>.<br>
                     </div>
                 </div>
             </div>
@@ -326,61 +316,8 @@ Math: {{ 1 + 1 }}") }}
                                 </li>
                             </ul>
                     </div>
-                    <div class="pure-control-group">
-                        {% set field = render_field(form.include_filters,
-                            rows=5,
-                            placeholder=has_tag_filters_extra+"#example
-xpath://body/div/span[contains(@class, 'example-class')]",
-                            class="m-d")
-                        %}
-                        {{ field }}
-                        {% if '/text()' in  field %}
-                          <span class="pure-form-message-inline"><strong>Note!: //text() function does not work where the &lt;element&gt; contains &lt;![CDATA[]]&gt;</strong></span><br>
-                        {% endif %}
-                        <span class="pure-form-message-inline">One CSS, xPath, JSON Path/JQ selector per line, <i>any</i> rules that matches will be used.<br>
-                        <span data-target="#advanced-help-selectors" class="toggle-show pure-button button-tag button-xsmall">Show advanced help and tips</span><br>
-                    <ul id="advanced-help-selectors" style="display: none;">
-                        <li>CSS - Limit text to this CSS rule, only text matching this CSS rule is included.</li>
-                        <li>JSON - Limit text to this JSON rule, using either <a href="https://pypi.org/project/jsonpath-ng/" target="new">JSONPath</a> or <a href="https://stedolan.github.io/jq/" target="new">jq</a> (if installed).
-                            <ul>
-                                <li>JSONPath: Prefix with <code>json:</code>, use <code>json:$</code> to force re-formatting if required,  <a href="https://jsonpath.com/" target="new">test your JSONPath here</a>.</li>
-                                {% if jq_support %}
-                                <li>jq: Prefix with <code>jq:</code> and <a href="https://jqplay.org/" target="new">test your jq here</a>. Using <a href="https://stedolan.github.io/jq/" target="new">jq</a> allows for complex filtering and processing of JSON data with built-in functions, regex, filtering, and more. See examples and documentation <a href="https://stedolan.github.io/jq/manual/" target="new">here</a>. Prefix <code>jqraw:</code> outputs the results as text instead of a JSON list.</li>
-                                {% else %}
-                                <li>jq support not installed</li>
-                                {% endif %}
-                            </ul>
-                        </li>
-                        <li>XPath - Limit text to this XPath rule, simply start with a forward-slash. To specify XPath to be used explicitly or the XPath rule starts with an XPath function: Prefix with <code>xpath:</code>
-                            <ul>
-                                <li>Example:  <code>//*[contains(@class, 'sametext')]</code> or <code>xpath:count(//*[contains(@class, 'sametext')])</code>, <a
-                                href="http://xpather.com/" target="new">test your XPath here</a></li>
-                                <li>Example: Get all titles from an RSS feed <code>//title/text()</code></li>
-                                <li>To use XPath1.0: Prefix with <code>xpath1:</code></li>
-                            </ul>
-                            </li>
-                    <li>
-                        Please be sure that you thoroughly understand how to write CSS, JSONPath, XPath{% if jq_support %}, or jq selector{%endif%} rules before filing an issue on GitHub! <a
-                                href="https://github.com/dgtlmoon/changedetection.io/wiki/CSS-Selector-help">here for more CSS selector help</a>.<br>
-                    </li>
-                    </ul>
 
-                </span>
-                    </div>
-                <fieldset class="pure-control-group">
-                    {{ render_field(form.subtractive_selectors, rows=5, placeholder=has_tag_filters_extra+"header
-footer
-nav
-.stockticker
-//*[contains(text(), 'Advertisement')]") }}
-                    <span class="pure-form-message-inline">
-                        <ul>
-                          <li> Remove HTML element(s) by CSS and XPath selectors before text conversion. </li>
-                          <li> Don't paste HTML here, use only CSS and XPath selectors </li>
-                          <li> Add multiple elements, CSS or XPath selectors per line to ignore multiple parts of the HTML. </li>
-                        </ul>
-                      </span>
-                </fieldset>
+{% include "edit/include_subtract.html" %}
                 <div class="text-filtering border-fieldset">
                 <fieldset class="pure-group" id="text-filtering-type-options">
                     <h3>Text filtering</h3>
@@ -408,76 +345,9 @@ nav
                     {{ render_checkbox_field(form.trim_text_whitespace) }}
                     <span class="pure-form-message-inline">Remove any whitespace before and after each line of text</span>
                 </fieldset>
-                <fieldset>
-                    <div class="pure-control-group">
-                        {{ render_field(form.trigger_text, rows=5, placeholder="Some text to wait for in a line
-/some.regex\d{2}/ for case-INsensitive regex
-") }}
-                        <span class="pure-form-message-inline">
-                    <ul>
-                        <li>Text to wait for before triggering a change/notification, all text and regex are tested <i>case-insensitive</i>.</li>
-                        <li>Trigger text is processed from the result-text that comes out of any CSS/JSON Filters for this watch</li>
-                        <li>Each line is processed separately (think of each line as "OR")</li>
-                        <li>Note: Wrap in forward slash / to use regex  example: <code>/foo\d/</code></li>
-                    </ul>
-                        </span>
-                    </div>
-                </fieldset>
-                <fieldset class="pure-group">
-                    {{ render_field(form.ignore_text, rows=5, placeholder="Some text to ignore in a line
-/some.regex\d{2}/ for case-INsensitive regex
-") }}
-                    <span class="pure-form-message-inline">
-                        <ul>
-                            <li>Matching text will be <strong>ignored</strong> in the text snapshot (you can still see it but it wont trigger a change)</li>
-                            <li>Each line processed separately, any line matching will be ignored (removed before creating the checksum)</li>
-                            <li>Regular Expression support, wrap the entire line in forward slash <code>/regex/</code></li>
-                            <li>Changing this will affect the comparison checksum which may trigger an alert</li>
-                        </ul>
-                </span>
-
-                </fieldset>
-
-                <fieldset>
-                    <div class="pure-control-group">
-                        {{ render_field(form.text_should_not_be_present, rows=5, placeholder="For example: Out of stock
-Sold out
-Not in stock
-Unavailable") }}
-                        <span class="pure-form-message-inline">
-                            <ul>
-                                <li>Block change-detection while this text is on the page, all text and regex are tested <i>case-insensitive</i>, good for waiting for when a product is available again</li>
-                                <li>Block text is processed from the result-text that comes out of any CSS/JSON Filters for this watch</li>
-                                <li>All lines here must not exist (think of each line as "OR")</li>
-                                <li>Note: Wrap in forward slash / to use regex  example: <code>/foo\d/</code></li>
-                            </ul>
-                        </span>
-                    </div>
-                </fieldset>
-                <fieldset>
-                    <div class="pure-control-group">
-                        {{ render_field(form.extract_text, rows=5, placeholder="/.+?\d+ comments.+?/
- or
-keyword") }}
-                        <span class="pure-form-message-inline">
-                    <ul>
-                        <li>Extracts text in the final output (line by line) after other filters using regular expressions or string match;
-                            <ul>
-                                <li>Regular expression &dash; example <code>/reports.+?2022/i</code></li>
-                                <li>Don't forget to consider the white-space at the start of a line <code>/.+?reports.+?2022/i</code></li>
-                                <li>Use <code>//(?aiLmsux))</code> type flags (more <a href="https://docs.python.org/3/library/re.html#index-15">information here</a>)<br></li>
-                                <li>Keyword example &dash; example <code>Out of stock</code></li>
-                                <li>Use groups to extract just that text &dash; example <code>/reports.+?(\d+)/i</code> returns a list of years only</li>
-                                <li>Example - match lines containing a keyword <code>/.*icecream.*/</code></li>
-                            </ul>
-                        </li>
-                        <li>One line per regular-expression/string match</li>
-                    </ul>
-                        </span>
-                    </div>
-                </fieldset>
+                {% include "edit/text-options.html" %}
                 </div>
-            </div>
+              </div>
               <div id="text-preview" style="display: none;" >
                     <script>
                         const preview_text_edit_filters_url="{{url_for('ui.ui_edit.watch_get_preview_rendered', uuid=uuid)}}";
@@ -511,7 +381,9 @@ keyword") }}
 
                 <fieldset>
                     <div class="pure-control-group">
-                        {% if playwright_enabled and watch_uses_webdriver %}
+                        {% if watch_needs_selenium_or_playwright %}
+                            {% if system_has_playwright_configured %}
+                                {%  if visual_selector_data_ready %}
                             <span class="pure-form-message-inline" id="visual-selector-heading">
                                 The Visual Selector tool lets you select the <i>text</i> elements that will be used for the change detection. It automatically fills-in the filters in the "CSS/JSONPath/JQ/XPath Filters" box of the <a href="#filters-and-triggers">Filters & Triggers</a> tab. Use <strong>Shift+Click</strong> to select multiple items.
                             </span>
@@ -528,14 +400,20 @@ keyword") }}
                                 <canvas id="selector-canvas"></canvas>
                             </div>
                             <div id="selector-current-xpath" style="overflow-x: hidden"><strong>Currently:</strong>&nbsp;<span class="text">Loading...</span></div>
+                            {% else %}
+                                <strong>Error, The Visual selector data is not ready, it needs to complete atleast one fetch, please queue the item and reload.</strong>
+                            {% endif %}
                         {% else %}
-                            {% if not watch_uses_webdriver %}
-                                {{ only_webdriver_type_watches_warning() }}
-                            {% endif %}
-                            {% if not playwright_enabled %}
-                                {{ playwright_warning() }}
-                            {% endif %}
+                            {# The watch needed chrome but system says that playwright is not ready #}
+                            {{ playwright_warning() }}
                         {% endif %}
+                            {% if system_has_webdriver_configured %}
+                                <strong>Selenium/Webdriver cant be used here because it wont fetch screenshots reliably.</strong>
+                            {% endif %}
+                    {% else %}
+                        {# "This functionality needs chrome.." #}
+                        {{ only_playwright_type_watches_warning() }}
+                    {% endif %}
                     </div>
                 </fieldset>
             </div>
@@ -588,10 +466,10 @@ keyword") }}
                     {{ render_button(form.save_button) }}
                     <a href="{{url_for('ui.form_delete', uuid=uuid)}}"
                        class="pure-button button-small button-error ">Delete</a>
-                    <a href="{{url_for('ui.clear_watch_history', uuid=uuid)}}"
-                       class="pure-button button-small button-error ">Clear History</a>
+                    {% if watch.history_n %}<a href="{{url_for('ui.clear_watch_history', uuid=uuid)}}"
+                       class="pure-button button-small button-error ">Clear History</a>{% endif %}
                     <a href="{{url_for('ui.form_clone', uuid=uuid)}}"
-                       class="pure-button button-small ">Create Copy</a>
+                       class="pure-button button-small ">Clone &amp; Edit</a>
                 </div>
             </div>
         </form>

changedetectionio/templates/edit/include_subtract.html

@@ -0,0 +1,55 @@
+                    <div class="pure-control-group">
+                        {% set field = render_field(form.include_filters,
+                            rows=5,
+                            placeholder=has_tag_filters_extra+"#example
+xpath://body/div/span[contains(@class, 'example-class')]",
+                            class="m-d")
+                        %}
+                        {{ field }}
+                        {% if '/text()' in  field %}
+                          <span class="pure-form-message-inline"><strong>Note!: //text() function does not work where the &lt;element&gt; contains &lt;![CDATA[]]&gt;</strong></span><br>
+                        {% endif %}
+                        <span class="pure-form-message-inline">One CSS, xPath 1 &amp; 2, JSON Path/JQ selector per line, <i>any</i> rules that matches will be used.<br>
+                        <span data-target="#advanced-help-selectors" class="toggle-show pure-button button-tag button-xsmall">Show advanced help and tips</span><br>
+                    <ul id="advanced-help-selectors" style="display: none;">
+                        <li>CSS - Limit text to this CSS rule, only text matching this CSS rule is included.</li>
+                        <li>JSON - Limit text to this JSON rule, using either <a href="https://pypi.org/project/jsonpath-ng/" target="new">JSONPath</a> or <a href="https://stedolan.github.io/jq/" target="new">jq</a> (if installed).
+                            <ul>
+                                <li>JSONPath: Prefix with <code>json:</code>, use <code>json:$</code> to force re-formatting if required,  <a href="https://jsonpath.com/" target="new">test your JSONPath here</a>.</li>
+                                {% if jq_support %}
+                                <li>jq: Prefix with <code>jq:</code> and <a href="https://jqplay.org/" target="new">test your jq here</a>. Using <a href="https://stedolan.github.io/jq/" target="new">jq</a> allows for complex filtering and processing of JSON data with built-in functions, regex, filtering, and more. See examples and documentation <a href="https://stedolan.github.io/jq/manual/" target="new">here</a>. Prefix <code>jqraw:</code> outputs the results as text instead of a JSON list.</li>
+                                {% else %}
+                                <li>jq support not installed</li>
+                                {% endif %}
+                            </ul>
+                        </li>
+                        <li>XPath - Limit text to this XPath rule, simply start with a forward-slash. To specify XPath to be used explicitly or the XPath rule starts with an XPath function: Prefix with <code>xpath:</code>
+                            <ul>
+                                <li>Example:  <code>//*[contains(@class, 'sametext')]</code> or <code>xpath:count(//*[contains(@class, 'sametext')])</code>, <a
+                                href="http://xpather.com/" target="new">test your XPath here</a></li>
+                                <li>Example: Get all titles from an RSS feed <code>//title/text()</code></li>
+                                <li>To use XPath1.0: Prefix with <code>xpath1:</code></li>
+                            </ul>
+                            </li>
+                    <li>
+                        Please be sure that you thoroughly understand how to write CSS, JSONPath, XPath{% if jq_support %}, or jq selector{%endif%} rules before filing an issue on GitHub! <a
+                                href="https://github.com/dgtlmoon/changedetection.io/wiki/CSS-Selector-help">here for more CSS selector help</a>.<br>
+                    </li>
+                    </ul>
+
+                </span>
+                    </div>
+                <fieldset class="pure-control-group">
+                    {{ render_field(form.subtractive_selectors, rows=5, placeholder=has_tag_filters_extra+"header
+footer
+nav
+.stockticker
+//*[contains(text(), 'Advertisement')]") }}
+                    <span class="pure-form-message-inline">
+                        <ul>
+                          <li> Remove HTML element(s) by CSS and XPath selectors before text conversion. </li>
+                          <li> Don't paste HTML here, use only CSS and XPath selectors </li>
+                          <li> Add multiple elements, CSS or XPath selectors per line to ignore multiple parts of the HTML. </li>
+                        </ul>
+                      </span>
+                </fieldset>

changedetectionio/templates/edit/text-options.html

@@ -0,0 +1,69 @@
+
+                <fieldset>
+                    <div class="pure-control-group">
+                        {{ render_field(form.trigger_text, rows=5, placeholder="Some text to wait for in a line
+/some.regex\d{2}/ for case-INsensitive regex
+") }}
+                        <span class="pure-form-message-inline">
+                    <ul>
+                        <li>Text to wait for before triggering a change/notification, all text and regex are tested <i>case-insensitive</i>.</li>
+                        <li>Trigger text is processed from the result-text that comes out of any CSS/JSON Filters for this watch</li>
+                        <li>Each line is processed separately (think of each line as "OR")</li>
+                        <li>Note: Wrap in forward slash / to use regex  example: <code>/foo\d/</code></li>
+                    </ul>
+                        </span>
+                    </div>
+                </fieldset>
+                <fieldset class="pure-group">
+                    {{ render_field(form.ignore_text, rows=5, placeholder="Some text to ignore in a line
+/some.regex\d{2}/ for case-INsensitive regex
+") }}
+                    <span class="pure-form-message-inline">
+                        <ul>
+                            <li>Matching text will be <strong>ignored</strong> in the text snapshot (you can still see it but it wont trigger a change)</li>
+                            <li>Each line processed separately, any line matching will be ignored (removed before creating the checksum)</li>
+                            <li>Regular Expression support, wrap the entire line in forward slash <code>/regex/</code></li>
+                            <li>Changing this will affect the comparison checksum which may trigger an alert</li>
+                        </ul>
+                </span>
+
+                </fieldset>
+
+                <fieldset>
+                    <div class="pure-control-group">
+                        {{ render_field(form.text_should_not_be_present, rows=5, placeholder="For example: Out of stock
+Sold out
+Not in stock
+Unavailable") }}
+                        <span class="pure-form-message-inline">
+                            <ul>
+                                <li>Block change-detection while this text is on the page, all text and regex are tested <i>case-insensitive</i>, good for waiting for when a product is available again</li>
+                                <li>Block text is processed from the result-text that comes out of any CSS/JSON Filters for this watch</li>
+                                <li>All lines here must not exist (think of each line as "OR")</li>
+                                <li>Note: Wrap in forward slash / to use regex  example: <code>/foo\d/</code></li>
+                            </ul>
+                        </span>
+                    </div>
+                </fieldset>
+                <fieldset>
+                    <div class="pure-control-group">
+                        {{ render_field(form.extract_text, rows=5, placeholder="/.+?\d+ comments.+?/
+ or
+keyword") }}
+                        <span class="pure-form-message-inline">
+                    <ul>
+                        <li>Extracts text in the final output (line by line) after other filters using regular expressions or string match;
+                            <ul>
+                                <li>Regular expression &dash; example <code>/reports.+?2022/i</code></li>
+                                <li>Don't forget to consider the white-space at the start of a line <code>/.+?reports.+?2022/i</code></li>
+                                <li>Use <code>//(?aiLmsux))</code> type flags (more <a href="https://docs.python.org/3/library/re.html#index-15">information here</a>)<br></li>
+                                <li>Keyword example &dash; example <code>Out of stock</code></li>
+                                <li>Use groups to extract just that text &dash; example <code>/reports.+?(\d+)/i</code> returns a list of years only</li>
+                                <li>Example - match lines containing a keyword <code>/.*icecream.*/</code></li>
+                            </ul>
+                        </li>
+                        <li>One line per regular-expression/string match</li>
+                    </ul>
+                        </span>
+                    </div>
+                </fieldset>

changedetectionio/tests/apprise/test_apprise_asset.py

@@ -0,0 +1,24 @@
+import pytest
+from apprise import AppriseAsset
+
+from changedetectionio.apprise_asset import (
+    APPRISE_APP_DESC,
+    APPRISE_APP_ID,
+    APPRISE_APP_URL,
+    APPRISE_AVATAR_URL,
+)
+
+
+@pytest.fixture(scope="function")
+def apprise_asset() -> AppriseAsset:
+    from changedetectionio.apprise_asset import apprise_asset
+
+    return apprise_asset
+
+
+def test_apprise_asset_init(apprise_asset: AppriseAsset):
+    assert isinstance(apprise_asset, AppriseAsset)
+    assert apprise_asset.app_id == APPRISE_APP_ID
+    assert apprise_asset.app_desc == APPRISE_APP_DESC
+    assert apprise_asset.app_url == APPRISE_APP_URL
+    assert apprise_asset.image_url_logo == APPRISE_AVATAR_URL

changedetectionio/tests/apprise/test_apprise_custom_api_call.py

@@ -0,0 +1,211 @@
+import json
+from unittest.mock import patch
+
+import pytest
+import requests
+from apprise.utils.parse import parse_url as apprise_parse_url
+
+from ...apprise_plugin.custom_handlers import (
+    _get_auth,
+    _get_headers,
+    _get_params,
+    apprise_http_custom_handler,
+    SUPPORTED_HTTP_METHODS,
+)
+
+
+@pytest.mark.parametrize(
+    "url,expected_auth",
+    [
+        ("get://user:pass@localhost:9999", ("user", "pass")),
+        ("get://user@localhost:9999", "user"),
+        ("get://localhost:9999", ""),
+        ("get://user%20name:pass%20word@localhost:9999", ("user name", "pass word")),
+    ],
+)
+def test_get_auth(url, expected_auth):
+    """Test authentication extraction with various URL formats."""
+    parsed_url = apprise_parse_url(url)
+    assert _get_auth(parsed_url) == expected_auth
+
+
+@pytest.mark.parametrize(
+    "url,body,expected_content_type",
+    [
+        (
+            "get://localhost:9999?+content-type=application/xml",
+            "test",
+            "application/xml",
+        ),
+        ("get://localhost:9999", '{"key": "value"}', "application/json; charset=utf-8"),
+        ("get://localhost:9999", "plain text", None),
+        ("get://localhost:9999?+content-type=text/plain", "test", "text/plain"),
+    ],
+)
+def test_get_headers(url, body, expected_content_type):
+    """Test header extraction and content type detection."""
+    parsed_url = apprise_parse_url(url)
+    headers = _get_headers(parsed_url, body)
+
+    if expected_content_type:
+        assert headers.get("Content-Type") == expected_content_type
+
+
+@pytest.mark.parametrize(
+    "url,expected_params",
+    [
+        ("get://localhost:9999?param1=value1", {"param1": "value1"}),
+        ("get://localhost:9999?param1=value1&-param2=ignored", {"param1": "value1"}),
+        ("get://localhost:9999?param1=value1&+header=test", {"param1": "value1"}),
+        (
+            "get://localhost:9999?encoded%20param=encoded%20value",
+            {"encoded param": "encoded value"},
+        ),
+    ],
+)
+def test_get_params(url, expected_params):
+    """Test parameter extraction with URL encoding and exclusion logic."""
+    parsed_url = apprise_parse_url(url)
+    params = _get_params(parsed_url)
+    assert dict(params) == expected_params
+
+
+@pytest.mark.parametrize(
+    "url,schema,method",
+    [
+        ("get://localhost:9999", "get", "GET"),
+        ("post://localhost:9999", "post", "POST"),
+        ("delete://localhost:9999", "delete", "DELETE"),
+    ],
+)
+@patch("requests.request")
+def test_apprise_custom_api_call_success(mock_request, url, schema, method):
+    """Test successful API calls with different HTTP methods and schemas."""
+    mock_request.return_value.raise_for_status.return_value = None
+
+    meta = {"url": url, "schema": schema}
+    result = apprise_http_custom_handler(
+        body="test body", title="Test Title", notify_type="info", meta=meta
+    )
+
+    assert result is True
+    mock_request.assert_called_once()
+
+    call_args = mock_request.call_args
+    assert call_args[1]["method"] == method.upper()
+    assert call_args[1]["url"].startswith("http")
+
+
+@patch("requests.request")
+def test_apprise_custom_api_call_with_auth(mock_request):
+    """Test API call with authentication."""
+    mock_request.return_value.raise_for_status.return_value = None
+
+    url = "get://user:pass@localhost:9999/secure"
+    meta = {"url": url, "schema": "get"}
+
+    result = apprise_http_custom_handler(
+        body=json.dumps({"key": "value"}),
+        title="Secure Test",
+        notify_type="info",
+        meta=meta,
+    )
+
+    assert result is True
+    mock_request.assert_called_once()
+    call_args = mock_request.call_args
+    assert call_args[1]["auth"] == ("user", "pass")
+
+
+@pytest.mark.parametrize(
+    "exception_type,expected_result",
+    [
+        (requests.RequestException, False),
+        (requests.HTTPError, False),
+        (Exception, False),
+    ],
+)
+@patch("requests.request")
+def test_apprise_custom_api_call_failure(mock_request, exception_type, expected_result):
+    """Test various failure scenarios."""
+    url = "get://localhost:9999/error"
+    meta = {"url": url, "schema": "get"}
+
+    # Simulate different types of exceptions
+    mock_request.side_effect = exception_type("Error occurred")
+
+    result = apprise_http_custom_handler(
+        body="error body", title="Error Test", notify_type="error", meta=meta
+    )
+
+    assert result == expected_result
+
+
+def test_invalid_url_parsing():
+    """Test handling of invalid URL parsing."""
+    meta = {"url": "invalid://url", "schema": "invalid"}
+    result = apprise_http_custom_handler(
+        body="test", title="Invalid URL", notify_type="info", meta=meta
+    )
+
+    assert result is False
+
+
+@pytest.mark.parametrize(
+    "schema,expected_method",
+    [
+        (http_method, http_method.upper())
+        for http_method in SUPPORTED_HTTP_METHODS
+    ],
+)
+@patch("requests.request")
+def test_http_methods(mock_request, schema, expected_method):
+    """Test all supported HTTP methods."""
+    mock_request.return_value.raise_for_status.return_value = None
+
+    url = f"{schema}://localhost:9999"
+
+    result = apprise_http_custom_handler(
+        body="test body",
+        title="Test Title",
+        notify_type="info",
+        meta={"url": url, "schema": schema},
+    )
+
+    assert result is True
+    mock_request.assert_called_once()
+
+    call_args = mock_request.call_args
+    assert call_args[1]["method"] == expected_method
+
+
+@pytest.mark.parametrize(
+    "input_schema,expected_method",
+    [
+        (f"{http_method}s", http_method.upper())
+        for http_method in SUPPORTED_HTTP_METHODS
+    ],
+)
+@patch("requests.request")
+def test_https_method_conversion(
+    mock_request, input_schema, expected_method
+):
+    """Validate that methods ending with 's' use HTTPS and correct HTTP method."""
+    mock_request.return_value.raise_for_status.return_value = None
+
+    url = f"{input_schema}://localhost:9999"
+
+    result = apprise_http_custom_handler(
+        body="test body",
+        title="Test Title",
+        notify_type="info",
+        meta={"url": url, "schema": input_schema},
+    )
+
+    assert result is True
+    mock_request.assert_called_once()
+
+    call_args = mock_request.call_args
+
+    assert call_args[1]["method"] == expected_method
+    assert call_args[1]["url"].startswith("https")

changedetectionio/tests/proxy_list/test_select_custom_proxy.py

@@ -36,7 +36,7 @@ def test_select_custom(client, live_server, measure_memory_usage):
     assert b"1 Imported" in res.data
     wait_for_all_checks(client)
 
-    res = client.get(url_for("index"))
+    res = client.get(url_for("watchlist.index"))
     assert b'Proxy Authentication Required' not in res.data
 
     res = client.get(

changedetectionio/tests/restock/test_restock.py

@@ -83,14 +83,14 @@ def test_restock_detection(client, live_server, measure_memory_usage):
 
     # Is it correctly show as NOT in stock?
     wait_for_all_checks(client)
-    res = client.get(url_for("index"))
+    res = client.get(url_for("watchlist.index"))
     assert b'not-in-stock' in res.data
 
     # Is it correctly shown as in stock
     set_back_in_stock_response()
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
-    res = client.get(url_for("index"))
+    res = client.get(url_for("watchlist.index"))
     assert b'not-in-stock' not in res.data
 
     # We should have a notification
@@ -107,6 +107,6 @@ def test_restock_detection(client, live_server, measure_memory_usage):
     assert not os.path.isfile("test-datastore/notification.txt"), "No notification should have fired when it went OUT OF STOCK by default"
 
     # BUT we should see that it correctly shows "not in stock"
-    res = client.get(url_for("index"))
+    res = client.get(url_for("watchlist.index"))
     assert b'not-in-stock' in res.data, "Correctly showing NOT IN STOCK in the list after it changed from IN STOCK"
 

changedetectionio/tests/smtp/test_notification_smtp.py

@@ -25,7 +25,6 @@ def test_setup(live_server):
 
 def get_last_message_from_smtp_server():
     import socket
-    global smtp_test_server
     port = 11080  # socket server port number
 
     client_socket = socket.socket()  # instantiate
@@ -44,7 +43,6 @@ def test_check_notification_email_formats_default_HTML(client, live_server, meas
     # live_server_setup(live_server)
     set_original_response()
 
-    global smtp_test_server
     notification_url = f'mailto://changedetection@{smtp_test_server}:11025/?to=fff@home.com'
 
     #####################
@@ -99,7 +97,6 @@ def test_check_notification_email_formats_default_Text_override_HTML(client, liv
     # https://github.com/caronc/apprise/issues/633
 
     set_original_response()
-    global smtp_test_server
     notification_url = f'mailto://changedetection@{smtp_test_server}:11025/?to=fff@home.com'
     notification_body = f"""<!DOCTYPE html>
 <html lang="en">

changedetectionio/tests/test_access_control.py

@@ -1,4 +1,4 @@
-from .util import live_server_setup
+from .util import live_server_setup, wait_for_all_checks
 from flask import url_for
 import time
 
@@ -44,7 +44,7 @@ def test_check_access_control(app, client, live_server):
         assert b"Password protection enabled." in res.data
 
         # Check we hit the login
-        res = c.get(url_for("index"), follow_redirects=True)
+        res = c.get(url_for("watchlist.index"), follow_redirects=True)
         # Should be logged out
         assert b"Login" in res.data
 
@@ -52,6 +52,19 @@ def test_check_access_control(app, client, live_server):
         res = c.get(url_for("ui.ui_views.diff_history_page", uuid="first"))
         assert b'Random content' in res.data
 
+        # access to assets should work (check_authentication)
+        res = c.get(url_for('static_content', group='js', filename='jquery-3.6.0.min.js'))
+        assert res.status_code == 200
+        res = c.get(url_for('static_content', group='styles', filename='styles.css'))
+        assert res.status_code == 200
+        res = c.get(url_for('static_content', group='styles', filename='404-testetest.css'))
+        assert res.status_code == 404
+
+        # Access to screenshots should be limited by 'shared_diff_access'
+        path = url_for('static_content', group='screenshot', filename='random-uuid-that-will-404.png', _external=True)
+        res = c.get(path)
+        assert res.status_code == 404
+
         # Check wrong password does not let us in
         res = c.post(
             url_for("login"),
@@ -155,7 +168,7 @@ def test_check_access_control(app, client, live_server):
             url_for("settings.settings_page"),
             data={"application-password": "foobar",
                   # Should be disabled
-#                  "application-shared_diff_access": "True",
+                  "application-shared_diff_access": "",
                   "requests-time_between_check-minutes": 180,
                   'application-fetch_backend': "html_requests"},
             follow_redirects=True
@@ -164,10 +177,14 @@ def test_check_access_control(app, client, live_server):
         assert b"Password protection enabled." in res.data
 
         # Check we hit the login
-        res = c.get(url_for("index"), follow_redirects=True)
+        res = c.get(url_for("watchlist.index"), follow_redirects=True)
         # Should be logged out
         assert b"Login" in res.data
 
+        # Access to screenshots should be limited by 'shared_diff_access'
+        res = c.get(url_for('static_content', group='screenshot', filename='random-uuid-that-will-403.png'))
+        assert res.status_code == 403
+
         # The diff page should return something valid when logged out
         res = c.get(url_for("ui.ui_views.diff_history_page", uuid="first"))
         assert b'Random content' not in res.data

changedetectionio/tests/test_add_replace_remove_filter.py

@@ -72,7 +72,7 @@ def test_check_removed_line_contains_trigger(client, live_server, measure_memory
     res = client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     assert b'Queued 1 watch for rechecking.' in res.data
     wait_for_all_checks(client)
-    res = client.get(url_for("index"))
+    res = client.get(url_for("watchlist.index"))
     assert b'unviewed' not in res.data
 
     # The trigger line is REMOVED,  this should trigger
@@ -81,7 +81,7 @@ def test_check_removed_line_contains_trigger(client, live_server, measure_memory
     # Check in the processor here what's going on, its triggering empty-reply and no change.
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
-    res = client.get(url_for("index"))
+    res = client.get(url_for("watchlist.index"))
     assert b'unviewed' in res.data
 
 
@@ -90,14 +90,14 @@ def test_check_removed_line_contains_trigger(client, live_server, measure_memory
     set_original(excluding=None)
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
-    res = client.get(url_for("index"))
+    res = client.get(url_for("watchlist.index"))
     assert b'unviewed' not in res.data
 
     # Remove it again, and we should get a trigger
     set_original(excluding='The golden line')
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
-    res = client.get(url_for("index"))
+    res = client.get(url_for("watchlist.index"))
     assert b'unviewed' in res.data
 
     res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True)
@@ -157,14 +157,14 @@ def test_check_add_line_contains_trigger(client, live_server, measure_memory_usa
     assert b'Queued 1 watch for rechecking.' in res.data
 
     wait_for_all_checks(client)
-    res = client.get(url_for("index"))
+    res = client.get(url_for("watchlist.index"))
     assert b'unviewed' not in res.data
 
     # The trigger line is ADDED,  this should trigger
     set_original(add_line='<p>Oh yes please</p>')
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
-    res = client.get(url_for("index"))
+    res = client.get(url_for("watchlist.index"))
 
     assert b'unviewed' in res.data
 

changedetectionio/tests/test_api.py

@@ -383,7 +383,7 @@ def test_api_import(client, live_server, measure_memory_usage):
 
     assert res.status_code == 200
     assert len(res.json) == 2
-    res = client.get(url_for("index"))
+    res = client.get(url_for("watchlist.index"))
     assert b"https://website1.com" in res.data
     assert b"https://website2.com" in res.data
 

changedetectionio/tests/test_api_notifications.py

@@ -0,0 +1,108 @@
+#!/usr/bin/env python3
+
+from flask import url_for
+from .util import live_server_setup
+import json
+
+def test_api_notifications_crud(client, live_server):
+    live_server_setup(live_server)
+    api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')
+
+    # Confirm notifications are initially empty
+    res = client.get(
+        url_for("notifications"),
+        headers={'x-api-key': api_key}
+    )
+    assert res.status_code == 200
+    assert res.json == {"notification_urls": []}
+
+    # Add notification URLs
+    test_urls = ["posts://example.com/notify1", "posts://example.com/notify2"]
+    res = client.post(
+        url_for("notifications"),
+        data=json.dumps({"notification_urls": test_urls}),
+        headers={'content-type': 'application/json', 'x-api-key': api_key}
+    )
+    assert res.status_code == 201
+    for url in test_urls:
+        assert url in res.json["notification_urls"]
+
+    # Confirm the notification URLs were added
+    res = client.get(
+        url_for("notifications"),
+        headers={'x-api-key': api_key}
+    )
+    assert res.status_code == 200
+    for url in test_urls:
+        assert url in res.json["notification_urls"]
+
+    # Delete one notification URL
+    res = client.delete(
+        url_for("notifications"),
+        data=json.dumps({"notification_urls": [test_urls[0]]}),
+        headers={'content-type': 'application/json', 'x-api-key': api_key}
+    )
+    assert res.status_code == 204
+
+    # Confirm it was removed and the other remains
+    res = client.get(
+        url_for("notifications"),
+        headers={'x-api-key': api_key}
+    )
+    assert res.status_code == 200
+    assert test_urls[0] not in res.json["notification_urls"]
+    assert test_urls[1] in res.json["notification_urls"]
+
+    # Try deleting a non-existent URL
+    res = client.delete(
+        url_for("notifications"),
+        data=json.dumps({"notification_urls": ["posts://nonexistent.com"]}),
+        headers={'content-type': 'application/json', 'x-api-key': api_key}
+    )
+    assert res.status_code == 400
+
+    res = client.post(
+        url_for("notifications"),
+        data=json.dumps({"notification_urls": test_urls}),
+        headers={'content-type': 'application/json', 'x-api-key': api_key}
+    )
+    assert res.status_code == 201
+
+    # Replace with a new list
+    replacement_urls = ["posts://new.example.com"]
+    res = client.put(
+        url_for("notifications"),
+        data=json.dumps({"notification_urls": replacement_urls}),
+        headers={'content-type': 'application/json', 'x-api-key': api_key}
+    )
+    assert res.status_code == 200
+    assert res.json["notification_urls"] == replacement_urls
+
+    # Replace with an empty list
+    res = client.put(
+        url_for("notifications"),
+        data=json.dumps({"notification_urls": []}),
+        headers={'content-type': 'application/json', 'x-api-key': api_key}
+    )
+    assert res.status_code == 200
+    assert res.json["notification_urls"] == []
+
+    # Provide an invalid AppRise URL to trigger validation error
+    invalid_urls = ["ftp://not-app-rise"]
+    res = client.post(
+        url_for("notifications"),
+        data=json.dumps({"notification_urls": invalid_urls}),
+        headers={'content-type': 'application/json', 'x-api-key': api_key}
+    )
+    assert res.status_code == 400
+    assert "is not a valid AppRise URL." in res.data.decode()
+
+    res = client.put(
+        url_for("notifications"),
+        data=json.dumps({"notification_urls": invalid_urls}),
+        headers={'content-type': 'application/json', 'x-api-key': api_key}
+    )
+    assert res.status_code == 400
+    assert "is not a valid AppRise URL." in res.data.decode()
+
+    

changedetectionio/tests/test_api_search.py

@@ -0,0 +1,101 @@
+from copy import copy
+
+from flask import url_for
+import json
+import time
+from .util import live_server_setup, wait_for_all_checks
+
+
+def test_api_search(client, live_server):
+    live_server_setup(live_server)
+    api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')
+
+    watch_data = {}
+    # Add some test watches
+    urls = [
+        'https://example.com/page1',
+        'https://example.org/testing',
+        'https://test-site.com/example'
+    ]
+
+    # Import the test URLs
+    res = client.post(
+        url_for("imports.import_page"),
+        data={"urls": "\r\n".join(urls)},
+        follow_redirects=True
+    )
+    assert b"3 Imported" in res.data
+    wait_for_all_checks(client)
+
+    # Get a listing, it will be the first one
+    watches_response = client.get(
+        url_for("createwatch"),
+        headers={'x-api-key': api_key}
+    )
+
+
+    # Add a title to one watch for title search testing
+    for uuid, watch in watches_response.json.items():
+
+        watch_data = client.get(url_for("watch", uuid=uuid),
+                                follow_redirects=True,
+                                headers={'x-api-key': api_key}
+                                )
+
+        if urls[0] == watch_data.json['url']:
+            # HTTP PUT ( UPDATE an existing watch )
+            client.put(
+                url_for("watch", uuid=uuid),
+                headers={'x-api-key': api_key, 'content-type': 'application/json'},
+                data=json.dumps({'title': 'Example Title Test'}),
+            )
+
+    # Test search by URL
+    res = client.get(url_for("search")+"?q=https://example.com/page1", headers={'x-api-key': api_key, 'content-type': 'application/json'})
+    assert len(res.json) == 1
+    assert list(res.json.values())[0]['url'] == urls[0]
+
+    # Test search by URL - partial should NOT match without ?partial=true flag
+    res = client.get(url_for("search")+"?q=https://example", headers={'x-api-key': api_key, 'content-type': 'application/json'})
+    assert len(res.json) == 0
+
+
+    # Test search by title
+    res = client.get(url_for("search")+"?q=Example Title Test", headers={'x-api-key': api_key, 'content-type': 'application/json'})
+    assert len(res.json) == 1
+    assert list(res.json.values())[0]['url'] == urls[0]
+    assert list(res.json.values())[0]['title'] == 'Example Title Test'
+
+    # Test search that should return multiple results (partial = true)
+    res = client.get(url_for("search")+"?q=https://example&partial=true", headers={'x-api-key': api_key, 'content-type': 'application/json'})
+    assert len(res.json) == 2
+
+    # Test empty search
+    res = client.get(url_for("search")+"?q=", headers={'x-api-key': api_key, 'content-type': 'application/json'})
+    assert res.status_code == 400
+
+    # Add a tag to test search with tag filter
+    tag_name = 'test-tag'
+    res = client.post(
+        url_for("tag"),
+        data=json.dumps({"title": tag_name}),
+        headers={'content-type': 'application/json', 'x-api-key': api_key}
+    )
+    assert res.status_code == 201
+    tag_uuid = res.json['uuid']
+
+    # Add the tag to one watch
+    for uuid, watch in watches_response.json.items():
+        if urls[2] == watch['url']:
+            client.put(
+                url_for("watch", uuid=uuid),
+                headers={'x-api-key': api_key, 'content-type': 'application/json'},
+                data=json.dumps({'tags': [tag_uuid]}),
+            )
+
+
+    # Test search with tag filter and q
+    res = client.get(url_for("search") + f"?q={urls[2]}&tag={tag_name}", headers={'x-api-key': api_key, 'content-type': 'application/json'})
+    assert len(res.json) == 1
+    assert list(res.json.values())[0]['url'] == urls[2]
+

changedetectionio/tests/test_automatic_follow_ldjson_price.py

@@ -95,7 +95,7 @@ def test_check_ldjson_price_autodetect(client, live_server, measure_memory_usage
     wait_for_all_checks(client)
 
     # Should get a notice that it's available
-    res = client.get(url_for("index"))
+    res = client.get(url_for("watchlist.index"))
     assert b'ldjson-price-track-offer' in res.data
 
     # Accept it
@@ -105,7 +105,7 @@ def test_check_ldjson_price_autodetect(client, live_server, measure_memory_usage
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
     # Offer should be gone
-    res = client.get(url_for("index"))
+    res = client.get(url_for("watchlist.index"))
     assert b'Embedded price data' not in res.data
     assert b'tracking-ldjson-price-data' in res.data
 
@@ -136,7 +136,7 @@ def test_check_ldjson_price_autodetect(client, live_server, measure_memory_usage
     )
     assert b"1 Imported" in res.data
     wait_for_all_checks(client)
-    res = client.get(url_for("index"))
+    res = client.get(url_for("watchlist.index"))
     assert b'ldjson-price-track-offer' not in res.data
     
     ##########################################################################################

changedetectionio/tests/test_backend.py

@@ -39,7 +39,7 @@ def test_check_basic_change_detection_functionality(client, live_server, measure
         wait_for_all_checks(client)
 
         # It should report nothing found (no new 'unviewed' class)
-        res = client.get(url_for("index"))
+        res = client.get(url_for("watchlist.index"))
         assert b'unviewed' not in res.data
         assert b'test-endpoint' in res.data
 
@@ -75,7 +75,7 @@ def test_check_basic_change_detection_functionality(client, live_server, measure
     assert b'which has this one new line' in res.data
 
     # Now something should be ready, indicated by having a 'unviewed' class
-    res = client.get(url_for("index"))
+    res = client.get(url_for("watchlist.index"))
     assert b'unviewed' in res.data
 
     # #75, and it should be in the RSS feed
@@ -112,7 +112,7 @@ def test_check_basic_change_detection_functionality(client, live_server, measure
         wait_for_all_checks(client)
 
         # It should report nothing found (no new 'unviewed' class)
-        res = client.get(url_for("index"))
+        res = client.get(url_for("watchlist.index"))
         assert b'unviewed' not in res.data
         assert b'Mark all viewed' not in res.data
         assert b'head title' not in res.data  # Should not be present because this is off by default
@@ -131,7 +131,7 @@ def test_check_basic_change_detection_functionality(client, live_server, measure
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
 
-    res = client.get(url_for("index"))
+    res = client.get(url_for("watchlist.index"))
     assert b'unviewed' in res.data
     assert b'Mark all viewed' in res.data
 
@@ -151,7 +151,7 @@ def test_check_basic_change_detection_functionality(client, live_server, measure
     client.get(url_for("ui.clear_watch_history", uuid=uuid))
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
-    res = client.get(url_for("index"))
+    res = client.get(url_for("watchlist.index"))
     assert b'preview/' in res.data
 
     #

changedetectionio/tests/test_block_while_text_present.py

@@ -107,7 +107,7 @@ def test_check_block_changedetection_text_NOT_present(client, live_server, measu
     wait_for_all_checks(client)
 
     # It should report nothing found (no new 'unviewed' class)
-    res = client.get(url_for("index"))
+    res = client.get(url_for("watchlist.index"))
     assert b'unviewed' not in res.data
     assert b'/test-endpoint' in res.data
 
@@ -120,7 +120,7 @@ def test_check_block_changedetection_text_NOT_present(client, live_server, measu
     wait_for_all_checks(client)
 
     # It should report nothing found (no new 'unviewed' class)
-    res = client.get(url_for("index"))
+    res = client.get(url_for("watchlist.index"))
     assert b'unviewed' not in res.data
     assert b'/test-endpoint' in res.data
 
@@ -129,7 +129,7 @@ def test_check_block_changedetection_text_NOT_present(client, live_server, measu
     set_original_ignore_response()
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
-    res = client.get(url_for("index"))
+    res = client.get(url_for("watchlist.index"))
     assert b'unviewed' not in res.data
 
 
@@ -137,7 +137,7 @@ def test_check_block_changedetection_text_NOT_present(client, live_server, measu
     set_modified_response_minus_block_text()
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
-    res = client.get(url_for("index"))
+    res = client.get(url_for("watchlist.index"))
     assert b'unviewed' in res.data
 
 

changedetectionio/tests/test_clone.py

@@ -2,29 +2,39 @@
 
 import time
 from flask import url_for
-from . util import live_server_setup
+from .util import live_server_setup, wait_for_all_checks
 
 
-
-def test_trigger_functionality(client, live_server, measure_memory_usage):
+def test_clone_functionality(client, live_server, measure_memory_usage):
 
     live_server_setup(live_server)
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write("<html><body>Some content</body></html>")
 
-    # Give the endpoint time to spin up
-    time.sleep(1)
+    test_url = url_for('test_endpoint', _external=True)
 
     # Add our URL to the import page
     res = client.post(
         url_for("imports.import_page"),
-        data={"urls": "https://changedetection.io"},
+        data={"urls": test_url},
         follow_redirects=True
     )
     assert b"1 Imported" in res.data
+    wait_for_all_checks(client)
 
+    # So that we can be sure the same history doesnt carry over
+    time.sleep(1)
 
     res = client.get(
         url_for("ui.form_clone", uuid="first"),
         follow_redirects=True
     )
+    existing_uuids = set()
 
-    assert b"Cloned." in res.data
+    for uuid, watch in live_server.app.config['DATASTORE'].data['watching'].items():
+        new_uuids = set(watch.history.keys())
+        duplicates = existing_uuids.intersection(new_uuids)
+        assert len(duplicates) == 0
+        existing_uuids.update(new_uuids)
+
+    assert b"Cloned" in res.data

changedetectionio/tests/test_conditions.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 import json
-import urllib
+import time
 
 from flask import url_for
 from .util import live_server_setup, wait_for_all_checks
@@ -113,8 +113,9 @@ def test_conditions_with_text_and_number(client, live_server):
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
 
+    time.sleep(2)
     # 75 is > 20 and < 100 and contains "5"
-    res = client.get(url_for("index"))
+    res = client.get(url_for("watchlist.index"))
     assert b'unviewed' in res.data
 
 
@@ -128,7 +129,7 @@ def test_conditions_with_text_and_number(client, live_server):
     wait_for_all_checks(client)
 
     # Should NOT be marked as having changes since not all conditions are met
-    res = client.get(url_for("index"))
+    res = client.get(url_for("watchlist.index"))
     assert b'unviewed' not in res.data
 
     res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True)

changedetectionio/tests/test_css_selector.py

@@ -119,7 +119,7 @@ def test_check_markup_include_filters_restriction(client, live_server, measure_m
 
     # It should have 'unviewed' still
     # Because it should be looking at only that 'sametext' id
-    res = client.get(url_for("index"))
+    res = client.get(url_for("watchlist.index"))
     assert b'unviewed' in res.data
 
 
@@ -218,7 +218,7 @@ def test_filter_is_empty_help_suggestion(client, live_server, measure_memory_usa
 
 
     res = client.get(
-        url_for("index"),
+        url_for("watchlist.index"),
         follow_redirects=True
     )
 
@@ -240,7 +240,7 @@ def test_filter_is_empty_help_suggestion(client, live_server, measure_memory_usa
     wait_for_all_checks(client)
 
     res = client.get(
-        url_for("index"),
+        url_for("watchlist.index"),
         follow_redirects=True
     )
 

changedetectionio/tests/test_element_removal.py

@@ -204,7 +204,7 @@ def test_element_removal_full(client, live_server, measure_memory_usage):
     wait_for_all_checks(client)
 
     # There should not be an unviewed change, as changes should be removed
-    res = client.get(url_for("index"))
+    res = client.get(url_for("watchlist.index"))
     assert b"unviewed" not in res.data
 
 # Re #2752

changedetectionio/tests/test_errorhandling.py

@@ -32,7 +32,7 @@ def _runner_test_http_errors(client, live_server, http_code, expected_text):
     # Give the thread time to pick it up
     wait_for_all_checks(client)
 
-    res = client.get(url_for("index"))
+    res = client.get(url_for("watchlist.index"))
     # no change
     assert b'unviewed' not in res.data
     assert bytes(expected_text.encode('utf-8')) in res.data
@@ -78,7 +78,7 @@ def test_DNS_errors(client, live_server, measure_memory_usage):
     # Give the thread time to pick it up
     wait_for_all_checks(client)
 
-    res = client.get(url_for("index"))
+    res = client.get(url_for("watchlist.index"))
     found_name_resolution_error = b"Temporary failure in name resolution" in res.data or b"Name or service not known" in res.data
     assert found_name_resolution_error
     # Should always record that we tried
@@ -107,7 +107,7 @@ def test_low_level_errors_clear_correctly(client, live_server, measure_memory_us
     wait_for_all_checks(client)
 
     # We should see the DNS error
-    res = client.get(url_for("index"))
+    res = client.get(url_for("watchlist.index"))
     found_name_resolution_error = b"Temporary failure in name resolution" in res.data or b"Name or service not known" in res.data
     assert found_name_resolution_error
 
@@ -122,7 +122,7 @@ def test_low_level_errors_clear_correctly(client, live_server, measure_memory_us
 
     # Now the error should be gone
     wait_for_all_checks(client)
-    res = client.get(url_for("index"))
+    res = client.get(url_for("watchlist.index"))
     found_name_resolution_error = b"Temporary failure in name resolution" in res.data or b"Name or service not known" in res.data
     assert not found_name_resolution_error
 

changedetectionio/tests/test_extract_regex.py

@@ -103,7 +103,7 @@ def test_check_filter_multiline(client, live_server, measure_memory_usage):
     assert b"Updated watch." in res.data
     wait_for_all_checks(client)
 
-    res = client.get(url_for("index"))
+    res = client.get(url_for("watchlist.index"))
 
     # Issue 1828
     assert b'not at the start of the expression' not in res.data
@@ -160,7 +160,7 @@ def test_check_filter_and_regex_extract(client, live_server, measure_memory_usag
     # Give the thread time to pick it up
     wait_for_all_checks(client)
 
-    res = client.get(url_for("index"))
+    res = client.get(url_for("watchlist.index"))
     #issue 1828
     assert b'not at the start of the expression' not in res.data
 
@@ -174,7 +174,7 @@ def test_check_filter_and_regex_extract(client, live_server, measure_memory_usag
 
     # It should have 'unviewed' still
     # Because it should be looking at only that 'sametext' id
-    res = client.get(url_for("index"))
+    res = client.get(url_for("watchlist.index"))
     assert b'unviewed' in res.data
 
     # Check HTML conversion detected and workd

changedetectionio/tests/test_filter_failure_notification.py

@@ -113,7 +113,7 @@ def run_filter_test(client, live_server, content_filter):
         checked += 1
         client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
         wait_for_all_checks(client)
-        res = client.get(url_for("index"))
+        res = client.get(url_for("watchlist.index"))
         assert b'Warning, no filters were found' in res.data
         assert not os.path.isfile("test-datastore/notification.txt")
         time.sleep(1)

changedetectionio/tests/test_group.py

@@ -77,7 +77,7 @@ def test_setup_group_tag(client, live_server, measure_memory_usage):
     )
     assert b"1 Imported" in res.data
 
-    res = client.get(url_for("index"))
+    res = client.get(url_for("watchlist.index"))
     assert b'import-tag' in res.data
     assert b'extra-import-tag' in res.data
 
@@ -90,7 +90,7 @@ def test_setup_group_tag(client, live_server, measure_memory_usage):
 
     wait_for_all_checks(client)
 
-    res = client.get(url_for("index"))
+    res = client.get(url_for("watchlist.index"))
     assert b'Warning, no filters were found' not in res.data
 
     res = client.get(
@@ -255,7 +255,7 @@ def test_limit_tag_ui(client, live_server, measure_memory_usage):
 
     assert b"40 Imported" in res.data
 
-    res = client.get(url_for("index"))
+    res = client.get(url_for("watchlist.index"))
     assert b'test-tag' in res.data
 
     # All should be here
@@ -263,7 +263,7 @@ def test_limit_tag_ui(client, live_server, measure_memory_usage):
 
     tag_uuid = get_UUID_for_tag_name(client, name="test-tag")
 
-    res = client.get(url_for("index", tag=tag_uuid))
+    res = client.get(url_for("watchlist.index", tag=tag_uuid))
 
     # Just a subset should be here
     assert b'test-tag' in res.data
@@ -273,6 +273,7 @@ def test_limit_tag_ui(client, live_server, measure_memory_usage):
     assert b'Deleted' in res.data
     res = client.get(url_for("tags.delete_all"), follow_redirects=True)
     assert b'All tags deleted' in res.data
+
 def test_clone_tag_on_import(client, live_server, measure_memory_usage):
     #live_server_setup(live_server)
     test_url = url_for('test_endpoint', _external=True)
@@ -284,7 +285,7 @@ def test_clone_tag_on_import(client, live_server, measure_memory_usage):
 
     assert b"1 Imported" in res.data
 
-    res = client.get(url_for("index"))
+    res = client.get(url_for("watchlist.index"))
     assert b'test-tag' in res.data
     assert b'another-tag' in res.data
 
@@ -292,6 +293,7 @@ def test_clone_tag_on_import(client, live_server, measure_memory_usage):
     res = client.get(url_for("ui.form_clone", uuid=watch_uuid), follow_redirects=True)
 
     assert b'Cloned' in res.data
+    res = client.get(url_for("watchlist.index"))
     # 2 times plus the top link to tag
     assert res.data.count(b'test-tag') == 3
     assert res.data.count(b'another-tag') == 3
@@ -311,14 +313,15 @@ def test_clone_tag_on_quickwatchform_add(client, live_server, measure_memory_usa
 
     assert b"Watch added" in res.data
 
-    res = client.get(url_for("index"))
+    res = client.get(url_for("watchlist.index"))
     assert b'test-tag' in res.data
     assert b'another-tag' in res.data
 
     watch_uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
     res = client.get(url_for("ui.form_clone", uuid=watch_uuid), follow_redirects=True)
-
     assert b'Cloned' in res.data
+
+    res = client.get(url_for("watchlist.index"))
     # 2 times plus the top link to tag
     assert res.data.count(b'test-tag') == 3
     assert res.data.count(b'another-tag') == 3

changedetectionio/tests/test_ignore_regex_text.py

@@ -32,7 +32,6 @@ def test_strip_regex_text_func():
     ]
 
     stripped_content = html_tools.strip_ignore_text(test_content, ignore_lines)
-
     assert "but 1 lines" in stripped_content
     assert "igNORe-cAse text" not in stripped_content
     assert "but 1234 lines" not in stripped_content
@@ -42,6 +41,46 @@ def test_strip_regex_text_func():
     # Check line number reporting
     stripped_content = html_tools.strip_ignore_text(test_content, ignore_lines, mode="line numbers")
     assert stripped_content == [2, 5, 6, 7, 8, 10]
+    
+    stripped_content = html_tools.strip_ignore_text(test_content, ['/but 1.+5 lines/s'])
+    assert "but 1 lines" not in stripped_content
+    assert "skip 5 lines" not in stripped_content
+    
+    stripped_content = html_tools.strip_ignore_text(test_content, ['/but 1.+5 lines/s'], mode="line numbers")
+    assert stripped_content == [4, 5]
+    
+    stripped_content = html_tools.strip_ignore_text(test_content, ['/.+/s'])
+    assert stripped_content == ""
+    
+    stripped_content = html_tools.strip_ignore_text(test_content, ['/.+/s'], mode="line numbers")
+    assert stripped_content == [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]
+
+    stripped_content = html_tools.strip_ignore_text(test_content, ['/^.+but.+\\n.+lines$/m'])
+    assert "but 1 lines" not in stripped_content
+    assert "skip 5 lines" not in stripped_content
+
+    stripped_content = html_tools.strip_ignore_text(test_content, ['/^.+but.+\\n.+lines$/m'], mode="line numbers")
+    assert stripped_content == [4, 5]
+
+    stripped_content = html_tools.strip_ignore_text(test_content, ['/^.+?\.$/m'])
+    assert "but sometimes we want to remove the lines." not in stripped_content
+    assert "but not always." not in stripped_content
+
+    stripped_content = html_tools.strip_ignore_text(test_content, ['/^.+?\.$/m'], mode="line numbers")
+    assert stripped_content == [2, 11]
+
+    stripped_content = html_tools.strip_ignore_text(test_content, ['/but.+?but/ms'])
+    assert "but sometimes we want to remove the lines." not in stripped_content
+    assert "but 1 lines" not in stripped_content
+    assert "but 1234 lines" not in stripped_content
+    assert "igNORe-cAse text we dont want to keep" not in stripped_content
+    assert "but not always." not in stripped_content
+
+    stripped_content = html_tools.strip_ignore_text(test_content, ['/but.+?but/ms'], mode="line numbers")
+    assert stripped_content == [2, 3, 4, 9, 10, 11]
+
+    stripped_content = html_tools.strip_ignore_text("\n\ntext\n\ntext\n\n", ['/^$/ms'], mode="line numbers")
+    assert stripped_content == [1, 2, 4, 6]
 
     # Check that linefeeds are preserved when there are is no matching ignores
     content = "some text\n\nand other text\n"

changedetectionio/tests/test_ignore_text.py

@@ -127,7 +127,7 @@ def test_check_ignore_text_functionality(client, live_server, measure_memory_usa
     wait_for_all_checks(client)
 
     # It should report nothing found (no new 'unviewed' class)
-    res = client.get(url_for("index"))
+    res = client.get(url_for("watchlist.index"))
     assert b'unviewed' not in res.data
     assert b'/test-endpoint' in res.data
 
@@ -140,7 +140,7 @@ def test_check_ignore_text_functionality(client, live_server, measure_memory_usa
     wait_for_all_checks(client)
 
     # It should report nothing found (no new 'unviewed' class)
-    res = client.get(url_for("index"))
+    res = client.get(url_for("watchlist.index"))
     assert b'unviewed' not in res.data
     assert b'/test-endpoint' in res.data
 
@@ -151,7 +151,7 @@ def test_check_ignore_text_functionality(client, live_server, measure_memory_usa
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
 
-    res = client.get(url_for("index"))
+    res = client.get(url_for("watchlist.index"))
     assert b'unviewed' in res.data
 
     res = client.get(url_for("ui.ui_views.preview_page", uuid="first"))
@@ -214,7 +214,7 @@ def test_check_global_ignore_text_functionality(client, live_server, measure_mem
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
     # It should report nothing found (no new 'unviewed' class), adding random ignore text should not cause a change
-    res = client.get(url_for("index"))
+    res = client.get(url_for("watchlist.index"))
     assert b'unviewed' not in res.data
     assert b'/test-endpoint' in res.data
 #####
@@ -229,7 +229,7 @@ def test_check_global_ignore_text_functionality(client, live_server, measure_mem
     wait_for_all_checks(client)
 
     # It should report nothing found (no new 'unviewed' class)
-    res = client.get(url_for("index"))
+    res = client.get(url_for("watchlist.index"))
 
     assert b'unviewed' not in res.data
     assert b'/test-endpoint' in res.data
@@ -238,7 +238,7 @@ def test_check_global_ignore_text_functionality(client, live_server, measure_mem
     set_modified_original_ignore_response()
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
-    res = client.get(url_for("index"))
+    res = client.get(url_for("watchlist.index"))
     assert b'unviewed' in res.data
 
     res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True)

changedetectionio/tests/test_ignorehyperlinks.py

@@ -114,7 +114,7 @@ def test_render_anchor_tag_content_true(client, live_server, measure_memory_usag
 
     # since the link has changed, and we chose to render anchor tag content,
     # we should detect a change (new 'unviewed' class)
-    res = client.get(url_for("index"))
+    res = client.get(url_for("watchlist.index"))
     assert b"unviewed" in res.data
     assert b"/test-endpoint" in res.data
 

changedetectionio/tests/test_ignorestatuscode.py

@@ -79,7 +79,7 @@ def test_normal_page_check_works_with_ignore_status_code(client, live_server, me
     wait_for_all_checks(client)
 
     # It should report nothing found (no new 'unviewed' class)
-    res = client.get(url_for("index"))
+    res = client.get(url_for("watchlist.index"))
     assert b'unviewed' in res.data
     assert b'/test-endpoint' in res.data
 
@@ -127,6 +127,6 @@ def test_403_page_check_works_with_ignore_status_code(client, live_server, measu
 
     # It should have 'unviewed' still
     # Because it should be looking at only that 'sametext' id
-    res = client.get(url_for("index"))
+    res = client.get(url_for("watchlist.index"))
     assert b'unviewed' in res.data