Merge branch 'master' into updating-js

.github/workflows/test-only.yml

@@ -210,7 +210,7 @@ jobs:
 
       - name: Store container log
         if: always()
-        uses: actions/upload-artifact@v1
+        uses: actions/upload-artifact@v4
         with:
           name: test-cdio-basic-tests-output
           path: output-logs

changedetectionio/blueprint/tags/__init__.py

@@ -11,9 +11,10 @@ def construct_blueprint(datastore: ChangeDetectionStore):
     def tags_overview_page():
         from .form import SingleTag
         add_form = SingleTag(request.form)
+        sorted_tags = sorted(datastore.data['settings']['application'].get('tags').items(), key=lambda x: x[1]['title'])
         output = render_template("groups-overview.html",
                                  form=add_form,
-                                 available_tags=datastore.data['settings']['application'].get('tags', {}),
+                                 available_tags=sorted_tags,
                                  )
 
         return output

changedetectionio/blueprint/tags/templates/groups-overview.html

@@ -40,7 +40,7 @@
                 <td colspan="3">No website organisational tags/groups configured</td>
             </tr>
             {% endif %}
-            {% for uuid, tag in available_tags.items()  %}
+            {% for uuid, tag in available_tags  %}
             <tr id="{{ uuid }}" class="{{ loop.cycle('pure-table-odd', 'pure-table-even') }}">
                 <td class="watch-controls">
                     <a class="link-mute state-{{'on' if tag.notification_muted else 'off'}}" href="{{url_for('tags.mute', uuid=tag.uuid)}}"><img src="{{url_for('static_content', group='images', filename='bell-off.svg')}}" alt="Mute notifications" title="Mute notifications" class="icon icon-mute" ></a>

changedetectionio/content_fetchers/__init__.py

@@ -4,7 +4,7 @@ from distutils.util import strtobool
 from changedetectionio.content_fetchers.exceptions import BrowserStepsStepException
 import os
 
-visualselector_xpath_selectors = 'div,span,form,table,tbody,tr,td,a,p,ul,li,h1,h2,h3,h4, header, footer, section, article, aside, details, main, nav, section, summary'
+visualselector_xpath_selectors = 'div,span,form,table,tbody,tr,td,a,p,ul,li,h1,h2,h3,h4,header,footer,section,article,aside,details,main,nav,section,summary'
 
 # available_fetchers() will scan this implementation looking for anything starting with html_
 # this information is used in the form selections

changedetectionio/content_fetchers/exceptions/__init__.py

@@ -36,6 +36,13 @@ class BrowserConnectError(Exception):
         logger.error(f"Browser connection error {msg}")
         return
 
+class BrowserFetchTimedOut(Exception):
+    msg = ''
+    def __init__(self, msg):
+        self.msg = msg
+        logger.error(f"Browser processing took too long - {msg}")
+        return
+
 class BrowserStepsStepException(Exception):
     def __init__(self, step_n, original_e):
         self.step_n = step_n

changedetectionio/content_fetchers/puppeteer.py

@@ -6,7 +6,7 @@ from urllib.parse import urlparse
 
 from loguru import logger
 from changedetectionio.content_fetchers.base import Fetcher
-from changedetectionio.content_fetchers.exceptions import PageUnloadable, Non200ErrorCodeReceived, EmptyReply, ScreenshotUnavailable, BrowserConnectError
+from changedetectionio.content_fetchers.exceptions import PageUnloadable, Non200ErrorCodeReceived, EmptyReply, BrowserFetchTimedOut, BrowserConnectError
 
 
 class fetcher(Fetcher):
@@ -41,15 +41,15 @@ class fetcher(Fetcher):
                 self.proxy = {'username': parsed.username, 'password': parsed.password}
                 # Add the proxy server chrome start option, the username and password never gets added here
                 # (It always goes in via await self.page.authenticate(self.proxy))
-                import urllib.parse
-                # @todo filter some injection attack?
-                # check /somepath?thisandthat
-                # check scheme when no scheme
-                h = urllib.parse.quote(parsed.scheme + "://") if parsed.scheme else ''
-                h += urllib.parse.quote(f"{parsed.hostname}:{parsed.port}{parsed.path}?{parsed.query}", safe='')
 
+                # @todo filter some injection attack?
+                # check scheme when no scheme
+                proxy_url = parsed.scheme + "://" if parsed.scheme else 'http://'
                 r = "?" if not '?' in self.browser_connection_url else '&'
-                self.browser_connection_url += f"{r}--proxy-server={h}"
+                port = ":"+str(parsed.port) if parsed.port else ''
+                q = "?"+parsed.query if parsed.query else ''
+                proxy_url += f"{parsed.hostname}{port}{parsed.path}{q}"
+                self.browser_connection_url += f"{r}--proxy-server={proxy_url}"
 
     # def screenshot_step(self, step_n=''):
     #     screenshot = self.page.screenshot(type='jpeg', full_page=True, quality=85)
@@ -80,6 +80,7 @@ class fetcher(Fetcher):
 
         from changedetectionio.content_fetchers import visualselector_xpath_selectors
         self.delete_browser_steps_screenshots()
+        extra_wait = int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)) + self.render_extract_delay
 
         from pyppeteer import Pyppeteer
         pyppeteer_instance = Pyppeteer()
@@ -88,7 +89,7 @@ class fetcher(Fetcher):
         # @todo timeout
         try:
             browser = await pyppeteer_instance.connect(browserWSEndpoint=self.browser_connection_url,
-                                                       defaultViewport={"width": 1024, "height": 768}
+                                                       ignoreHTTPSErrors=True
                                                        )
         except websockets.exceptions.InvalidStatusCode as e:
             raise BrowserConnectError(msg=f"Error while trying to connect the browser, Code {e.status_code} (check your access)")
@@ -107,8 +108,8 @@ class fetcher(Fetcher):
         # SOCKS5 with authentication is not supported (yet)
         # https://github.com/microsoft/playwright/issues/10567
         self.page.setDefaultNavigationTimeout(0)
-
+        await self.page.setCacheEnabled(True)
-        if self.proxy:
+        if self.proxy and self.proxy.get('username'):
             # Setting Proxy-Authentication header is deprecated, and doing so can trigger header change errors from Puppeteer
             # https://github.com/puppeteer/puppeteer/issues/676 ?
             # https://help.brightdata.com/hc/en-us/articles/12632549957649-Proxy-Manager-How-to-Guides#h_01HAKWR4Q0AFS8RZTNYWRDFJC2
@@ -123,7 +124,7 @@ class fetcher(Fetcher):
         #            browsersteps_interface.page = self.page
 
         response = await self.page.goto(url, waitUntil="load")
-        self.headers = response.headers
+
 
         if response is None:
             await self.page.close()
@@ -131,6 +132,8 @@ class fetcher(Fetcher):
             logger.warning("Content Fetcher > Response object was none")
             raise EmptyReply(url=url, status_code=None)
 
+        self.headers = response.headers
+
         try:
             if self.webdriver_js_execute_code is not None and len(self.webdriver_js_execute_code):
                 await self.page.evaluate(self.webdriver_js_execute_code)
@@ -142,9 +145,6 @@ class fetcher(Fetcher):
             # This can be ok, we will try to grab what we could retrieve
             raise PageUnloadable(url=url, status_code=None, message=str(e))
 
-        extra_wait = int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)) + self.render_extract_delay
-        await asyncio.sleep(1 + extra_wait)
-
         try:
             self.status_code = response.status
         except Exception as e:
@@ -221,14 +221,21 @@ class fetcher(Fetcher):
     def run(self, url, timeout, request_headers, request_body, request_method, ignore_status_codes=False,
             current_include_filters=None, is_binary=False):
 
+        #@todo make update_worker async which could run any of these content_fetchers within memory and time constraints
+        max_time = os.getenv('PUPPETEER_MAX_PROCESSING_TIMEOUT_SECONDS', 180)
+
         # This will work in 3.10 but not >= 3.11 because 3.11 wants tasks only
-        asyncio.run(self.main(
+        try:
-            url=url,
+            asyncio.run(asyncio.wait_for(self.main(
-            timeout=timeout,
+                url=url,
-            request_headers=request_headers,
+                timeout=timeout,
-            request_body=request_body,
+                request_headers=request_headers,
-            request_method=request_method,
+                request_body=request_body,
-            ignore_status_codes=ignore_status_codes,
+                request_method=request_method,
-            current_include_filters=current_include_filters,
+                ignore_status_codes=ignore_status_codes,
-            is_binary=is_binary
+                current_include_filters=current_include_filters,
-        ))
+                is_binary=is_binary
+            ), timeout=max_time))
+        except asyncio.TimeoutError:
+            raise(BrowserFetchTimedOut(msg=f"Browser connected but was unable to process the page in {max_time} seconds."))
+

changedetectionio/content_fetchers/res/stock-not-in-stock.js

@@ -10,6 +10,7 @@ function isItemInStock() {
     const outOfStockTexts = [
         ' أخبرني عندما يتوفر',
         '0 in stock',
+        'actuellement indisponible',
         'agotado',
         'article épuisé',
         'artikel zurzeit vergriffen',
@@ -17,6 +18,7 @@ function isItemInStock() {
         'ausverkauft', // sold out
         'available for back order',
         'back-order or out of stock',
+        'back in stock soon',
         'backordered',
         'benachrichtigt mich', // notify me
         'brak na stanie',
@@ -24,6 +26,7 @@ function isItemInStock() {
         'coming soon',
         'currently have any tickets for this',
         'currently unavailable',
+        'dieser artikel ist bald wieder verfügbar',
         'dostępne wkrótce',
         'en rupture de stock',
         'ist derzeit nicht auf lager',
@@ -42,9 +45,9 @@ function isItemInStock() {
         'no tickets available',
         'not available',
         'not currently available',
-        'not in stock',        
+        'not in stock',
         'notify me when available',
-        'notify when available',            
+        'notify when available',
         'não estamos a aceitar encomendas',
         'out of stock',
         'out-of-stock',
@@ -57,15 +60,19 @@ function isItemInStock() {
         'tickets unavailable',
         'tijdelijk uitverkocht',
         'unavailable tickets',
+        'vorbestellung ist bald möglich',
         'we do not currently have an estimate of when this product will be back in stock.',
         'we don\'t know when or if this item will be back in stock.',
         'zur zeit nicht an lager',
         '品切れ',
         '已售完',
+        '已售',
         '품절'
     ];
 
+
     const vh = Math.max(document.documentElement.clientHeight || 0, window.innerHeight || 0);
+
     function getElementBaseText(element) {
         // .textContent can include text from children which may give the wrong results
         // scan only immediate TEXT_NODEs, which will be a child of the element
@@ -76,29 +83,69 @@ function isItemInStock() {
         return text.toLowerCase().trim();
     }
 
-    const negateOutOfStockRegex = new RegExp('([0-9] in stock|add to cart)', 'ig');
+    const negateOutOfStockRegex = new RegExp('^([0-9] in stock|add to cart|in stock)', 'ig');
 
     // The out-of-stock or in-stock-text is generally always above-the-fold
     // and often below-the-fold is a list of related products that may or may not contain trigger text
     // so it's good to filter to just the 'above the fold' elements
     // and it should be atleast 100px from the top to ignore items in the toolbar, sometimes menu items like "Coming soon" exist
-    const elementsToScan = Array.from(document.getElementsByTagName('*')).filter(element => element.getBoundingClientRect().top + window.scrollY <= vh && element.getBoundingClientRect().top + window.scrollY >= 100);
+
+
+// @todo - if it's SVG or IMG, go into image diff mode
+// %ELEMENTS% replaced at injection time because different interfaces use it with different settings
+
+    console.log("Scanning %ELEMENTS%");
+
+    function collectVisibleElements(parent, visibleElements) {
+        if (!parent) return; // Base case: if parent is null or undefined, return
+
+        // Add the parent itself to the visible elements array if it's of the specified types
+        visibleElements.push(parent);
+
+        // Iterate over the parent's children
+        const children = parent.children;
+        for (let i = 0; i < children.length; i++) {
+            const child = children[i];
+            if (
+                child.nodeType === Node.ELEMENT_NODE &&
+                window.getComputedStyle(child).display !== 'none' &&
+                window.getComputedStyle(child).visibility !== 'hidden' &&
+                child.offsetWidth >= 0 &&
+                child.offsetHeight >= 0 &&
+                window.getComputedStyle(child).contentVisibility !== 'hidden'
+            ) {
+                // If the child is an element and is visible, recursively collect visible elements
+                collectVisibleElements(child, visibleElements);
+            }
+        }
+    }
+
+    const elementsToScan = [];
+    collectVisibleElements(document.body, elementsToScan);
 
     var elementText = "";
 
     // REGEXS THAT REALLY MEAN IT'S IN STOCK
     for (let i = elementsToScan.length - 1; i >= 0; i--) {
         const element = elementsToScan[i];
+
+        // outside the 'fold' or some weird text in the heading area
+        // .getBoundingClientRect() was causing a crash in chrome 119, can only be run on contentVisibility != hidden
+        if (element.getBoundingClientRect().top + window.scrollY >= vh || element.getBoundingClientRect().top + window.scrollY <= 100) {
+            continue
+        }
+
         elementText = "";
         if (element.tagName.toLowerCase() === "input") {
-            elementText = element.value.toLowerCase();
+            elementText = element.value.toLowerCase().trim();
         } else {
             elementText = getElementBaseText(element);
         }
 
         if (elementText.length) {
             // try which ones could mean its in stock
-            if (negateOutOfStockRegex.test(elementText)) {
+            if (negateOutOfStockRegex.test(elementText) && !elementText.includes('(0 products)')) {
+                console.log(`Negating/overriding 'Out of Stock' back to "Possibly in stock" found "${elementText}"`)
                 return 'Possibly in stock';
             }
         }
@@ -107,28 +154,34 @@ function isItemInStock() {
     // OTHER STUFF THAT COULD BE THAT IT'S OUT OF STOCK
     for (let i = elementsToScan.length - 1; i >= 0; i--) {
         const element = elementsToScan[i];
-        if (element.offsetWidth > 0 || element.offsetHeight > 0 || element.getClientRects().length > 0) {
+        // outside the 'fold' or some weird text in the heading area
-            elementText = "";
+        // .getBoundingClientRect() was causing a crash in chrome 119, can only be run on contentVisibility != hidden
-            if (element.tagName.toLowerCase() === "input") {
+        if (element.getBoundingClientRect().top + window.scrollY >= vh + 150 || element.getBoundingClientRect().top + window.scrollY <= 100) {
-                elementText = element.value.toLowerCase();
+            continue
-            } else {
+        }
-                elementText = getElementBaseText(element);
+        elementText = "";
-            }
+        if (element.tagName.toLowerCase() === "input") {
+            elementText = element.value.toLowerCase().trim();
+        } else {
+            elementText = getElementBaseText(element);
+        }
 
-            if (elementText.length) {
+        if (elementText.length) {
-                // and these mean its out of stock
+            // and these mean its out of stock
-                for (const outOfStockText of outOfStockTexts) {
+            for (const outOfStockText of outOfStockTexts) {
-                    if (elementText.includes(outOfStockText)) {
+                if (elementText.includes(outOfStockText)) {
-                        return outOfStockText; // item is out of stock
+                    console.log(`Selected 'Out of Stock' - found text "${outOfStockText}" - "${elementText}"`)
-                    }
+                    return outOfStockText; // item is out of stock
                 }
             }
         }
     }
 
+    console.log(`Returning 'Possibly in stock' - cant' find any useful matching text`)
     return 'Possibly in stock'; // possibly in stock, cant decide otherwise.
 }
 
 // returns the element text that makes it think it's out of stock
 return isItemInStock().trim()
 
+

changedetectionio/content_fetchers/res/xpath_element_scraper.js

@@ -16,24 +16,23 @@ try {
 }
 
 
-
 // Include the getXpath script directly, easier than fetching
 function getxpath(e) {
-        var n = e;
+    var n = e;
-        if (n && n.id) return '//*[@id="' + n.id + '"]';
+    if (n && n.id) return '//*[@id="' + n.id + '"]';
-        for (var o = []; n && Node.ELEMENT_NODE === n.nodeType;) {
+    for (var o = []; n && Node.ELEMENT_NODE === n.nodeType;) {
-            for (var i = 0, r = !1, d = n.previousSibling; d;) d.nodeType !== Node.DOCUMENT_TYPE_NODE && d.nodeName === n.nodeName && i++, d = d.previousSibling;
+        for (var i = 0, r = !1, d = n.previousSibling; d;) d.nodeType !== Node.DOCUMENT_TYPE_NODE && d.nodeName === n.nodeName && i++, d = d.previousSibling;
-            for (d = n.nextSibling; d;) {
+        for (d = n.nextSibling; d;) {
-                if (d.nodeName === n.nodeName) {
+            if (d.nodeName === n.nodeName) {
-                    r = !0;
+                r = !0;
-                    break
+                break
-                }
-                d = d.nextSibling
             }
-            o.push((n.prefix ? n.prefix + ":" : "") + n.localName + (i || r ? "[" + (i + 1) + "]" : "")), n = n.parentNode
+            d = d.nextSibling
         }
-        return o.length ? "/" + o.reverse().join("/") : ""
+        o.push((n.prefix ? n.prefix + ":" : "") + n.localName + (i || r ? "[" + (i + 1) + "]" : "")), n = n.parentNode
     }
+    return o.length ? "/" + o.reverse().join("/") : ""
+}
 
 const findUpTag = (el) => {
     let r = el
@@ -59,14 +58,14 @@ const findUpTag = (el) => {
 
     // Strategy 2: Keep going up until we hit an ID tag, imagine it's like  #list-widget div h4
     while (r.parentNode) {
-        if (depth == 5) {
+        if (depth === 5) {
             break;
         }
         if ('' !== r.id) {
             chained_css.unshift("#" + CSS.escape(r.id));
             final_selector = chained_css.join(' > ');
             // Be sure theres only one, some sites have multiples of the same ID tag :-(
-            if (window.document.querySelectorAll(final_selector).length == 1) {
+            if (window.document.querySelectorAll(final_selector).length === 1) {
                 return final_selector;
             }
             return null;
@@ -82,30 +81,60 @@ const findUpTag = (el) => {
 
 // @todo - if it's SVG or IMG, go into image diff mode
 // %ELEMENTS% replaced at injection time because different interfaces use it with different settings
-var elements = window.document.querySelectorAll("%ELEMENTS%");
+
 var size_pos = [];
 // after page fetch, inject this JS
 // build a map of all elements and their positions (maybe that only include text?)
 var bbox;
-for (var i = 0; i < elements.length; i++) {
+console.log("Scanning %ELEMENTS%");
-    bbox = elements[i].getBoundingClientRect();
 
-    // Exclude items that are not interactable or visible
+function collectVisibleElements(parent, visibleElements) {
-    if(elements[i].style.opacity === "0") {
+    if (!parent) return; // Base case: if parent is null or undefined, return
-        continue
+
+
+    // Add the parent itself to the visible elements array if it's of the specified types
+    const tagName = parent.tagName.toLowerCase();
+    if ("%ELEMENTS%".split(',').includes(tagName)) {
+        visibleElements.push(parent);
     }
-    if(elements[i].style.display === "none" || elements[i].style.pointerEvents === "none" ) {
+
-        continue
+    // Iterate over the parent's children
+    const children = parent.children;
+    for (let i = 0; i < children.length; i++) {
+        const child = children[i];
+        if (
+            child.nodeType === Node.ELEMENT_NODE &&
+            window.getComputedStyle(child).display !== 'none' &&
+            window.getComputedStyle(child).visibility !== 'hidden' &&
+            child.offsetWidth >= 0 &&
+            child.offsetHeight >= 0 &&
+            window.getComputedStyle(child).contentVisibility !== 'hidden'
+        ) {
+            // If the child is an element and is visible, recursively collect visible elements
+            collectVisibleElements(child, visibleElements);
+        }
     }
+}
+
+// Create an array to hold the visible elements
+const visibleElementsArray = [];
+
+// Call collectVisibleElements with the starting parent element
+collectVisibleElements(document.body, visibleElementsArray);
+
+
+visibleElementsArray.forEach(function (element) {
+
+    bbox = element.getBoundingClientRect();
 
     // Skip really small ones, and where width or height ==0
-    if (bbox['width'] * bbox['height'] < 100) {
+    if (bbox['width'] * bbox['height'] < 10) {
-        continue;
+        return
     }
 
     // Don't include elements that are offset from canvas
-    if (bbox['top']+scroll_y < 0 || bbox['left'] < 0) {
+    if (bbox['top'] + scroll_y < 0 || bbox['left'] < 0) {
-        continue;
+        return
     }
 
     // @todo the getXpath kind of sucks, it doesnt know when there is for example just one ID sometimes
@@ -114,46 +143,41 @@ for (var i = 0; i < elements.length; i++) {
 
     // 1st primitive - if it has class, try joining it all and select, if theres only one.. well thats us.
     xpath_result = false;
-
     try {
-        var d = findUpTag(elements[i]);
+        var d = findUpTag(element);
         if (d) {
             xpath_result = d;
         }
     } catch (e) {
         console.log(e);
     }
-
     // You could swap it and default to getXpath and then try the smarter one
     // default back to the less intelligent one
     if (!xpath_result) {
         try {
             // I've seen on FB and eBay that this doesnt work
             // ReferenceError: getXPath is not defined at eval (eval at evaluate (:152:29), <anonymous>:67:20) at UtilityScript.evaluate (<anonymous>:159:18) at UtilityScript.<anonymous> (<anonymous>:1:44)
-            xpath_result = getxpath(elements[i]);
+            xpath_result = getxpath(element);
         } catch (e) {
             console.log(e);
-            continue;
+            return
         }
     }
 
-    if (window.getComputedStyle(elements[i]).visibility === "hidden") {
-        continue;
-    }
 
-    // @todo Possible to ONLY list where it's clickable to save JSON xfer size
     size_pos.push({
         xpath: xpath_result,
         width: Math.round(bbox['width']),
         height: Math.round(bbox['height']),
         left: Math.floor(bbox['left']),
-        top: Math.floor(bbox['top'])+scroll_y,
+        top: Math.floor(bbox['top']) + scroll_y,
-        tagName: (elements[i].tagName) ? elements[i].tagName.toLowerCase() : '',
+        tagName: (element.tagName) ? element.tagName.toLowerCase() : '',
-        tagtype: (elements[i].tagName == 'INPUT' && elements[i].type) ? elements[i].type.toLowerCase() : '',
+        tagtype: (element.tagName.toLowerCase() === 'input' && element.type) ? element.type.toLowerCase() : '',
-        isClickable: (elements[i].onclick) || window.getComputedStyle(elements[i]).cursor == "pointer"
+        isClickable: window.getComputedStyle(element).cursor == "pointer"
     });
 
-}
+});
+
 
 // Inject the current one set in the include_filters, which may be a CSS rule
 // used for displaying the current one in VisualSelector, where its not one we generated.
@@ -180,7 +204,7 @@ if (include_filters.length) {
             }
         } catch (e) {
             // Maybe catch DOMException and alert?
-            console.log("xpath_element_scraper: Exception selecting element from filter "+f);
+            console.log("xpath_element_scraper: Exception selecting element from filter " + f);
             console.log(e);
         }
 
@@ -210,8 +234,8 @@ if (include_filters.length) {
                 }
             }
         }
-        
+
-        if(!q) {
+        if (!q) {
             console.log("xpath_element_scraper: filter element " + f + " was not found");
         }
 
@@ -221,7 +245,7 @@ if (include_filters.length) {
                 width: parseInt(bbox['width']),
                 height: parseInt(bbox['height']),
                 left: parseInt(bbox['left']),
-                top: parseInt(bbox['top'])+scroll_y
+                top: parseInt(bbox['top']) + scroll_y
             });
         }
     }
@@ -229,7 +253,7 @@ if (include_filters.length) {
 
 // Sort the elements so we find the smallest one first, in other words, we find the smallest one matching in that area
 // so that we dont select the wrapping element by mistake and be unable to select what we want
-size_pos.sort((a, b) => (a.width*a.height > b.width*b.height) ? 1 : -1)
+size_pos.sort((a, b) => (a.width * a.height > b.width * b.height) ? 1 : -1)
 
 // Window.width required for proper scaling in the frontend
 return {'size_pos': size_pos, 'browser_width': window.innerWidth};

changedetectionio/flask_app.py

@@ -404,17 +404,21 @@ def changedetection_app(config=None, datastore_o=None):
         global datastore
         from changedetectionio import forms
 
-        limit_tag = request.args.get('tag', '').lower().strip()
+        active_tag_req = request.args.get('tag', '').lower().strip()
+        active_tag_uuid = active_tag = None
 
         # Be sure limit_tag is a uuid
-        for uuid, tag in datastore.data['settings']['application'].get('tags', {}).items():
+        if active_tag_req:
-            if limit_tag == tag.get('title', '').lower().strip():
+            for uuid, tag in datastore.data['settings']['application'].get('tags', {}).items():
-                limit_tag = uuid
+                if active_tag_req == tag.get('title', '').lower().strip() or active_tag_req == uuid:
+                    active_tag = tag
+                    active_tag_uuid = uuid
+                    break
 
 
         # Redirect for the old rss path which used the /?rss=true
         if request.args.get('rss'):
-            return redirect(url_for('rss', tag=limit_tag))
+            return redirect(url_for('rss', tag=active_tag_uuid))
 
         op = request.args.get('op')
         if op:
@@ -425,7 +429,7 @@ def changedetection_app(config=None, datastore_o=None):
                 datastore.data['watching'][uuid].toggle_mute()
 
             datastore.needs_write = True
-            return redirect(url_for('index', tag = limit_tag))
+            return redirect(url_for('index', tag = active_tag_uuid))
 
         # Sort by last_changed and add the uuid which is usually the key..
         sorted_watches = []
@@ -436,7 +440,7 @@ def changedetection_app(config=None, datastore_o=None):
             if with_errors and not watch.get('last_error'):
                 continue
 
-            if limit_tag and not limit_tag in watch['tags']:
+            if active_tag_uuid and not active_tag_uuid in watch['tags']:
                     continue
             if watch.get('last_error'):
                 errored_count += 1
@@ -455,11 +459,12 @@ def changedetection_app(config=None, datastore_o=None):
                                 total=total_count,
                                 per_page=datastore.data['settings']['application'].get('pager_size', 50), css_framework="semantic")
 
-
+        sorted_tags = sorted(datastore.data['settings']['application'].get('tags').items(), key=lambda x: x[1]['title'])
         output = render_template(
             "watch-overview.html",
                                  # Don't link to hosting when we're on the hosting environment
-                                 active_tag=limit_tag,
+                                 active_tag=active_tag,
+                                 active_tag_uuid=active_tag_uuid,
                                  app_rss_token=datastore.data['settings']['application']['rss_access_token'],
                                  datastore=datastore,
                                  errored_count=errored_count,
@@ -474,7 +479,7 @@ def changedetection_app(config=None, datastore_o=None):
                                  sort_attribute=request.args.get('sort') if request.args.get('sort') else request.cookies.get('sort'),
                                  sort_order=request.args.get('order') if request.args.get('order') else request.cookies.get('order'),
                                  system_default_fetcher=datastore.data['settings']['application'].get('fetch_backend'),
-                                 tags=datastore.data['settings']['application'].get('tags'),
+                                 tags=sorted_tags,
                                  watches=sorted_watches
                                  )
 

changedetectionio/processors/__init__.py

@@ -75,8 +75,12 @@ class difference_detection_processor():
 
         proxy_url = None
         if preferred_proxy_id:
-            proxy_url = self.datastore.proxy_list.get(preferred_proxy_id).get('url')
+            # Custom browser endpoints should NOT have a proxy added
-            logger.debug(f"Selected proxy key '{preferred_proxy_id}' as proxy URL '{proxy_url}' for {url}")
+            if not prefer_fetch_backend.startswith('extra_browser_'):
+                proxy_url = self.datastore.proxy_list.get(preferred_proxy_id).get('url')
+                logger.debug(f"Selected proxy key '{preferred_proxy_id}' as proxy URL '{proxy_url}' for {url}")
+            else:
+                logger.debug(f"Skipping adding proxy data when custom Browser endpoint is specified. ")
 
         # Now call the fetcher (playwright/requests/etc) with arguments that only a fetcher would need.
         # When browser_connection_url is None, it method should default to working out whats the best defaults (os env vars etc)

changedetectionio/templates/watch-overview.html

@@ -1,6 +1,6 @@
 {% extends 'base.html' %}
 {% block content %}
-{% from '_helpers.jinja' import render_simple_field, render_field, render_nolabel_field %}
+{% from '_helpers.jinja' import render_simple_field, render_field, render_nolabel_field, sort_by_title %}
 <script src="{{url_for('static_content', group='js', filename='jquery-3.6.0.min.js')}}"></script>
 <script src="{{url_for('static_content', group='js', filename='watch-overview.js')}}" defer></script>
 
@@ -13,7 +13,7 @@
             <div id="watch-add-wrapper-zone">
 
                     {{ render_nolabel_field(form.url, placeholder="https://...", required=true) }}
-                    {{ render_nolabel_field(form.tags, value=tags[active_tag].title if active_tag else '', placeholder="watch label / tag") }}
+                    {{ render_nolabel_field(form.tags, value=active_tag.title if active_tag else '', placeholder="watch label / tag") }}
                     {{ render_nolabel_field(form.watch_submit_button, title="Watch this URL!" ) }}
                     {{ render_nolabel_field(form.edit_and_watch_submit_button, title="Edit first then Watch") }}
             </div>
@@ -46,11 +46,13 @@
     {% if search_q %}<div id="search-result-info">Searching "<strong><i>{{search_q}}</i></strong>"</div>{% endif %}
     <div>
         <a href="{{url_for('index')}}" class="pure-button button-tag {{'active' if not active_tag }}">All</a>
-        {% for uuid, tag in tags.items() %}
+
-            {% if tag != "" %}
+    <!-- tag list -->
-                <a href="{{url_for('index', tag=uuid) }}" class="pure-button button-tag {{'active' if active_tag == uuid }}">{{ tag.title }}</a>
+    {% for uuid, tag in tags %}
-            {% endif %}
+        {% if tag != "" %}
-        {% endfor %}
+            <a href="{{url_for('index', tag=uuid) }}" class="pure-button button-tag {{'active' if active_tag_uuid == uuid }}">{{ tag.title }}</a>
+        {% endif %}
+    {% endfor %}
     </div>
 
     {% set sort_order = sort_order or 'asc' %}
@@ -197,8 +199,8 @@
             </li>
             {% endif %}
             <li>
-               <a href="{{ url_for('form_watch_checknow', tag=active_tag, with_errors=request.args.get('with_errors',0)) }}" class="pure-button button-tag ">Recheck
+               <a href="{{ url_for('form_watch_checknow', tag=active_tag_uuid, with_errors=request.args.get('with_errors',0)) }}" class="pure-button button-tag ">Recheck
-                all {% if active_tag%} in "{{tags[active_tag].title}}"{%endif%}</a>
+                all {% if active_tag_uuid %} in "{{active_tag.title}}"{%endif%}</a>
             </li>
             <li>
                 <a href="{{ url_for('rss', tag=active_tag , token=app_rss_token)}}"><img alt="RSS Feed" id="feed-icon" src="{{url_for('static_content', group='images', filename='Generic_Feed-icon.svg')}}" height="15"></a>

changedetectionio/update_worker.py

@@ -369,6 +369,12 @@ class update_worker(threading.Thread):
                                                                 }
                                                     )
                         process_changedetection_results = False
+                    except content_fetchers.exceptions.BrowserFetchTimedOut as e:
+                        self.datastore.update_watch(uuid=uuid,
+                                                    update_obj={'last_error': e.msg
+                                                                }
+                                                    )
+                        process_changedetection_results = False
                     except content_fetchers.exceptions.BrowserStepsStepException as e:
 
                         if not self.datastore.data['watching'].get(uuid):

requirements.txt

@@ -1,7 +1,7 @@
 # Used by Pyppeteer
 pyee
 
-eventlet>=0.33.3 # related to dnspython fixes
+eventlet==0.33.3 # related to dnspython fixes
 feedgen~=0.9
 flask-compress
 # 0.6.3 included compatibility fix for werkzeug 3.x (2.x had deprecation of url handlers)
@@ -22,13 +22,15 @@ validators~=0.21
 brotli~=1.0
 requests[socks]
 
-urllib3>1.26
+urllib3==1.26.18
 chardet>2.3.0
 
 wtforms~=3.0
 jsonpath-ng~=1.5.3
 
-dnspython~=2.4 # related to eventlet fixes
+# Pinned: module 'eventlet.green.select' has no attribute 'epoll'
+# https://github.com/eventlet/eventlet/issues/805#issuecomment-1640463482
+dnspython==2.3.0 # related to eventlet fixes
 
 # jq not available on Windows so must be installed manually