Improve job names in graph

changedetectionio/blueprint/browser_steps/__init__.py

@@ -228,8 +228,10 @@ def construct_blueprint(datastore: ChangeDetectionStore):
                     watch.save_screenshot(screenshot=screenshot)
                     watch.save_xpath_data(data=xpath_data)
 
+        except playwright._impl._api_types.Error as e:
+            return make_response("Browser session ran out of time :( Please reload this page."+str(e), 401)
         except Exception as e:
-            return make_response(f"Error fetching screenshot and element data - {str(e)}", 401)
+            return make_response("Error fetching screenshot and element data - " + str(e), 401)
 
         # SEND THIS BACK TO THE BROWSER
         output = {

changedetectionio/blueprint/browser_steps/browser_steps.py

@@ -1,6 +1,8 @@
 import os
 import time
 import re
+import sys
+import traceback
 from random import randint
 from loguru import logger
 
@@ -90,32 +92,8 @@ class steppable_browser_interface():
         if optional_value and ('{%' in optional_value or '{{' in optional_value):
             optional_value = jinja_render(template_str=optional_value)
 
-        # Trigger click and cautiously handle potential navigation
-        # This means the page redirects/reloads/changes JS etc etc
-        if call_action_name.startswith('click_'):
-            try:
-                # Set up navigation expectation before the click (like sync version)
-                async with self.page.expect_event("framenavigated", timeout=3000) as navigation_info:
-                    await action_handler(selector, optional_value)
-                
-                # Check if navigation actually occurred
-                try:
-                    await navigation_info.value  # This waits for the navigation promise
-                    logger.debug(f"Navigation occurred on {call_action_name}.")
-                except Exception:
-                    logger.debug(f"No navigation occurred within timeout when calling {call_action_name}, that's OK, continuing.")
-                    
-            except Exception as e:
-                # If expect_event itself times out, that means no navigation occurred - that's OK
-                if "framenavigated" in str(e) and "exceeded" in str(e):
-                    logger.debug(f"No navigation occurred within timeout when calling {call_action_name}, that's OK, continuing.")
-                else:
-                    raise e
-        else:
-            # Some other action that probably a navigation is not expected
-            await action_handler(selector, optional_value)
-
 
+        await action_handler(selector, optional_value)
         # Safely wait for timeout
         await self.page.wait_for_timeout(1.5 * 1000)
         logger.debug(f"Call action done in {time.time()-now:.2f}s")
@@ -132,7 +110,7 @@ class steppable_browser_interface():
 
     # Incase they request to go back to the start
     async def action_goto_site(self, selector=None, value=None):
-        return await self.action_goto_url(value=re.sub(r'^source:', '', self.start_url, flags=re.IGNORECASE))
+        return await self.action_goto_url(value=self.start_url)
 
     async def action_click_element_containing_text(self, selector=None, value=''):
         logger.debug("Clicking element containing text")
@@ -450,9 +428,6 @@ class browsersteps_live_ui(steppable_browser_interface):
         try:
             # Get screenshot first
             screenshot = await capture_full_page_async(page=self.page)
-            if not screenshot:
-                logger.error("No screenshot was retrieved :((")
-
             logger.debug(f"Time to get screenshot from browser {time.time() - now:.2f}s")
 
             # Then get interactive elements
@@ -475,12 +450,6 @@ class browsersteps_live_ui(steppable_browser_interface):
             
         except Exception as e:
             logger.error(f"Error getting current state: {str(e)}")
-            # If the page has navigated (common with logins) then the context is destroyed on navigation, continue
-            # I'm not sure that this is required anymore because we have the "expect navigation wrapper" at the top
-            if "Execution context was destroyed" in str(e):
-                logger.debug("Execution context was destroyed, most likely because of navigation, continuing...")
-            pass
-
             # Attempt recovery - force garbage collection
             try:
                 await self.page.request_gc()

changedetectionio/content_fetchers/puppeteer.py

@@ -51,15 +51,7 @@ async def capture_full_page(page):
         await page.setViewport({'width': page.viewport['width'], 'height': step_size})
 
     while y < min(page_height, SCREENSHOT_MAX_TOTAL_HEIGHT):
-        # better than scrollTo incase they override it in the page
-        await page.evaluate(
-            """(y) => {
-                document.documentElement.scrollTop = y;
-                document.body.scrollTop = y;
-            }""",
-            y
-        )
-
+        await page.evaluate(f"window.scrollTo(0, {y})")
         screenshot_chunks.append(await page.screenshot(type_='jpeg',
                                                        fullPage=False,
                                                        quality=int(os.getenv("SCREENSHOT_QUALITY", 72))))
@@ -157,11 +149,7 @@ class fetcher(Fetcher):
                          ):
         import re
         self.delete_browser_steps_screenshots()
-
-        n = int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)) + self.render_extract_delay
-        extra_wait = min(n, 15)
-
-        logger.debug(f"Extra wait set to {extra_wait}s, requested was {n}s.")
+        extra_wait = int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)) + self.render_extract_delay
 
         from pyppeteer import Pyppeteer
         pyppeteer_instance = Pyppeteer()
@@ -177,7 +165,7 @@ class fetcher(Fetcher):
         except websockets.exceptions.InvalidURI:
             raise BrowserConnectError(msg=f"Error connecting to the browser, check your browser connection address (should be ws:// or wss://")
         except Exception as e:
-            raise BrowserConnectError(msg=f"Error connecting to the browser - Exception '{str(e)}'")
+            raise BrowserConnectError(msg=f"Error connecting to the browser {str(e)}")
 
         # Better is to launch chrome with the URL as arg
         # non-headless - newPage() will launch an extra tab/window, .browser should already contain 1 page/tab
@@ -239,35 +227,13 @@ class fetcher(Fetcher):
         #            browsersteps_interface = steppable_browser_interface()
         #            browsersteps_interface.page = self.page
 
-        async def handle_frame_navigation(event):
-            logger.debug(f"Frame navigated: {event}")
-            w = extra_wait - 2 if extra_wait > 4 else 2
-            logger.debug(f"Waiting {w} seconds before calling Page.stopLoading...")
-            await asyncio.sleep(w)
-            logger.debug("Issuing stopLoading command...")
-            await self.page._client.send('Page.stopLoading')
-            logger.debug("stopLoading command sent!")
+        response = await self.page.goto(url, waitUntil="load")
 
-        self.page._client.on('Page.frameStartedNavigating', lambda event: asyncio.create_task(handle_frame_navigation(event)))
-        self.page._client.on('Page.frameStartedLoading', lambda event: asyncio.create_task(handle_frame_navigation(event)))
-        self.page._client.on('Page.frameStoppedLoading', lambda event: logger.debug(f"Frame stopped loading: {event}"))
-
-        response = None
-        attempt=0
-        while not response:
-            logger.debug(f"Attempting page fetch {url} attempt {attempt}")
-            response = await self.page.goto(url)
-            await asyncio.sleep(1 + extra_wait)
-            if response:
-                break
-            if not response:
-                logger.warning("Page did not fetch! trying again!")
-            if response is None and attempt>=2:
-                await self.page.close()
-                await browser.close()
-                logger.warning(f"Content Fetcher > Response object was none (as in, the response from the browser was empty, not just the content) exiting attmpt {attempt}")
-                raise EmptyReply(url=url, status_code=None)
-            attempt+=1
+        if response is None:
+            await self.page.close()
+            await browser.close()
+            logger.warning("Content Fetcher > Response object was none (as in, the response from the browser was empty, not just the content)")
+            raise EmptyReply(url=url, status_code=None)
 
         self.headers = response.headers
 
@@ -310,6 +276,7 @@ class fetcher(Fetcher):
         #            if self.browser_steps_get_valid_steps():
         #                self.iterate_browser_steps()
 
+        await asyncio.sleep(1 + extra_wait)
 
         # So we can find an element on the page where its selector was entered manually (maybe not xPath etc)
         # Setup the xPath/VisualSelector scraper

changedetectionio/content_fetchers/res/stock-not-in-stock.js

@@ -18,7 +18,6 @@ async () => {
             'back-order or out of stock',
             'backordered',
             'benachrichtigt mich', // notify me
-            'binnenkort leverbaar', // coming soon
             'brak na stanie',
             'brak w magazynie',
             'coming soon',
@@ -86,7 +85,6 @@ async () => {
             'tidak tersedia',
             'tijdelijk uitverkocht',
             'tiket tidak tersedia',
-            'to subscribe to back in stock',
             'tükendi',
             'unavailable nearby',
             'unavailable tickets',
@@ -121,7 +119,8 @@ async () => {
             return text.toLowerCase().trim();
         }
 
-        const negateOutOfStockRegex = new RegExp('^([0-9] in stock|add to cart|in stock|arrives approximately)', 'ig');
+        const negateOutOfStockRegex = new RegExp('^([0-9] in stock|add to cart|in stock)', 'ig');
+
         // The out-of-stock or in-stock-text is generally always above-the-fold
         // and often below-the-fold is a list of related products that may or may not contain trigger text
         // so it's good to filter to just the 'above the fold' elements

changedetectionio/custom_queue.py

@@ -3,41 +3,6 @@ import asyncio
 from blinker import signal
 from loguru import logger
 
-
-class NotificationQueue(queue.Queue):
-    """
-    Extended Queue that sends a 'notification_event' signal when notifications are added.
-    
-    This class extends the standard Queue and adds a signal emission after a notification
-    is put into the queue. The signal includes the watch UUID if available.
-    """
-    
-    def __init__(self, maxsize=0):
-        super().__init__(maxsize)
-        try:
-            self.notification_event_signal = signal('notification_event')
-        except Exception as e:
-            logger.critical(f"Exception creating notification_event signal: {e}")
-
-    def put(self, item, block=True, timeout=None):
-        # Call the parent's put method first
-        super().put(item, block, timeout)
-        
-        # After putting the notification in the queue, emit signal with watch UUID
-        try:
-            if self.notification_event_signal and isinstance(item, dict):
-                watch_uuid = item.get('uuid')
-                if watch_uuid:
-                    # Send the notification_event signal with the watch UUID
-                    self.notification_event_signal.send(watch_uuid=watch_uuid)
-                    logger.trace(f"NotificationQueue: Emitted notification_event signal for watch UUID {watch_uuid}")
-                else:
-                    # Send signal without UUID for system notifications
-                    self.notification_event_signal.send()
-                    logger.trace("NotificationQueue: Emitted notification_event signal for system notification")
-        except Exception as e:
-            logger.error(f"Exception emitting notification_event signal: {e}")
-
 class SignalPriorityQueue(queue.PriorityQueue):
     """
     Extended PriorityQueue that sends a signal when items with a UUID are added.

changedetectionio/flask_app.py

@@ -12,7 +12,7 @@ from blinker import signal
 
 from changedetectionio.strtobool import strtobool
 from threading import Event
-from changedetectionio.custom_queue import SignalPriorityQueue, AsyncSignalPriorityQueue, NotificationQueue
+from changedetectionio.custom_queue import SignalPriorityQueue, AsyncSignalPriorityQueue
 from changedetectionio import worker_handler
 
 from flask import (
@@ -52,7 +52,7 @@ extra_stylesheets = []
 
 # Use async queue by default, keep sync for backward compatibility  
 update_q = AsyncSignalPriorityQueue() if worker_handler.USE_ASYNC_WORKERS else SignalPriorityQueue()
-notification_q = NotificationQueue()
+notification_q = queue.Queue()
 MAX_QUEUE_SIZE = 2000
 
 app = Flask(__name__,

changedetectionio/realtime/socket_server.py

@@ -29,11 +29,6 @@ class SignalHandler:
         watch_delete_signal = signal('watch_deleted')
         watch_delete_signal.connect(self.handle_deleted_signal, weak=False)
 
-        # Connect to the notification_event signal
-        notification_event_signal = signal('notification_event')
-        notification_event_signal.connect(self.handle_notification_event, weak=False)
-        logger.info("SignalHandler: Connected to notification_event signal")
-
         # Create and start the queue update thread using standard threading
         import threading
         self.polling_emitter_thread = threading.Thread(
@@ -94,23 +89,6 @@ class SignalHandler:
         except Exception as e:
             logger.error(f"Socket.IO error in handle_queue_length: {str(e)}")
 
-    def handle_notification_event(self, *args, **kwargs):
-        """Handle notification_event signal and emit to all clients"""
-        try:
-            watch_uuid = kwargs.get('watch_uuid')
-            logger.debug(f"SignalHandler: Notification event received for watch UUID: {watch_uuid}")
-
-            # Emit the notification event to all connected clients
-            self.socketio_instance.emit("notification_event", {
-                "watch_uuid": watch_uuid,
-                "event_timestamp": time.time()
-            })
-            
-            logger.trace(f"Socket.IO: Emitted notification_event for watch UUID {watch_uuid}")
-
-        except Exception as e:
-            logger.error(f"Socket.IO error in handle_notification_event: {str(e)}")
-
 
     def polling_emit_running_or_queued_watches_threaded(self):
         """Threading version of polling for Windows compatibility"""

changedetectionio/static/js/realtime.js

@@ -55,7 +55,6 @@ $(document).ready(function () {
 
             // Connection status logging
             socket.on('connect', function () {
-                $('#realtime-conn-error').hide();
                 console.log('Socket.IO connected with path:', socketio_url);
                 console.log('Socket transport:', socket.io.engine.transport.name);
                 bindSocketHandlerButtonsEvents(socket);
@@ -75,8 +74,7 @@ $(document).ready(function () {
 
             socket.on('disconnect', function (reason) {
                 console.log('Socket.IO disconnected, reason:', reason);
-                $('.ajax-op').off('.socketHandlerNamespace');
-                $('#realtime-conn-error').show();
+                $('.ajax-op').off('.socketHandlerNamespace')
             });
 
             socket.on('queue_size', function (data) {
@@ -94,10 +92,6 @@ $(document).ready(function () {
                 }
             });
 
-            socket.on('notification_event', function (data) {
-                console.log(`Stub handler for notification_event ${data.watch_uuid}`)
-            });
-
             // Listen for periodically emitted watch data
             console.log('Adding watch_update event listener');
 

changedetectionio/static/styles/scss/parts/_watch_table.scss

@@ -17,13 +17,11 @@
     &.title-col {
       word-break: break-all;
       white-space: normal;
+      a::after {
+        content: url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAYAAACNMs+9AAAAQElEQVR42qXKwQkAIAxDUUdxtO6/RBQkQZvSi8I/pL4BoGw/XPkh4XigPmsUgh0626AjRsgxHTkUThsG2T/sIlzdTsp52kSS1wAAAABJRU5ErkJggg==);
+        margin: 0 3px 0 5px;
+      }
     }
-
-    a.external::after {
-      content: url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAYAAACNMs+9AAAAQElEQVR42qXKwQkAIAxDUUdxtO6/RBQkQZvSi8I/pL4BoGw/XPkh4XigPmsUgh0626AjRsgxHTkUThsG2T/sIlzdTsp52kSS1wAAAABJRU5ErkJggg==);
-      margin: 0 3px 0 5px;
-    }
-
   }
 
 

changedetectionio/static/styles/scss/styles.scss

@@ -1190,12 +1190,3 @@ ul {
   vertical-align: middle;
 }
 
-#realtime-conn-error {
-  position: absolute;
-  bottom: 0;
-  left: 30px;
-  background: var(--color-warning);
-  padding: 10px;
-  font-size: 0.8rem;
-  color: #fff;
-}

changedetectionio/static/styles/styles.css

@@ -537,9 +537,9 @@ body.preview-text-enabled {
     .watch-table td.title-col {
       word-break: break-all;
       white-space: normal; }
-    .watch-table td a.external::after {
-      content: url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAYAAACNMs+9AAAAQElEQVR42qXKwQkAIAxDUUdxtO6/RBQkQZvSi8I/pL4BoGw/XPkh4XigPmsUgh0626AjRsgxHTkUThsG2T/sIlzdTsp52kSS1wAAAABJRU5ErkJggg==);
-      margin: 0 3px 0 5px; }
+      .watch-table td.title-col a::after {
+        content: url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAYAAACNMs+9AAAAQElEQVR42qXKwQkAIAxDUUdxtO6/RBQkQZvSi8I/pL4BoGw/XPkh4XigPmsUgh0626AjRsgxHTkUThsG2T/sIlzdTsp52kSS1wAAAABJRU5ErkJggg==);
+        margin: 0 3px 0 5px; }
   .watch-table th {
     white-space: nowrap; }
     .watch-table th a {
@@ -1535,12 +1535,3 @@ ul {
     height: 21px;
     padding: 2px;
     vertical-align: middle; }
-
-#realtime-conn-error {
-  position: absolute;
-  bottom: 0;
-  left: 30px;
-  background: var(--color-warning);
-  padding: 10px;
-  font-size: 0.8rem;
-  color: #fff; }

changedetectionio/store.py

@@ -412,8 +412,12 @@ class ChangeDetectionStore:
                 # system was out of memory, out of RAM etc
                 with open(self.json_store_path+".tmp", 'w') as json_file:
                     # Use compact JSON in production for better performance
-                    json.dump(data, json_file, indent=2)
-                    os.replace(self.json_store_path+".tmp", self.json_store_path)
+                    debug_mode = os.environ.get('CHANGEDETECTION_DEBUG', 'false').lower() == 'true'
+                    if debug_mode:
+                        json.dump(data, json_file, indent=4)
+                    else:
+                        json.dump(data, json_file, separators=(',', ':'))
+                os.replace(self.json_store_path+".tmp", self.json_store_path)
             except Exception as e:
                 logger.error(f"Error writing JSON!! (Main JSON file save was skipped) : {str(e)}")
 

changedetectionio/templates/base.html

@@ -236,7 +236,6 @@
     <script src="{{url_for('static_content', group='js', filename='toggle-theme.js')}}" defer></script>
 
     <div id="checking-now-fixed-tab" style="display: none;"><span class="spinner"></span><span>&nbsp;Checking now</span></div>
-    <div id="realtime-conn-error" style="display:none">Offline</div>
   </body>
 
 </html>