Adding random tips

changedetectionio/blueprint/backups/templates/overview.html

@@ -3,10 +3,10 @@
     {% from '_helpers.html' import render_simple_field, render_field %}
     <div class="edit-form">
         <div class="box-wrap inner">
-            <h2>{{ _('Backups') }}</h2>
+            <h4>{{ _('Backups') }}</h4>
             {% if backup_running %}
                 <p>
-                    <span class="spinner"></span>&nbsp;<strong>{{ _('A backup is running!') }}</strong>
+                    <strong>{{ _('A backup is running!') }}</strong>
                 </p>
             {% endif %}
             <p>

changedetectionio/blueprint/settings/templates/settings.html

@@ -25,7 +25,7 @@
             <li class="tab"><a href="#ui-options">{{ _('UI Options') }}</a></li>
             <li class="tab"><a href="#api">{{ _('API') }}</a></li>
             <li class="tab"><a href="#rss">{{ _('RSS') }}</a></li>
-            <li class="tab"><a href="{{ url_for('backups.index') }}">{{ _('Backups') }}</a></li>
+            <li class="tab"><a href="{{ url_for('backups.index') }}" class="pure-menu-link">{{ _('Backups') }}</a></li>
             <li class="tab"><a href="#timedate">{{ _('Time & Date') }}</a></li>
             <li class="tab"><a href="#proxies">{{ _('CAPTCHA & Proxies') }}</a></li>
             {% if plugin_tabs %}

changedetectionio/blueprint/ui/views.py

@@ -26,7 +26,7 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMe
         add_paused = request.form.get('edit_and_watch_submit_button') != None
         from changedetectionio import processors
         processor = request.form.get('processor', processors.get_default_processor())
-        new_uuid = datastore.add_watch(url=url, tag=request.form.get('tags','').strip(), extras={'paused': add_paused, 'processor': processor})
+        new_uuid = datastore.add_watch(url=url, tag=request.form.get('tags').strip(), extras={'paused': add_paused, 'processor': processor})
 
         if new_uuid:
             if add_paused:
@@ -39,4 +39,4 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMe
 
         return redirect(url_for('watchlist.index', tag=request.args.get('tag','')))
 
-    return views_blueprint
+    return views_blueprint

changedetectionio/content_fetchers/playwright.py

@@ -1,4 +1,3 @@
-import asyncio
 import gc
 import json
 import os
@@ -350,7 +349,12 @@ class fetcher(Fetcher):
 
             if self.status_code != 200 and not ignore_status_codes:
                 screenshot = await capture_full_page_async(self.page, screenshot_format=self.screenshot_format, watch_uuid=watch_uuid, lock_viewport_elements=self.lock_viewport_elements)
-                # Finally block will handle cleanup
+                # Cleanup before raising to prevent memory leak
+                await self.page.close()
+                await context.close()
+                await browser.close()
+                # Force garbage collection to release Playwright resources immediately
+                gc.collect()
                 raise Non200ErrorCodeReceived(url=url, status_code=self.status_code, screenshot=screenshot)
 
             if not empty_pages_are_a_change and len((await self.page.content()).strip()) == 0:
@@ -366,7 +370,12 @@ class fetcher(Fetcher):
                     try:
                         await self.iterate_browser_steps(start_url=url)
                     except BrowserStepsStepException:
-                        # Finally block will handle cleanup
+                        try:
+                            await context.close()
+                            await browser.close()
+                        except Exception as e:
+                            # Fine, could be messy situation
+                            pass
                         raise
 
                     await self.page.wait_for_timeout(extra_wait * 1000)
@@ -415,40 +424,35 @@ class fetcher(Fetcher):
                 raise ScreenshotUnavailable(url=url, status_code=self.status_code)
 
             finally:
-                # Clean up resources properly with timeouts to prevent hanging
+                # Request garbage collection one more time before closing
                 try:
-                    if hasattr(self, 'page') and self.page:
-                        await self.page.request_gc()
-                        await asyncio.wait_for(self.page.close(), timeout=5.0)
-                        logger.debug(f"Successfully closed page for {url}")
-                except asyncio.TimeoutError:
-                    logger.warning(f"Timed out closing page for {url} (5s)")
-                except Exception as e:
-                    logger.warning(f"Error closing page for {url}: {e}")
-                finally:
-                    self.page = None
+                    await self.page.request_gc()
+                except:
+                    pass
+                
+                # Clean up resources properly
+                try:
+                    await self.page.request_gc()
+                except:
+                    pass
 
                 try:
-                    if context:
-                        await asyncio.wait_for(context.close(), timeout=5.0)
-                        logger.debug(f"Successfully closed context for {url}")
-                except asyncio.TimeoutError:
-                    logger.warning(f"Timed out closing context for {url} (5s)")
-                except Exception as e:
-                    logger.warning(f"Error closing context for {url}: {e}")
-                finally:
-                    context = None
+                    await self.page.close()
+                except:
+                    pass
+                self.page = None
 
                 try:
-                    if browser:
-                        await asyncio.wait_for(browser.close(), timeout=5.0)
-                        logger.debug(f"Successfully closed browser connection for {url}")
-                except asyncio.TimeoutError:
-                    logger.warning(f"Timed out closing browser connection for {url} (5s)")
-                except Exception as e:
-                    logger.warning(f"Error closing browser for {url}: {e}")
-                finally:
-                    browser = None
+                    await context.close()
+                except:
+                    pass
+                context = None
+
+                try:
+                    await browser.close()
+                except:
+                    pass
+                browser = None
 
                 # Force Python GC to release Playwright resources immediately
                 # Playwright objects can have circular references that delay cleanup

changedetectionio/content_fetchers/puppeteer.py

@@ -1,5 +1,4 @@
 import asyncio
-import gc
 import json
 import os
 import websockets.exceptions
@@ -222,36 +221,19 @@ class fetcher(Fetcher):
                 self.browser_connection_url += f"{r}--proxy-server={proxy_url}"
 
     async def quit(self, watch=None):
-        watch_uuid = watch.get('uuid') if watch else 'unknown'
-
-        # Close page
         try:
-            if hasattr(self, 'page') and self.page:
-                await asyncio.wait_for(self.page.close(), timeout=5.0)
-                logger.debug(f"[{watch_uuid}] Page closed successfully")
-        except asyncio.TimeoutError:
-            logger.warning(f"[{watch_uuid}] Timed out closing page (5s)")
+            await self.page.close()
+            del self.page
         except Exception as e:
-            logger.warning(f"[{watch_uuid}] Error closing page: {e}")
-        finally:
-            self.page = None
+            pass
 
-        # Close browser connection
         try:
-            if hasattr(self, 'browser') and self.browser:
-                await asyncio.wait_for(self.browser.close(), timeout=5.0)
-                logger.debug(f"[{watch_uuid}] Browser closed successfully")
-        except asyncio.TimeoutError:
-            logger.warning(f"[{watch_uuid}] Timed out closing browser (5s)")
+            await self.browser.close()
+            del self.browser
         except Exception as e:
-            logger.warning(f"[{watch_uuid}] Error closing browser: {e}")
-        finally:
-            self.browser = None
+            pass
 
-        logger.info(f"[{watch_uuid}] Cleanup puppeteer complete")
-
-        # Force garbage collection to release resources
-        gc.collect()
+        logger.info("Cleanup puppeteer complete.")
 
     async def fetch_page(self,
                          current_include_filters,
@@ -281,11 +263,9 @@ class fetcher(Fetcher):
         # Connect directly using the specified browser_ws_endpoint
         # @todo timeout
         try:
-            logger.debug(f"[{watch_uuid}] Connecting to browser at {self.browser_connection_url}")
             self.browser = await pyppeteer_instance.connect(browserWSEndpoint=self.browser_connection_url,
                                                             ignoreHTTPSErrors=True
                                                             )
-            logger.debug(f"[{watch_uuid}] Browser connected successfully")
         except websockets.exceptions.InvalidStatusCode as e:
             raise BrowserConnectError(msg=f"Error while trying to connect the browser, Code {e.status_code} (check your access, whitelist IP, password etc)")
         except websockets.exceptions.InvalidURI:
@@ -294,18 +274,7 @@ class fetcher(Fetcher):
             raise BrowserConnectError(msg=f"Error connecting to the browser - Exception '{str(e)}'")
 
         # more reliable is to just request a new page
-        try:
-            logger.debug(f"[{watch_uuid}] Creating new page")
-            self.page = await self.browser.newPage()
-            logger.debug(f"[{watch_uuid}] Page created successfully")
-        except Exception as e:
-            logger.error(f"[{watch_uuid}] Failed to create new page: {e}")
-            # Browser is connected but page creation failed - must cleanup browser
-            try:
-                await asyncio.wait_for(self.browser.close(), timeout=3.0)
-            except Exception as cleanup_error:
-                logger.error(f"[{watch_uuid}] Failed to cleanup browser after page creation failure: {cleanup_error}")
-            raise
+        self.page = await self.browser.newPage()
         
         # Add console handler to capture console.log from favicon fetcher
         #self.page.on('console', lambda msg: logger.debug(f"Browser console [{msg.type}]: {msg.text}"))
@@ -374,12 +343,6 @@ class fetcher(Fetcher):
             w = extra_wait - 2 if extra_wait > 4 else 2
             logger.debug(f"Waiting {w} seconds before calling Page.stopLoading...")
             await asyncio.sleep(w)
-
-            # Check if page still exists (might have been closed due to error during sleep)
-            if not self.page or not hasattr(self.page, '_client'):
-                logger.debug("Page already closed, skipping stopLoading")
-                return
-
             logger.debug("Issuing stopLoading command...")
             await self.page._client.send('Page.stopLoading')
             logger.debug("stopLoading command sent!")
@@ -405,9 +368,7 @@ class fetcher(Fetcher):
             asyncio.create_task(handle_frame_navigation())
             response = await self.page.goto(url, timeout=0)
             await asyncio.sleep(1 + extra_wait)
-            # Check if page still exists before sending command
-            if self.page and hasattr(self.page, '_client'):
-                await self.page._client.send('Page.stopLoading')
+            await self.page._client.send('Page.stopLoading')
 
             if response:
                 break
@@ -476,9 +437,15 @@ class fetcher(Fetcher):
         logger.debug(f"Screenshot format {self.screenshot_format}")
         self.screenshot = await capture_full_page(page=self.page, screenshot_format=self.screenshot_format, watch_uuid=watch_uuid, lock_viewport_elements=self.lock_viewport_elements)
 
-        # Force garbage collection - pyppeteer base64 decode creates temporary buffers
+        # Force aggressive memory cleanup - pyppeteer base64 decode creates temporary buffers
         import gc
         gc.collect()
+        # Release C-level memory from base64 decode back to OS
+        try:
+            import ctypes
+            ctypes.CDLL('libc.so.6').malloc_trim(0)
+        except Exception:
+            pass
         self.xpath_data = await self.page.evaluate(XPATH_ELEMENT_JS, {
             "visualselector_xpath_selectors": visualselector_xpath_selectors,
             "max_height": MAX_TOTAL_HEIGHT

changedetectionio/jinja2_custom/safe_jinja.py

@@ -52,13 +52,7 @@ def render(template_str, **args: t.Any) -> str:
     return output[:JINJA2_MAX_RETURN_PAYLOAD_SIZE]
 
 def render_fully_escaped(content):
-    """
-    Escape HTML content safely.
-
-    MEMORY LEAK FIX: Use markupsafe.escape() directly instead of creating
-    Jinja2 environments (was causing 1M+ compilations per page load).
-    Simpler, faster, and no concerns about environment state.
-    """
-    from markupsafe import escape
-    return str(escape(content))
+    env = jinja2.sandbox.ImmutableSandboxedEnvironment(autoescape=True)
+    template = env.from_string("{{ some_html|e }}")
+    return template.render(some_html=content)
 

changedetectionio/model/Watch.py

@@ -613,11 +613,6 @@ class model(watch_base):
                 try:
                     with open(fname, 'wb') as f:
                         f.write(decoded)
-
-                    # Invalidate favicon filename cache
-                    if hasattr(self, '_favicon_filename_cache'):
-                        delattr(self, '_favicon_filename_cache')
-
                     # A signal that could trigger the socket server to update the browser also
                     watch_check_update = signal('watch_favicon_bump')
                     if watch_check_update:
@@ -634,32 +629,20 @@ class model(watch_base):
         Find any favicon.* file in the current working directory
         and return the contents of the newest one.
 
-        MEMORY LEAK FIX: Cache the result to avoid repeated glob.glob() operations.
-        glob.glob() causes millions of fnmatch allocations when called for every watch on page load.
-
         Returns:
-            str: Basename of the newest favicon file, or None if not found.
+            bytes: Contents of the newest favicon file, or None if not found.
         """
-        # Check cache first (prevents 26M+ allocations from repeated glob operations)
-        cache_key = '_favicon_filename_cache'
-        if hasattr(self, cache_key):
-            return getattr(self, cache_key)
-
         import glob
 
         # Search for all favicon.* files
         files = glob.glob(os.path.join(self.watch_data_dir, "favicon.*"))
 
         if not files:
-            result = None
-        else:
-            # Find the newest by modification time
-            newest_file = max(files, key=os.path.getmtime)
-            result = os.path.basename(newest_file)
+            return None
 
-        # Cache the result
-        setattr(self, cache_key, result)
-        return result
+        # Find the newest by modification time
+        newest_file = max(files, key=os.path.getmtime)
+        return os.path.basename(newest_file)
 
     def get_screenshot_as_thumbnail(self, max_age=3200):
         """Return path to a square thumbnail of the most recent screenshot.

changedetectionio/worker.py

@@ -480,9 +480,14 @@ async def async_update_worker(worker_id, q, notification_q, app, datastore, exec
                         del update_handler
                         update_handler = None
 
-                # Force garbage collection
+                # Force aggressive memory cleanup after clearing
                 import gc
                 gc.collect()
+                try:
+                    import ctypes
+                    ctypes.CDLL('libc.so.6').malloc_trim(0)
+                except Exception:
+                    pass
 
         except Exception as e:
             import traceback
@@ -498,7 +503,6 @@ async def async_update_worker(worker_id, q, notification_q, app, datastore, exec
         finally:
             # Always cleanup - this runs whether there was an exception or not
             if uuid:
-                # Call quit() as backup (Puppeteer/Playwright have internal cleanup, but this acts as safety net)
                 try:
                     if update_handler and hasattr(update_handler, 'fetcher') and update_handler.fetcher:
                         await update_handler.fetcher.quit(watch=watch)
@@ -507,25 +511,35 @@ async def async_update_worker(worker_id, q, notification_q, app, datastore, exec
                 try:
                     # Release UUID from processing (thread-safe)
                     worker_pool.release_uuid_from_processing(uuid, worker_id=worker_id)
-
+                    
                     # Send completion signal
                     if watch:
+                        #logger.info(f"Worker {worker_id} sending completion signal for UUID {watch['uuid']}")
                         watch_check_update.send(watch_uuid=watch['uuid'])
 
-                    # Clean up all memory references BEFORE garbage collection
+                    # Explicitly clean up update_handler and all its references
                     if update_handler:
+                        # Clear fetcher content using the proper method
                         if hasattr(update_handler, 'fetcher') and update_handler.fetcher:
                             update_handler.fetcher.clear_content()
+
+                        # Clear processor references
                         if hasattr(update_handler, 'content_processor'):
                             update_handler.content_processor = None
-                        del update_handler
+
                         update_handler = None
 
-                    # Clear large content variables
+                    # Clear local contents variable if it still exists
                     if 'contents' in locals():
                         del contents
 
-                    # Force garbage collection after all references are cleared
+                    # Note: We don't set watch = None here because:
+                    # 1. watch is just a local reference to datastore.data['watching'][uuid]
+                    # 2. Setting it to None doesn't affect the datastore
+                    # 3. GC can't collect the object anyway (still referenced by datastore)
+                    # 4. It would just cause confusion
+
+                    # Force garbage collection after cleanup
                     import gc
                     gc.collect()
 

requirements.txt

@@ -70,7 +70,7 @@ lxml >=4.8.0,!=5.2.0,!=5.2.1,<7
 
 # XPath 2.0-3.1 support - 4.2.0 had issues, 4.1.5 stable
 # Consider updating to latest stable version periodically
-elementpath==5.1.1
+elementpath==5.1.0
 
 # For fast image comparison in screenshot change detection
 # opencv-python-headless is OPTIONAL (excluded from requirements.txt)