From bc614e16519fc3c346ad58ff982d2c602a205919 Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Sun, 12 Oct 2025 15:51:44 +0200 Subject: [PATCH] Helper to give python GC a better chance of cleaning up --- changedetectionio/async_update_worker.py | 33 ++++++++++++++++++- changedetectionio/content_fetchers/base.py | 13 ++++++++ .../processors/text_json_diff/processor.py | 14 ++++++++ 3 files changed, 59 insertions(+), 1 deletion(-) diff --git a/changedetectionio/async_update_worker.py b/changedetectionio/async_update_worker.py index a607ca09..ed090320 100644 --- a/changedetectionio/async_update_worker.py +++ b/changedetectionio/async_update_worker.py @@ -334,6 +334,10 @@ async def async_update_worker(worker_id, q, notification_q, app, datastore): if update_handler.fetcher.content or (not update_handler.fetcher.content and empty_pages_are_a_change): watch.save_last_fetched_html(contents=update_handler.fetcher.content, timestamp=int(fetch_start_time)) + # Explicitly delete large content variables to free memory IMMEDIATELY after saving + # These are no longer needed after being saved to history + del contents + # Send notifications on second+ check if watch.history_n >= 2: logger.info(f"Change detected in UUID {uuid} - {watch['url']}") @@ -372,6 +376,12 @@ async def async_update_worker(worker_id, q, notification_q, app, datastore): datastore.update_watch(uuid=uuid, update_obj={'fetch_time': round(time.time() - fetch_start_time, 3), 'check_count': count}) + # NOW clear fetcher content - after all processing is complete + # This is the last point where we need the fetcher data + if update_handler and hasattr(update_handler, 'fetcher') and update_handler.fetcher: + update_handler.fetcher.clear_content() + logger.debug(f"Cleared fetcher content for UUID {uuid}") + except Exception as e: logger.error(f"Worker {worker_id} unexpected error processing {uuid}: {e}") logger.error(f"Worker {worker_id} traceback:", exc_info=True) @@ -392,7 +402,28 @@ async def async_update_worker(worker_id, q, notification_q, app, datastore): #logger.info(f"Worker {worker_id} sending completion signal for UUID {watch['uuid']}") watch_check_update.send(watch_uuid=watch['uuid']) - update_handler = None + # Explicitly clean up update_handler and all its references + if update_handler: + # Clear fetcher content using the proper method + if hasattr(update_handler, 'fetcher') and update_handler.fetcher: + update_handler.fetcher.clear_content() + + # Clear processor references + if hasattr(update_handler, 'content_processor'): + update_handler.content_processor = None + + update_handler = None + + # Clear local contents variable if it still exists + if 'contents' in locals(): + del contents + + # Note: We don't set watch = None here because: + # 1. watch is just a local reference to datastore.data['watching'][uuid] + # 2. Setting it to None doesn't affect the datastore + # 3. GC can't collect the object anyway (still referenced by datastore) + # 4. It would just cause confusion + logger.debug(f"Worker {worker_id} completed watch {uuid} in {time.time()-fetch_start_time:.2f}s") except Exception as cleanup_error: logger.error(f"Worker {worker_id} error during cleanup: {cleanup_error}") diff --git a/changedetectionio/content_fetchers/base.py b/changedetectionio/content_fetchers/base.py index 8be939ef..6cffe5aa 100644 --- a/changedetectionio/content_fetchers/base.py +++ b/changedetectionio/content_fetchers/base.py @@ -64,6 +64,19 @@ class Fetcher(): # Time ONTOP of the system defined env minimum time render_extract_delay = 0 + def clear_content(self): + """ + Explicitly clear all content from memory to free up heap space. + Call this after content has been saved to disk. + """ + self.content = None + if hasattr(self, 'raw_content'): + self.raw_content = None + self.screenshot = None + self.xpath_data = None + # Keep headers and status_code as they're small + logger.trace("Fetcher content cleared from memory") + @abstractmethod def get_error(self): return self.error diff --git a/changedetectionio/processors/text_json_diff/processor.py b/changedetectionio/processors/text_json_diff/processor.py index 32b646a9..c9baa6a5 100644 --- a/changedetectionio/processors/text_json_diff/processor.py +++ b/changedetectionio/processors/text_json_diff/processor.py @@ -556,6 +556,20 @@ class perform_site_check(difference_detection_processor): else: logger.debug(f"check_unique_lines: UUID {watch.get('uuid')} had unique content") + # Note: Explicit cleanup is only needed here because text_json_diff handles + # large strings (100KB-300KB for RSS/HTML). The other processors work with + # small strings and don't need this. + # + # Python would clean these up automatically, but explicit `del` frees memory + # immediately rather than waiting for function return, reducing peak memory usage. + del content + if 'html_content' in locals() and html_content is not stripped_text: + del html_content + if 'text_content_before_ignored_filter' in locals() and text_content_before_ignored_filter is not stripped_text: + del text_content_before_ignored_filter + if 'text_for_checksuming' in locals() and text_for_checksuming is not stripped_text: + del text_for_checksuming + return changed_detected, update_obj, stripped_text def _apply_diff_filtering(self, watch, stripped_text, text_before_filter):