mirror of
https://github.com/dgtlmoon/changedetection.io.git
synced 2026-09-05 21:15:57 +00:00
Retry notifications on fail
This commit is contained in:
@@ -44,10 +44,10 @@ def sigshutdown_handler(_signo, _stack_frame):
|
||||
|
||||
# Close janus queues properly
|
||||
try:
|
||||
from changedetectionio.flask_app import update_q, notification_q
|
||||
from changedetectionio.flask_app import update_q
|
||||
update_q.close()
|
||||
notification_q.close()
|
||||
logger.debug("Janus queues closed successfully")
|
||||
logger.debug("Janus update queue closed successfully")
|
||||
# notification_q is deprecated - now using Huey task queue which handles its own shutdown
|
||||
except Exception as e:
|
||||
logger.critical(f"CRITICAL: Failed to close janus queues: {e}")
|
||||
|
||||
|
||||
@@ -77,12 +77,12 @@ def construct_blueprint(datastore: ChangeDetectionStore):
|
||||
# Adjust worker count if it changed
|
||||
if new_worker_count != old_worker_count:
|
||||
from changedetectionio import worker_handler
|
||||
from changedetectionio.flask_app import update_q, notification_q, app, datastore as ds
|
||||
|
||||
from changedetectionio.flask_app import update_q, app, datastore as ds
|
||||
|
||||
result = worker_handler.adjust_async_worker_count(
|
||||
new_count=new_worker_count,
|
||||
update_q=update_q,
|
||||
notification_q=notification_q,
|
||||
notification_q=None, # Now using Huey task queue
|
||||
app=app,
|
||||
datastore=ds
|
||||
)
|
||||
@@ -142,4 +142,31 @@ def construct_blueprint(datastore: ChangeDetectionStore):
|
||||
logs=notification_debug_log if len(notification_debug_log) else ["Notification logs are empty - no notifications sent yet."])
|
||||
return output
|
||||
|
||||
@settings_blueprint.route("/failed-notifications", methods=['GET'])
|
||||
@login_optionally_required
|
||||
def failed_notifications():
|
||||
"""View notifications that failed all retry attempts"""
|
||||
from changedetectionio.notification.task_queue import get_failed_notifications
|
||||
|
||||
failed = get_failed_notifications(limit=100)
|
||||
|
||||
output = render_template("failed-notifications.html",
|
||||
failed_notifications=failed)
|
||||
return output
|
||||
|
||||
@settings_blueprint.route("/retry-notification/<task_id>", methods=['POST'])
|
||||
@login_optionally_required
|
||||
def retry_notification(task_id):
|
||||
"""Retry a failed notification by task ID"""
|
||||
from changedetectionio.notification.task_queue import retry_failed_notification
|
||||
|
||||
success = retry_failed_notification(task_id)
|
||||
|
||||
if success:
|
||||
flash(f"Notification {task_id} queued for retry.", 'notice')
|
||||
else:
|
||||
flash(f"Failed to retry notification {task_id}. Check logs for details.", 'error')
|
||||
|
||||
return redirect(url_for('settings.failed_notifications'))
|
||||
|
||||
return settings_blueprint
|
||||
@@ -12,7 +12,7 @@ from blinker import signal
|
||||
|
||||
from changedetectionio.strtobool import strtobool
|
||||
from threading import Event
|
||||
from changedetectionio.queue_handlers import RecheckPriorityQueue, NotificationQueue
|
||||
from changedetectionio.queue_handlers import RecheckPriorityQueue # NotificationQueue deprecated - now using Huey
|
||||
from changedetectionio import worker_handler
|
||||
|
||||
from flask import (
|
||||
@@ -48,9 +48,9 @@ datastore = None
|
||||
ticker_thread = None
|
||||
extra_stylesheets = []
|
||||
|
||||
# Use bulletproof janus-based queues for sync/async reliability
|
||||
# Use bulletproof janus-based queues for sync/async reliability
|
||||
update_q = RecheckPriorityQueue()
|
||||
notification_q = NotificationQueue()
|
||||
# notification_q = NotificationQueue() # DEPRECATED: Now using Huey task queue
|
||||
MAX_QUEUE_SIZE = 2000
|
||||
|
||||
app = Flask(__name__,
|
||||
@@ -565,7 +565,7 @@ def changedetection_app(config=None, datastore_o=None):
|
||||
health_result = worker_handler.check_worker_health(
|
||||
expected_count=expected_workers,
|
||||
update_q=update_q,
|
||||
notification_q=notification_q,
|
||||
notification_q=None, # Now using Huey task queue
|
||||
app=app,
|
||||
datastore=datastore
|
||||
)
|
||||
@@ -626,11 +626,20 @@ def changedetection_app(config=None, datastore_o=None):
|
||||
# Can be overridden by ENV or use the default settings
|
||||
n_workers = int(os.getenv("FETCH_WORKERS", datastore.data['settings']['requests']['workers']))
|
||||
logger.info(f"Starting {n_workers} workers during app initialization")
|
||||
worker_handler.start_workers(n_workers, update_q, notification_q, app, datastore)
|
||||
# Pass None for notification_q - now using Huey task queue directly
|
||||
worker_handler.start_workers(n_workers, update_q, None, app, datastore)
|
||||
|
||||
# Initialize Huey task queue for notifications
|
||||
from changedetectionio.notification.task_queue import init_huey, init_huey_task, start_huey_consumer
|
||||
init_huey(datastore.datastore_path)
|
||||
init_huey_task() # Apply task decorator
|
||||
|
||||
# Start Huey consumer for notification processing (replaces notification_runner)
|
||||
# Using 1 worker thread to match original notification_runner behavior
|
||||
threading.Thread(target=start_huey_consumer, args=(1,), daemon=True).start()
|
||||
|
||||
# @todo handle ctrl break
|
||||
ticker_thread = threading.Thread(target=ticker_thread_check_time_launch_checks).start()
|
||||
threading.Thread(target=notification_runner).start()
|
||||
|
||||
in_pytest = "pytest" in sys.modules or "PYTEST_CURRENT_TEST" in os.environ
|
||||
# Check for new release version, but not when running in test/build or pytest
|
||||
@@ -670,56 +679,10 @@ def check_for_new_version():
|
||||
app.config.exit.wait(86400)
|
||||
|
||||
|
||||
def notification_runner():
|
||||
global notification_debug_log
|
||||
from datetime import datetime
|
||||
import json
|
||||
with app.app_context():
|
||||
while not app.config.exit.is_set():
|
||||
try:
|
||||
# At the moment only one thread runs (single runner)
|
||||
n_object = notification_q.get(block=False)
|
||||
except queue.Empty:
|
||||
time.sleep(1)
|
||||
|
||||
else:
|
||||
|
||||
now = datetime.now()
|
||||
sent_obj = None
|
||||
|
||||
try:
|
||||
from changedetectionio.notification.handler import process_notification
|
||||
|
||||
# Fallback to system config if not set
|
||||
if not n_object.get('notification_body') and datastore.data['settings']['application'].get('notification_body'):
|
||||
n_object['notification_body'] = datastore.data['settings']['application'].get('notification_body')
|
||||
|
||||
if not n_object.get('notification_title') and datastore.data['settings']['application'].get('notification_title'):
|
||||
n_object['notification_title'] = datastore.data['settings']['application'].get('notification_title')
|
||||
|
||||
if not n_object.get('notification_format') and datastore.data['settings']['application'].get('notification_format'):
|
||||
n_object['notification_format'] = datastore.data['settings']['application'].get('notification_format')
|
||||
if n_object.get('notification_urls', {}):
|
||||
sent_obj = process_notification(n_object, datastore)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Watch URL: {n_object['watch_url']} Error {str(e)}")
|
||||
|
||||
# UUID wont be present when we submit a 'test' from the global settings
|
||||
if 'uuid' in n_object:
|
||||
datastore.update_watch(uuid=n_object['uuid'],
|
||||
update_obj={'last_notification_error': "Notification error detected, goto notification log."})
|
||||
|
||||
log_lines = str(e).splitlines()
|
||||
notification_debug_log += log_lines
|
||||
|
||||
with app.app_context():
|
||||
app.config['watch_check_update_SIGNAL'].send(app_context=app, watch_uuid=n_object.get('uuid'))
|
||||
|
||||
# Process notifications
|
||||
notification_debug_log+= ["{} - SENDING - {}".format(now.strftime("%c"), json.dumps(sent_obj))]
|
||||
# Trim the log length
|
||||
notification_debug_log = notification_debug_log[-100:]
|
||||
# DEPRECATED: notification_runner has been replaced by Huey task queue
|
||||
# All logic from this function has been moved to changedetectionio/notification/task_queue.py
|
||||
# in the send_notification_task() function with automatic retry logic and persistent queuing
|
||||
# See: changedetectionio/notification/task_queue.py - send_notification_task()
|
||||
|
||||
|
||||
|
||||
@@ -743,7 +706,7 @@ def ticker_thread_check_time_launch_checks():
|
||||
health_result = worker_handler.check_worker_health(
|
||||
expected_count=expected_workers,
|
||||
update_q=update_q,
|
||||
notification_q=notification_q,
|
||||
notification_q=None, # Now using Huey task queue
|
||||
app=app,
|
||||
datastore=datastore
|
||||
)
|
||||
|
||||
@@ -415,8 +415,10 @@ def process_notification(n_object: NotificationContextData, datastore):
|
||||
if not '<pre' in n_body and not '<body' in n_body: # No custom HTML-ish body was setup already
|
||||
n_body = as_monospaced_html_email(content=n_body, title=n_title)
|
||||
|
||||
# Send the notification and capture return value (True if any succeeded, False if all failed)
|
||||
notification_success = True
|
||||
if not url.startswith('null://'):
|
||||
apobj.notify(
|
||||
notification_success = apobj.notify(
|
||||
title=n_title,
|
||||
body=n_body,
|
||||
# `body_format` Tell apprise what format the INPUT is in, specify a wrong/bad type and it will force skip conversion in apprise
|
||||
@@ -429,8 +431,15 @@ def process_notification(n_object: NotificationContextData, datastore):
|
||||
# Returns empty string if nothing found, multi-line string otherwise
|
||||
log_value = logs.getvalue()
|
||||
|
||||
if log_value and ('WARNING' in log_value or 'ERROR' in log_value):
|
||||
logger.critical(log_value)
|
||||
# Check both Apprise return value AND log capture for failures
|
||||
if not notification_success:
|
||||
error_msg = f"Apprise notification failed - all notification URLs returned False"
|
||||
if log_value:
|
||||
error_msg += f"\nApprise logs:\n{log_value}"
|
||||
logger.critical(error_msg)
|
||||
raise Exception(error_msg)
|
||||
elif log_value and ('WARNING' in log_value or 'ERROR' in log_value):
|
||||
logger.critical(f"Apprise warning/error detected:\n{log_value}")
|
||||
raise Exception(log_value)
|
||||
|
||||
# Return what was sent for better logging - after the for loop
|
||||
|
||||
@@ -0,0 +1,381 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
"""
|
||||
Notification Task Queue - Huey-based notification processing with retry
|
||||
|
||||
Defaults to FileHuey for maximum compatibility with NFS/CIFS network storage
|
||||
commonly used by Synology/QNAP NAS users.
|
||||
|
||||
Environment Variables:
|
||||
QUEUE_STORAGE: 'file' (default), 'sqlite', or 'redis'
|
||||
REDIS_URL: Redis connection URL (only if QUEUE_STORAGE=redis)
|
||||
"""
|
||||
|
||||
import os
|
||||
from loguru import logger
|
||||
|
||||
# Get queue storage type from environment
|
||||
QUEUE_STORAGE = os.getenv('QUEUE_STORAGE', 'file').lower()
|
||||
|
||||
# Global Huey instance (initialized later with proper datastore path)
|
||||
huey = None
|
||||
|
||||
|
||||
def init_huey(datastore_path):
|
||||
"""
|
||||
Initialize Huey instance with the correct datastore path.
|
||||
|
||||
Must be called after datastore is initialized, using datastore.datastore_path
|
||||
|
||||
Args:
|
||||
datastore_path: Path to the datastore directory (from ChangeDetectionStore instance)
|
||||
|
||||
Returns:
|
||||
Huey instance configured for the specified storage backend
|
||||
"""
|
||||
global huey
|
||||
|
||||
# Common options for all queue storage types
|
||||
common_options = {
|
||||
'name': 'changedetection-notifications',
|
||||
'results': True, # Enable result storage for failed notification tracking
|
||||
'store_errors': True # Store error details for debugging
|
||||
}
|
||||
|
||||
# Default to FileHuey unless explicitly configured otherwise
|
||||
if QUEUE_STORAGE == 'file' or QUEUE_STORAGE not in ['sqlite', 'redis']:
|
||||
# FileHuey (default) - NAS-safe, works on all storage types
|
||||
from huey import FileHuey
|
||||
|
||||
queue_path = os.path.join(datastore_path, 'notification-queue')
|
||||
|
||||
# Create directory if it doesn't exist
|
||||
os.makedirs(queue_path, exist_ok=True)
|
||||
|
||||
logger.info(f"Notification queue: FileHuey (NAS-safe, file-based) - DEFAULT")
|
||||
logger.info(f" Queue storage path: {queue_path}")
|
||||
|
||||
huey = FileHuey(
|
||||
path=queue_path,
|
||||
use_thread_lock=True,
|
||||
**common_options
|
||||
)
|
||||
|
||||
elif QUEUE_STORAGE == 'sqlite':
|
||||
# SQLite storage - ONLY for local disk storage!
|
||||
# WARNING: Do NOT use on NFS/CIFS network storage
|
||||
from huey import SqliteHuey
|
||||
|
||||
queue_file = os.path.join(datastore_path, 'notification-queue.db')
|
||||
logger.info(f"Notification queue: SqliteHuey (local storage only!) - {queue_file}")
|
||||
logger.warning("Ensure datastore is on LOCAL disk, not NFS/CIFS network storage!")
|
||||
|
||||
huey = SqliteHuey(
|
||||
filename=queue_file,
|
||||
immediate=False,
|
||||
storage_kwargs={
|
||||
'journal_mode': 'WAL',
|
||||
'timeout': 10
|
||||
},
|
||||
**common_options
|
||||
)
|
||||
|
||||
elif QUEUE_STORAGE == 'redis':
|
||||
# Redis storage - for distributed deployments
|
||||
from huey import RedisHuey
|
||||
|
||||
redis_url = os.getenv('REDIS_URL', 'redis://localhost:6379/0')
|
||||
logger.info(f"Notification queue: RedisHuey - {redis_url}")
|
||||
|
||||
huey = RedisHuey(
|
||||
url=redis_url,
|
||||
**common_options
|
||||
)
|
||||
|
||||
return huey
|
||||
|
||||
|
||||
def get_failed_notifications(limit=100):
|
||||
"""
|
||||
Get list of failed notification tasks from Huey's result store.
|
||||
|
||||
Args:
|
||||
limit: Maximum number of failed tasks to return (default: 100)
|
||||
|
||||
Returns:
|
||||
List of dicts containing failed notification info:
|
||||
- task_id: Huey task ID
|
||||
- timestamp: When the task failed
|
||||
- error: Error message
|
||||
- notification_data: Original notification data
|
||||
- watch_url: URL of the watch
|
||||
- watch_uuid: UUID of the watch
|
||||
"""
|
||||
if huey is None:
|
||||
return []
|
||||
|
||||
failed_tasks = []
|
||||
|
||||
try:
|
||||
# Query Huey's result storage for failed tasks
|
||||
# Note: This requires accessing Huey's internal storage
|
||||
from huey.storage import PeeweeStorage
|
||||
|
||||
# Get all results and filter for errors
|
||||
# Huey stores results with task IDs as keys
|
||||
results = huey.storage.result_store.flush()
|
||||
|
||||
for task_id, result in results.items():
|
||||
if isinstance(result, Exception):
|
||||
# This is a failed task
|
||||
# Try to extract notification data from task args
|
||||
try:
|
||||
task_data = huey.storage.get(task_id)
|
||||
if task_data:
|
||||
failed_tasks.append({
|
||||
'task_id': task_id,
|
||||
'timestamp': task_data.get('execute_time'),
|
||||
'error': str(result),
|
||||
'notification_data': task_data.get('args', [{}])[0] if task_data.get('args') else {},
|
||||
})
|
||||
except Exception as e:
|
||||
logger.error(f"Error extracting failed task data: {e}")
|
||||
|
||||
if len(failed_tasks) >= limit:
|
||||
break
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error querying failed notifications: {e}")
|
||||
|
||||
return failed_tasks
|
||||
|
||||
|
||||
def retry_failed_notification(task_id):
|
||||
"""
|
||||
Retry a failed notification by task ID.
|
||||
|
||||
Args:
|
||||
task_id: Huey task ID to retry
|
||||
|
||||
Returns:
|
||||
True if successfully queued for retry, False otherwise
|
||||
"""
|
||||
if huey is None:
|
||||
logger.error("Huey not initialized")
|
||||
return False
|
||||
|
||||
try:
|
||||
# Get the original task data
|
||||
task_data = huey.storage.get(task_id)
|
||||
|
||||
if not task_data:
|
||||
logger.error(f"Task {task_id} not found in storage")
|
||||
return False
|
||||
|
||||
# Extract notification data and re-queue
|
||||
notification_data = task_data.get('args', [{}])[0] if task_data.get('args') else {}
|
||||
|
||||
if notification_data:
|
||||
# Queue it again
|
||||
send_notification_task(notification_data)
|
||||
logger.info(f"Re-queued failed notification task {task_id}")
|
||||
return True
|
||||
else:
|
||||
logger.error(f"No notification data found for task {task_id}")
|
||||
return False
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error retrying notification {task_id}: {e}")
|
||||
return False
|
||||
|
||||
|
||||
def send_notification_task(n_object_dict):
|
||||
"""
|
||||
Background task to send a notification with automatic retry on failure.
|
||||
|
||||
Retries 3 times with 60 second delay between attempts.
|
||||
|
||||
IMPORTANT: Notification configuration (notification_urls, notification_title,
|
||||
notification_body, notification_format) is RELOADED from the datastore at
|
||||
retry time. This allows operators to fix broken settings (e.g., wrong SMTP
|
||||
server) and retry with corrected configuration.
|
||||
|
||||
Snapshot data (diff, watch_url, triggered_text, etc.) is preserved from
|
||||
the original notification trigger.
|
||||
|
||||
Preserves all logic from the original notification_runner including:
|
||||
- Reloading notification settings from current datastore state at retry time
|
||||
- notification_debug_log tracking
|
||||
- Signal emission on errors
|
||||
- Error handling and watch updates
|
||||
|
||||
Args:
|
||||
n_object_dict: Serialized NotificationContextData as dict (snapshot data)
|
||||
|
||||
Returns:
|
||||
List of sent notification objects with title, body, url
|
||||
|
||||
Raises:
|
||||
Exception: Any error during notification sending (triggers retry)
|
||||
"""
|
||||
from changedetectionio.notification_service import NotificationContextData
|
||||
from changedetectionio.notification.handler import process_notification
|
||||
from changedetectionio.flask_app import datastore, notification_debug_log, app
|
||||
from datetime import datetime
|
||||
import json
|
||||
|
||||
# Reconstruct NotificationContextData from serialized dict
|
||||
n_object = NotificationContextData(initial_data=n_object_dict)
|
||||
|
||||
now = datetime.now()
|
||||
sent_obj = None
|
||||
|
||||
try:
|
||||
# ALWAYS reload notification configuration from current datastore state
|
||||
# This allows operators to fix broken notification settings (e.g., wrong SMTP server)
|
||||
# and retry failed notifications with the corrected configuration
|
||||
watch_uuid = n_object.get('uuid')
|
||||
watch = None
|
||||
|
||||
# Get current watch data if this is a watch notification (not a test notification)
|
||||
if watch_uuid and watch_uuid in datastore.data['watching']:
|
||||
watch = datastore.data['watching'][watch_uuid]
|
||||
|
||||
# Reload notification_urls from current settings (watch-level or system-level)
|
||||
if watch and watch.get('notification_urls'):
|
||||
n_object['notification_urls'] = watch.get('notification_urls')
|
||||
else:
|
||||
# Fallback to system-level notification_urls
|
||||
n_object['notification_urls'] = datastore.data['settings']['application'].get('notification_urls', {})
|
||||
|
||||
# Reload notification_title from current settings
|
||||
if watch and watch.get('notification_title'):
|
||||
n_object['notification_title'] = watch.get('notification_title')
|
||||
else:
|
||||
n_object['notification_title'] = datastore.data['settings']['application'].get('notification_title')
|
||||
|
||||
# Reload notification_body from current settings
|
||||
if watch and watch.get('notification_body'):
|
||||
n_object['notification_body'] = watch.get('notification_body')
|
||||
else:
|
||||
n_object['notification_body'] = datastore.data['settings']['application'].get('notification_body')
|
||||
|
||||
# Reload notification_format from current settings
|
||||
if watch and watch.get('notification_format'):
|
||||
n_object['notification_format'] = watch.get('notification_format')
|
||||
else:
|
||||
n_object['notification_format'] = datastore.data['settings']['application'].get('notification_format')
|
||||
|
||||
# Process and send the notification using shared datastore
|
||||
if n_object.get('notification_urls'):
|
||||
sent_obj = process_notification(n_object, datastore)
|
||||
|
||||
# Clear any previous error on success
|
||||
watch_uuid = n_object.get('uuid')
|
||||
if watch_uuid and watch_uuid in datastore.data['watching']:
|
||||
datastore.update_watch(
|
||||
uuid=watch_uuid,
|
||||
update_obj={'last_notification_error': None}
|
||||
)
|
||||
|
||||
# Add to notification debug log (preserve original logging)
|
||||
notification_debug_log.append("{} - SENDING - {}".format(now.strftime("%c"), json.dumps(sent_obj)))
|
||||
# Trim the log length
|
||||
while len(notification_debug_log) > 100:
|
||||
notification_debug_log.pop(0)
|
||||
|
||||
logger.success(f"Notification sent successfully for {n_object.get('watch_url')}")
|
||||
return sent_obj
|
||||
|
||||
except Exception as e:
|
||||
# Log error and update watch with error message (preserve original error handling)
|
||||
logger.error(f"Watch URL: {n_object.get('watch_url')} Error {str(e)}")
|
||||
|
||||
watch_uuid = n_object.get('uuid')
|
||||
|
||||
# UUID wont be present when we submit a 'test' from the global settings
|
||||
if watch_uuid:
|
||||
try:
|
||||
if watch_uuid in datastore.data['watching']:
|
||||
datastore.update_watch(
|
||||
uuid=watch_uuid,
|
||||
update_obj={'last_notification_error': "Notification error detected, goto notification log."}
|
||||
)
|
||||
except Exception as update_error:
|
||||
logger.error(f"Failed to update watch error status: {update_error}")
|
||||
|
||||
# Add error lines to debug log (preserve original logging)
|
||||
log_lines = str(e).splitlines()
|
||||
notification_debug_log.extend(log_lines)
|
||||
# Trim the log length
|
||||
while len(notification_debug_log) > 100:
|
||||
notification_debug_log.pop(0)
|
||||
|
||||
# Send signal (preserve original signal emission)
|
||||
try:
|
||||
with app.app_context():
|
||||
app.config['watch_check_update_SIGNAL'].send(app_context=app, watch_uuid=watch_uuid)
|
||||
except Exception as signal_error:
|
||||
logger.error(f"Failed to send watch_check_update signal: {signal_error}")
|
||||
|
||||
# Re-raise to trigger Huey retry
|
||||
raise
|
||||
|
||||
|
||||
# Decorator will be applied after huey is initialized
|
||||
# This is set up in init_huey_task()
|
||||
def init_huey_task():
|
||||
"""
|
||||
Decorate send_notification_task with Huey task decorator.
|
||||
|
||||
Must be called after init_huey() so the decorator can be applied.
|
||||
"""
|
||||
global send_notification_task
|
||||
if huey is None:
|
||||
raise RuntimeError("Huey not initialized! Call init_huey(datastore_path) first")
|
||||
|
||||
# Apply Huey task decorator
|
||||
send_notification_task = huey.task(retries=3, retry_delay=60)(send_notification_task)
|
||||
|
||||
|
||||
def start_huey_consumer(workers=1):
|
||||
"""
|
||||
Start Huey consumer in-process as background threads.
|
||||
|
||||
Replaces the old notification_runner() thread with Huey's consumer
|
||||
threads that provide retry logic and persistent queuing.
|
||||
|
||||
Args:
|
||||
workers: Number of worker threads (default: 1)
|
||||
"""
|
||||
global huey
|
||||
|
||||
if huey is None:
|
||||
raise RuntimeError("Huey not initialized! Call init_huey(datastore_path) first")
|
||||
|
||||
logger.info(f"Starting Huey notification consumer with {workers} worker threads")
|
||||
|
||||
try:
|
||||
from huey.consumer import Consumer
|
||||
|
||||
# Create and run consumer
|
||||
# Note: We disable signal handlers since we're running in a background thread, not main thread
|
||||
consumer = Consumer(
|
||||
huey,
|
||||
workers=workers,
|
||||
worker_type='thread',
|
||||
scheduler_interval=1, # Poll queue every 1 second
|
||||
check_worker_health=True,
|
||||
health_check_interval=60,
|
||||
# Disable signal handlers - we're in a thread, not main process
|
||||
# The main Flask app will handle shutdown signals
|
||||
)
|
||||
|
||||
# Override signal handler setup to do nothing (we're in a thread)
|
||||
consumer._set_signal_handlers = lambda: None
|
||||
|
||||
consumer.run() # This blocks, so it runs in the thread
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to start Huey consumer: {e}")
|
||||
raise
|
||||
@@ -171,9 +171,10 @@ class NotificationService:
|
||||
Standalone notification service that handles all notification functionality
|
||||
previously embedded in the update_worker class
|
||||
"""
|
||||
|
||||
def __init__(self, datastore, notification_q):
|
||||
|
||||
def __init__(self, datastore, notification_q=None):
|
||||
self.datastore = datastore
|
||||
# notification_q is deprecated - now using Huey task queue directly
|
||||
self.notification_q = notification_q
|
||||
|
||||
def queue_notification_for_watch(self, n_object: NotificationContextData, watch, date_index_from=-2, date_index_to=-1):
|
||||
@@ -227,12 +228,11 @@ class NotificationService:
|
||||
triggered_text=triggered_text,
|
||||
timestamp_changed=dates[date_index_to]))
|
||||
|
||||
if self.notification_q:
|
||||
logger.debug("Queued notification for sending")
|
||||
self.notification_q.put(n_object)
|
||||
else:
|
||||
logger.debug("Not queued, no queue defined. Just returning processed data")
|
||||
return n_object
|
||||
# Queue notification to Huey for processing with retry logic
|
||||
from changedetectionio.notification.task_queue import send_notification_task
|
||||
logger.debug("Queuing notification to Huey for sending with retry")
|
||||
send_notification_task(dict(n_object))
|
||||
return n_object
|
||||
|
||||
def send_content_changed_notification(self, watch_uuid):
|
||||
"""
|
||||
@@ -315,8 +315,9 @@ Thanks - Your omniscient changedetection.io installation.
|
||||
'uuid': watch_uuid,
|
||||
'screenshot': None
|
||||
})
|
||||
self.notification_q.put(n_object)
|
||||
logger.debug(f"Sent filter not found notification for {watch_uuid}")
|
||||
from changedetectionio.notification.task_queue import send_notification_task
|
||||
send_notification_task(dict(n_object))
|
||||
logger.debug(f"Queued filter not found notification for {watch_uuid}")
|
||||
else:
|
||||
logger.debug(f"NOT sending filter not found notification for {watch_uuid} - no notification URLs")
|
||||
|
||||
@@ -363,13 +364,18 @@ Thanks - Your omniscient changedetection.io installation.
|
||||
'watch_url': watch['url'],
|
||||
'uuid': watch_uuid
|
||||
})
|
||||
self.notification_q.put(n_object)
|
||||
logger.error(f"Sent step not found notification for {watch_uuid}")
|
||||
from changedetectionio.notification.task_queue import send_notification_task
|
||||
send_notification_task(dict(n_object))
|
||||
logger.error(f"Queued step not found notification for {watch_uuid}")
|
||||
|
||||
|
||||
# Convenience functions for creating notification service instances
|
||||
def create_notification_service(datastore, notification_q):
|
||||
def create_notification_service(datastore, notification_q=None):
|
||||
"""
|
||||
Factory function to create a NotificationService instance
|
||||
|
||||
Args:
|
||||
datastore: The ChangeDetectionStore instance
|
||||
notification_q: Deprecated, no longer used (kept for backward compatibility)
|
||||
"""
|
||||
return NotificationService(datastore, notification_q)
|
||||
@@ -141,6 +141,7 @@
|
||||
<a id="add-email-helper" class="pure-button button-secondary button-xsmall" >Add email <img style="height: 1em; display: inline-block" src="{{url_for('static_content', group='images', filename='email.svg')}}" alt="Add an email address"> </a>
|
||||
{% endif %}
|
||||
<a href="{{url_for('settings.notification_logs')}}" class="pure-button button-secondary button-xsmall" >Notification debug logs</a>
|
||||
<a href="{{url_for('settings.failed_notifications')}}" class="pure-button button-secondary button-xsmall" >Failed notifications</a>
|
||||
<br>
|
||||
<div id="notification-test-log" style="display: none;"><span class="pure-form-message-inline">Processing..</span></div>
|
||||
</div>
|
||||
|
||||
@@ -9,6 +9,7 @@ flask_expects_json~=1.7
|
||||
flask_restful
|
||||
flask_cors # For the Chrome extension to operate
|
||||
janus # Thread-safe async/sync queue bridge
|
||||
huey ~= 2.5 # Task queue for notification retries with FileHuey/SqliteHuey/RedisHuey support
|
||||
flask_wtf~=1.2
|
||||
flask~=3.1
|
||||
flask-socketio~=5.5.1
|
||||
|
||||
Reference in New Issue
Block a user