mirror of
https://github.com/dgtlmoon/changedetection.io.git
synced 2026-08-29 01:27:30 +00:00
385 lines
13 KiB
Python
385 lines
13 KiB
Python
"""
|
|
FileStorage backend task manager for Huey notifications.
|
|
|
|
This is the default backend, optimized for NAS/CIFS compatibility.
|
|
|
|
INDUSTRIAL-GRADE ENHANCEMENTS:
|
|
- Atomic file writes (prevents corruption on crash/power failure)
|
|
- JSON corruption detection and recovery
|
|
- Lost-found directory for corrupted files
|
|
"""
|
|
|
|
from loguru import logger
|
|
|
|
from .base import HueyTaskManager
|
|
|
|
import os
|
|
|
|
|
|
def _atomic_json_write(filepath, data):
|
|
"""
|
|
Atomic JSON write - never leaves partial/corrupted files.
|
|
|
|
Critical for aerospace-grade reliability:
|
|
- Writes to temp file first
|
|
- Forces data to disk (fsync)
|
|
- Atomic rename (POSIX guarantees atomicity)
|
|
|
|
If process crashes mid-write, you either get:
|
|
- Old complete file (rename didn't happen)
|
|
- New complete file (rename succeeded)
|
|
Never a partial/corrupted file.
|
|
|
|
Args:
|
|
filepath: Target file path
|
|
data: Data to write (will be JSON encoded)
|
|
|
|
Returns:
|
|
True on success
|
|
|
|
Raises:
|
|
Exception on write failure
|
|
"""
|
|
import tempfile
|
|
import json
|
|
|
|
directory = os.path.dirname(filepath)
|
|
|
|
# Write to temp file in same directory (same filesystem = atomic rename)
|
|
fd, temp_path = tempfile.mkstemp(dir=directory, prefix='.tmp_', suffix='.json')
|
|
|
|
try:
|
|
with os.fdopen(fd, 'w') as f:
|
|
json.dump(data, f, indent=2)
|
|
f.flush()
|
|
os.fsync(f.fileno()) # Force to disk before rename
|
|
|
|
# Atomic rename (POSIX guarantees atomicity)
|
|
# If crash happens here, worst case is orphaned temp file
|
|
os.replace(temp_path, filepath)
|
|
return True
|
|
|
|
except Exception as e:
|
|
# Clean up temp file on error
|
|
try:
|
|
os.unlink(temp_path)
|
|
except:
|
|
pass
|
|
raise
|
|
|
|
|
|
def _safe_json_load(filepath, schema_name, storage_path):
|
|
"""
|
|
Load JSON with corruption detection and recovery.
|
|
|
|
Corrupted files are moved to lost-found for forensic analysis.
|
|
|
|
Args:
|
|
filepath: JSON file to load
|
|
schema_name: Schema type (for lost-found directory)
|
|
storage_path: Root storage path
|
|
|
|
Returns:
|
|
Parsed JSON data or None if corrupted
|
|
"""
|
|
import json
|
|
import shutil
|
|
import time
|
|
|
|
try:
|
|
with open(filepath, 'r') as f:
|
|
return json.load(f)
|
|
|
|
except (json.JSONDecodeError, EOFError) as e:
|
|
# Corrupted or incomplete JSON file
|
|
file_size = os.path.getsize(filepath) if os.path.exists(filepath) else 0
|
|
filename = os.path.basename(filepath)
|
|
logger.warning(f"Corrupted {schema_name} file: {filename} ({file_size} bytes) - moving to lost-found")
|
|
|
|
try:
|
|
lost_found_dir = os.path.join(storage_path, 'lost-found', schema_name)
|
|
os.makedirs(lost_found_dir, exist_ok=True)
|
|
|
|
timestamp = int(time.time())
|
|
lost_found_path = os.path.join(lost_found_dir, f"{filename}.{timestamp}.corrupted")
|
|
|
|
shutil.move(filepath, lost_found_path)
|
|
logger.info(f"Moved corrupted {schema_name} file to {lost_found_path}")
|
|
|
|
except Exception as move_err:
|
|
logger.error(f"Unable to move corrupted {schema_name} file: {move_err}")
|
|
|
|
return None
|
|
|
|
except Exception as e:
|
|
logger.debug(f"Unable to load {schema_name} file {filepath}: {e}")
|
|
return None
|
|
|
|
|
|
class FileStorageTaskManager(HueyTaskManager):
|
|
"""Task manager for FileStorage backend (default, NAS-safe)."""
|
|
|
|
def enumerate_results(self):
|
|
"""Enumerate results by walking filesystem directories."""
|
|
import os
|
|
import pickle
|
|
import struct
|
|
import time
|
|
|
|
results = {}
|
|
|
|
if not self.storage_path:
|
|
return results
|
|
|
|
results_dir = os.path.join(self.storage_path, 'results')
|
|
|
|
if not os.path.exists(results_dir):
|
|
return results
|
|
|
|
# Walk through all subdirectories to find result files
|
|
for root, dirs, files in os.walk(results_dir):
|
|
for filename in files:
|
|
if filename.startswith('.'):
|
|
continue
|
|
|
|
filepath = os.path.join(root, filename)
|
|
try:
|
|
# Read and unpickle the result
|
|
# Huey FileStorage format: 4-byte length + task_id + pickled data
|
|
with open(filepath, 'rb') as f:
|
|
# Read the task ID header (length-prefixed)
|
|
task_id_len_bytes = f.read(4)
|
|
if len(task_id_len_bytes) < 4:
|
|
raise EOFError("Incomplete header")
|
|
task_id_len = struct.unpack('>I', task_id_len_bytes)[0]
|
|
task_id_bytes = f.read(task_id_len)
|
|
if len(task_id_bytes) < task_id_len:
|
|
raise EOFError("Incomplete task ID")
|
|
task_id = task_id_bytes.decode('utf-8')
|
|
|
|
# Now unpickle the result data
|
|
result_data = pickle.load(f)
|
|
results[task_id] = result_data
|
|
except (pickle.UnpicklingError, EOFError) as e:
|
|
# Corrupted or incomplete result file
|
|
file_size = os.path.getsize(filepath)
|
|
logger.warning(f"Corrupted result file {filename} ({file_size} bytes) - moving to lost-found.")
|
|
try:
|
|
import shutil
|
|
lost_found_dir = os.path.join(self.storage_path, 'lost-found', 'results')
|
|
os.makedirs(lost_found_dir, exist_ok=True)
|
|
|
|
timestamp = int(time.time())
|
|
lost_found_path = os.path.join(lost_found_dir, f"{filename}.{timestamp}.corrupted")
|
|
|
|
shutil.move(filepath, lost_found_path)
|
|
logger.info(f"Moved corrupted file to {lost_found_path}")
|
|
except Exception as move_err:
|
|
logger.error(f"Unable to move corrupted file: {move_err}")
|
|
except Exception as e:
|
|
logger.debug(f"Unable to read result file {filename}: {e}")
|
|
|
|
return results
|
|
|
|
def delete_result(self, task_id):
|
|
"""Delete result file from filesystem."""
|
|
import hashlib
|
|
|
|
if not self.storage_path:
|
|
return False
|
|
|
|
results_dir = os.path.join(self.storage_path, 'results')
|
|
|
|
# Huey uses MD5 hash to create subdirectories
|
|
task_id_bytes = task_id.encode('utf-8')
|
|
hex_hash = hashlib.md5(task_id_bytes).hexdigest()
|
|
|
|
# FileStorage creates subdirectories based on first 2 chars of hash
|
|
subdir = hex_hash[:2]
|
|
result_file = os.path.join(results_dir, subdir, task_id)
|
|
|
|
if os.path.exists(result_file):
|
|
os.remove(result_file)
|
|
logger.debug(f"Deleted result file for task {task_id}")
|
|
return True
|
|
else:
|
|
logger.debug(f"Result file not found for task {task_id}")
|
|
return False
|
|
|
|
def count_storage_items(self):
|
|
"""Count items by walking filesystem directories."""
|
|
queue_count = 0
|
|
schedule_count = 0
|
|
|
|
if not self.storage_path:
|
|
return queue_count, schedule_count
|
|
|
|
try:
|
|
# Count queue files
|
|
queue_dir = os.path.join(self.storage_path, 'queue')
|
|
if os.path.exists(queue_dir):
|
|
for root, dirs, files in os.walk(queue_dir):
|
|
queue_count += len([f for f in files if not f.startswith('.')])
|
|
|
|
# Count schedule files
|
|
schedule_dir = os.path.join(self.storage_path, 'schedule')
|
|
if os.path.exists(schedule_dir):
|
|
for root, dirs, files in os.walk(schedule_dir):
|
|
schedule_count += len([f for f in files if not f.startswith('.')])
|
|
except Exception as e:
|
|
logger.debug(f"FileStorage count error: {e}")
|
|
|
|
return queue_count, schedule_count
|
|
|
|
def clear_all_notifications(self):
|
|
"""Clear all notification files from filesystem."""
|
|
cleared = {
|
|
'queue': 0,
|
|
'schedule': 0,
|
|
'results': 0,
|
|
'retry_attempts': 0,
|
|
'task_metadata': 0,
|
|
'delivered': 0
|
|
}
|
|
|
|
if not self.storage_path:
|
|
return cleared
|
|
|
|
# Clear queue
|
|
queue_dir = os.path.join(self.storage_path, 'queue')
|
|
if os.path.exists(queue_dir):
|
|
for root, dirs, files in os.walk(queue_dir):
|
|
for f in files:
|
|
if not f.startswith('.'):
|
|
os.remove(os.path.join(root, f))
|
|
cleared['queue'] += 1
|
|
|
|
# Clear schedule
|
|
schedule_dir = os.path.join(self.storage_path, 'schedule')
|
|
if os.path.exists(schedule_dir):
|
|
for root, dirs, files in os.walk(schedule_dir):
|
|
for f in files:
|
|
if not f.startswith('.'):
|
|
os.remove(os.path.join(root, f))
|
|
cleared['schedule'] += 1
|
|
|
|
# Clear results
|
|
results_dir = os.path.join(self.storage_path, 'results')
|
|
if os.path.exists(results_dir):
|
|
for root, dirs, files in os.walk(results_dir):
|
|
for f in files:
|
|
if not f.startswith('.'):
|
|
os.remove(os.path.join(root, f))
|
|
cleared['results'] += 1
|
|
|
|
# Clear retry attempts
|
|
attempts_dir = os.path.join(self.storage_path, 'retry_attempts')
|
|
if os.path.exists(attempts_dir):
|
|
for f in os.listdir(attempts_dir):
|
|
if f.endswith('.json'):
|
|
os.remove(os.path.join(attempts_dir, f))
|
|
cleared['retry_attempts'] += 1
|
|
|
|
# Clear task metadata
|
|
metadata_dir = os.path.join(self.storage_path, 'task_metadata')
|
|
if os.path.exists(metadata_dir):
|
|
for f in os.listdir(metadata_dir):
|
|
if f.endswith('.json'):
|
|
os.remove(os.path.join(metadata_dir, f))
|
|
cleared['task_metadata'] += 1
|
|
|
|
# Clear delivered (success) notifications
|
|
success_dir = os.path.join(self.storage_path, 'success')
|
|
if os.path.exists(success_dir):
|
|
for f in os.listdir(success_dir):
|
|
if f.startswith('success-') and f.endswith('.json'):
|
|
os.remove(os.path.join(success_dir, f))
|
|
cleared['delivered'] += 1
|
|
|
|
return cleared
|
|
|
|
def store_task_metadata(self, task_id, metadata):
|
|
"""Store task metadata as JSON file with atomic write."""
|
|
import time
|
|
|
|
if not self.storage_path:
|
|
return False
|
|
|
|
try:
|
|
metadata_dir = os.path.join(self.storage_path, 'task_metadata')
|
|
os.makedirs(metadata_dir, exist_ok=True)
|
|
|
|
metadata_file = os.path.join(metadata_dir, f"{task_id}.json")
|
|
metadata_with_id = {
|
|
'task_id': task_id,
|
|
'timestamp': time.time(),
|
|
**metadata
|
|
}
|
|
|
|
# Use atomic write to prevent corruption
|
|
_atomic_json_write(metadata_file, metadata_with_id)
|
|
return True
|
|
|
|
except Exception as e:
|
|
logger.debug(f"Unable to store task metadata: {e}")
|
|
return False
|
|
|
|
def get_task_metadata(self, task_id):
|
|
"""Retrieve task metadata from JSON file with corruption handling."""
|
|
if not self.storage_path:
|
|
return None
|
|
|
|
try:
|
|
metadata_dir = os.path.join(self.storage_path, 'task_metadata')
|
|
metadata_file = os.path.join(metadata_dir, f"{task_id}.json")
|
|
|
|
if os.path.exists(metadata_file):
|
|
# Use safe JSON load with corruption handling
|
|
return _safe_json_load(metadata_file, 'task_metadata', self.storage_path)
|
|
|
|
except Exception as e:
|
|
logger.debug(f"Unable to load task metadata for {task_id}: {e}")
|
|
|
|
return None
|
|
|
|
def delete_task_metadata(self, task_id):
|
|
"""Delete task metadata JSON file."""
|
|
if not self.storage_path:
|
|
return False
|
|
|
|
try:
|
|
metadata_dir = os.path.join(self.storage_path, 'task_metadata')
|
|
metadata_file = os.path.join(metadata_dir, f"{task_id}.json")
|
|
|
|
if os.path.exists(metadata_file):
|
|
os.remove(metadata_file)
|
|
return True
|
|
return False
|
|
except Exception as e:
|
|
logger.debug(f"Unable to delete task metadata for {task_id}: {e}")
|
|
return False
|
|
|
|
def cleanup_old_retry_attempts(self, cutoff_time):
|
|
"""Clean up old retry attempt files from filesystem."""
|
|
if not self.storage_path:
|
|
return 0
|
|
|
|
deleted_count = 0
|
|
try:
|
|
attempts_dir = os.path.join(self.storage_path, 'retry_attempts')
|
|
if os.path.exists(attempts_dir):
|
|
for filename in os.listdir(attempts_dir):
|
|
if filename.endswith('.json'):
|
|
filepath = os.path.join(attempts_dir, filename)
|
|
try:
|
|
file_mtime = os.path.getmtime(filepath)
|
|
if file_mtime < cutoff_time:
|
|
os.remove(filepath)
|
|
deleted_count += 1
|
|
except Exception as fe:
|
|
logger.debug(f"Unable to delete old retry attempt file {filename}: {fe}")
|
|
except Exception as e:
|
|
logger.debug(f"Error cleaning up old retry attempts: {e}")
|
|
|
|
return deleted_count
|