feat: add recording viewer and vnc (#78)

* feat: add recording viewer and vnc

* feat: add recording viewer and vnc

* feat: add recording viewer and vnc

* feat: add recording viewer and vnc

* chore: update gitignore [skip ci]

* chore: update dev compose [skip ci]

* fix: only run manually

api/backend/app.py

@@ -2,6 +2,7 @@
 import os
 import logging
 import apscheduler  # type: ignore
+from contextlib import asynccontextmanager
 
 # PDM
 import apscheduler.schedulers
@@ -33,7 +34,30 @@ logging.basicConfig(
 
 LOG = logging.getLogger(__name__)
 
-app = FastAPI(title="api", root_path="/api")
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    # Startup
+    LOG.info("Starting application...")
+
+    init_database()
+
+    LOG.info("Starting cron scheduler...")
+    start_cron_scheduler(scheduler)
+    scheduler.start()
+    LOG.info("Cron scheduler started successfully")
+
+    yield
+
+    # Shutdown
+    LOG.info("Shutting down application...")
+    LOG.info("Stopping cron scheduler...")
+    scheduler.shutdown(wait=False)  # Set wait=False to not block shutdown
+    LOG.info("Cron scheduler stopped")
+    LOG.info("Application shutdown complete")
+
+
+app = FastAPI(title="api", root_path="/api", lifespan=lifespan)
 
 app.add_middleware(
     CORSMiddleware,
@@ -43,28 +67,12 @@ app.add_middleware(
     allow_headers=["*"],
 )
 
-
 app.include_router(auth_router)
 app.include_router(ai_router)
 app.include_router(job_router)
 app.include_router(stats_router)
 
 
-@app.on_event("startup")
-async def startup_event():
-    start_cron_scheduler(scheduler)
-    scheduler.start()
-
-    if os.getenv("ENV") != "test":
-        init_database()
-        LOG.info("Starting up...")
-
-
-@app.on_event("shutdown")
-def shutdown_scheduler():
-    scheduler.shutdown(wait=False)  # Set wait=False to not block shutdown
-
-
 @app.exception_handler(RequestValidationError)
 async def validation_exception_handler(request: Request, exc: RequestValidationError):
     exc_str = f"{exc}".replace("\n", " ").replace("   ", " ")

api/backend/auth/auth_router.py

@@ -66,4 +66,8 @@ async def read_users_me(current_user: User = Depends(get_current_user)):
 
 @auth_router.get("/auth/check")
 async def check_auth():
-    return {"registration": os.environ.get("REGISTRATION_ENABLED", "True") == "True"}
+    return {
+        "registration": os.environ.get("REGISTRATION_ENABLED", "True") == "True",
+        "recordings_enabled": os.environ.get("RECORDINGS_ENABLED", "true").lower()
+        == "true",
+    }

api/backend/constants.py

@@ -1 +1,6 @@
+from pathlib import Path
+import os
+
 DATABASE_PATH = "data/database.db"
+RECORDINGS_DIR = Path("media/recordings")
+RECORDINGS_ENABLED = os.getenv("RECORDINGS_ENABLED", "true").lower() == "true"

api/backend/routers/job_router.py

@@ -10,7 +10,7 @@ import random
 # PDM
 from fastapi import Depends, APIRouter
 from fastapi.encoders import jsonable_encoder
-from fastapi.responses import JSONResponse, StreamingResponse
+from fastapi.responses import FileResponse, JSONResponse, StreamingResponse
 from api.backend.scheduler import scheduler
 from apscheduler.triggers.cron import CronTrigger  # type: ignore
 
@@ -42,6 +42,8 @@ from api.backend.job.cron_scheduling.cron_scheduling import (
 from api.backend.job.utils.clean_job_format import clean_job_format
 from api.backend.job.utils.stream_md_from_job_results import stream_md_from_job_results
 
+from api.backend.constants import RECORDINGS_DIR
+
 LOG = logging.getLogger(__name__)
 
 job_router = APIRouter()
@@ -231,3 +233,14 @@ async def delete_cron_job_request(request: DeleteCronJob):
 async def get_cron_jobs_request(user: User = Depends(get_current_user)):
     cron_jobs = get_cron_jobs(user.email)
     return JSONResponse(content=jsonable_encoder(cron_jobs))
+
+
+@job_router.get("/recordings/{id}")
+async def get_recording(id: str):
+    path = RECORDINGS_DIR / f"{id}.mp4"
+    if not path.exists():
+        return JSONResponse(content={"error": "Recording not found."}, status_code=404)
+
+    return FileResponse(
+        path, headers={"Content-Type": "video/mp4", "Accept-Ranges": "bytes"}
+    )

api/backend/scraping.py

@@ -1,4 +1,5 @@
 import logging
+from pickle import FALSE
 import random
 from typing import Any, Optional, cast
 
@@ -14,6 +15,8 @@ from api.backend.job.site_mapping.site_mapping import handle_site_mapping
 
 from api.backend.job.scraping.add_custom import add_custom_items
 
+from api.backend.constants import RECORDINGS_ENABLED
+
 LOG = logging.getLogger(__name__)
 
 
@@ -57,8 +60,9 @@ async def make_site_request(
         proxy = random.choice(proxies)
         LOG.info(f"Using proxy: {proxy}")
 
-    async with AsyncCamoufox(headless=True, proxy=proxy) as browser:
+    async with AsyncCamoufox(headless=not RECORDINGS_ENABLED, proxy=proxy) as browser:
         page: Page = await browser.new_page()
+        await page.set_viewport_size({"width": 1920, "height": 1080})
 
         # Add cookies and headers
         await add_custom_items(url, page, custom_cookies, headers)

api/backend/worker/job_worker.py

@@ -1,10 +1,12 @@
 import os
 import json
+from pathlib import Path
 
 from api.backend.job import get_queued_job, update_job
 from api.backend.scraping import scrape
 from api.backend.models import Element
 from fastapi.encoders import jsonable_encoder
+import subprocess
 
 import asyncio
 import traceback
@@ -26,14 +28,42 @@ SMTP_USER = os.getenv("SMTP_USER", "")
 SMTP_PASSWORD = os.getenv("SMTP_PASSWORD", "")
 USE_TLS = os.getenv("USE_TLS", "false").lower() == "true"
 
+RECORDINGS_ENABLED = os.getenv("RECORDINGS_ENABLED", "true").lower() == "true"
+RECORDINGS_DIR = Path("/project/app/media/recordings")
+
 
 async def process_job():
     job = await get_queued_job()
+    ffmpeg_proc = None
     status = "Queued"
 
     if job:
         LOG.info(f"Beginning processing job: {job}.")
+
         try:
+            output_path = RECORDINGS_DIR / f"{job['id']}.mp4"
+
+            if RECORDINGS_ENABLED:
+                ffmpeg_proc = subprocess.Popen(
+                    [
+                        "ffmpeg",
+                        "-y",
+                        "-video_size",
+                        "1280x1024",
+                        "-framerate",
+                        "15",
+                        "-f",
+                        "x11grab",
+                        "-i",
+                        ":99",
+                        "-codec:v",
+                        "libx264",
+                        "-preset",
+                        "ultrafast",
+                        output_path,
+                    ]
+                )
+
             _ = await update_job([job["id"]], field="status", value="Scraping")
 
             proxies = job["job_options"]["proxies"]
@@ -87,12 +117,18 @@ async def process_job():
                 },
             )
 
+            if ffmpeg_proc:
+                ffmpeg_proc.terminate()
+                ffmpeg_proc.wait()
+
 
 async def main():
     LOG.info("Starting job worker...")
 
     init_database()
 
+    RECORDINGS_DIR.mkdir(parents=True, exist_ok=True)
+
     while True:
         await process_job()
         await asyncio.sleep(5)