mirror of
https://github.com/jaypyles/Scraperr.git
synced 2026-08-24 13:26:30 +00:00
feat: add recording viewer and vnc (#78)
* feat: add recording viewer and vnc * feat: add recording viewer and vnc * feat: add recording viewer and vnc * feat: add recording viewer and vnc * chore: update gitignore [skip ci] * chore: update dev compose [skip ci] * fix: only run manually
This commit is contained in:
@@ -0,0 +1,4 @@
|
||||
node_modules
|
||||
npm-debug.log
|
||||
Dockerfile
|
||||
.dockerignore
|
||||
@@ -19,7 +19,7 @@ runs:
|
||||
|
||||
- name: Install dependencies
|
||||
shell: bash
|
||||
run: npm install
|
||||
run: yarn install
|
||||
|
||||
- name: Wait for frontend to be ready
|
||||
shell: bash
|
||||
|
||||
@@ -1,14 +1,9 @@
|
||||
name: Docker Image
|
||||
on:
|
||||
workflow_run:
|
||||
workflows: ["Unit Tests"]
|
||||
types:
|
||||
- completed
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
build:
|
||||
if: ${{ github.event.workflow_run.conclusion == 'success' && github.ref == 'refs/heads/master' }}
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
|
||||
+3
-1
@@ -188,4 +188,6 @@ postgres_data
|
||||
.vscode
|
||||
ollama
|
||||
data
|
||||
media
|
||||
media
|
||||
cypress/screenshots
|
||||
cypress/videos
|
||||
@@ -17,6 +17,7 @@ help:
|
||||
@echo " make down - Stop and remove containers, networks, images, and volumes"
|
||||
@echo " make setup - Setup server with dependencies and clone repo"
|
||||
@echo " make deploy - Deploy site onto server"
|
||||
@echo " make cypress-start - Start Cypress"
|
||||
@echo ""
|
||||
|
||||
logs:
|
||||
@@ -51,3 +52,6 @@ setup:
|
||||
|
||||
deploy:
|
||||
ansible-playbook -i ./ansible/inventory.yaml ./ansible/deploy_site.yaml -v
|
||||
|
||||
cypress-start:
|
||||
DISPLAY=:0 npx cypress open
|
||||
+25
-17
@@ -2,6 +2,7 @@
|
||||
import os
|
||||
import logging
|
||||
import apscheduler # type: ignore
|
||||
from contextlib import asynccontextmanager
|
||||
|
||||
# PDM
|
||||
import apscheduler.schedulers
|
||||
@@ -33,7 +34,30 @@ logging.basicConfig(
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
app = FastAPI(title="api", root_path="/api")
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
# Startup
|
||||
LOG.info("Starting application...")
|
||||
|
||||
init_database()
|
||||
|
||||
LOG.info("Starting cron scheduler...")
|
||||
start_cron_scheduler(scheduler)
|
||||
scheduler.start()
|
||||
LOG.info("Cron scheduler started successfully")
|
||||
|
||||
yield
|
||||
|
||||
# Shutdown
|
||||
LOG.info("Shutting down application...")
|
||||
LOG.info("Stopping cron scheduler...")
|
||||
scheduler.shutdown(wait=False) # Set wait=False to not block shutdown
|
||||
LOG.info("Cron scheduler stopped")
|
||||
LOG.info("Application shutdown complete")
|
||||
|
||||
|
||||
app = FastAPI(title="api", root_path="/api", lifespan=lifespan)
|
||||
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
@@ -43,28 +67,12 @@ app.add_middleware(
|
||||
allow_headers=["*"],
|
||||
)
|
||||
|
||||
|
||||
app.include_router(auth_router)
|
||||
app.include_router(ai_router)
|
||||
app.include_router(job_router)
|
||||
app.include_router(stats_router)
|
||||
|
||||
|
||||
@app.on_event("startup")
|
||||
async def startup_event():
|
||||
start_cron_scheduler(scheduler)
|
||||
scheduler.start()
|
||||
|
||||
if os.getenv("ENV") != "test":
|
||||
init_database()
|
||||
LOG.info("Starting up...")
|
||||
|
||||
|
||||
@app.on_event("shutdown")
|
||||
def shutdown_scheduler():
|
||||
scheduler.shutdown(wait=False) # Set wait=False to not block shutdown
|
||||
|
||||
|
||||
@app.exception_handler(RequestValidationError)
|
||||
async def validation_exception_handler(request: Request, exc: RequestValidationError):
|
||||
exc_str = f"{exc}".replace("\n", " ").replace(" ", " ")
|
||||
|
||||
@@ -66,4 +66,8 @@ async def read_users_me(current_user: User = Depends(get_current_user)):
|
||||
|
||||
@auth_router.get("/auth/check")
|
||||
async def check_auth():
|
||||
return {"registration": os.environ.get("REGISTRATION_ENABLED", "True") == "True"}
|
||||
return {
|
||||
"registration": os.environ.get("REGISTRATION_ENABLED", "True") == "True",
|
||||
"recordings_enabled": os.environ.get("RECORDINGS_ENABLED", "true").lower()
|
||||
== "true",
|
||||
}
|
||||
|
||||
@@ -1 +1,6 @@
|
||||
from pathlib import Path
|
||||
import os
|
||||
|
||||
DATABASE_PATH = "data/database.db"
|
||||
RECORDINGS_DIR = Path("media/recordings")
|
||||
RECORDINGS_ENABLED = os.getenv("RECORDINGS_ENABLED", "true").lower() == "true"
|
||||
|
||||
@@ -10,7 +10,7 @@ import random
|
||||
# PDM
|
||||
from fastapi import Depends, APIRouter
|
||||
from fastapi.encoders import jsonable_encoder
|
||||
from fastapi.responses import JSONResponse, StreamingResponse
|
||||
from fastapi.responses import FileResponse, JSONResponse, StreamingResponse
|
||||
from api.backend.scheduler import scheduler
|
||||
from apscheduler.triggers.cron import CronTrigger # type: ignore
|
||||
|
||||
@@ -42,6 +42,8 @@ from api.backend.job.cron_scheduling.cron_scheduling import (
|
||||
from api.backend.job.utils.clean_job_format import clean_job_format
|
||||
from api.backend.job.utils.stream_md_from_job_results import stream_md_from_job_results
|
||||
|
||||
from api.backend.constants import RECORDINGS_DIR
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
job_router = APIRouter()
|
||||
@@ -231,3 +233,14 @@ async def delete_cron_job_request(request: DeleteCronJob):
|
||||
async def get_cron_jobs_request(user: User = Depends(get_current_user)):
|
||||
cron_jobs = get_cron_jobs(user.email)
|
||||
return JSONResponse(content=jsonable_encoder(cron_jobs))
|
||||
|
||||
|
||||
@job_router.get("/recordings/{id}")
|
||||
async def get_recording(id: str):
|
||||
path = RECORDINGS_DIR / f"{id}.mp4"
|
||||
if not path.exists():
|
||||
return JSONResponse(content={"error": "Recording not found."}, status_code=404)
|
||||
|
||||
return FileResponse(
|
||||
path, headers={"Content-Type": "video/mp4", "Accept-Ranges": "bytes"}
|
||||
)
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import logging
|
||||
from pickle import FALSE
|
||||
import random
|
||||
from typing import Any, Optional, cast
|
||||
|
||||
@@ -14,6 +15,8 @@ from api.backend.job.site_mapping.site_mapping import handle_site_mapping
|
||||
|
||||
from api.backend.job.scraping.add_custom import add_custom_items
|
||||
|
||||
from api.backend.constants import RECORDINGS_ENABLED
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@@ -57,8 +60,9 @@ async def make_site_request(
|
||||
proxy = random.choice(proxies)
|
||||
LOG.info(f"Using proxy: {proxy}")
|
||||
|
||||
async with AsyncCamoufox(headless=True, proxy=proxy) as browser:
|
||||
async with AsyncCamoufox(headless=not RECORDINGS_ENABLED, proxy=proxy) as browser:
|
||||
page: Page = await browser.new_page()
|
||||
await page.set_viewport_size({"width": 1920, "height": 1080})
|
||||
|
||||
# Add cookies and headers
|
||||
await add_custom_items(url, page, custom_cookies, headers)
|
||||
|
||||
@@ -1,10 +1,12 @@
|
||||
import os
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from api.backend.job import get_queued_job, update_job
|
||||
from api.backend.scraping import scrape
|
||||
from api.backend.models import Element
|
||||
from fastapi.encoders import jsonable_encoder
|
||||
import subprocess
|
||||
|
||||
import asyncio
|
||||
import traceback
|
||||
@@ -26,14 +28,42 @@ SMTP_USER = os.getenv("SMTP_USER", "")
|
||||
SMTP_PASSWORD = os.getenv("SMTP_PASSWORD", "")
|
||||
USE_TLS = os.getenv("USE_TLS", "false").lower() == "true"
|
||||
|
||||
RECORDINGS_ENABLED = os.getenv("RECORDINGS_ENABLED", "true").lower() == "true"
|
||||
RECORDINGS_DIR = Path("/project/app/media/recordings")
|
||||
|
||||
|
||||
async def process_job():
|
||||
job = await get_queued_job()
|
||||
ffmpeg_proc = None
|
||||
status = "Queued"
|
||||
|
||||
if job:
|
||||
LOG.info(f"Beginning processing job: {job}.")
|
||||
|
||||
try:
|
||||
output_path = RECORDINGS_DIR / f"{job['id']}.mp4"
|
||||
|
||||
if RECORDINGS_ENABLED:
|
||||
ffmpeg_proc = subprocess.Popen(
|
||||
[
|
||||
"ffmpeg",
|
||||
"-y",
|
||||
"-video_size",
|
||||
"1280x1024",
|
||||
"-framerate",
|
||||
"15",
|
||||
"-f",
|
||||
"x11grab",
|
||||
"-i",
|
||||
":99",
|
||||
"-codec:v",
|
||||
"libx264",
|
||||
"-preset",
|
||||
"ultrafast",
|
||||
output_path,
|
||||
]
|
||||
)
|
||||
|
||||
_ = await update_job([job["id"]], field="status", value="Scraping")
|
||||
|
||||
proxies = job["job_options"]["proxies"]
|
||||
@@ -87,12 +117,18 @@ async def process_job():
|
||||
},
|
||||
)
|
||||
|
||||
if ffmpeg_proc:
|
||||
ffmpeg_proc.terminate()
|
||||
ffmpeg_proc.wait()
|
||||
|
||||
|
||||
async def main():
|
||||
LOG.info("Starting job worker...")
|
||||
|
||||
init_database()
|
||||
|
||||
RECORDINGS_DIR.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
while True:
|
||||
await process_job()
|
||||
await asyncio.sleep(5)
|
||||
|
||||
@@ -14,3 +14,5 @@ services:
|
||||
- LOG_LEVEL=INFO
|
||||
volumes:
|
||||
- "$PWD/api:/project/app/api"
|
||||
ports:
|
||||
- "5900:5900"
|
||||
|
||||
@@ -1,7 +1,5 @@
|
||||
services:
|
||||
scraperr:
|
||||
depends_on:
|
||||
- scraperr_api
|
||||
image: jpyles0524/scraperr:1.0.13
|
||||
build:
|
||||
context: .
|
||||
|
||||
@@ -3,7 +3,7 @@ FROM python:3.10.12-slim as pybuilder
|
||||
|
||||
RUN apt-get update && \
|
||||
apt-get install -y curl && \
|
||||
apt-get install -y uvicorn wget gnupg supervisor libgl1 libglx-mesa0 libglx0 vainfo libva-dev libva-glx2 libva-drm2 && \
|
||||
apt-get install -y x11vnc xvfb uvicorn wget gnupg supervisor libgl1 libglx-mesa0 libglx0 vainfo libva-dev libva-glx2 libva-drm2 ffmpeg && \
|
||||
curl -LsSf https://astral.sh/uv/install.sh | sh && \
|
||||
apt-get remove -y curl && \
|
||||
apt-get autoremove -y && \
|
||||
@@ -14,7 +14,8 @@ RUN pdm config python.use_venv false
|
||||
|
||||
WORKDIR /project/app
|
||||
COPY pyproject.toml pdm.lock /project/app/
|
||||
RUN pdm install
|
||||
|
||||
RUN pdm install -v --frozen-lockfile
|
||||
|
||||
RUN pdm run playwright install --with-deps
|
||||
|
||||
@@ -30,7 +31,12 @@ EXPOSE 8000
|
||||
|
||||
WORKDIR /project/app
|
||||
|
||||
RUN mkdir -p /project/app/media
|
||||
RUN mkdir -p /project/app/data
|
||||
RUN touch /project/app/data/database.db
|
||||
|
||||
EXPOSE 5900
|
||||
|
||||
COPY start.sh /project/app/start.sh
|
||||
|
||||
CMD [ "supervisord", "-c", "/etc/supervisor/conf.d/supervisord.conf" ]
|
||||
@@ -1,10 +1,14 @@
|
||||
# Build next dependencies
|
||||
FROM node:23.1
|
||||
FROM node:23.1-slim
|
||||
WORKDIR /app
|
||||
|
||||
COPY package*.json ./
|
||||
RUN npm install
|
||||
# Copy package files first to leverage Docker cache
|
||||
COPY package.json yarn.lock ./
|
||||
|
||||
# Install dependencies in a separate layer
|
||||
RUN yarn install --frozen-lockfile
|
||||
|
||||
# Copy the rest of the application
|
||||
COPY tsconfig.json /app/tsconfig.json
|
||||
COPY tailwind.config.js /app/tailwind.config.js
|
||||
COPY next.config.mjs /app/next.config.mjs
|
||||
@@ -13,6 +17,7 @@ COPY postcss.config.js /app/postcss.config.js
|
||||
COPY public /app/public
|
||||
COPY src /app/src
|
||||
|
||||
RUN npm run build
|
||||
# Build the application
|
||||
RUN yarn build
|
||||
|
||||
EXPOSE 3000
|
||||
Generated
-11371
File diff suppressed because it is too large
Load Diff
+7
-3
@@ -12,9 +12,11 @@
|
||||
"@minchat/react-chat-ui": "^0.16.2",
|
||||
"@mui/icons-material": "^5.15.3",
|
||||
"@mui/material": "^5.16.0",
|
||||
"@reduxjs/toolkit": "^2.8.2",
|
||||
"@testing-library/jest-dom": "^5.16.5",
|
||||
"@testing-library/react": "^13.4.0",
|
||||
"@testing-library/user-event": "^13.5.0",
|
||||
"@types/react": "^18.3.21",
|
||||
"axios": "^1.7.2",
|
||||
"bootstrap": "^5.3.0",
|
||||
"chart.js": "^4.4.3",
|
||||
@@ -30,16 +32,18 @@
|
||||
"react-dom": "^18.3.1",
|
||||
"react-markdown": "^9.0.0",
|
||||
"react-modal-image": "^2.6.0",
|
||||
"react-redux": "^9.2.0",
|
||||
"react-router": "^6.14.1",
|
||||
"react-router-dom": "^6.14.1",
|
||||
"react-spinners": "^0.14.1",
|
||||
"redux-persist": "^6.0.0",
|
||||
"typescript": "^4.9.5",
|
||||
"web-vitals": "^2.1.4"
|
||||
},
|
||||
"scripts": {
|
||||
"dev": "next dev",
|
||||
"build": "next build",
|
||||
"start": "next start",
|
||||
"dev": "yarn next dev",
|
||||
"build": "yarn next build",
|
||||
"start": "yarn next start",
|
||||
"serve": "serve -s ./dist",
|
||||
"cy:open": "cypress open",
|
||||
"cy:run": "cypress run"
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
import React, { useState, useEffect, Dispatch, useRef } from "react";
|
||||
import React, { useState, Dispatch } from "react";
|
||||
import { Job } from "../../types";
|
||||
import { fetchJobs } from "../../lib";
|
||||
import Box from "@mui/material/Box";
|
||||
import InputLabel from "@mui/material/InputLabel";
|
||||
import FormControl from "@mui/material/FormControl";
|
||||
@@ -11,7 +10,9 @@ import { SxProps } from "@mui/material";
|
||||
|
||||
interface Props {
|
||||
sxProps: SxProps;
|
||||
setSelectedJob: Dispatch<React.SetStateAction<Job | null>>;
|
||||
setSelectedJob:
|
||||
| Dispatch<React.SetStateAction<Job | null>>
|
||||
| ((job: Job) => void);
|
||||
selectedJob: Job | null;
|
||||
setJobs: Dispatch<React.SetStateAction<Job[]>>;
|
||||
jobs: Job[];
|
||||
@@ -55,9 +56,11 @@ export const JobSelector = ({
|
||||
value={selectedJob?.id || ""}
|
||||
label="Job"
|
||||
onChange={(e) => {
|
||||
setSelectedJob(
|
||||
jobs.find((job) => job.id === e.target.value) || null
|
||||
);
|
||||
const job = jobs.find((job) => job.id === e.target.value);
|
||||
|
||||
if (job) {
|
||||
setSelectedJob(job);
|
||||
}
|
||||
}}
|
||||
>
|
||||
{jobs.map((job) => (
|
||||
|
||||
@@ -7,7 +7,7 @@ import TerminalIcon from "@mui/icons-material/Terminal";
|
||||
import BarChart from "@mui/icons-material/BarChart";
|
||||
import AutoAwesomeIcon from "@mui/icons-material/AutoAwesome";
|
||||
import { List } from "@mui/material";
|
||||
import { Schedule } from "@mui/icons-material";
|
||||
import { Schedule, VideoFile } from "@mui/icons-material";
|
||||
|
||||
const items = [
|
||||
{
|
||||
@@ -35,6 +35,11 @@ const items = [
|
||||
text: "Cron Jobs",
|
||||
href: "/cron-jobs",
|
||||
},
|
||||
{
|
||||
icon: <VideoFile />,
|
||||
text: "Recordings",
|
||||
href: "/recordings",
|
||||
},
|
||||
];
|
||||
|
||||
export const NavItems = () => {
|
||||
|
||||
@@ -20,7 +20,7 @@ import {
|
||||
import ExpandMoreIcon from "@mui/icons-material/ExpandMore";
|
||||
import StarIcon from "@mui/icons-material/Star";
|
||||
import { Job } from "../../types";
|
||||
import { AutoAwesome } from "@mui/icons-material";
|
||||
import { AutoAwesome, VideoCameraBack } from "@mui/icons-material";
|
||||
import { useRouter } from "next/router";
|
||||
|
||||
interface stringMap {
|
||||
@@ -106,6 +106,22 @@ export const JobQueue = ({
|
||||
</IconButton>
|
||||
</span>
|
||||
</Tooltip>
|
||||
<Tooltip title="View Recording">
|
||||
<span>
|
||||
<IconButton
|
||||
onClick={() => {
|
||||
router.push({
|
||||
pathname: "/recordings",
|
||||
query: {
|
||||
id: row.id,
|
||||
},
|
||||
});
|
||||
}}
|
||||
>
|
||||
<VideoCameraBack />
|
||||
</IconButton>
|
||||
</span>
|
||||
</Tooltip>
|
||||
</TableCell>
|
||||
<TableCell sx={{ maxWidth: 100, overflow: "auto" }}>
|
||||
<Box
|
||||
|
||||
@@ -11,7 +11,7 @@ import {
|
||||
import { JobSelector } from "../../ai";
|
||||
import { Job, Message } from "../../../types";
|
||||
import { useSearchParams } from "next/navigation";
|
||||
import { checkAI, fetchJob, fetchJobs, updateJob } from "../../../lib";
|
||||
import { fetchJob, fetchJobs, updateJob, checkAI } from "../../../lib";
|
||||
import SendIcon from "@mui/icons-material/Send";
|
||||
import EditNoteIcon from "@mui/icons-material/EditNote";
|
||||
|
||||
|
||||
@@ -0,0 +1,204 @@
|
||||
import { JobSelector } from "@/components/ai";
|
||||
import { fetchJobs } from "@/lib";
|
||||
import { useUserSettings } from "@/store/hooks";
|
||||
import { Job } from "@/types";
|
||||
import {
|
||||
Box,
|
||||
useTheme,
|
||||
Typography,
|
||||
CircularProgress,
|
||||
Alert,
|
||||
Paper,
|
||||
} from "@mui/material";
|
||||
import { useRouter, useSearchParams } from "next/navigation";
|
||||
import { useState, useEffect } from "react";
|
||||
|
||||
export const RecordingId = () => {
|
||||
const searchParams = useSearchParams();
|
||||
const theme = useTheme();
|
||||
const { userSettings } = useUserSettings();
|
||||
const router = useRouter();
|
||||
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const [videoUrl, setVideoUrl] = useState<string | null>(null);
|
||||
const [loading, setLoading] = useState(true);
|
||||
const [jobs, setJobs] = useState<Job[]>([]);
|
||||
const [selectedJob, setSelectedJob] = useState<Job | null>(null);
|
||||
|
||||
const currentId = searchParams.get("id");
|
||||
|
||||
const handleSelectJob = (job: Job | null) => {
|
||||
if (job) {
|
||||
router.push(`/recordings?id=${job.id}`);
|
||||
}
|
||||
};
|
||||
|
||||
useEffect(() => {
|
||||
fetchJobs(setJobs);
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
if (!userSettings.recordingsEnabled) {
|
||||
setError("Recordings are disabled");
|
||||
setLoading(false);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!currentId) {
|
||||
setError("No recording ID provided");
|
||||
setLoading(false);
|
||||
return;
|
||||
}
|
||||
|
||||
setLoading(true);
|
||||
setError(null);
|
||||
|
||||
const url = `/api/recordings/${currentId}`;
|
||||
fetch(url, { method: "HEAD" })
|
||||
.then((res) => {
|
||||
if (!res.ok) {
|
||||
throw new Error(`Video not found (status: ${res.status})`);
|
||||
}
|
||||
setVideoUrl(url);
|
||||
})
|
||||
.catch(() => {
|
||||
setError("404 recording not found");
|
||||
})
|
||||
.finally(() => {
|
||||
setLoading(false);
|
||||
});
|
||||
}, [currentId, userSettings.recordingsEnabled]);
|
||||
|
||||
useEffect(() => {
|
||||
if (!currentId) {
|
||||
setSelectedJob(null);
|
||||
return;
|
||||
}
|
||||
|
||||
const job = jobs.find((j) => j.id === currentId);
|
||||
setSelectedJob(job || null);
|
||||
}, [currentId, jobs]);
|
||||
|
||||
return (
|
||||
<Box
|
||||
sx={{
|
||||
height: "100%",
|
||||
width: "100%",
|
||||
display: "flex",
|
||||
flexDirection: "column",
|
||||
position: "relative",
|
||||
borderRadius: 2,
|
||||
overflow: "hidden",
|
||||
border: `1px solid ${theme.palette.divider}`,
|
||||
backgroundColor: theme.palette.background.paper,
|
||||
}}
|
||||
>
|
||||
<Box
|
||||
sx={{
|
||||
display: "flex",
|
||||
justifyContent: "flex-end",
|
||||
p: 1,
|
||||
borderBottom: `1px solid ${theme.palette.divider}`,
|
||||
backgroundColor:
|
||||
theme.palette.mode === "light"
|
||||
? theme.palette.grey[50]
|
||||
: theme.palette.grey[900],
|
||||
zIndex: 10,
|
||||
}}
|
||||
>
|
||||
<Box sx={{ width: "300px" }}>
|
||||
<JobSelector
|
||||
setSelectedJob={handleSelectJob}
|
||||
selectedJob={selectedJob}
|
||||
setJobs={setJobs}
|
||||
jobs={jobs}
|
||||
sxProps={{}}
|
||||
/>
|
||||
</Box>
|
||||
</Box>
|
||||
|
||||
<Box
|
||||
sx={{
|
||||
flex: 1,
|
||||
display: "flex",
|
||||
justifyContent: "center",
|
||||
alignItems: "center",
|
||||
position: "relative",
|
||||
backgroundColor:
|
||||
theme.palette.mode === "light"
|
||||
? theme.palette.grey[100]
|
||||
: theme.palette.grey[900],
|
||||
p: 2,
|
||||
overflow: "hidden",
|
||||
}}
|
||||
>
|
||||
{loading ? (
|
||||
<Box
|
||||
display="flex"
|
||||
flexDirection="column"
|
||||
alignItems="center"
|
||||
gap={2}
|
||||
>
|
||||
<CircularProgress />
|
||||
<Typography variant="body2" color="textSecondary">
|
||||
Loading recording...
|
||||
</Typography>
|
||||
</Box>
|
||||
) : error ? (
|
||||
<Paper
|
||||
elevation={3}
|
||||
sx={{
|
||||
p: 3,
|
||||
maxWidth: "500px",
|
||||
width: "100%",
|
||||
backgroundColor: theme.palette.background.paper,
|
||||
borderRadius: 2,
|
||||
}}
|
||||
>
|
||||
<Alert
|
||||
severity="error"
|
||||
variant="filled"
|
||||
sx={{
|
||||
mb: 2,
|
||||
backgroundColor: theme.palette.error.main,
|
||||
}}
|
||||
>
|
||||
{error}
|
||||
</Alert>
|
||||
<Typography variant="body2" color="textSecondary" sx={{ mt: 2 }}>
|
||||
Please select a different recording from the dropdown menu above
|
||||
or check if recordings are enabled.
|
||||
</Typography>
|
||||
</Paper>
|
||||
) : (
|
||||
<Box
|
||||
sx={{
|
||||
width: "100%",
|
||||
height: "100%",
|
||||
display: "flex",
|
||||
justifyContent: "center",
|
||||
alignItems: "center",
|
||||
overflow: "hidden",
|
||||
borderRadius: 1,
|
||||
}}
|
||||
>
|
||||
<video
|
||||
className="h-full w-full object-contain"
|
||||
controls
|
||||
onError={() => setError("Error loading video")}
|
||||
style={{
|
||||
maxHeight: "100%",
|
||||
maxWidth: "100%",
|
||||
borderRadius: "4px",
|
||||
boxShadow: theme.shadows[4],
|
||||
}}
|
||||
>
|
||||
<source src={videoUrl ?? undefined} type="video/mp4" />
|
||||
Your browser does not support the video tag.
|
||||
</video>
|
||||
</Box>
|
||||
)}
|
||||
</Box>
|
||||
</Box>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1 @@
|
||||
export { RecordingId } from "./id";
|
||||
@@ -80,3 +80,22 @@ export const updateJob = async (ids: string[], field: string, value: any) => {
|
||||
console.error("Error fetching jobs:", error);
|
||||
});
|
||||
};
|
||||
|
||||
export const getUserSettings = async () => {
|
||||
const token = Cookies.get("token");
|
||||
|
||||
try {
|
||||
const response = await fetch("/api/check", {
|
||||
headers: {
|
||||
"content-type": "application/json",
|
||||
Authorization: `Bearer ${token}`,
|
||||
},
|
||||
});
|
||||
|
||||
const data = await response.json();
|
||||
return data;
|
||||
} catch (error) {
|
||||
console.error("Error fetching jobs:", error);
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
|
||||
+27
-20
@@ -8,6 +8,9 @@ import { ThemeProvider, CssBaseline, Box } from "@mui/material";
|
||||
import { NavDrawer } from "../components/common";
|
||||
import { darkTheme, lightTheme } from "../styles/themes";
|
||||
import { AuthProvider } from "../contexts/AuthContext";
|
||||
import { Provider } from "react-redux";
|
||||
import { PersistGate } from "redux-persist/integration/react";
|
||||
import { store, persistor } from "@/store/store";
|
||||
|
||||
const App: React.FC<AppProps> = ({ Component, pageProps }) => {
|
||||
const [isDarkMode, setIsDarkMode] = useState(false);
|
||||
@@ -35,26 +38,30 @@ const App: React.FC<AppProps> = ({ Component, pageProps }) => {
|
||||
<Head>
|
||||
<title>Scraperr</title>
|
||||
</Head>
|
||||
<AuthProvider>
|
||||
<ThemeProvider theme={isDarkMode ? darkTheme : lightTheme}>
|
||||
<CssBaseline />
|
||||
<Box sx={{ height: "100%", display: "flex" }}>
|
||||
<NavDrawer isDarkMode={isDarkMode} toggleTheme={toggleTheme} />
|
||||
<Box
|
||||
component="main"
|
||||
sx={{
|
||||
p: 3,
|
||||
bgcolor: "background.default",
|
||||
overflow: "hidden",
|
||||
height: "100%",
|
||||
width: "100%",
|
||||
}}
|
||||
>
|
||||
<Component {...pageProps} />
|
||||
</Box>
|
||||
</Box>
|
||||
</ThemeProvider>
|
||||
</AuthProvider>
|
||||
<Provider store={store}>
|
||||
<PersistGate loading={null} persistor={persistor}>
|
||||
<AuthProvider>
|
||||
<ThemeProvider theme={isDarkMode ? darkTheme : lightTheme}>
|
||||
<CssBaseline />
|
||||
<Box sx={{ height: "100%", display: "flex" }}>
|
||||
<NavDrawer isDarkMode={isDarkMode} toggleTheme={toggleTheme} />
|
||||
<Box
|
||||
component="main"
|
||||
sx={{
|
||||
p: 3,
|
||||
bgcolor: "background.default",
|
||||
overflow: "hidden",
|
||||
height: "100%",
|
||||
width: "100%",
|
||||
}}
|
||||
>
|
||||
<Component {...pageProps} />
|
||||
</Box>
|
||||
</Box>
|
||||
</ThemeProvider>
|
||||
</AuthProvider>
|
||||
</PersistGate>
|
||||
</Provider>
|
||||
</>
|
||||
);
|
||||
};
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
import { NextApiRequest, NextApiResponse } from "next";
|
||||
|
||||
export default async function handler(
|
||||
req: NextApiRequest,
|
||||
res: NextApiResponse
|
||||
) {
|
||||
const { id } = req.query;
|
||||
|
||||
try {
|
||||
const response = await fetch(
|
||||
`${process.env.NEXT_PUBLIC_API_URL}/recordings/${id}`
|
||||
);
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`Error: ${response.statusText}`);
|
||||
}
|
||||
|
||||
res.setHeader("Content-Type", "video/mp4");
|
||||
res.setHeader("Accept-Ranges", "bytes");
|
||||
|
||||
const reader = response.body?.getReader();
|
||||
|
||||
if (!reader) {
|
||||
res.status(404).json({ error: "Recording not found" });
|
||||
return;
|
||||
}
|
||||
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
res.write(value);
|
||||
}
|
||||
|
||||
res.end();
|
||||
} catch (error) {
|
||||
console.error("Error streaming video:", error);
|
||||
res.status(404).json({ error: "Error streaming video" });
|
||||
}
|
||||
}
|
||||
+8
-1
@@ -6,7 +6,8 @@ import { Button, TextField, Typography, Box } from "@mui/material";
|
||||
import { useTheme } from "@mui/material/styles";
|
||||
import { useRouter } from "next/router";
|
||||
import { useAuth } from "../contexts/AuthContext";
|
||||
import { Constants } from "../lib";
|
||||
import { Constants, getUserSettings } from "../lib";
|
||||
import { useUserSettings } from "@/store/hooks";
|
||||
|
||||
type Mode = "login" | "signup";
|
||||
|
||||
@@ -19,6 +20,7 @@ const AuthForm: React.FC = () => {
|
||||
const router = useRouter();
|
||||
const { login } = useAuth();
|
||||
const [registrationEnabled, setRegistrationEnabled] = useState<boolean>(true);
|
||||
const { setUserSettings } = useUserSettings();
|
||||
|
||||
const checkRegistrationEnabled = async () => {
|
||||
const response = await axios.get(`/api/check`);
|
||||
@@ -28,12 +30,17 @@ const AuthForm: React.FC = () => {
|
||||
useEffect(() => {
|
||||
checkRegistrationEnabled();
|
||||
}, []);
|
||||
|
||||
const handleSubmit = async (event: React.FormEvent) => {
|
||||
event.preventDefault();
|
||||
try {
|
||||
if (mode === "login") {
|
||||
await login(email, password);
|
||||
alert("Login successful");
|
||||
|
||||
const userSettings = await getUserSettings();
|
||||
setUserSettings(userSettings);
|
||||
|
||||
router.push("/");
|
||||
} else {
|
||||
await axios.post(`/api/signup`, {
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
export { RecordingId as default } from "@/components/pages/recordings/id";
|
||||
@@ -0,0 +1,23 @@
|
||||
import { TypedUseSelectorHook, useDispatch, useSelector } from "react-redux";
|
||||
import type { RootState, AppDispatch } from "./store";
|
||||
import {
|
||||
SettingsState,
|
||||
setAiEnabled,
|
||||
setRecordingsEnabled,
|
||||
} from "./slices/settingsSlice";
|
||||
|
||||
export const useAppDispatch = () => useDispatch<AppDispatch>();
|
||||
export const useAppSelector: TypedUseSelectorHook<RootState> = useSelector;
|
||||
|
||||
export const useUserSettings = () => {
|
||||
const userSettings = useAppSelector((state) => state.settings);
|
||||
const dispatch = useAppDispatch();
|
||||
|
||||
const setUserSettings = (userSettings: any) => {
|
||||
dispatch(setAiEnabled(userSettings.ai_enabled));
|
||||
dispatch(setRecordingsEnabled(userSettings.recordings_enabled));
|
||||
return userSettings;
|
||||
};
|
||||
|
||||
return { userSettings, setUserSettings };
|
||||
};
|
||||
@@ -0,0 +1,28 @@
|
||||
import { createSlice, PayloadAction } from "@reduxjs/toolkit";
|
||||
|
||||
export interface SettingsState {
|
||||
aiEnabled: boolean;
|
||||
recordingsEnabled: boolean;
|
||||
}
|
||||
|
||||
const initialState: SettingsState = {
|
||||
aiEnabled: false,
|
||||
recordingsEnabled: false,
|
||||
};
|
||||
|
||||
const settingsSlice = createSlice({
|
||||
name: "settings",
|
||||
initialState,
|
||||
reducers: {
|
||||
setAiEnabled: (state, action: PayloadAction<boolean>) => {
|
||||
state.aiEnabled = action.payload;
|
||||
},
|
||||
setRecordingsEnabled: (state, action: PayloadAction<boolean>) => {
|
||||
state.recordingsEnabled = action.payload;
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
export const { setAiEnabled, setRecordingsEnabled } = settingsSlice.actions;
|
||||
|
||||
export default settingsSlice.reducer;
|
||||
@@ -0,0 +1,32 @@
|
||||
import { configureStore } from "@reduxjs/toolkit";
|
||||
import { persistStore, persistReducer } from "redux-persist";
|
||||
import storage from "redux-persist/lib/storage";
|
||||
import { combineReducers } from "@reduxjs/toolkit";
|
||||
import settingsReducer from "./slices/settingsSlice";
|
||||
|
||||
const persistConfig = {
|
||||
key: "root",
|
||||
storage,
|
||||
whitelist: ["settings"], // only settings will be persisted
|
||||
};
|
||||
|
||||
const rootReducer = combineReducers({
|
||||
settings: settingsReducer,
|
||||
});
|
||||
|
||||
const persistedReducer = persistReducer(persistConfig, rootReducer);
|
||||
|
||||
export const store = configureStore({
|
||||
reducer: persistedReducer,
|
||||
middleware: (getDefaultMiddleware) =>
|
||||
getDefaultMiddleware({
|
||||
serializableCheck: {
|
||||
ignoredActions: ["persist/PERSIST", "persist/REHYDRATE"],
|
||||
},
|
||||
}),
|
||||
});
|
||||
|
||||
export const persistor = persistStore(store);
|
||||
|
||||
export type RootState = ReturnType<typeof store.getState>;
|
||||
export type AppDispatch = typeof store.dispatch;
|
||||
@@ -0,0 +1,14 @@
|
||||
#!/bin/bash
|
||||
|
||||
RECORDINGS_ENABLED=${RECORDINGS_ENABLED:-true}
|
||||
|
||||
if [ "$RECORDINGS_ENABLED" == "false" ]; then
|
||||
pdm run python -m api.backend.worker.job_worker
|
||||
else
|
||||
Xvfb :99 -screen 0 1280x1024x24 &
|
||||
XVFB_PID=$!
|
||||
sleep 2
|
||||
x11vnc -display :99 -rfbport 5900 -forever -nopw &
|
||||
VNC_PID=$!
|
||||
DISPLAY=:99 pdm run python -m api.backend.worker.job_worker
|
||||
fi
|
||||
+1
-1
@@ -12,7 +12,7 @@ stdout_logfile_maxbytes=0
|
||||
stderr_logfile_maxbytes=0
|
||||
|
||||
[program:worker]
|
||||
command=pdm run python -m api.backend.worker.job_worker
|
||||
command=/project/app/start.sh
|
||||
directory=/project/app
|
||||
autostart=true
|
||||
autorestart=true
|
||||
|
||||
Reference in New Issue
Block a user