Add cron jobs (#60)

* feat: finish up cron jobs

* feat: clean up
This commit is contained in:
Jayden Pyles
2025-04-24 22:03:28 -05:00
committed by GitHub
parent 186b4a0231
commit 3475d66995
21 changed files with 717 additions and 16 deletions

View File

@@ -1,9 +1,13 @@
# STL
import os
import logging
import apscheduler # type: ignore
# PDM
from fastapi import FastAPI
import apscheduler.schedulers
import apscheduler.schedulers.background
from fastapi import FastAPI, Request, status
from fastapi.exceptions import RequestValidationError
from fastapi.middleware.cors import CORSMiddleware
# LOCAL
@@ -14,6 +18,10 @@ from api.backend.routers.job_router import job_router
from api.backend.routers.log_router import log_router
from api.backend.routers.stats_router import stats_router
from api.backend.database.startup import init_database
from fastapi.responses import JSONResponse
from api.backend.job.cron_scheduling.cron_scheduling import start_cron_scheduler
from api.backend.scheduler import scheduler
log_level = os.getenv("LOG_LEVEL")
LOG_LEVEL = get_log_level(log_level)
@@ -46,6 +54,24 @@ app.include_router(stats_router)
@app.on_event("startup")
async def startup_event():
start_cron_scheduler(scheduler)
scheduler.start()
if os.getenv("ENV") != "test":
init_database()
LOG.info("Starting up...")
@app.on_event("shutdown")
def shutdown_scheduler():
scheduler.shutdown(wait=False) # Set wait=False to not block shutdown
@app.exception_handler(RequestValidationError)
async def validation_exception_handler(request: Request, exc: RequestValidationError):
exc_str = f"{exc}".replace("\n", " ").replace(" ", " ")
logging.error(f"{request}: {exc_str}")
content = {"status_code": 10422, "message": exc_str, "data": None}
return JSONResponse(
content=content, status_code=status.HTTP_422_UNPROCESSABLE_ENTITY
)

View File

@@ -1,7 +1,5 @@
# STL
import os
from gc import disable
from queue import Empty
from typing import Any, Optional
from datetime import datetime, timedelta
import logging
@@ -78,10 +76,10 @@ def create_access_token(
async def get_current_user(token: str = Depends(oauth2_scheme)):
LOG.info(f"Getting current user with token: {token}")
LOG.debug(f"Getting current user with token: {token}")
if not token:
LOG.error("No token provided")
LOG.debug("No token provided")
return EMPTY_USER
if len(token.split(".")) != 3:
@@ -89,7 +87,7 @@ async def get_current_user(token: str = Depends(oauth2_scheme)):
return EMPTY_USER
try:
LOG.info(
LOG.debug(
f"Decoding token: {token} with secret key: {SECRET_KEY} and algorithm: {ALGORITHM}"
)

View File

@@ -17,4 +17,14 @@ CREATE TABLE IF NOT EXISTS users (
full_name STRING,
disabled BOOLEAN
);
CREATE TABLE IF NOT EXISTS cron_jobs (
id STRING PRIMARY KEY NOT NULL,
user_email STRING NOT NULL,
job_id STRING NOT NULL,
cron_expression STRING NOT NULL,
time_created DATETIME NOT NULL,
time_updated DATETIME NOT NULL,
FOREIGN KEY (job_id) REFERENCES jobs(id)
);
"""

View File

@@ -0,0 +1,100 @@
import datetime
from typing import Any
import uuid
from api.backend.database.common import insert, query
from api.backend.models import CronJob
from apscheduler.schedulers.background import BackgroundScheduler # type: ignore
from apscheduler.triggers.cron import CronTrigger # type: ignore
from api.backend.job import insert as insert_job
import logging
LOG = logging.getLogger("Cron Scheduler")
def insert_cron_job(cron_job: CronJob):
query = """
INSERT INTO cron_jobs (id, user_email, job_id, cron_expression, time_created, time_updated)
VALUES (?, ?, ?, ?, ?, ?)
"""
values = (
cron_job.id,
cron_job.user_email,
cron_job.job_id,
cron_job.cron_expression,
cron_job.time_created,
cron_job.time_updated,
)
insert(query, values)
return True
def delete_cron_job(id: str, user_email: str):
query = """
DELETE FROM cron_jobs
WHERE id = ? AND user_email = ?
"""
values = (id, user_email)
insert(query, values)
return True
def get_cron_jobs(user_email: str):
cron_jobs = query("SELECT * FROM cron_jobs WHERE user_email = ?", (user_email,))
return cron_jobs
def get_all_cron_jobs():
cron_jobs = query("SELECT * FROM cron_jobs")
return cron_jobs
def insert_job_from_cron_job(job: dict[str, Any]):
insert_job(
{
**job,
"id": uuid.uuid4().hex,
"status": "Queued",
"result": "",
"chat": None,
"time_created": datetime.datetime.now(),
"time_updated": datetime.datetime.now(),
}
)
def get_cron_job_trigger(cron_expression: str):
expression_parts = cron_expression.split()
if len(expression_parts) != 5:
print(f"Invalid cron expression: {cron_expression}")
return None
minute, hour, day, month, day_of_week = expression_parts
return CronTrigger(
minute=minute, hour=hour, day=day, month=month, day_of_week=day_of_week
)
def start_cron_scheduler(scheduler: BackgroundScheduler):
cron_jobs = get_all_cron_jobs()
LOG.info(f"Cron jobs: {cron_jobs}")
for job in cron_jobs:
queried_job = query("SELECT * FROM jobs WHERE id = ?", (job["job_id"],))
LOG.info(f"Adding job: {queried_job}")
scheduler.add_job(
insert_job_from_cron_job,
get_cron_job_trigger(job["cron_expression"]),
id=job["id"],
args=[queried_job[0]],
)

View File

@@ -14,7 +14,7 @@ from api.backend.database.common import (
LOG = logging.getLogger(__name__)
async def insert(item: dict[str, Any]) -> None:
def insert(item: dict[str, Any]) -> None:
common_insert(
QUERIES["insert_job"],
(

View File

@@ -57,3 +57,17 @@ class Job(pydantic.BaseModel):
job_options: JobOptions
status: str = "Queued"
chat: Optional[str] = None
class CronJob(pydantic.BaseModel):
id: Optional[str] = None
user_email: str
job_id: str
cron_expression: str
time_created: Optional[Union[datetime, str]] = None
time_updated: Optional[Union[datetime, str]] = None
class DeleteCronJob(pydantic.BaseModel):
id: str
user_email: str

View File

@@ -1,4 +1,5 @@
# STL
import datetime
import uuid
import traceback
from io import StringIO
@@ -10,14 +11,18 @@ import random
from fastapi import Depends, APIRouter
from fastapi.encoders import jsonable_encoder
from fastapi.responses import JSONResponse, StreamingResponse
from api.backend.scheduler import scheduler
from apscheduler.triggers.cron import CronTrigger # type: ignore
# LOCAL
from api.backend.job import insert, update_job, delete_jobs
from api.backend.models import (
DeleteCronJob,
UpdateJobs,
DownloadJob,
DeleteScrapeJobs,
Job,
CronJob,
)
from api.backend.schemas import User
from api.backend.auth.auth_utils import get_current_user
@@ -26,6 +31,14 @@ from api.backend.job.models.job_options import FetchOptions
from api.backend.database.common import query
from api.backend.job.cron_scheduling.cron_scheduling import (
delete_cron_job,
get_cron_job_trigger,
insert_cron_job,
get_cron_jobs,
insert_job_from_cron_job,
)
LOG = logging.getLogger(__name__)
job_router = APIRouter()
@@ -44,7 +57,7 @@ async def submit_scrape_job(job: Job):
job.id = uuid.uuid4().hex
job_dict = job.model_dump()
await insert(job_dict)
insert(job_dict)
return JSONResponse(content={"id": job.id})
except Exception as e:
@@ -140,3 +153,47 @@ async def delete(delete_scrape_jobs: DeleteScrapeJobs):
if result
else JSONResponse({"error": "Jobs not deleted."})
)
@job_router.post("/schedule-cron-job")
async def schedule_cron_job(cron_job: CronJob):
if not cron_job.id:
cron_job.id = uuid.uuid4().hex
if not cron_job.time_created:
cron_job.time_created = datetime.datetime.now()
if not cron_job.time_updated:
cron_job.time_updated = datetime.datetime.now()
insert_cron_job(cron_job)
queried_job = query("SELECT * FROM jobs WHERE id = ?", (cron_job.job_id,))
scheduler.add_job(
insert_job_from_cron_job,
get_cron_job_trigger(cron_job.cron_expression),
id=cron_job.id,
args=[queried_job[0]],
)
return JSONResponse(content={"message": "Cron job scheduled successfully."})
@job_router.post("/delete-cron-job")
async def delete_cron_job_request(request: DeleteCronJob):
if not request.id:
return JSONResponse(
content={"error": "Cron job id is required."}, status_code=400
)
delete_cron_job(request.id, request.user_email)
scheduler.remove_job(request.id)
return JSONResponse(content={"message": "Cron job deleted successfully."})
@job_router.get("/cron-jobs")
async def get_cron_jobs_request(user: User = Depends(get_current_user)):
cron_jobs = get_cron_jobs(user.email)
return JSONResponse(content=jsonable_encoder(cron_jobs))

3
api/backend/scheduler.py Normal file
View File

@@ -0,0 +1,3 @@
from apscheduler.schedulers.background import BackgroundScheduler # type: ignore
scheduler = BackgroundScheduler()

31
pdm.lock generated
View File

@@ -5,7 +5,7 @@
groups = ["default", "dev"]
strategy = ["inherit_metadata"]
lock_version = "4.5.0"
content_hash = "sha256:d3c8eb4d20f8aaddc2f44cea1629b3ee4fe1efa3aad65b6700d085bd9f31558b"
content_hash = "sha256:1d142e8b44e3a6a04135c54e1967b7c19c5c7ccd6b2ff8ec8bca8792bf961bb9"
[[metadata.targets]]
requires_python = ">=3.10"
@@ -171,6 +171,21 @@ files = [
{file = "appdirs-1.4.4.tar.gz", hash = "sha256:7d5d0167b2b1ba821647616af46a749d1c653740dd0d2415100fe26e27afdf41"},
]
[[package]]
name = "apscheduler"
version = "3.11.0"
requires_python = ">=3.8"
summary = "In-process task scheduler with Cron-like capabilities"
groups = ["default"]
dependencies = [
"backports-zoneinfo; python_version < \"3.9\"",
"tzlocal>=3.0",
]
files = [
{file = "APScheduler-3.11.0-py3-none-any.whl", hash = "sha256:fc134ca32e50f5eadcc4938e3a4545ab19131435e851abb40b34d63d5141c6da"},
{file = "apscheduler-3.11.0.tar.gz", hash = "sha256:4c622d250b0955a65d5d0eb91c33e6d43fd879834bf541e0a18661ae60460133"},
]
[[package]]
name = "asgiref"
version = "3.8.1"
@@ -2883,6 +2898,20 @@ files = [
{file = "tzdata-2025.2.tar.gz", hash = "sha256:b60a638fcc0daffadf82fe0f57e53d06bdec2f36c4df66280ae79bce6bd6f2b9"},
]
[[package]]
name = "tzlocal"
version = "5.3.1"
requires_python = ">=3.9"
summary = "tzinfo object for the local timezone"
groups = ["default"]
dependencies = [
"tzdata; platform_system == \"Windows\"",
]
files = [
{file = "tzlocal-5.3.1-py3-none-any.whl", hash = "sha256:eb1a66c3ef5847adf7a834f1be0800581b683b5608e74f86ecbcef8ab91bb85d"},
{file = "tzlocal-5.3.1.tar.gz", hash = "sha256:cceffc7edecefea1f595541dbd6e990cb1ea3d19bf01b2809f362a03dd7921fd"},
]
[[package]]
name = "urllib3"
version = "2.4.0"

View File

@@ -39,6 +39,7 @@ dependencies = [
"pytest-asyncio>=0.24.0",
"python-multipart>=0.0.1",
"bcrypt==4.0.1",
"apscheduler>=3.11.0",
]
requires-python = ">=3.10"
readme = "README.md"
@@ -56,14 +57,42 @@ ignore = []
defineConstant = { DEBUG = true }
stubPath = ""
reportUnknownMemberType = false
reportMissingImports = true
reportMissingTypeStubs = false
reportAny = false
reportCallInDefaultInitializer = false
# Type checking strictness
typeCheckingMode = "strict" # Enables strict type checking mode
reportPrivateUsage = "error"
reportMissingTypeStubs = "error"
reportUntypedFunctionDecorator = "error"
reportUntypedClassDecorator = "error"
reportUntypedBaseClass = "error"
reportInvalidTypeVarUse = "error"
reportUnnecessaryTypeIgnoreComment = "information"
reportUnknownVariableType = "none"
reportUnknownMemberType = "none"
reportUnknownParameterType = "none"
pythonVersion = "3.9"
pythonPlatform = "Linux"
# Additional checks
reportImplicitStringConcatenation = "error"
reportInvalidStringEscapeSequence = "error"
reportMissingImports = "error"
reportMissingModuleSource = "error"
reportOptionalCall = "error"
reportOptionalIterable = "error"
reportOptionalMemberAccess = "error"
reportOptionalOperand = "error"
reportOptionalSubscript = "error"
reportTypedDictNotRequiredAccess = "error"
# Function return type checking
reportIncompleteStub = "error"
reportIncompatibleMethodOverride = "error"
reportInvalidStubStatement = "error"
reportInconsistentOverload = "error"
# Misc settings
pythonVersion = "3.10" # Matches your Python version from pyproject.toml
strictListInference = true
strictDictionaryInference = true
strictSetInference = true
[tool.isort]

View File

@@ -7,6 +7,7 @@ import TerminalIcon from "@mui/icons-material/Terminal";
import BarChart from "@mui/icons-material/BarChart";
import AutoAwesomeIcon from "@mui/icons-material/AutoAwesome";
import { List } from "@mui/material";
import { Schedule } from "@mui/icons-material";
const items = [
{
@@ -34,6 +35,11 @@ const items = [
text: "View App Logs",
href: "/logs",
},
{
icon: <Schedule />,
text: "Cron Jobs",
href: "/cron-jobs",
},
];
export const NavItems = () => {

View File

@@ -0,0 +1,182 @@
import { Job } from "@/types";
import {
Button,
Dialog,
DialogTitle,
DialogContent,
TextField,
Snackbar,
Alert,
} from "@mui/material";
import Cookies from "js-cookie";
import { useState } from "react";
export type CreateCronJobsProps = {
availableJobs: Job[];
user: any;
};
export const CreateCronJobs = ({
availableJobs,
user,
}: CreateCronJobsProps) => {
const [open, setOpen] = useState(false);
return (
<>
<Button
variant="contained"
color="primary"
onClick={() => setOpen(true)}
sx={{ borderRadius: 2 }}
>
Create Cron Job
</Button>
<CreateCronJobDialog
open={open}
onClose={() => setOpen(false)}
availableJobs={availableJobs}
user={user}
/>
</>
);
};
const CreateCronJobDialog = ({
open,
onClose,
availableJobs,
user,
}: {
open: boolean;
onClose: () => void;
availableJobs: Job[];
user: any;
}) => {
const [cronExpression, setCronExpression] = useState("");
const [jobId, setJobId] = useState("");
const [successOpen, setSuccessOpen] = useState(false);
const [isSubmitting, setIsSubmitting] = useState(false);
const [error, setError] = useState("");
const handleSubmit = async () => {
if (!cronExpression || !jobId) {
setError("Please fill in all fields");
return;
}
setIsSubmitting(true);
const token = Cookies.get("token");
try {
const response = await fetch("/api/schedule-cron-job", {
method: "POST",
headers: {
"Content-Type": "application/json",
Authorization: `Bearer ${token}`,
},
body: JSON.stringify({
data: {
cron_expression: cronExpression,
job_id: jobId,
user_email: user.email,
},
}),
});
if (!response.ok) {
throw new Error("Failed to schedule job");
}
setSuccessOpen(true);
setCronExpression("");
setJobId("");
setTimeout(() => {
onClose();
}, 1500);
window.location.reload();
} catch (error) {
console.error(error);
setError("Failed to create cron job");
} finally {
setIsSubmitting(false);
}
};
const handleClose = () => {
setSuccessOpen(false);
};
return (
<>
<Dialog
open={open}
onClose={onClose}
PaperProps={{
sx: { borderRadius: 2, minWidth: "400px" },
}}
>
<DialogTitle sx={{ fontWeight: 500 }}>Create Cron Job</DialogTitle>
<DialogContent>
<div className="flex flex-col gap-1 mt0">
<TextField
label="Cron Expression"
fullWidth
value={cronExpression}
onChange={(e) => setCronExpression(e.target.value)}
variant="outlined"
placeholder="* * * * *"
margin="normal"
helperText="Format: minute hour day month day-of-week"
/>
<TextField
label="Job ID"
fullWidth
value={jobId}
onChange={(e) => setJobId(e.target.value)}
variant="outlined"
margin="normal"
/>
{error && (
<Alert severity="error" sx={{ mt: 2 }}>
{error}
</Alert>
)}
<div className="flex justify-end gap-2 mt-4">
<Button
variant="outlined"
onClick={onClose}
sx={{ borderRadius: 2 }}
>
Cancel
</Button>
<Button
variant="contained"
color="primary"
onClick={handleSubmit}
disabled={isSubmitting}
sx={{ borderRadius: 2 }}
>
{isSubmitting ? "Submitting..." : "Create Job"}
</Button>
</div>
</div>
</DialogContent>
</Dialog>
<Snackbar
open={successOpen}
autoHideDuration={4000}
onClose={handleClose}
anchorOrigin={{ vertical: "bottom", horizontal: "right" }}
>
<Alert onClose={handleClose} severity="success" sx={{ width: "100%" }}>
Cron job created successfully!
</Alert>
</Snackbar>
</>
);
};

View File

@@ -0,0 +1 @@
export * from "./create-cron-jobs";

View File

@@ -0,0 +1,92 @@
import { Job, CronJob } from "@/types/job";
import { useState, useEffect } from "react";
import { CreateCronJobs } from "./create-cron-jobs";
import {
Table,
TableHead,
TableRow,
TableCell,
TableBody,
Button,
} from "@mui/material";
import Cookies from "js-cookie";
export type CronJobsProps = {
initialJobs: Job[];
initialCronJobs: CronJob[];
initialUser: any;
};
export const CronJobs = ({
initialJobs,
initialCronJobs,
initialUser,
}: CronJobsProps) => {
const [jobs, setJobs] = useState<Job[]>(initialJobs);
const [cronJobs, setCronJobs] = useState<CronJob[]>(initialCronJobs);
const [user, setUser] = useState<any>(initialUser);
useEffect(() => {
setJobs(initialJobs);
setCronJobs(initialCronJobs);
setUser(initialUser);
}, [initialJobs, initialCronJobs, initialUser]);
const handleDeleteCronJob = async (id: string) => {
const token = Cookies.get("token");
const response = await fetch("/api/delete-cron-job", {
method: "POST",
headers: {
"Content-Type": "application/json",
Authorization: `Bearer ${token}`,
},
body: JSON.stringify({ data: { id, user_email: user.email } }),
});
if (response.ok) {
console.log("Cron job deleted successfully");
setCronJobs(cronJobs.filter((cronJob) => cronJob.id !== id));
} else {
console.error("Failed to delete cron job");
}
};
return (
<div>
<CreateCronJobs availableJobs={jobs} user={user} />
<Table>
<TableHead>
<TableRow>
<TableCell>Cron Expression</TableCell>
<TableCell>Job ID</TableCell>
<TableCell>User Email</TableCell>
<TableCell>Created At</TableCell>
<TableCell>Updated At</TableCell>
<TableCell>Actions</TableCell>
</TableRow>
</TableHead>
<TableBody>
{cronJobs.map((cronJob) => (
<TableRow key={cronJob.id}>
<TableCell>{cronJob.cron_expression}</TableCell>
<TableCell>{cronJob.job_id}</TableCell>
<TableCell>{cronJob.user_email}</TableCell>
<TableCell>
{new Date(cronJob.time_created).toLocaleString()}
</TableCell>
<TableCell>
{new Date(cronJob.time_updated).toLocaleString()}
</TableCell>
<TableCell>
<Button onClick={() => handleDeleteCronJob(cronJob.id)}>
Delete
</Button>
</TableCell>
</TableRow>
))}
</TableBody>
</Table>
</div>
);
};

View File

@@ -0,0 +1,62 @@
import axios from "axios";
import { GetServerSideProps } from "next";
import { parseCookies } from "nookies";
import { CronJob, Job } from "../../../types";
export const getServerSideProps: GetServerSideProps = async (context) => {
const { req } = context;
const cookies = parseCookies({ req });
const token = cookies.token;
let user = null;
let initialJobs: Job[] = [];
let initialCronJobs: CronJob[] = [];
if (token) {
try {
const userResponse = await axios.get(
`${process.env.NEXT_PUBLIC_API_URL}/api/auth/users/me`,
{
headers: { Authorization: `Bearer ${token}` },
}
);
user = userResponse.data;
const jobsResponse = await fetch(
`${process.env.NEXT_PUBLIC_API_URL}/api/retrieve-scrape-jobs`,
{
method: "POST",
body: JSON.stringify({ user: user.email }),
headers: {
"content-type": "application/json",
Authorization: `Bearer ${token}`,
},
}
);
initialJobs = await jobsResponse.json();
console.log(initialJobs);
const cronJobsResponse = await fetch(
`${process.env.NEXT_PUBLIC_API_URL}/api/cron-jobs`,
{
headers: {
"content-type": "application/json",
Authorization: `Bearer ${token}`,
},
}
);
initialCronJobs = await cronJobsResponse.json();
} catch (error) {
console.error("Error fetching user or jobs:", error);
}
}
return {
props: {
initialJobs,
initialUser: user,
initialCronJobs,
},
};
};

View File

@@ -0,0 +1 @@
export { CronJobs } from "./cron-jobs";

View File

@@ -0,0 +1,39 @@
import { NextApiRequest, NextApiResponse } from "next";
export default async function handler(
req: NextApiRequest,
res: NextApiResponse
) {
if (req.method === "POST") {
const { data } = req.body;
console.log("Data", data);
const headers = new Headers();
headers.set("content-type", "application/json");
try {
const response = await fetch(
`${global.process.env.NEXT_PUBLIC_API_URL}/api/delete-cron-job`,
{
method: "POST",
headers,
body: JSON.stringify(data),
}
);
if (!response.ok) {
console.error(response);
throw new Error(`Error: ${response.statusText}`);
}
const result = await response.json();
res.status(200).json(result);
} catch (error) {
console.error("Error deleting cron job:", error);
res.status(500).json({ error: "Internal Server Error" });
}
} else {
res.setHeader("Allow", ["POST"]);
res.status(405).end(`Method ${req.method} Not Allowed`);
}
}

View File

@@ -0,0 +1,39 @@
import { NextApiRequest, NextApiResponse } from "next";
export default async function handler(
req: NextApiRequest,
res: NextApiResponse
) {
if (req.method === "POST") {
const { data } = req.body;
console.log("Data", data);
const headers = new Headers();
headers.set("content-type", "application/json");
try {
const response = await fetch(
`${global.process.env.NEXT_PUBLIC_API_URL}/api/schedule-cron-job`,
{
method: "POST",
headers,
body: JSON.stringify(data),
}
);
if (!response.ok) {
console.error(response);
throw new Error(`Error: ${response.statusText}`);
}
const result = await response.json();
res.status(200).json(result);
} catch (error) {
console.error("Error scheduling cron job:", error);
res.status(500).json({ error: "Internal Server Error" });
}
} else {
res.setHeader("Allow", ["POST"]);
res.status(405).end(`Method ${req.method} Not Allowed`);
}
}

4
src/pages/cron-jobs.tsx Normal file
View File

@@ -0,0 +1,4 @@
import { CronJobs } from "../components/pages/cron-jobs";
import { getServerSideProps } from "../components/pages/cron-jobs/get-server-side-props";
export { getServerSideProps };
export default CronJobs;

View File

@@ -38,3 +38,12 @@ export type Action = {
export type SiteMap = {
actions: Action[];
};
export type CronJob = {
id: string;
user_email: string;
job_id: string;
cron_expression: string;
time_created: Date;
time_updated: Date;
};