mirror of
https://github.com/jaypyles/Scraperr.git
synced 2025-12-14 11:46:17 +00:00
@@ -1,9 +1,13 @@
|
|||||||
# STL
|
# STL
|
||||||
import os
|
import os
|
||||||
import logging
|
import logging
|
||||||
|
import apscheduler # type: ignore
|
||||||
|
|
||||||
# PDM
|
# PDM
|
||||||
from fastapi import FastAPI
|
import apscheduler.schedulers
|
||||||
|
import apscheduler.schedulers.background
|
||||||
|
from fastapi import FastAPI, Request, status
|
||||||
|
from fastapi.exceptions import RequestValidationError
|
||||||
from fastapi.middleware.cors import CORSMiddleware
|
from fastapi.middleware.cors import CORSMiddleware
|
||||||
|
|
||||||
# LOCAL
|
# LOCAL
|
||||||
@@ -14,6 +18,10 @@ from api.backend.routers.job_router import job_router
|
|||||||
from api.backend.routers.log_router import log_router
|
from api.backend.routers.log_router import log_router
|
||||||
from api.backend.routers.stats_router import stats_router
|
from api.backend.routers.stats_router import stats_router
|
||||||
from api.backend.database.startup import init_database
|
from api.backend.database.startup import init_database
|
||||||
|
from fastapi.responses import JSONResponse
|
||||||
|
|
||||||
|
from api.backend.job.cron_scheduling.cron_scheduling import start_cron_scheduler
|
||||||
|
from api.backend.scheduler import scheduler
|
||||||
|
|
||||||
log_level = os.getenv("LOG_LEVEL")
|
log_level = os.getenv("LOG_LEVEL")
|
||||||
LOG_LEVEL = get_log_level(log_level)
|
LOG_LEVEL = get_log_level(log_level)
|
||||||
@@ -46,6 +54,24 @@ app.include_router(stats_router)
|
|||||||
|
|
||||||
@app.on_event("startup")
|
@app.on_event("startup")
|
||||||
async def startup_event():
|
async def startup_event():
|
||||||
|
start_cron_scheduler(scheduler)
|
||||||
|
scheduler.start()
|
||||||
|
|
||||||
if os.getenv("ENV") != "test":
|
if os.getenv("ENV") != "test":
|
||||||
init_database()
|
init_database()
|
||||||
LOG.info("Starting up...")
|
LOG.info("Starting up...")
|
||||||
|
|
||||||
|
|
||||||
|
@app.on_event("shutdown")
|
||||||
|
def shutdown_scheduler():
|
||||||
|
scheduler.shutdown(wait=False) # Set wait=False to not block shutdown
|
||||||
|
|
||||||
|
|
||||||
|
@app.exception_handler(RequestValidationError)
|
||||||
|
async def validation_exception_handler(request: Request, exc: RequestValidationError):
|
||||||
|
exc_str = f"{exc}".replace("\n", " ").replace(" ", " ")
|
||||||
|
logging.error(f"{request}: {exc_str}")
|
||||||
|
content = {"status_code": 10422, "message": exc_str, "data": None}
|
||||||
|
return JSONResponse(
|
||||||
|
content=content, status_code=status.HTTP_422_UNPROCESSABLE_ENTITY
|
||||||
|
)
|
||||||
|
|||||||
@@ -1,7 +1,5 @@
|
|||||||
# STL
|
# STL
|
||||||
import os
|
import os
|
||||||
from gc import disable
|
|
||||||
from queue import Empty
|
|
||||||
from typing import Any, Optional
|
from typing import Any, Optional
|
||||||
from datetime import datetime, timedelta
|
from datetime import datetime, timedelta
|
||||||
import logging
|
import logging
|
||||||
@@ -78,10 +76,10 @@ def create_access_token(
|
|||||||
|
|
||||||
|
|
||||||
async def get_current_user(token: str = Depends(oauth2_scheme)):
|
async def get_current_user(token: str = Depends(oauth2_scheme)):
|
||||||
LOG.info(f"Getting current user with token: {token}")
|
LOG.debug(f"Getting current user with token: {token}")
|
||||||
|
|
||||||
if not token:
|
if not token:
|
||||||
LOG.error("No token provided")
|
LOG.debug("No token provided")
|
||||||
return EMPTY_USER
|
return EMPTY_USER
|
||||||
|
|
||||||
if len(token.split(".")) != 3:
|
if len(token.split(".")) != 3:
|
||||||
@@ -89,7 +87,7 @@ async def get_current_user(token: str = Depends(oauth2_scheme)):
|
|||||||
return EMPTY_USER
|
return EMPTY_USER
|
||||||
|
|
||||||
try:
|
try:
|
||||||
LOG.info(
|
LOG.debug(
|
||||||
f"Decoding token: {token} with secret key: {SECRET_KEY} and algorithm: {ALGORITHM}"
|
f"Decoding token: {token} with secret key: {SECRET_KEY} and algorithm: {ALGORITHM}"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -17,4 +17,14 @@ CREATE TABLE IF NOT EXISTS users (
|
|||||||
full_name STRING,
|
full_name STRING,
|
||||||
disabled BOOLEAN
|
disabled BOOLEAN
|
||||||
);
|
);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS cron_jobs (
|
||||||
|
id STRING PRIMARY KEY NOT NULL,
|
||||||
|
user_email STRING NOT NULL,
|
||||||
|
job_id STRING NOT NULL,
|
||||||
|
cron_expression STRING NOT NULL,
|
||||||
|
time_created DATETIME NOT NULL,
|
||||||
|
time_updated DATETIME NOT NULL,
|
||||||
|
FOREIGN KEY (job_id) REFERENCES jobs(id)
|
||||||
|
);
|
||||||
"""
|
"""
|
||||||
|
|||||||
100
api/backend/job/cron_scheduling/cron_scheduling.py
Normal file
100
api/backend/job/cron_scheduling/cron_scheduling.py
Normal file
@@ -0,0 +1,100 @@
|
|||||||
|
import datetime
|
||||||
|
from typing import Any
|
||||||
|
import uuid
|
||||||
|
from api.backend.database.common import insert, query
|
||||||
|
from api.backend.models import CronJob
|
||||||
|
from apscheduler.schedulers.background import BackgroundScheduler # type: ignore
|
||||||
|
from apscheduler.triggers.cron import CronTrigger # type: ignore
|
||||||
|
|
||||||
|
from api.backend.job import insert as insert_job
|
||||||
|
import logging
|
||||||
|
|
||||||
|
LOG = logging.getLogger("Cron Scheduler")
|
||||||
|
|
||||||
|
|
||||||
|
def insert_cron_job(cron_job: CronJob):
|
||||||
|
query = """
|
||||||
|
INSERT INTO cron_jobs (id, user_email, job_id, cron_expression, time_created, time_updated)
|
||||||
|
VALUES (?, ?, ?, ?, ?, ?)
|
||||||
|
"""
|
||||||
|
values = (
|
||||||
|
cron_job.id,
|
||||||
|
cron_job.user_email,
|
||||||
|
cron_job.job_id,
|
||||||
|
cron_job.cron_expression,
|
||||||
|
cron_job.time_created,
|
||||||
|
cron_job.time_updated,
|
||||||
|
)
|
||||||
|
|
||||||
|
insert(query, values)
|
||||||
|
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def delete_cron_job(id: str, user_email: str):
|
||||||
|
query = """
|
||||||
|
DELETE FROM cron_jobs
|
||||||
|
WHERE id = ? AND user_email = ?
|
||||||
|
"""
|
||||||
|
values = (id, user_email)
|
||||||
|
insert(query, values)
|
||||||
|
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def get_cron_jobs(user_email: str):
|
||||||
|
cron_jobs = query("SELECT * FROM cron_jobs WHERE user_email = ?", (user_email,))
|
||||||
|
|
||||||
|
return cron_jobs
|
||||||
|
|
||||||
|
|
||||||
|
def get_all_cron_jobs():
|
||||||
|
cron_jobs = query("SELECT * FROM cron_jobs")
|
||||||
|
|
||||||
|
return cron_jobs
|
||||||
|
|
||||||
|
|
||||||
|
def insert_job_from_cron_job(job: dict[str, Any]):
|
||||||
|
insert_job(
|
||||||
|
{
|
||||||
|
**job,
|
||||||
|
"id": uuid.uuid4().hex,
|
||||||
|
"status": "Queued",
|
||||||
|
"result": "",
|
||||||
|
"chat": None,
|
||||||
|
"time_created": datetime.datetime.now(),
|
||||||
|
"time_updated": datetime.datetime.now(),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def get_cron_job_trigger(cron_expression: str):
|
||||||
|
expression_parts = cron_expression.split()
|
||||||
|
|
||||||
|
if len(expression_parts) != 5:
|
||||||
|
print(f"Invalid cron expression: {cron_expression}")
|
||||||
|
return None
|
||||||
|
|
||||||
|
minute, hour, day, month, day_of_week = expression_parts
|
||||||
|
|
||||||
|
return CronTrigger(
|
||||||
|
minute=minute, hour=hour, day=day, month=month, day_of_week=day_of_week
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def start_cron_scheduler(scheduler: BackgroundScheduler):
|
||||||
|
cron_jobs = get_all_cron_jobs()
|
||||||
|
|
||||||
|
LOG.info(f"Cron jobs: {cron_jobs}")
|
||||||
|
|
||||||
|
for job in cron_jobs:
|
||||||
|
queried_job = query("SELECT * FROM jobs WHERE id = ?", (job["job_id"],))
|
||||||
|
|
||||||
|
LOG.info(f"Adding job: {queried_job}")
|
||||||
|
|
||||||
|
scheduler.add_job(
|
||||||
|
insert_job_from_cron_job,
|
||||||
|
get_cron_job_trigger(job["cron_expression"]),
|
||||||
|
id=job["id"],
|
||||||
|
args=[queried_job[0]],
|
||||||
|
)
|
||||||
@@ -14,7 +14,7 @@ from api.backend.database.common import (
|
|||||||
LOG = logging.getLogger(__name__)
|
LOG = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
async def insert(item: dict[str, Any]) -> None:
|
def insert(item: dict[str, Any]) -> None:
|
||||||
common_insert(
|
common_insert(
|
||||||
QUERIES["insert_job"],
|
QUERIES["insert_job"],
|
||||||
(
|
(
|
||||||
|
|||||||
@@ -57,3 +57,17 @@ class Job(pydantic.BaseModel):
|
|||||||
job_options: JobOptions
|
job_options: JobOptions
|
||||||
status: str = "Queued"
|
status: str = "Queued"
|
||||||
chat: Optional[str] = None
|
chat: Optional[str] = None
|
||||||
|
|
||||||
|
|
||||||
|
class CronJob(pydantic.BaseModel):
|
||||||
|
id: Optional[str] = None
|
||||||
|
user_email: str
|
||||||
|
job_id: str
|
||||||
|
cron_expression: str
|
||||||
|
time_created: Optional[Union[datetime, str]] = None
|
||||||
|
time_updated: Optional[Union[datetime, str]] = None
|
||||||
|
|
||||||
|
|
||||||
|
class DeleteCronJob(pydantic.BaseModel):
|
||||||
|
id: str
|
||||||
|
user_email: str
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
# STL
|
# STL
|
||||||
|
import datetime
|
||||||
import uuid
|
import uuid
|
||||||
import traceback
|
import traceback
|
||||||
from io import StringIO
|
from io import StringIO
|
||||||
@@ -10,14 +11,18 @@ import random
|
|||||||
from fastapi import Depends, APIRouter
|
from fastapi import Depends, APIRouter
|
||||||
from fastapi.encoders import jsonable_encoder
|
from fastapi.encoders import jsonable_encoder
|
||||||
from fastapi.responses import JSONResponse, StreamingResponse
|
from fastapi.responses import JSONResponse, StreamingResponse
|
||||||
|
from api.backend.scheduler import scheduler
|
||||||
|
from apscheduler.triggers.cron import CronTrigger # type: ignore
|
||||||
|
|
||||||
# LOCAL
|
# LOCAL
|
||||||
from api.backend.job import insert, update_job, delete_jobs
|
from api.backend.job import insert, update_job, delete_jobs
|
||||||
from api.backend.models import (
|
from api.backend.models import (
|
||||||
|
DeleteCronJob,
|
||||||
UpdateJobs,
|
UpdateJobs,
|
||||||
DownloadJob,
|
DownloadJob,
|
||||||
DeleteScrapeJobs,
|
DeleteScrapeJobs,
|
||||||
Job,
|
Job,
|
||||||
|
CronJob,
|
||||||
)
|
)
|
||||||
from api.backend.schemas import User
|
from api.backend.schemas import User
|
||||||
from api.backend.auth.auth_utils import get_current_user
|
from api.backend.auth.auth_utils import get_current_user
|
||||||
@@ -26,6 +31,14 @@ from api.backend.job.models.job_options import FetchOptions
|
|||||||
|
|
||||||
from api.backend.database.common import query
|
from api.backend.database.common import query
|
||||||
|
|
||||||
|
from api.backend.job.cron_scheduling.cron_scheduling import (
|
||||||
|
delete_cron_job,
|
||||||
|
get_cron_job_trigger,
|
||||||
|
insert_cron_job,
|
||||||
|
get_cron_jobs,
|
||||||
|
insert_job_from_cron_job,
|
||||||
|
)
|
||||||
|
|
||||||
LOG = logging.getLogger(__name__)
|
LOG = logging.getLogger(__name__)
|
||||||
|
|
||||||
job_router = APIRouter()
|
job_router = APIRouter()
|
||||||
@@ -44,7 +57,7 @@ async def submit_scrape_job(job: Job):
|
|||||||
job.id = uuid.uuid4().hex
|
job.id = uuid.uuid4().hex
|
||||||
|
|
||||||
job_dict = job.model_dump()
|
job_dict = job.model_dump()
|
||||||
await insert(job_dict)
|
insert(job_dict)
|
||||||
|
|
||||||
return JSONResponse(content={"id": job.id})
|
return JSONResponse(content={"id": job.id})
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
@@ -140,3 +153,47 @@ async def delete(delete_scrape_jobs: DeleteScrapeJobs):
|
|||||||
if result
|
if result
|
||||||
else JSONResponse({"error": "Jobs not deleted."})
|
else JSONResponse({"error": "Jobs not deleted."})
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@job_router.post("/schedule-cron-job")
|
||||||
|
async def schedule_cron_job(cron_job: CronJob):
|
||||||
|
if not cron_job.id:
|
||||||
|
cron_job.id = uuid.uuid4().hex
|
||||||
|
|
||||||
|
if not cron_job.time_created:
|
||||||
|
cron_job.time_created = datetime.datetime.now()
|
||||||
|
|
||||||
|
if not cron_job.time_updated:
|
||||||
|
cron_job.time_updated = datetime.datetime.now()
|
||||||
|
|
||||||
|
insert_cron_job(cron_job)
|
||||||
|
|
||||||
|
queried_job = query("SELECT * FROM jobs WHERE id = ?", (cron_job.job_id,))
|
||||||
|
|
||||||
|
scheduler.add_job(
|
||||||
|
insert_job_from_cron_job,
|
||||||
|
get_cron_job_trigger(cron_job.cron_expression),
|
||||||
|
id=cron_job.id,
|
||||||
|
args=[queried_job[0]],
|
||||||
|
)
|
||||||
|
|
||||||
|
return JSONResponse(content={"message": "Cron job scheduled successfully."})
|
||||||
|
|
||||||
|
|
||||||
|
@job_router.post("/delete-cron-job")
|
||||||
|
async def delete_cron_job_request(request: DeleteCronJob):
|
||||||
|
if not request.id:
|
||||||
|
return JSONResponse(
|
||||||
|
content={"error": "Cron job id is required."}, status_code=400
|
||||||
|
)
|
||||||
|
|
||||||
|
delete_cron_job(request.id, request.user_email)
|
||||||
|
scheduler.remove_job(request.id)
|
||||||
|
|
||||||
|
return JSONResponse(content={"message": "Cron job deleted successfully."})
|
||||||
|
|
||||||
|
|
||||||
|
@job_router.get("/cron-jobs")
|
||||||
|
async def get_cron_jobs_request(user: User = Depends(get_current_user)):
|
||||||
|
cron_jobs = get_cron_jobs(user.email)
|
||||||
|
return JSONResponse(content=jsonable_encoder(cron_jobs))
|
||||||
|
|||||||
3
api/backend/scheduler.py
Normal file
3
api/backend/scheduler.py
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
from apscheduler.schedulers.background import BackgroundScheduler # type: ignore
|
||||||
|
|
||||||
|
scheduler = BackgroundScheduler()
|
||||||
31
pdm.lock
generated
31
pdm.lock
generated
@@ -5,7 +5,7 @@
|
|||||||
groups = ["default", "dev"]
|
groups = ["default", "dev"]
|
||||||
strategy = ["inherit_metadata"]
|
strategy = ["inherit_metadata"]
|
||||||
lock_version = "4.5.0"
|
lock_version = "4.5.0"
|
||||||
content_hash = "sha256:d3c8eb4d20f8aaddc2f44cea1629b3ee4fe1efa3aad65b6700d085bd9f31558b"
|
content_hash = "sha256:1d142e8b44e3a6a04135c54e1967b7c19c5c7ccd6b2ff8ec8bca8792bf961bb9"
|
||||||
|
|
||||||
[[metadata.targets]]
|
[[metadata.targets]]
|
||||||
requires_python = ">=3.10"
|
requires_python = ">=3.10"
|
||||||
@@ -171,6 +171,21 @@ files = [
|
|||||||
{file = "appdirs-1.4.4.tar.gz", hash = "sha256:7d5d0167b2b1ba821647616af46a749d1c653740dd0d2415100fe26e27afdf41"},
|
{file = "appdirs-1.4.4.tar.gz", hash = "sha256:7d5d0167b2b1ba821647616af46a749d1c653740dd0d2415100fe26e27afdf41"},
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "apscheduler"
|
||||||
|
version = "3.11.0"
|
||||||
|
requires_python = ">=3.8"
|
||||||
|
summary = "In-process task scheduler with Cron-like capabilities"
|
||||||
|
groups = ["default"]
|
||||||
|
dependencies = [
|
||||||
|
"backports-zoneinfo; python_version < \"3.9\"",
|
||||||
|
"tzlocal>=3.0",
|
||||||
|
]
|
||||||
|
files = [
|
||||||
|
{file = "APScheduler-3.11.0-py3-none-any.whl", hash = "sha256:fc134ca32e50f5eadcc4938e3a4545ab19131435e851abb40b34d63d5141c6da"},
|
||||||
|
{file = "apscheduler-3.11.0.tar.gz", hash = "sha256:4c622d250b0955a65d5d0eb91c33e6d43fd879834bf541e0a18661ae60460133"},
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "asgiref"
|
name = "asgiref"
|
||||||
version = "3.8.1"
|
version = "3.8.1"
|
||||||
@@ -2883,6 +2898,20 @@ files = [
|
|||||||
{file = "tzdata-2025.2.tar.gz", hash = "sha256:b60a638fcc0daffadf82fe0f57e53d06bdec2f36c4df66280ae79bce6bd6f2b9"},
|
{file = "tzdata-2025.2.tar.gz", hash = "sha256:b60a638fcc0daffadf82fe0f57e53d06bdec2f36c4df66280ae79bce6bd6f2b9"},
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "tzlocal"
|
||||||
|
version = "5.3.1"
|
||||||
|
requires_python = ">=3.9"
|
||||||
|
summary = "tzinfo object for the local timezone"
|
||||||
|
groups = ["default"]
|
||||||
|
dependencies = [
|
||||||
|
"tzdata; platform_system == \"Windows\"",
|
||||||
|
]
|
||||||
|
files = [
|
||||||
|
{file = "tzlocal-5.3.1-py3-none-any.whl", hash = "sha256:eb1a66c3ef5847adf7a834f1be0800581b683b5608e74f86ecbcef8ab91bb85d"},
|
||||||
|
{file = "tzlocal-5.3.1.tar.gz", hash = "sha256:cceffc7edecefea1f595541dbd6e990cb1ea3d19bf01b2809f362a03dd7921fd"},
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "urllib3"
|
name = "urllib3"
|
||||||
version = "2.4.0"
|
version = "2.4.0"
|
||||||
|
|||||||
@@ -39,6 +39,7 @@ dependencies = [
|
|||||||
"pytest-asyncio>=0.24.0",
|
"pytest-asyncio>=0.24.0",
|
||||||
"python-multipart>=0.0.1",
|
"python-multipart>=0.0.1",
|
||||||
"bcrypt==4.0.1",
|
"bcrypt==4.0.1",
|
||||||
|
"apscheduler>=3.11.0",
|
||||||
]
|
]
|
||||||
requires-python = ">=3.10"
|
requires-python = ">=3.10"
|
||||||
readme = "README.md"
|
readme = "README.md"
|
||||||
@@ -56,14 +57,42 @@ ignore = []
|
|||||||
defineConstant = { DEBUG = true }
|
defineConstant = { DEBUG = true }
|
||||||
stubPath = ""
|
stubPath = ""
|
||||||
|
|
||||||
reportUnknownMemberType = false
|
# Type checking strictness
|
||||||
reportMissingImports = true
|
typeCheckingMode = "strict" # Enables strict type checking mode
|
||||||
reportMissingTypeStubs = false
|
reportPrivateUsage = "error"
|
||||||
reportAny = false
|
reportMissingTypeStubs = "error"
|
||||||
reportCallInDefaultInitializer = false
|
reportUntypedFunctionDecorator = "error"
|
||||||
|
reportUntypedClassDecorator = "error"
|
||||||
|
reportUntypedBaseClass = "error"
|
||||||
|
reportInvalidTypeVarUse = "error"
|
||||||
|
reportUnnecessaryTypeIgnoreComment = "information"
|
||||||
|
reportUnknownVariableType = "none"
|
||||||
|
reportUnknownMemberType = "none"
|
||||||
|
reportUnknownParameterType = "none"
|
||||||
|
|
||||||
pythonVersion = "3.9"
|
# Additional checks
|
||||||
pythonPlatform = "Linux"
|
reportImplicitStringConcatenation = "error"
|
||||||
|
reportInvalidStringEscapeSequence = "error"
|
||||||
|
reportMissingImports = "error"
|
||||||
|
reportMissingModuleSource = "error"
|
||||||
|
reportOptionalCall = "error"
|
||||||
|
reportOptionalIterable = "error"
|
||||||
|
reportOptionalMemberAccess = "error"
|
||||||
|
reportOptionalOperand = "error"
|
||||||
|
reportOptionalSubscript = "error"
|
||||||
|
reportTypedDictNotRequiredAccess = "error"
|
||||||
|
|
||||||
|
# Function return type checking
|
||||||
|
reportIncompleteStub = "error"
|
||||||
|
reportIncompatibleMethodOverride = "error"
|
||||||
|
reportInvalidStubStatement = "error"
|
||||||
|
reportInconsistentOverload = "error"
|
||||||
|
|
||||||
|
# Misc settings
|
||||||
|
pythonVersion = "3.10" # Matches your Python version from pyproject.toml
|
||||||
|
strictListInference = true
|
||||||
|
strictDictionaryInference = true
|
||||||
|
strictSetInference = true
|
||||||
|
|
||||||
|
|
||||||
[tool.isort]
|
[tool.isort]
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ import TerminalIcon from "@mui/icons-material/Terminal";
|
|||||||
import BarChart from "@mui/icons-material/BarChart";
|
import BarChart from "@mui/icons-material/BarChart";
|
||||||
import AutoAwesomeIcon from "@mui/icons-material/AutoAwesome";
|
import AutoAwesomeIcon from "@mui/icons-material/AutoAwesome";
|
||||||
import { List } from "@mui/material";
|
import { List } from "@mui/material";
|
||||||
|
import { Schedule } from "@mui/icons-material";
|
||||||
|
|
||||||
const items = [
|
const items = [
|
||||||
{
|
{
|
||||||
@@ -34,6 +35,11 @@ const items = [
|
|||||||
text: "View App Logs",
|
text: "View App Logs",
|
||||||
href: "/logs",
|
href: "/logs",
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
icon: <Schedule />,
|
||||||
|
text: "Cron Jobs",
|
||||||
|
href: "/cron-jobs",
|
||||||
|
},
|
||||||
];
|
];
|
||||||
|
|
||||||
export const NavItems = () => {
|
export const NavItems = () => {
|
||||||
|
|||||||
@@ -0,0 +1,182 @@
|
|||||||
|
import { Job } from "@/types";
|
||||||
|
import {
|
||||||
|
Button,
|
||||||
|
Dialog,
|
||||||
|
DialogTitle,
|
||||||
|
DialogContent,
|
||||||
|
TextField,
|
||||||
|
Snackbar,
|
||||||
|
Alert,
|
||||||
|
} from "@mui/material";
|
||||||
|
import Cookies from "js-cookie";
|
||||||
|
import { useState } from "react";
|
||||||
|
|
||||||
|
export type CreateCronJobsProps = {
|
||||||
|
availableJobs: Job[];
|
||||||
|
user: any;
|
||||||
|
};
|
||||||
|
|
||||||
|
export const CreateCronJobs = ({
|
||||||
|
availableJobs,
|
||||||
|
user,
|
||||||
|
}: CreateCronJobsProps) => {
|
||||||
|
const [open, setOpen] = useState(false);
|
||||||
|
|
||||||
|
return (
|
||||||
|
<>
|
||||||
|
<Button
|
||||||
|
variant="contained"
|
||||||
|
color="primary"
|
||||||
|
onClick={() => setOpen(true)}
|
||||||
|
sx={{ borderRadius: 2 }}
|
||||||
|
>
|
||||||
|
Create Cron Job
|
||||||
|
</Button>
|
||||||
|
<CreateCronJobDialog
|
||||||
|
open={open}
|
||||||
|
onClose={() => setOpen(false)}
|
||||||
|
availableJobs={availableJobs}
|
||||||
|
user={user}
|
||||||
|
/>
|
||||||
|
</>
|
||||||
|
);
|
||||||
|
};
|
||||||
|
|
||||||
|
const CreateCronJobDialog = ({
|
||||||
|
open,
|
||||||
|
onClose,
|
||||||
|
availableJobs,
|
||||||
|
user,
|
||||||
|
}: {
|
||||||
|
open: boolean;
|
||||||
|
onClose: () => void;
|
||||||
|
availableJobs: Job[];
|
||||||
|
user: any;
|
||||||
|
}) => {
|
||||||
|
const [cronExpression, setCronExpression] = useState("");
|
||||||
|
const [jobId, setJobId] = useState("");
|
||||||
|
const [successOpen, setSuccessOpen] = useState(false);
|
||||||
|
const [isSubmitting, setIsSubmitting] = useState(false);
|
||||||
|
const [error, setError] = useState("");
|
||||||
|
|
||||||
|
const handleSubmit = async () => {
|
||||||
|
if (!cronExpression || !jobId) {
|
||||||
|
setError("Please fill in all fields");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
setIsSubmitting(true);
|
||||||
|
const token = Cookies.get("token");
|
||||||
|
|
||||||
|
try {
|
||||||
|
const response = await fetch("/api/schedule-cron-job", {
|
||||||
|
method: "POST",
|
||||||
|
headers: {
|
||||||
|
"Content-Type": "application/json",
|
||||||
|
Authorization: `Bearer ${token}`,
|
||||||
|
},
|
||||||
|
body: JSON.stringify({
|
||||||
|
data: {
|
||||||
|
cron_expression: cronExpression,
|
||||||
|
job_id: jobId,
|
||||||
|
user_email: user.email,
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!response.ok) {
|
||||||
|
throw new Error("Failed to schedule job");
|
||||||
|
}
|
||||||
|
|
||||||
|
setSuccessOpen(true);
|
||||||
|
setCronExpression("");
|
||||||
|
setJobId("");
|
||||||
|
setTimeout(() => {
|
||||||
|
onClose();
|
||||||
|
}, 1500);
|
||||||
|
window.location.reload();
|
||||||
|
} catch (error) {
|
||||||
|
console.error(error);
|
||||||
|
setError("Failed to create cron job");
|
||||||
|
} finally {
|
||||||
|
setIsSubmitting(false);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const handleClose = () => {
|
||||||
|
setSuccessOpen(false);
|
||||||
|
};
|
||||||
|
|
||||||
|
return (
|
||||||
|
<>
|
||||||
|
<Dialog
|
||||||
|
open={open}
|
||||||
|
onClose={onClose}
|
||||||
|
PaperProps={{
|
||||||
|
sx: { borderRadius: 2, minWidth: "400px" },
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
<DialogTitle sx={{ fontWeight: 500 }}>Create Cron Job</DialogTitle>
|
||||||
|
<DialogContent>
|
||||||
|
<div className="flex flex-col gap-1 mt0">
|
||||||
|
<TextField
|
||||||
|
label="Cron Expression"
|
||||||
|
fullWidth
|
||||||
|
value={cronExpression}
|
||||||
|
onChange={(e) => setCronExpression(e.target.value)}
|
||||||
|
variant="outlined"
|
||||||
|
placeholder="* * * * *"
|
||||||
|
margin="normal"
|
||||||
|
helperText="Format: minute hour day month day-of-week"
|
||||||
|
/>
|
||||||
|
|
||||||
|
<TextField
|
||||||
|
label="Job ID"
|
||||||
|
fullWidth
|
||||||
|
value={jobId}
|
||||||
|
onChange={(e) => setJobId(e.target.value)}
|
||||||
|
variant="outlined"
|
||||||
|
margin="normal"
|
||||||
|
/>
|
||||||
|
|
||||||
|
{error && (
|
||||||
|
<Alert severity="error" sx={{ mt: 2 }}>
|
||||||
|
{error}
|
||||||
|
</Alert>
|
||||||
|
)}
|
||||||
|
|
||||||
|
<div className="flex justify-end gap-2 mt-4">
|
||||||
|
<Button
|
||||||
|
variant="outlined"
|
||||||
|
onClick={onClose}
|
||||||
|
sx={{ borderRadius: 2 }}
|
||||||
|
>
|
||||||
|
Cancel
|
||||||
|
</Button>
|
||||||
|
<Button
|
||||||
|
variant="contained"
|
||||||
|
color="primary"
|
||||||
|
onClick={handleSubmit}
|
||||||
|
disabled={isSubmitting}
|
||||||
|
sx={{ borderRadius: 2 }}
|
||||||
|
>
|
||||||
|
{isSubmitting ? "Submitting..." : "Create Job"}
|
||||||
|
</Button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</DialogContent>
|
||||||
|
</Dialog>
|
||||||
|
|
||||||
|
<Snackbar
|
||||||
|
open={successOpen}
|
||||||
|
autoHideDuration={4000}
|
||||||
|
onClose={handleClose}
|
||||||
|
anchorOrigin={{ vertical: "bottom", horizontal: "right" }}
|
||||||
|
>
|
||||||
|
<Alert onClose={handleClose} severity="success" sx={{ width: "100%" }}>
|
||||||
|
Cron job created successfully!
|
||||||
|
</Alert>
|
||||||
|
</Snackbar>
|
||||||
|
</>
|
||||||
|
);
|
||||||
|
};
|
||||||
1
src/components/pages/cron-jobs/create-cron-jobs/index.ts
Normal file
1
src/components/pages/cron-jobs/create-cron-jobs/index.ts
Normal file
@@ -0,0 +1 @@
|
|||||||
|
export * from "./create-cron-jobs";
|
||||||
0
src/components/pages/cron-jobs/cron-jobs.module.css
Normal file
0
src/components/pages/cron-jobs/cron-jobs.module.css
Normal file
92
src/components/pages/cron-jobs/cron-jobs.tsx
Normal file
92
src/components/pages/cron-jobs/cron-jobs.tsx
Normal file
@@ -0,0 +1,92 @@
|
|||||||
|
import { Job, CronJob } from "@/types/job";
|
||||||
|
import { useState, useEffect } from "react";
|
||||||
|
import { CreateCronJobs } from "./create-cron-jobs";
|
||||||
|
import {
|
||||||
|
Table,
|
||||||
|
TableHead,
|
||||||
|
TableRow,
|
||||||
|
TableCell,
|
||||||
|
TableBody,
|
||||||
|
Button,
|
||||||
|
} from "@mui/material";
|
||||||
|
import Cookies from "js-cookie";
|
||||||
|
|
||||||
|
export type CronJobsProps = {
|
||||||
|
initialJobs: Job[];
|
||||||
|
initialCronJobs: CronJob[];
|
||||||
|
initialUser: any;
|
||||||
|
};
|
||||||
|
|
||||||
|
export const CronJobs = ({
|
||||||
|
initialJobs,
|
||||||
|
initialCronJobs,
|
||||||
|
initialUser,
|
||||||
|
}: CronJobsProps) => {
|
||||||
|
const [jobs, setJobs] = useState<Job[]>(initialJobs);
|
||||||
|
const [cronJobs, setCronJobs] = useState<CronJob[]>(initialCronJobs);
|
||||||
|
const [user, setUser] = useState<any>(initialUser);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
setJobs(initialJobs);
|
||||||
|
setCronJobs(initialCronJobs);
|
||||||
|
setUser(initialUser);
|
||||||
|
}, [initialJobs, initialCronJobs, initialUser]);
|
||||||
|
|
||||||
|
const handleDeleteCronJob = async (id: string) => {
|
||||||
|
const token = Cookies.get("token");
|
||||||
|
const response = await fetch("/api/delete-cron-job", {
|
||||||
|
method: "POST",
|
||||||
|
headers: {
|
||||||
|
"Content-Type": "application/json",
|
||||||
|
Authorization: `Bearer ${token}`,
|
||||||
|
},
|
||||||
|
body: JSON.stringify({ data: { id, user_email: user.email } }),
|
||||||
|
});
|
||||||
|
|
||||||
|
if (response.ok) {
|
||||||
|
console.log("Cron job deleted successfully");
|
||||||
|
setCronJobs(cronJobs.filter((cronJob) => cronJob.id !== id));
|
||||||
|
} else {
|
||||||
|
console.error("Failed to delete cron job");
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div>
|
||||||
|
<CreateCronJobs availableJobs={jobs} user={user} />
|
||||||
|
|
||||||
|
<Table>
|
||||||
|
<TableHead>
|
||||||
|
<TableRow>
|
||||||
|
<TableCell>Cron Expression</TableCell>
|
||||||
|
<TableCell>Job ID</TableCell>
|
||||||
|
<TableCell>User Email</TableCell>
|
||||||
|
<TableCell>Created At</TableCell>
|
||||||
|
<TableCell>Updated At</TableCell>
|
||||||
|
<TableCell>Actions</TableCell>
|
||||||
|
</TableRow>
|
||||||
|
</TableHead>
|
||||||
|
<TableBody>
|
||||||
|
{cronJobs.map((cronJob) => (
|
||||||
|
<TableRow key={cronJob.id}>
|
||||||
|
<TableCell>{cronJob.cron_expression}</TableCell>
|
||||||
|
<TableCell>{cronJob.job_id}</TableCell>
|
||||||
|
<TableCell>{cronJob.user_email}</TableCell>
|
||||||
|
<TableCell>
|
||||||
|
{new Date(cronJob.time_created).toLocaleString()}
|
||||||
|
</TableCell>
|
||||||
|
<TableCell>
|
||||||
|
{new Date(cronJob.time_updated).toLocaleString()}
|
||||||
|
</TableCell>
|
||||||
|
<TableCell>
|
||||||
|
<Button onClick={() => handleDeleteCronJob(cronJob.id)}>
|
||||||
|
Delete
|
||||||
|
</Button>
|
||||||
|
</TableCell>
|
||||||
|
</TableRow>
|
||||||
|
))}
|
||||||
|
</TableBody>
|
||||||
|
</Table>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
};
|
||||||
62
src/components/pages/cron-jobs/get-server-side-props.ts
Normal file
62
src/components/pages/cron-jobs/get-server-side-props.ts
Normal file
@@ -0,0 +1,62 @@
|
|||||||
|
import axios from "axios";
|
||||||
|
import { GetServerSideProps } from "next";
|
||||||
|
import { parseCookies } from "nookies";
|
||||||
|
import { CronJob, Job } from "../../../types";
|
||||||
|
|
||||||
|
export const getServerSideProps: GetServerSideProps = async (context) => {
|
||||||
|
const { req } = context;
|
||||||
|
const cookies = parseCookies({ req });
|
||||||
|
const token = cookies.token;
|
||||||
|
let user = null;
|
||||||
|
let initialJobs: Job[] = [];
|
||||||
|
let initialCronJobs: CronJob[] = [];
|
||||||
|
if (token) {
|
||||||
|
try {
|
||||||
|
const userResponse = await axios.get(
|
||||||
|
`${process.env.NEXT_PUBLIC_API_URL}/api/auth/users/me`,
|
||||||
|
{
|
||||||
|
headers: { Authorization: `Bearer ${token}` },
|
||||||
|
}
|
||||||
|
);
|
||||||
|
|
||||||
|
user = userResponse.data;
|
||||||
|
|
||||||
|
const jobsResponse = await fetch(
|
||||||
|
`${process.env.NEXT_PUBLIC_API_URL}/api/retrieve-scrape-jobs`,
|
||||||
|
{
|
||||||
|
method: "POST",
|
||||||
|
body: JSON.stringify({ user: user.email }),
|
||||||
|
headers: {
|
||||||
|
"content-type": "application/json",
|
||||||
|
Authorization: `Bearer ${token}`,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
);
|
||||||
|
|
||||||
|
initialJobs = await jobsResponse.json();
|
||||||
|
console.log(initialJobs);
|
||||||
|
|
||||||
|
const cronJobsResponse = await fetch(
|
||||||
|
`${process.env.NEXT_PUBLIC_API_URL}/api/cron-jobs`,
|
||||||
|
{
|
||||||
|
headers: {
|
||||||
|
"content-type": "application/json",
|
||||||
|
Authorization: `Bearer ${token}`,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
);
|
||||||
|
|
||||||
|
initialCronJobs = await cronJobsResponse.json();
|
||||||
|
} catch (error) {
|
||||||
|
console.error("Error fetching user or jobs:", error);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
props: {
|
||||||
|
initialJobs,
|
||||||
|
initialUser: user,
|
||||||
|
initialCronJobs,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
};
|
||||||
1
src/components/pages/cron-jobs/index.ts
Normal file
1
src/components/pages/cron-jobs/index.ts
Normal file
@@ -0,0 +1 @@
|
|||||||
|
export { CronJobs } from "./cron-jobs";
|
||||||
39
src/pages/api/delete-cron-job.ts
Normal file
39
src/pages/api/delete-cron-job.ts
Normal file
@@ -0,0 +1,39 @@
|
|||||||
|
import { NextApiRequest, NextApiResponse } from "next";
|
||||||
|
|
||||||
|
export default async function handler(
|
||||||
|
req: NextApiRequest,
|
||||||
|
res: NextApiResponse
|
||||||
|
) {
|
||||||
|
if (req.method === "POST") {
|
||||||
|
const { data } = req.body;
|
||||||
|
console.log("Data", data);
|
||||||
|
|
||||||
|
const headers = new Headers();
|
||||||
|
headers.set("content-type", "application/json");
|
||||||
|
|
||||||
|
try {
|
||||||
|
const response = await fetch(
|
||||||
|
`${global.process.env.NEXT_PUBLIC_API_URL}/api/delete-cron-job`,
|
||||||
|
{
|
||||||
|
method: "POST",
|
||||||
|
headers,
|
||||||
|
body: JSON.stringify(data),
|
||||||
|
}
|
||||||
|
);
|
||||||
|
|
||||||
|
if (!response.ok) {
|
||||||
|
console.error(response);
|
||||||
|
throw new Error(`Error: ${response.statusText}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const result = await response.json();
|
||||||
|
res.status(200).json(result);
|
||||||
|
} catch (error) {
|
||||||
|
console.error("Error deleting cron job:", error);
|
||||||
|
res.status(500).json({ error: "Internal Server Error" });
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
res.setHeader("Allow", ["POST"]);
|
||||||
|
res.status(405).end(`Method ${req.method} Not Allowed`);
|
||||||
|
}
|
||||||
|
}
|
||||||
39
src/pages/api/schedule-cron-job.ts
Normal file
39
src/pages/api/schedule-cron-job.ts
Normal file
@@ -0,0 +1,39 @@
|
|||||||
|
import { NextApiRequest, NextApiResponse } from "next";
|
||||||
|
|
||||||
|
export default async function handler(
|
||||||
|
req: NextApiRequest,
|
||||||
|
res: NextApiResponse
|
||||||
|
) {
|
||||||
|
if (req.method === "POST") {
|
||||||
|
const { data } = req.body;
|
||||||
|
console.log("Data", data);
|
||||||
|
|
||||||
|
const headers = new Headers();
|
||||||
|
headers.set("content-type", "application/json");
|
||||||
|
|
||||||
|
try {
|
||||||
|
const response = await fetch(
|
||||||
|
`${global.process.env.NEXT_PUBLIC_API_URL}/api/schedule-cron-job`,
|
||||||
|
{
|
||||||
|
method: "POST",
|
||||||
|
headers,
|
||||||
|
body: JSON.stringify(data),
|
||||||
|
}
|
||||||
|
);
|
||||||
|
|
||||||
|
if (!response.ok) {
|
||||||
|
console.error(response);
|
||||||
|
throw new Error(`Error: ${response.statusText}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const result = await response.json();
|
||||||
|
res.status(200).json(result);
|
||||||
|
} catch (error) {
|
||||||
|
console.error("Error scheduling cron job:", error);
|
||||||
|
res.status(500).json({ error: "Internal Server Error" });
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
res.setHeader("Allow", ["POST"]);
|
||||||
|
res.status(405).end(`Method ${req.method} Not Allowed`);
|
||||||
|
}
|
||||||
|
}
|
||||||
4
src/pages/cron-jobs.tsx
Normal file
4
src/pages/cron-jobs.tsx
Normal file
@@ -0,0 +1,4 @@
|
|||||||
|
import { CronJobs } from "../components/pages/cron-jobs";
|
||||||
|
import { getServerSideProps } from "../components/pages/cron-jobs/get-server-side-props";
|
||||||
|
export { getServerSideProps };
|
||||||
|
export default CronJobs;
|
||||||
@@ -38,3 +38,12 @@ export type Action = {
|
|||||||
export type SiteMap = {
|
export type SiteMap = {
|
||||||
actions: Action[];
|
actions: Action[];
|
||||||
};
|
};
|
||||||
|
|
||||||
|
export type CronJob = {
|
||||||
|
id: string;
|
||||||
|
user_email: string;
|
||||||
|
job_id: string;
|
||||||
|
cron_expression: string;
|
||||||
|
time_created: Date;
|
||||||
|
time_updated: Date;
|
||||||
|
};
|
||||||
|
|||||||
Reference in New Issue
Block a user