lee/Scraperr
1
0
Fork 0
mirror of https://github.com/jaypyles/Scraperr.git synced 2026-08-26 14:26:31 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

wip: mutli page scraping and worker

Browse Source
This commit is contained in:
Jayden
2024-07-21 00:22:33 -05:00
parent 2b2de523d4
commit 14b229e07e
14 changed files with 2521 additions and 1672 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Dockerfile
+4 -2
View File
@@ -32,7 +32,7 @@ RUN npm run build
FROM python:3.10-slim
RUN apt-get update
RUN apt-get install -y wget gnupg
RUN apt-get install -y wget gnupg supervisor
RUN wget -q -O - https://dl.google.com/linux/linux_signing_key.pub | apt-key add -
RUN sh -c 'echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list'
RUN apt-get update
@@ -44,9 +44,11 @@ COPY --from=pybuilder /usr/local/bin /usr/local/bin
COPY --from=pybuilder /project/app /project/
COPY --from=jsbuilder /app/dist /project/dist
COPY supervisord.conf /etc/supervisor/conf.d/supervisord.conf
EXPOSE 8000
WORKDIR /project/
CMD [ "pdm", "run", "python", "-m", "uvicorn", "api.backend.app:app", "--reload", "--host", "0.0.0.0", "--port", "8000"]
CMD [ "supervisord", "-c", "/etc/supervisor/conf.d/supervisord.conf" ]
api/backend/app.py
+22 -25
View File
@@ -1,18 +1,19 @@
# STL
from functools import partial
import uuid
import logging
from io import StringIO
# PDM
import pandas as pd
from fastapi import FastAPI
from fastapi import BackgroundTasks, FastAPI
from fastapi.encoders import jsonable_encoder
from fastapi.responses import FileResponse, JSONResponse, StreamingResponse
from fastapi.staticfiles import StaticFiles
from fastapi.middleware.cors import CORSMiddleware
# LOCAL
from api.backend.job import query, insert, delete_jobs
from api.backend.job import query, insert, delete_jobs, update_job
from api.backend.models import (
DownloadJob,
SubmitScrapeJob,
@@ -21,6 +22,7 @@ from api.backend.models import (
)
from api.backend.scraping import scrape
from api.backend.auth.auth_router import auth_router
from seleniumwire.thirdparty.mitmproxy.master import traceback
logging.basicConfig(
level=logging.INFO,
@@ -55,23 +57,15 @@ def read_favicon():
@app.post("/api/submit-scrape-job")
async def submit_scrape_job(job: SubmitScrapeJob):
async def submit_scrape_job(job: SubmitScrapeJob, background_tasks: BackgroundTasks):
LOG.info(f"Recieved job: {job}")
try:
scraped = await scrape(job.url, job.elements)
LOG.info(
f"Scraped result for url: {job.url}, with elements: {job.elements}\n{scraped}"
)
json_scraped = jsonable_encoder(scraped)
job.result = json_scraped
job.id = uuid.uuid4().hex
if job.user:
await insert(jsonable_encoder(job))
return JSONResponse(content=json_scraped)
return JSONResponse(content=f"Job queued for scraping: {job.id}")
except Exception as e:
return JSONResponse(content={"error": str(e)}, status_code=500)
@@ -95,19 +89,21 @@ async def download(download_job: DownloadJob):
flattened_results = []
for result in results:
for key, values in result["result"].items():
for value in values:
flattened_results.append(
{
"id": result["id"],
"url": result["url"],
"element_name": key,
"xpath": value["xpath"],
"text": value["text"],
"user": result["user"],
"time_created": result["time_created"],
}
)
for res in result["result"]:
for url, elements in res.items():
for element_name, values in elements.items():
for value in values:
flattened_results.append(
{
"id": result.get("id", None),
"url": url,
"element_name": element_name,
"xpath": value.get("xpath", ""),
"text": value.get("text", ""),
"user": result.get("user", ""),
"time_created": result.get("time_created", ""),
}
)
df = pd.DataFrame(flattened_results)
@@ -120,6 +116,7 @@ async def download(download_job: DownloadJob):
except Exception as e:
LOG.error(f"Exception occurred: {e}")
traceback.print_exc()
return {"error": str(e)}
api/backend/job.py
+17
View File
@@ -1,6 +1,7 @@
# STL
import logging
from typing import Any
from pymongo import DESCENDING
# LOCAL
from api.backend.database import get_job_collection
@@ -14,6 +15,13 @@ async def insert(item: dict[str, Any]) -> None:
LOG.info(f"Inserted item: {i}")
async def get_queued_job():
collection = get_job_collection()
return await collection.find_one(
{"status": "Queued"}, sort=[("created_at", DESCENDING)]
)
async def query(filter: dict[str, Any]) -> list[dict[str, Any]]:
collection = get_job_collection()
cursor = collection.find(filter)
@@ -26,6 +34,15 @@ async def query(filter: dict[str, Any]) -> list[dict[str, Any]]:
return results
async def update_job(id: str, field: str, value: Any):
collection = get_job_collection()
result = await collection.update_one(
{"id": id},
{"$set": {field: value}},
)
return result.modified_count
async def delete_jobs(jobs: list[str]):
collection = get_job_collection()
result = await collection.delete_many({"id": {"$in": jobs}})
api/backend/models.py
+7
View File
@@ -17,6 +17,11 @@ class CapturedElement(pydantic.BaseModel):
name: str
class JobOptions(pydantic.BaseModel):
multi_page_scrape: bool
custom_headers: Optional[dict[str, Any]]
class SubmitScrapeJob(pydantic.BaseModel):
id: Optional[str] = None
url: str
@@ -24,6 +29,8 @@ class SubmitScrapeJob(pydantic.BaseModel):
user: Optional[str] = None
time_created: Optional[str] = None
result: Optional[dict[str, Any]] = None
job_options: JobOptions
status: str = "Queued"
class RetrieveScrapeJobs(pydantic.BaseModel):
api/backend/scraping.py
+99 -13
View File
@@ -1,10 +1,11 @@
# STL
import logging
from typing import Any, Optional
# PDM
from bs4 import BeautifulSoup
from lxml import etree
from selenium import webdriver
from seleniumwire import webdriver
from lxml.etree import _Element # type: ignore [reportPrivateImport]
from fake_useragent import UserAgent
from webdriver_manager.chrome import ChromeDriverManager
@@ -13,6 +14,7 @@ from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.chrome.options import Options as ChromeOptions
from selenium.webdriver.chrome.service import Service
from urllib.parse import urlparse, urljoin
# LOCAL
from api.backend.models import Element, CapturedElement
@@ -23,6 +25,14 @@ LOG = logging.getLogger(__name__)
class HtmlElement(_Element): ...
def is_same_domain(url: str, original_url: str) -> bool:
parsed_url = urlparse(url)
parsed_original_url = urlparse(original_url)
LOG.info(f"PARSED: {parsed_url.netloc}")
LOG.info(f"PARSED_ORIGINAL: {parsed_original_url.netloc}")
return parsed_url.netloc == parsed_original_url.netloc or parsed_url.netloc == ""
def clean_xpath(xpath: str) -> str:
parts = xpath.split("/")
clean_parts: list[str] = []
@@ -41,10 +51,25 @@ def sxpath(context: _Element, xpath: str) -> list[HtmlElement]:
return context.xpath(xpath) # type: ignore [reportReturnType]
async def make_site_request(url: str) -> str:
"""Make basic `GET` request to site using Selenium."""
ua = UserAgent()
def interceptor(headers: dict[str, Any]):
def _interceptor(request: Any):
for key, val in headers.items():
if request.headers.get(key):
del request.headers[key]
request.headers[key] = val
if "sec-ch-ua" in request.headers:
original_value = request.headers["sec-ch-ua"]
del request.headers["sec-ch-ua"]
modified_value = original_value.replace("HeadlessChrome", "Chrome")
request.headers["sec-ch-ua"] = modified_value
return _interceptor
def create_driver():
ua = UserAgent()
chrome_options = ChromeOptions()
chrome_options.add_argument("--headless")
chrome_options.add_argument("--no-sandbox")
@@ -52,23 +77,65 @@ async def make_site_request(url: str) -> str:
chrome_options.add_argument(f"user-agent={ua.random}")
service = Service(ChromeDriverManager().install())
driver = webdriver.Chrome(options=chrome_options, service=service)
return webdriver.Chrome(options=chrome_options, service=service)
async def make_site_request(
url: str,
headers: Optional[dict[str, Any]],
multi_page_scrape: bool = False,
visited_urls: set[str] = set(),
pages: set[tuple[str, str]] = set(),
original_url: str = "",
) -> None:
"""Make basic `GET` request to site using Selenium."""
# Check if URL has already been visited
if url in visited_urls:
return
driver = create_driver()
if headers:
driver.request_interceptor = interceptor(headers)
try:
driver.get(url)
visited_urls.add(url)
_ = WebDriverWait(driver, 10).until(
EC.presence_of_element_located((By.TAG_NAME, "body"))
)
page_source = driver.page_source
LOG.debug(f"Page source for url: {url}\n{page_source}")
pages.add((page_source, url))
finally:
driver.quit()
LOG.debug(f"Page source for url: {url}\n{page_source}")
return page_source
if not multi_page_scrape:
return
soup = BeautifulSoup(page_source, "html.parser")
for a_tag in soup.find_all("a"):
link = a_tag.get("href")
if link:
if not urlparse(link).netloc:
base_url = "{0.scheme}://{0.netloc}".format(urlparse(original_url))
link = urljoin(base_url, link)
if link not in visited_urls and is_same_domain(link, original_url):
await make_site_request(
link,
headers=headers,
multi_page_scrape=multi_page_scrape,
visited_urls=visited_urls,
pages=pages,
original_url=original_url,
)
async def collect_scraped_elements(page: str, xpaths: list[Element]):
soup = BeautifulSoup(page, "lxml")
async def collect_scraped_elements(page: tuple[str, str], xpaths: list[Element]):
soup = BeautifulSoup(page[0], "lxml")
root = etree.HTML(str(soup))
elements: dict[str, list[CapturedElement]] = dict()
@@ -86,11 +153,30 @@ async def collect_scraped_elements(page: str, xpaths: list[Element]):
elements[elem.name] = [captured_element]
return elements
return {page[1]: elements}
async def scrape(url: str, xpaths: list[Element]):
page = await make_site_request(url)
elements = await collect_scraped_elements(page, xpaths)
async def scrape(
url: str,
xpaths: list[Element],
headers: Optional[dict[str, Any]],
multi_page_scrape: bool = False,
):
visited_urls: set[str] = set()
pages: set[tuple[str, str]] = set()
_ = await make_site_request(
url,
headers,
multi_page_scrape=multi_page_scrape,
visited_urls=visited_urls,
pages=pages,
original_url=url,
)
elements: list[dict[str, dict[str, list[CapturedElement]]]] = list()
for page in pages:
elements.append(await collect_scraped_elements(page, xpaths))
return elements
api/backend/worker/job_worker.py
+48
View File
@@ -0,0 +1,48 @@
from api.backend.job import get_queued_job, update_job
from api.backend.scraping import scrape
from api.backend.models import Element
from fastapi.encoders import jsonable_encoder
import asyncio
import logging
import sys
import traceback
logging.basicConfig(stream=sys.stdout, level=logging.INFO)
LOG = logging.getLogger(__name__)
async def process_job():
job = await get_queued_job()
if job:
LOG.info(f"Beginning processing job: {job}.")
try:
_ = await update_job(job["id"], field="status", value="Scraping")
scraped = await scrape(
job["url"],
[Element(**j) for j in job["elements"]],
job["job_options"]["custom_headers"],
job["job_options"]["multi_page_scrape"],
)
LOG.info(
f"Scraped result for url: {job['url']}, with elements: {job['elements']}\n{scraped}"
)
_ = await update_job(
job["id"], field="result", value=jsonable_encoder(scraped)
)
_ = await update_job(job["id"], field="status", value="Completed")
except Exception as e:
_ = await update_job(job["id"], field="status", value="Failed")
_ = await update_job(job["id"], field="result", value=e)
LOG.error(f"Exception as occured: {e}\n{traceback.print_exc()}")
async def main():
LOG.info("Starting job worker...")
while True:
await process_job()
await asyncio.sleep(5) # Sleep for 5 seconds before checking for new jobs
if __name__ == "__main__":
asyncio.run(main())
docker-compose.yml
+1
View File
@@ -1,6 +1,7 @@
services:
scraperr:
image: jpyles0524/scraperr:latest
init: True
build:
context: ./
container_name: scraperr
pdm.lock
Generated
+2144 -1580
View File
File diff suppressed because it is too large Load Diff
pyproject.toml
+3
View File
@@ -30,6 +30,9 @@ dependencies = [
"motor[asyncio]>=3.5.0",
"python-jose[cryptography]>=3.3.0",
"passlib[bcrypt]>=1.7.4",
"selenium-wire>=5.1.0",
"blinker<1.8.0",
"setuptools>=71.0.4",
]
requires-python = ">=3.10"
readme = "README.md"
src/components/JobTable.tsx
+29 -2
View File
@@ -33,6 +33,8 @@ interface Job {
elements: Object[];
result: Object;
time_created: Date;
status: string;
job_options: Object;
}
interface JobTableProps {
@@ -40,6 +42,16 @@ interface JobTableProps {
fetchJobs: () => void;
}
interface ColorMap {
[key: string]: string;
}
const COLOR_MAP: ColorMap = {
Queued: "rgba(255,201,5,0.5)",
Scraping: "rgba(3,104,255,0.5)",
Completed: "rgba(5,255,51,0.5)",
};
const JobTable: React.FC<JobTableProps> = ({ jobs, fetchJobs }) => {
const [selectedJobs, setSelectedJobs] = useState<Set<string>>(new Set());
const [allSelected, setAllSelected] = useState(false);
@@ -70,12 +82,14 @@ const JobTable: React.FC<JobTableProps> = ({ jobs, fetchJobs }) => {
}
};
const handleNavigate = (elements: Object[], url: string) => {
const handleNavigate = (elements: Object[], url: string, options: any) => {
console.log(options);
router.push({
pathname: "/",
query: {
elements: JSON.stringify(elements),
url: url,
job_options: JSON.stringify(options),
},
});
};
@@ -212,6 +226,7 @@ const JobTable: React.FC<JobTableProps> = ({ jobs, fetchJobs }) => {
<TableCell>Elements</TableCell>
<TableCell>Result</TableCell>
<TableCell>Time Created</TableCell>
<TableCell>Status</TableCell>
<TableCell>Actions</TableCell>
</TableRow>
</TableHead>
@@ -283,6 +298,16 @@ const JobTable: React.FC<JobTableProps> = ({ jobs, fetchJobs }) => {
{new Date(row.time_created).toLocaleString()}
</Box>
</TableCell>
<TableCell sx={{ maxWidth: 150, overflow: "auto" }}>
<Box sx={{ maxHeight: 100, overflow: "auto" }}>
<Box
className="rounded-md p-2 text-center"
sx={{ bgcolor: COLOR_MAP[row.status], opactity: "50%" }}
>
{row.status}
</Box>
</Box>
</TableCell>
<TableCell sx={{ maxWidth: 100, overflow: "auto" }}>
<Button
onClick={() => {
@@ -292,7 +317,9 @@ const JobTable: React.FC<JobTableProps> = ({ jobs, fetchJobs }) => {
Download
</Button>
<Button
onClick={() => handleNavigate(row.elements, row.url)}
onClick={() =>
handleNavigate(row.elements, row.url, row.job_options)
}
>
Rerun
</Button>
src/components/NavDrawer.tsx
+5 -2
View File
@@ -125,9 +125,12 @@ const NavDrawer: React.FC<NavDrawerProps> = ({ toggleTheme, isDarkMode }) => {
<Typography>Settings</Typography>
</AccordionSummary>
<AccordionDetails>
<Tooltip title="Dark Theme Toggle" placement="bottom">
<div className="flex flex-row mr-1">
<Typography className="mr-2">
<p className="text-sm">Dark Theme Toggle</p>
</Typography>
<Switch checked={isDarkMode} onChange={toggleTheme} />
</Tooltip>
</div>
</AccordionDetails>
</Accordion>
</Box>
src/components/submit/JobSubmitter.tsx
+96 -40
View File
@@ -1,39 +1,40 @@
import React, { useState, useEffect, useRef, Dispatch } from "react";
import React, { useEffect, useState, Dispatch } from "react";
import {
TextField,
Button,
Box,
Checkbox,
FormControlLabel,
Accordion,
AccordionSummary,
AccordionDetails,
CircularProgress,
} from "@mui/material";
import CircularProgress from "@mui/material/CircularProgress";
import ExpandMoreIcon from "@mui/icons-material/ExpandMore";
import { Element, Result } from "../../types";
import { useAuth } from "../../contexts/AuthContext";
import { useRouter } from "next/router";
interface stateProps {
interface StateProps {
submittedURL: string;
setSubmittedURL: Dispatch<React.SetStateAction<string>>;
rows: Element[];
setResults: Dispatch<React.SetStateAction<Result>>;
setSnackbarMessage: Dispatch<React.SetStateAction<string>>;
setSnackbarOpen: Dispatch<React.SetStateAction<boolean>>;
setSnackbarSeverity: Dispatch<React.SetStateAction<string>>;
}
interface Props {
stateProps: stateProps;
stateProps: StateProps;
}
interface JobOptions {
multi_page_scrape: boolean;
custom_headers: null | Object;
custom_headers: null | string;
}
export const JobSubmitter = ({ stateProps }: Props) => {
const { user } = useAuth();
const router = useRouter();
const { job_options } = router.query;
const {
submittedURL,
@@ -42,6 +43,7 @@ export const JobSubmitter = ({ stateProps }: Props) => {
setResults,
setSnackbarMessage,
setSnackbarOpen,
setSnackbarSeverity,
} = stateProps;
const [isValidURL, setIsValidUrl] = useState<boolean>(true);
@@ -51,6 +53,7 @@ export const JobSubmitter = ({ stateProps }: Props) => {
multi_page_scrape: false,
custom_headers: null,
});
const [customJSONSelected, setCustomJSONSelected] = useState<boolean>(false);
function validateURL(url: string): boolean {
try {
@@ -72,6 +75,19 @@ export const JobSubmitter = ({ stateProps }: Props) => {
setUrlError(null);
setLoading(true);
let customHeaders;
try {
customHeaders = jobOptions.custom_headers
? JSON.parse(jobOptions.custom_headers)
: null;
} catch (error) {
setSnackbarMessage("Invalid JSON in custom headers.");
setSnackbarOpen(true);
setSnackbarSeverity("error");
setLoading(false);
return;
}
fetch("/api/submit-scrape-job", {
method: "POST",
headers: { "content-type": "application/json" },
@@ -80,6 +96,10 @@ export const JobSubmitter = ({ stateProps }: Props) => {
elements: rows,
user: user?.email,
time_created: new Date().toISOString(),
job_options: {
...jobOptions,
custom_headers: customHeaders,
},
}),
})
.then((response) => {
@@ -90,14 +110,43 @@ export const JobSubmitter = ({ stateProps }: Props) => {
}
return response.json();
})
.then((data) => setResults(data))
.then((data) => {
console.log(data);
setSnackbarMessage(data);
setSnackbarSeverity("info");
setSnackbarOpen(true);
})
.catch((error) => {
setSnackbarMessage(error.message || "An error occurred.");
setSnackbarSeverity("error");
setSnackbarOpen(true);
})
.finally(() => setLoading(false));
};
useEffect(() => {
if (job_options) {
const jsonOptions = JSON.parse(job_options as string);
const newJobOptions: JobOptions = {
multi_page_scrape: false,
custom_headers: null,
};
if (
jsonOptions.custom_headers &&
Object.keys(jsonOptions.custom_headers).length
) {
setCustomJSONSelected(true);
newJobOptions.custom_headers = JSON.stringify(
jsonOptions.custom_headers
);
}
newJobOptions.multi_page_scrape = jsonOptions.multi_page_scrape;
setJobOptions(newJobOptions);
}
}, [job_options]);
return (
<>
<div className="flex flex-row space-x-4 items-center mb-2">
@@ -120,40 +169,47 @@ export const JobSubmitter = ({ stateProps }: Props) => {
{loading ? <CircularProgress size={24} /> : "Submit"}
</Button>
</div>
<Box
bgcolor="background.paper"
className="p-2 mb-2 flex flex-row space-x-2"
>
<FormControlLabel
label="Multi-Page Scrape"
control={
<Checkbox
checked={jobOptions.multi_page_scrape}
onChange={() =>
setJobOptions((prevJobOptions) => ({
...prevJobOptions,
multi_page_scrape: !prevJobOptions.multi_page_scrape,
}))
}
/>
}
></FormControlLabel>
<Accordion style={{ padding: 1 }}>
<AccordionSummary
expandIcon={<ExpandMoreIcon />}
aria-controls="panel1-content"
id="panel1-header"
>
Custom Headers (JSON)
</AccordionSummary>
<AccordionDetails>
<Box bgcolor="background.paper" className="flex flex-col mb-2">
<div id="options" className="p-2 flex flex-row space-x-2">
<FormControlLabel
label="Multi-Page Scrape"
control={
<Checkbox
checked={jobOptions.multi_page_scrape}
onChange={() =>
setJobOptions((prevJobOptions) => ({
...prevJobOptions,
multi_page_scrape: !prevJobOptions.multi_page_scrape,
}))
}
/>
}
></FormControlLabel>
<FormControlLabel
label="Custom Headers (JSON)"
control={
<Checkbox
checked={customJSONSelected}
onChange={() => {
setCustomJSONSelected(!customJSONSelected);
setJobOptions((prevJobOptions) => ({
...prevJobOptions,
custom_headers: "",
}));
}}
/>
}
></FormControlLabel>
</div>
{customJSONSelected ? (
<div id="custom-json" className="pl-2 pr-2 pb-2">
<TextField
InputLabelProps={{ shrink: false }}
fullWidth
multiline
minRows={4}
variant="outlined"
value={jobOptions.custom_headers}
value={jobOptions.custom_headers || ""}
onChange={(e) =>
setJobOptions((prevJobOptions) => ({
...prevJobOptions,
@@ -163,8 +219,8 @@ export const JobSubmitter = ({ stateProps }: Props) => {
style={{ maxHeight: "20vh", overflow: "auto" }}
className="mt-2"
/>
</AccordionDetails>
</Accordion>
</div>
) : null}
</Box>
</>
);
src/pages/index.tsx
+24 -8
View File
@@ -6,14 +6,14 @@ import { ElementTable, JobSubmitter, ResultsTable } from "../components/submit";
const Home = () => {
const router = useRouter();
const { elements, url } = router.query;
const [submittedURL, setSubmittedURL] = useState("");
const [submittedURL, setSubmittedURL] = useState<string>("");
const [rows, setRows] = useState<Element[]>([]);
const [results, setResults] = useState<Result>({});
const [snackbarOpen, setSnackbarOpen] = useState(false);
const [snackbarMessage, setSnackbarMessage] = useState("");
const [snackbarOpen, setSnackbarOpen] = useState<boolean>(false);
const [snackbarMessage, setSnackbarMessage] = useState<string>("");
const [snackbarSeverity, setSnackbarSeverity] = useState<string>("alert");
const resultsRef = useRef<HTMLTableElement | null>(null);
@@ -36,6 +36,22 @@ const Home = () => {
setSnackbarOpen(false);
};
const ErrorSnackbar = () => {
return (
<Alert onClose={handleCloseSnackbar} severity="error">
{snackbarMessage}
</Alert>
);
};
const NotifySnackbar = () => {
return (
<Alert onClose={handleCloseSnackbar} severity="info">
{snackbarMessage}
</Alert>
);
};
return (
<Box
bgcolor="background.default"
@@ -58,6 +74,7 @@ const Home = () => {
setResults,
setSnackbarMessage,
setSnackbarOpen,
setSnackbarSeverity,
}}
/>
<ElementTable
@@ -65,16 +82,15 @@ const Home = () => {
setRows={setRows}
submittedURL={submittedURL}
/>
<ResultsTable stateProps={{ results }} resultsRef={resultsRef} />
{/* <ResultsTable stateProps={{ results }} resultsRef={resultsRef} /> */}
</Container>
<Snackbar
open={snackbarOpen}
autoHideDuration={6000}
onClose={handleCloseSnackbar}
>
<Alert onClose={handleCloseSnackbar} severity="error">
{snackbarMessage}
</Alert>
<p>Woo</p>
{/* {snackbarSeverity === "info" ? <NotifySnackbar /> : <ErrorSnackbar />} */}
</Snackbar>
</Box>
);
supervisord.conf
+22
View File
@@ -0,0 +1,22 @@
[supervisord]
nodaemon=true
[program:api]
command=pdm run python -m uvicorn api.backend.app:app --reload --host 0.0.0.0 --port 8000
directory=/project
autostart=true
autorestart=true
stdout_logfile=/dev/stdout
stderr_logfile=/dev/stderr
stdout_logfile_maxbytes=0
stderr_logfile_maxbytes=0
[program:worker]
command=pdm run python -m api.backend.worker.job_worker
directory=/project
autostart=true
autorestart=true
stdout_logfile=/dev/stdout
stderr_logfile=/dev/stderr
stdout_logfile_maxbytes=0
stderr_logfile_maxbytes=0