lee/Scraperr
1
0
Fork 0
mirror of https://github.com/jaypyles/Scraperr.git synced 2025-12-11 18:26:08 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

wip: fix table

Browse Source
This commit is contained in:
Jayden Pyles
2024-07-07 12:55:39 -05:00
parent 67c0b125be
commit e1f6aaf03a
4 changed files with 135 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
api/backend/app.py
View File

@@ -18,7 +18,7 @@ from api.backend.scraping import scrape
from api.backend.auth.auth_router import auth_router from api.backend.auth.auth_router import auth_router
logging.basicConfig( logging.basicConfig(
level=logging.INFO, level=logging.DEBUG,
format="%(levelname)s: %(asctime)s - %(name)s - %(message)s", format="%(levelname)s: %(asctime)s - %(name)s - %(message)s",
handlers=[logging.StreamHandler()], handlers=[logging.StreamHandler()],
) )
@@ -71,7 +71,7 @@ async def retrieve_scrape_jobs(retrieve: RetrieveScrapeJobs):
LOG.info(f"Retrieving jobs for account: {retrieve.user}") LOG.info(f"Retrieving jobs for account: {retrieve.user}")
try: try:
results = await query({"user": retrieve.user}) results = await query({"user": retrieve.user})
return JSONResponse(content=results) return JSONResponse(content=results[::-1])
except Exception as e: except Exception as e:
LOG.error(f"Exception occurred: {e}") LOG.error(f"Exception occurred: {e}")
return JSONResponse(content={"error": str(e)}, status_code=500) return JSONResponse(content={"error": str(e)}, status_code=500)
@@ -83,7 +83,23 @@ async def download(download_job: DownloadJob):
try: try:
results = await query({"id": download_job.id}) results = await query({"id": download_job.id})
df = pd.DataFrame(results) flattened_results = []
for result in results:
for key, values in result["result"].items():
for value in values:
flattened_results.append(
{
"id": result["id"],
"url": result["url"],
"element_name": key,
"xpath": value["xpath"],
"text": value["text"],
"user": result["user"],
"time_created": result["time_created"],
}
)
df = pd.DataFrame(flattened_results)
csv_buffer = StringIO() csv_buffer = StringIO()
df.to_csv(csv_buffer, index=False) df.to_csv(csv_buffer, index=False)
@@ -94,3 +110,4 @@ async def download(download_job: DownloadJob):
except Exception as e: except Exception as e:
LOG.error(f"Exception occurred: {e}") LOG.error(f"Exception occurred: {e}")
return {"error": str(e)}

4
api/backend/models.py
View File

@@ -7,8 +7,8 @@ import pydantic
class Element(pydantic.BaseModel): class Element(pydantic.BaseModel):
name: str name: str
url: str
xpath: str xpath: str
url: Optional[str] = None
class CapturedElement(pydantic.BaseModel): class CapturedElement(pydantic.BaseModel):
@@ -22,7 +22,7 @@ class SubmitScrapeJob(pydantic.BaseModel):
url: str url: str
elements: list[Element] elements: list[Element]
user: Optional[str] = None user: Optional[str] = None
time_created: str time_created: Optional[str] = None
result: Optional[dict[str, Any]] = None result: Optional[dict[str, Any]] = None

128
src/components/JobTable.tsx
View File

@@ -1,6 +1,5 @@
import React from "react"; import React from "react";
import { import {
TextField,
Table, Table,
TableBody, TableBody,
TableCell, TableCell,
@@ -9,8 +8,12 @@ import {
Button, Button,
Box, Box,
Typography, Typography,
Accordion,
AccordionSummary,
AccordionDetails,
} from "@mui/material"; } from "@mui/material";
import { useRouter } from "next/router"; import { useRouter } from "next/router";
import ExpandMoreIcon from "@mui/icons-material/ExpandMore";
interface Job { interface Job {
id: string; id: string;
@@ -26,6 +29,7 @@ interface JobTableProps {
const JobTable: React.FC<JobTableProps> = ({ jobs }) => { const JobTable: React.FC<JobTableProps> = ({ jobs }) => {
const router = useRouter(); const router = useRouter();
const handleDownload = async (id: string) => { const handleDownload = async (id: string) => {
console.log(id); console.log(id);
const response = await fetch("/api/download", { const response = await fetch("/api/download", {
@@ -60,60 +64,98 @@ const JobTable: React.FC<JobTableProps> = ({ jobs }) => {
}; };
return ( return (
<> <Box
width="100%"
bgcolor="background.default"
display="flex"
justifyContent="center"
>
<Box <Box
width="fullWidth" className="flex flex-col justify-center align-center items-center"
bgcolor="background.paper" width="100%"
className="flex justify-center" maxWidth="100%"
bgcolor="background.default"
p={3}
overflow="auto"
> >
<Box <Typography variant="h4" gutterBottom>
maxWidth="lg" Scrape Jobs
minHeight="100vh" </Typography>
bgcolor="background.paper" <Box sx={{ overflow: "auto", width: "75%" }}>
className="p-4" <Table sx={{ tableLayout: "fixed", width: "100%" }}>
>
<Typography variant="h4">Scrape Jobs</Typography>
<Table>
<TableHead> <TableHead>
<TableRow> <TableRow>
<TableCell>id</TableCell> <TableCell>Id</TableCell>
<TableCell>url</TableCell> <TableCell>Url</TableCell>
<TableCell>elements</TableCell> <TableCell>Elements</TableCell>
<TableCell>result</TableCell> <TableCell>Result</TableCell>
<TableCell>time_created</TableCell> <TableCell>Time Created</TableCell>
<TableCell>Actions</TableCell>
</TableRow> </TableRow>
</TableHead> </TableHead>
<TableBody> <TableBody>
{jobs.map((row, index) => ( {jobs.map((row, index) => (
<TableRow key={index}> <TableRow key={index}>
<TableCell> <TableCell sx={{ maxWidth: 100, overflow: "auto" }}>
<TextField variant="outlined" fullWidth value={row.id} /> <Box sx={{ maxHeight: 100, overflow: "auto" }}>
{row.id}
</Box>
</TableCell> </TableCell>
<TableCell> <TableCell sx={{ maxWidth: 200, overflow: "auto" }}>
<TextField variant="outlined" fullWidth value={row.url} /> <Box sx={{ maxHeight: 100, overflow: "auto" }}>
{row.url}
</Box>
</TableCell> </TableCell>
<TableCell> <TableCell sx={{ maxWidth: 150, overflow: "auto" }}>
<TextField <Box sx={{ maxHeight: 100, overflow: "auto" }}>
variant="outlined" {JSON.stringify(row.elements)}
fullWidth </Box>
value={JSON.stringify(row.elements)}
/>
</TableCell> </TableCell>
<TableCell> <TableCell
<TextField sx={{ maxWidth: 150, overflow: "auto", padding: 0 }}
variant="outlined" >
fullWidth <Accordion sx={{ margin: 0, padding: 0.5 }}>
value={JSON.stringify(row.result)} <AccordionSummary
/> expandIcon={<ExpandMoreIcon />}
aria-controls="panel1a-content"
id="panel1a-header"
sx={{
minHeight: 0,
"&.Mui-expanded": { minHeight: 0 },
}}
>
<Box
sx={{
maxHeight: 150,
overflow: "auto",
width: "100%",
}}
>
<Typography sx={{ fontSize: "0.875rem" }}>
Show Result
</Typography>
</Box>
</AccordionSummary>
<AccordionDetails sx={{ padding: 1 }}>
<Box sx={{ maxHeight: 200, overflow: "auto" }}>
<Typography
sx={{
fontSize: "0.875rem",
whiteSpace: "pre-wrap",
}}
>
{JSON.stringify(row.result, null, 2)}
</Typography>
</Box>
</AccordionDetails>
</Accordion>
</TableCell> </TableCell>
<TableCell> <TableCell sx={{ maxWidth: 150, overflow: "auto" }}>
<TextField <Box sx={{ maxHeight: 100, overflow: "auto" }}>
variant="outlined" {new Date(row.time_created).toLocaleString()}
fullWidth </Box>
value={row.time_created}
/>
</TableCell> </TableCell>
<TableCell> <TableCell sx={{ maxWidth: 100, overflow: "auto" }}>
<Button <Button
onClick={() => { onClick={() => {
handleDownload(row.id); handleDownload(row.id);
@@ -121,8 +163,6 @@ const JobTable: React.FC<JobTableProps> = ({ jobs }) => {
> >
Download Download
</Button> </Button>
</TableCell>
<TableCell>
<Button <Button
onClick={() => handleNavigate(row.elements, row.url)} onClick={() => handleNavigate(row.elements, row.url)}
> >
@@ -135,7 +175,7 @@ const JobTable: React.FC<JobTableProps> = ({ jobs }) => {
</Table> </Table>
</Box> </Box>
</Box> </Box>
</> </Box>
); );
}; };

31
src/pages/index.tsx
View File

@@ -12,6 +12,8 @@ import {
Box, Box,
IconButton, IconButton,
Tooltip, Tooltip,
Snackbar,
Alert,
} from "@mui/material"; } from "@mui/material";
import AddIcon from "@mui/icons-material/Add"; import AddIcon from "@mui/icons-material/Add";
import { useAuth } from "../contexts/AuthContext"; import { useAuth } from "../contexts/AuthContext";
@@ -60,7 +62,8 @@ const Home = () => {
url: "", url: "",
}); });
const [loading, setLoading] = useState(false); const [loading, setLoading] = useState(false);
const [snackbarOpen, setSnackbarOpen] = useState(false);
const [snackbarMessage, setSnackbarMessage] = useState("");
const resultsRef = useRef<HTMLTableElement | null>(null); const resultsRef = useRef<HTMLTableElement | null>(null);
useEffect(() => { useEffect(() => {
@@ -113,11 +116,26 @@ const Home = () => {
time_created: new Date().toISOString(), time_created: new Date().toISOString(),
}), }),
}) })
.then((response) => response.json()) .then((response) => {
if (!response.ok) {
return response.json().then((error) => {
throw new Error(error.error);
});
}
return response.json();
})
.then((data) => setResults(data)) .then((data) => setResults(data))
.catch((error) => {
setSnackbarMessage(error.message || "An error occurred.");
setSnackbarOpen(true);
})
.finally(() => setLoading(false)); .finally(() => setLoading(false));
}; };
const handleCloseSnackbar = () => {
setSnackbarOpen(false);
};
return ( return (
<Box <Box
bgcolor="background.default" bgcolor="background.default"
@@ -245,6 +263,15 @@ const Home = () => {
</> </>
)} )}
</Container> </Container>
<Snackbar
open={snackbarOpen}
autoHideDuration={6000}
onClose={handleCloseSnackbar}
>
<Alert onClose={handleCloseSnackbar} severity="error">
{snackbarMessage}
</Alert>
</Snackbar>
</Box> </Box>
); );
}; };