lee/Scraperr
1
0
Fork 0
mirror of https://github.com/jaypyles/Scraperr.git synced 2025-12-11 18:26:08 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

wip: fix table

Browse Source
This commit is contained in:
Jayden Pyles
2024-07-07 12:55:39 -05:00
parent 67c0b125be
commit e1f6aaf03a
4 changed files with 135 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
api/backend/app.py
View File

@@ -18,7 +18,7 @@ from api.backend.scraping import scrape
from api.backend.auth.auth_router import auth_router
logging.basicConfig(
level=logging.INFO,
level=logging.DEBUG,
format="%(levelname)s: %(asctime)s - %(name)s - %(message)s",
handlers=[logging.StreamHandler()],
)
@@ -71,7 +71,7 @@ async def retrieve_scrape_jobs(retrieve: RetrieveScrapeJobs):
LOG.info(f"Retrieving jobs for account: {retrieve.user}")
try:
results = await query({"user": retrieve.user})
return JSONResponse(content=results)
return JSONResponse(content=results[::-1])
except Exception as e:
LOG.error(f"Exception occurred: {e}")
return JSONResponse(content={"error": str(e)}, status_code=500)
@@ -83,7 +83,23 @@ async def download(download_job: DownloadJob):
try:
results = await query({"id": download_job.id})
df = pd.DataFrame(results)
flattened_results = []
for result in results:
for key, values in result["result"].items():
for value in values:
flattened_results.append(
{
"id": result["id"],
"url": result["url"],
"element_name": key,
"xpath": value["xpath"],
"text": value["text"],
"user": result["user"],
"time_created": result["time_created"],
}
)
df = pd.DataFrame(flattened_results)
csv_buffer = StringIO()
df.to_csv(csv_buffer, index=False)
@@ -94,3 +110,4 @@ async def download(download_job: DownloadJob):
except Exception as e:
LOG.error(f"Exception occurred: {e}")
return {"error": str(e)}

4
api/backend/models.py
View File

@@ -7,8 +7,8 @@ import pydantic
class Element(pydantic.BaseModel):
name: str
url: str
xpath: str
url: Optional[str] = None
class CapturedElement(pydantic.BaseModel):
@@ -22,7 +22,7 @@ class SubmitScrapeJob(pydantic.BaseModel):
url: str
elements: list[Element]
user: Optional[str] = None
time_created: str
time_created: Optional[str] = None
result: Optional[dict[str, Any]] = None

128
src/components/JobTable.tsx
View File

@@ -1,6 +1,5 @@
import React from "react";
import {
TextField,
Table,
TableBody,
TableCell,
@@ -9,8 +8,12 @@ import {
Button,
Box,
Typography,
Accordion,
AccordionSummary,
AccordionDetails,
} from "@mui/material";
import { useRouter } from "next/router";
import ExpandMoreIcon from "@mui/icons-material/ExpandMore";
interface Job {
id: string;
@@ -26,6 +29,7 @@ interface JobTableProps {
const JobTable: React.FC<JobTableProps> = ({ jobs }) => {
const router = useRouter();
const handleDownload = async (id: string) => {
console.log(id);
const response = await fetch("/api/download", {
@@ -60,60 +64,98 @@ const JobTable: React.FC<JobTableProps> = ({ jobs }) => {
};
return (
<>
<Box
width="100%"
bgcolor="background.default"
display="flex"
justifyContent="center"
>
<Box
width="fullWidth"
bgcolor="background.paper"
className="flex justify-center"
className="flex flex-col justify-center align-center items-center"
width="100%"
maxWidth="100%"
bgcolor="background.default"
p={3}
overflow="auto"
>
<Box
maxWidth="lg"
minHeight="100vh"
bgcolor="background.paper"
className="p-4"
>
<Typography variant="h4">Scrape Jobs</Typography>
<Table>
<Typography variant="h4" gutterBottom>
Scrape Jobs
</Typography>
<Box sx={{ overflow: "auto", width: "75%" }}>
<Table sx={{ tableLayout: "fixed", width: "100%" }}>
<TableHead>
<TableRow>
<TableCell>id</TableCell>
<TableCell>url</TableCell>
<TableCell>elements</TableCell>
<TableCell>result</TableCell>
<TableCell>time_created</TableCell>
<TableCell>Id</TableCell>
<TableCell>Url</TableCell>
<TableCell>Elements</TableCell>
<TableCell>Result</TableCell>
<TableCell>Time Created</TableCell>
<TableCell>Actions</TableCell>
</TableRow>
</TableHead>
<TableBody>
{jobs.map((row, index) => (
<TableRow key={index}>
<TableCell>
<TextField variant="outlined" fullWidth value={row.id} />
<TableCell sx={{ maxWidth: 100, overflow: "auto" }}>
<Box sx={{ maxHeight: 100, overflow: "auto" }}>
{row.id}
</Box>
</TableCell>
<TableCell>
<TextField variant="outlined" fullWidth value={row.url} />
<TableCell sx={{ maxWidth: 200, overflow: "auto" }}>
<Box sx={{ maxHeight: 100, overflow: "auto" }}>
{row.url}
</Box>
</TableCell>
<TableCell>
<TextField
variant="outlined"
fullWidth
value={JSON.stringify(row.elements)}
/>
<TableCell sx={{ maxWidth: 150, overflow: "auto" }}>
<Box sx={{ maxHeight: 100, overflow: "auto" }}>
{JSON.stringify(row.elements)}
</Box>
</TableCell>
<TableCell>
<TextField
variant="outlined"
fullWidth
value={JSON.stringify(row.result)}
/>
<TableCell
sx={{ maxWidth: 150, overflow: "auto", padding: 0 }}
>
<Accordion sx={{ margin: 0, padding: 0.5 }}>
<AccordionSummary
expandIcon={<ExpandMoreIcon />}
aria-controls="panel1a-content"
id="panel1a-header"
sx={{
minHeight: 0,
"&.Mui-expanded": { minHeight: 0 },
}}
>
<Box
sx={{
maxHeight: 150,
overflow: "auto",
width: "100%",
}}
>
<Typography sx={{ fontSize: "0.875rem" }}>
Show Result
</Typography>
</Box>
</AccordionSummary>
<AccordionDetails sx={{ padding: 1 }}>
<Box sx={{ maxHeight: 200, overflow: "auto" }}>
<Typography
sx={{
fontSize: "0.875rem",
whiteSpace: "pre-wrap",
}}
>
{JSON.stringify(row.result, null, 2)}
</Typography>
</Box>
</AccordionDetails>
</Accordion>
</TableCell>
<TableCell>
<TextField
variant="outlined"
fullWidth
value={row.time_created}
/>
<TableCell sx={{ maxWidth: 150, overflow: "auto" }}>
<Box sx={{ maxHeight: 100, overflow: "auto" }}>
{new Date(row.time_created).toLocaleString()}
</Box>
</TableCell>
<TableCell>
<TableCell sx={{ maxWidth: 100, overflow: "auto" }}>
<Button
onClick={() => {
handleDownload(row.id);
@@ -121,8 +163,6 @@ const JobTable: React.FC<JobTableProps> = ({ jobs }) => {
>
Download
</Button>
</TableCell>
<TableCell>
<Button
onClick={() => handleNavigate(row.elements, row.url)}
>
@@ -135,7 +175,7 @@ const JobTable: React.FC<JobTableProps> = ({ jobs }) => {
</Table>
</Box>
</Box>
</>
</Box>
);
};

31
src/pages/index.tsx
View File

@@ -12,6 +12,8 @@ import {
Box,
IconButton,
Tooltip,
Snackbar,
Alert,
} from "@mui/material";
import AddIcon from "@mui/icons-material/Add";
import { useAuth } from "../contexts/AuthContext";
@@ -60,7 +62,8 @@ const Home = () => {
url: "",
});
const [loading, setLoading] = useState(false);
const [snackbarOpen, setSnackbarOpen] = useState(false);
const [snackbarMessage, setSnackbarMessage] = useState("");
const resultsRef = useRef<HTMLTableElement | null>(null);
useEffect(() => {
@@ -113,11 +116,26 @@ const Home = () => {
time_created: new Date().toISOString(),
}),
})
.then((response) => response.json())
.then((response) => {
if (!response.ok) {
return response.json().then((error) => {
throw new Error(error.error);
});
}
return response.json();
})
.then((data) => setResults(data))
.catch((error) => {
setSnackbarMessage(error.message || "An error occurred.");
setSnackbarOpen(true);
})
.finally(() => setLoading(false));
};
const handleCloseSnackbar = () => {
setSnackbarOpen(false);
};
return (
<Box
bgcolor="background.default"
@@ -245,6 +263,15 @@ const Home = () => {
</>
)}
</Container>
<Snackbar
open={snackbarOpen}
autoHideDuration={6000}
onClose={handleCloseSnackbar}
>
<Alert onClose={handleCloseSnackbar} severity="error">
{snackbarMessage}
</Alert>
</Snackbar>
</Box>
);
};