diff --git a/api/backend/app.py b/api/backend/app.py index e9ab3e9..a0759d5 100644 --- a/api/backend/app.py +++ b/api/backend/app.py @@ -18,7 +18,7 @@ from api.backend.scraping import scrape from api.backend.auth.auth_router import auth_router logging.basicConfig( - level=logging.INFO, + level=logging.DEBUG, format="%(levelname)s: %(asctime)s - %(name)s - %(message)s", handlers=[logging.StreamHandler()], ) @@ -71,7 +71,7 @@ async def retrieve_scrape_jobs(retrieve: RetrieveScrapeJobs): LOG.info(f"Retrieving jobs for account: {retrieve.user}") try: results = await query({"user": retrieve.user}) - return JSONResponse(content=results) + return JSONResponse(content=results[::-1]) except Exception as e: LOG.error(f"Exception occurred: {e}") return JSONResponse(content={"error": str(e)}, status_code=500) @@ -83,7 +83,23 @@ async def download(download_job: DownloadJob): try: results = await query({"id": download_job.id}) - df = pd.DataFrame(results) + flattened_results = [] + for result in results: + for key, values in result["result"].items(): + for value in values: + flattened_results.append( + { + "id": result["id"], + "url": result["url"], + "element_name": key, + "xpath": value["xpath"], + "text": value["text"], + "user": result["user"], + "time_created": result["time_created"], + } + ) + + df = pd.DataFrame(flattened_results) csv_buffer = StringIO() df.to_csv(csv_buffer, index=False) @@ -94,3 +110,4 @@ async def download(download_job: DownloadJob): except Exception as e: LOG.error(f"Exception occurred: {e}") + return {"error": str(e)} diff --git a/api/backend/models.py b/api/backend/models.py index 4fdf348..25eb5ea 100644 --- a/api/backend/models.py +++ b/api/backend/models.py @@ -7,8 +7,8 @@ import pydantic class Element(pydantic.BaseModel): name: str - url: str xpath: str + url: Optional[str] = None class CapturedElement(pydantic.BaseModel): @@ -22,7 +22,7 @@ class SubmitScrapeJob(pydantic.BaseModel): url: str elements: list[Element] user: Optional[str] = None - time_created: str + time_created: Optional[str] = None result: Optional[dict[str, Any]] = None diff --git a/src/components/JobTable.tsx b/src/components/JobTable.tsx index 2bb14bf..6632a5a 100644 --- a/src/components/JobTable.tsx +++ b/src/components/JobTable.tsx @@ -1,6 +1,5 @@ import React from "react"; import { - TextField, Table, TableBody, TableCell, @@ -9,8 +8,12 @@ import { Button, Box, Typography, + Accordion, + AccordionSummary, + AccordionDetails, } from "@mui/material"; import { useRouter } from "next/router"; +import ExpandMoreIcon from "@mui/icons-material/ExpandMore"; interface Job { id: string; @@ -26,6 +29,7 @@ interface JobTableProps { const JobTable: React.FC = ({ jobs }) => { const router = useRouter(); + const handleDownload = async (id: string) => { console.log(id); const response = await fetch("/api/download", { @@ -60,60 +64,98 @@ const JobTable: React.FC = ({ jobs }) => { }; return ( - <> + - - Scrape Jobs - + + Scrape Jobs + + +
- id - url - elements - result - time_created + Id + Url + Elements + Result + Time Created + Actions {jobs.map((row, index) => ( - - + + + {row.id} + - - + + + {row.url} + - - + + + {JSON.stringify(row.elements)} + - - + + + } + aria-controls="panel1a-content" + id="panel1a-header" + sx={{ + minHeight: 0, + "&.Mui-expanded": { minHeight: 0 }, + }} + > + + + Show Result + + + + + + + {JSON.stringify(row.result, null, 2)} + + + + - - + + + {new Date(row.time_created).toLocaleString()} + - + - -
- +
); }; diff --git a/src/pages/index.tsx b/src/pages/index.tsx index 247e6a0..677ed9f 100644 --- a/src/pages/index.tsx +++ b/src/pages/index.tsx @@ -12,6 +12,8 @@ import { Box, IconButton, Tooltip, + Snackbar, + Alert, } from "@mui/material"; import AddIcon from "@mui/icons-material/Add"; import { useAuth } from "../contexts/AuthContext"; @@ -60,7 +62,8 @@ const Home = () => { url: "", }); const [loading, setLoading] = useState(false); - + const [snackbarOpen, setSnackbarOpen] = useState(false); + const [snackbarMessage, setSnackbarMessage] = useState(""); const resultsRef = useRef(null); useEffect(() => { @@ -113,11 +116,26 @@ const Home = () => { time_created: new Date().toISOString(), }), }) - .then((response) => response.json()) + .then((response) => { + if (!response.ok) { + return response.json().then((error) => { + throw new Error(error.error); + }); + } + return response.json(); + }) .then((data) => setResults(data)) + .catch((error) => { + setSnackbarMessage(error.message || "An error occurred."); + setSnackbarOpen(true); + }) .finally(() => setLoading(false)); }; + const handleCloseSnackbar = () => { + setSnackbarOpen(false); + }; + return ( { )} + + + {snackbarMessage} + + ); };