diff --git a/README.md b/README.md index 6f61990..484938c 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,7 @@ From the table, users can download an excel sheet of the job's results, along wi - Download csv containing results - Rerun jobs - View status of queued jobs +- Favorite and view favorited jobs ![job_page](https://github.com/jaypyles/www-scrape/blob/master/docs/job_page.png) diff --git a/api/backend/app.py b/api/backend/app.py index 052bb19..1dbbdf8 100644 --- a/api/backend/app.py +++ b/api/backend/app.py @@ -24,6 +24,7 @@ from api.backend.job import ( query, insert, delete_jobs, + update_job, ) from api.backend.models import ( DownloadJob, @@ -31,6 +32,7 @@ from api.backend.models import ( SubmitScrapeJob, DeleteScrapeJobs, RetrieveScrapeJobs, + UpdateJobs, ) from api.backend.auth.auth_router import auth_router import traceback @@ -67,6 +69,12 @@ def read_favicon(): return FileResponse("dist/favicon.ico") +@app.post("/api/update") +async def update(update_jobs: UpdateJobs): + """Used to update jobs""" + await update_job(update_jobs.ids, update_jobs.field, update_jobs.value) + + @app.post("/api/submit-scrape-job") async def submit_scrape_job(job: SubmitScrapeJob, background_tasks: BackgroundTasks): LOG.info(f"Recieved job: {job}") diff --git a/api/backend/job.py b/api/backend/job.py index 69992a9..33d1cc1 100644 --- a/api/backend/job.py +++ b/api/backend/job.py @@ -34,13 +34,13 @@ async def query(filter: dict[str, Any]) -> list[dict[str, Any]]: return results -async def update_job(id: str, field: str, value: Any): +async def update_job(ids: list[str], field: str, value: Any): collection = get_job_collection() - result = await collection.update_one( - {"id": id}, - {"$set": {field: value}}, - ) - return result.modified_count + for id in ids: + _ = await collection.update_one( + {"id": id}, + {"$set": {field: value}}, + ) async def delete_jobs(jobs: list[str]): diff --git a/api/backend/models.py b/api/backend/models.py index b46065c..97acf0e 100644 --- a/api/backend/models.py +++ b/api/backend/models.py @@ -48,3 +48,9 @@ class DeleteScrapeJobs(pydantic.BaseModel): class GetStatistics(pydantic.BaseModel): user: str + + +class UpdateJobs(pydantic.BaseModel): + ids: list[str] + field: str + value: Any diff --git a/docs/job_page.png b/docs/job_page.png index 83ab909..732acba 100644 Binary files a/docs/job_page.png and b/docs/job_page.png differ diff --git a/docs/log_page.png b/docs/log_page.png index fd7cc5d..7c4e06f 100644 Binary files a/docs/log_page.png and b/docs/log_page.png differ diff --git a/docs/login.png b/docs/login.png index cc94522..9d61aff 100644 Binary files a/docs/login.png and b/docs/login.png differ diff --git a/docs/main_page.png b/docs/main_page.png index 46dc341..e34a137 100644 Binary files a/docs/main_page.png and b/docs/main_page.png differ diff --git a/src/components/JobTable.tsx b/src/components/JobTable.tsx index ce3bc38..e603e92 100644 --- a/src/components/JobTable.tsx +++ b/src/components/JobTable.tsx @@ -1,19 +1,9 @@ import React, { useState } from "react"; import { - Table, - TableBody, - TableCell, - TableHead, - TableRow, IconButton, Box, Typography, - Accordion, - AccordionSummary, - AccordionDetails, - Checkbox, Tooltip, - Button, TextField, FormControl, InputLabel, @@ -23,19 +13,11 @@ import { } from "@mui/material"; import DeleteIcon from "@mui/icons-material/Delete"; import SelectAllIcon from "@mui/icons-material/SelectAll"; -import ExpandMoreIcon from "@mui/icons-material/ExpandMore"; import DownloadIcon from "@mui/icons-material/Download"; +import StarIcon from "@mui/icons-material/Star"; import { useRouter } from "next/router"; - -interface Job { - id: string; - url: string; - elements: Object[]; - result: Object; - time_created: Date; - status: string; - job_options: Object; -} +import { Favorites, JobQueue } from "./jobs"; +import { Job } from "../types"; interface JobTableProps { jobs: Job[]; @@ -57,6 +39,7 @@ const JobTable: React.FC = ({ jobs, fetchJobs }) => { const [allSelected, setAllSelected] = useState(false); const [searchQuery, setSearchQuery] = useState(""); const [searchMode, setSearchMode] = useState("url"); + const [favoriteView, setFavoriteView] = useState(false); const router = useRouter(); @@ -139,6 +122,22 @@ const JobTable: React.FC = ({ jobs, fetchJobs }) => { return true; }); + const favoriteJob = async (ids: string[], field: string, value: any) => { + const postBody = { + ids: ids, + field: field, + value: value, + }; + + await fetch("/api/update", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(postBody), + }); + + await fetchJobs(); + }; + return ( = ({ jobs, fetchJobs }) => { + + + setFavoriteView(!favoriteView)} + > + + + +
= ({ jobs, fetchJobs }) => {
- - - - Select - Id - Url - Elements - Result - Time Created - Status - Actions - - - - {filteredJobs.map((row, index) => ( - - - handleSelectJob(row.id)} - /> - - - - {row.id} - - - - - {row.url} - - - - - {JSON.stringify(row.elements)} - - - - - } - aria-controls="panel1a-content" - id="panel1a-header" - sx={{ - minHeight: 0, - "&.Mui-expanded": { minHeight: 0 }, - }} - > - - - Show Result - - - - - - - {JSON.stringify(row.result, null, 2)} - - - - - - - - {new Date(row.time_created).toLocaleString()} - - - - - - {row.status} - - - - - - - - - ))} - -
+ {!favoriteView ? ( + + ) : ( + + )} diff --git a/src/components/jobs/Favorites.tsx b/src/components/jobs/Favorites.tsx new file mode 100644 index 0000000..5f47ec2 --- /dev/null +++ b/src/components/jobs/Favorites.tsx @@ -0,0 +1,102 @@ +import React from "react"; +import { + Tooltip, + IconButton, + Table, + TableBody, + TableCell, + TableHead, + TableRow, + Box, + Checkbox, + Button, +} from "@mui/material"; +import { Job } from "../../types"; +import StarIcon from "@mui/icons-material/Star"; + +interface stateProps { + selectedJobs: Set; + filteredJobs: Job[]; +} + +interface Props { + onSelectJob: (job: string) => void; + onNavigate: (elements: Object[], url: string, options: any) => void; + onFavorite: (ids: string[], field: string, value: any) => void; + stateProps: stateProps; +} + +export const Favorites = ({ + stateProps, + onSelectJob, + onNavigate, + onFavorite, +}: Props) => { + const { selectedJobs, filteredJobs } = stateProps; + const favoritedJobs = filteredJobs.filter((job) => job.favorite); + + return ( + + + + Select + Id + Url + Elements + Time Created + Actions + + + + {favoritedJobs.map((row, index) => ( + + + onSelectJob(row.id)} + /> + + + { + onFavorite([row.id], "favorite", !row.favorite); + row.favorite = !row.favorite; + }} + > + + + + + + + {row.id} + + + {row.url} + + + + {JSON.stringify(row.elements)} + + + + + {new Date(row.time_created).toLocaleString()} + + + + + + + ))} + +
+ ); +}; diff --git a/src/components/jobs/JobQueue.tsx b/src/components/jobs/JobQueue.tsx new file mode 100644 index 0000000..7badba6 --- /dev/null +++ b/src/components/jobs/JobQueue.tsx @@ -0,0 +1,170 @@ +import React from "react"; +import { + Table, + TableBody, + TableCell, + TableHead, + TableRow, + Box, + Typography, + Accordion, + AccordionSummary, + AccordionDetails, + Checkbox, + Button, + Tooltip, + IconButton, +} from "@mui/material"; +import ExpandMoreIcon from "@mui/icons-material/ExpandMore"; +import StarIcon from "@mui/icons-material/Star"; +import { Job } from "../../types"; + +interface stringMap { + [key: string]: string; +} + +interface stateProps { + selectedJobs: Set; + filteredJobs: Job[]; +} + +interface Props { + colors: stringMap; + onSelectJob: (job: string) => void; + onDownload: (job: string[]) => void; + onNavigate: (elements: Object[], url: string, options: any) => void; + onFavorite: (ids: string[], field: string, value: any) => void; + stateProps: stateProps; +} + +export const JobQueue = ({ + stateProps, + colors, + onSelectJob, + onDownload, + onNavigate, + onFavorite, +}: Props) => { + const { selectedJobs, filteredJobs } = stateProps; + + return ( + + + + Select + Id + Url + Elements + Result + Time Created + Status + Actions + + + + {filteredJobs.map((row, index) => ( + + + onSelectJob(row.id)} + /> + + + { + onFavorite([row.id], "favorite", !row.favorite); + row.favorite = !row.favorite; + }} + > + + + + + + + {row.id} + + + {row.url} + + + + {JSON.stringify(row.elements)} + + + + + } + aria-controls="panel1a-content" + id="panel1a-header" + sx={{ + minHeight: 0, + "&.Mui-expanded": { minHeight: 0 }, + }} + > + + + Show Result + + + + + + + {JSON.stringify(row.result, null, 2)} + + + + + + + + {new Date(row.time_created).toLocaleString()} + + + + + + {row.status} + + + + + + + + + ))} + +
+ ); +}; diff --git a/src/components/jobs/index.tsx b/src/components/jobs/index.tsx new file mode 100644 index 0000000..9a39e71 --- /dev/null +++ b/src/components/jobs/index.tsx @@ -0,0 +1,2 @@ +export * from "./JobQueue"; +export * from "./Favorites"; diff --git a/src/pages/jobs.tsx b/src/pages/jobs.tsx index f93d554..ef37850 100644 --- a/src/pages/jobs.tsx +++ b/src/pages/jobs.tsx @@ -7,8 +7,8 @@ const Jobs = () => { const { user } = useAuth(); const [jobs, setJobs] = useState([]); - const fetchJobs = () => { - fetch("/api/retrieve-scrape-jobs", { + const fetchJobs = async () => { + await fetch("/api/retrieve-scrape-jobs", { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ user: user?.email }), diff --git a/src/types/index.ts b/src/types/index.ts index 3f12449..01406a9 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -1,2 +1,3 @@ export * from "./element"; export * from "./result"; +export * from "./job"; diff --git a/src/types/job.tsx b/src/types/job.tsx new file mode 100644 index 0000000..adf1dd1 --- /dev/null +++ b/src/types/job.tsx @@ -0,0 +1,10 @@ +export interface Job { + id: string; + url: string; + elements: Object[]; + result: Object; + time_created: Date; + status: string; + job_options: Object; + favorite: boolean; +}