diff --git a/src/components/submit/ElementTable.tsx b/src/components/submit/ElementTable.tsx new file mode 100644 index 0000000..4b3d9f2 --- /dev/null +++ b/src/components/submit/ElementTable.tsx @@ -0,0 +1,120 @@ +import React, { useState, Dispatch, SetStateAction } from "react"; +import { + Typography, + TextField, + Button, + Table, + TableBody, + TableContainer, + TableCell, + TableHead, + TableRow, + Box, + IconButton, + Tooltip, +} from "@mui/material"; +import AddIcon from "@mui/icons-material/Add"; +import { Element } from "../../types"; + +interface Props { + rows: Element[]; + setRows: Dispatch>; + submittedURL: string; +} + +export const ElementTable = ({ rows, setRows, submittedURL }: Props) => { + const [newRow, setNewRow] = useState({ + name: "", + xpath: "", + url: "", + }); + + const handleAddRow = () => { + const updatedRow = { ...newRow, url: submittedURL }; + setRows([...rows, updatedRow]); + setNewRow({ name: "", xpath: "", url: "" }); + }; + + const handleDeleteRow = (elementName: string) => { + setRows( + rows.filter((r) => { + return elementName !== r.name; + }) + ); + }; + return ( + <> + + setNewRow({ ...newRow, name: e.target.value })} + /> + setNewRow({ ...newRow, xpath: e.target.value })} + /> + 0 && newRow.name.length > 0 + ? "Add Element" + : "Fill out all fields to add an element" + } + placement="top" + > + + 0 && newRow.name.length > 0)} + > + + + + + + Elements + + + + + + Name + + + XPath + + + + + {rows.map((row, index) => ( + + + {row.name} + + + {row.xpath} + + + + + + ))} + +
+
+ + ); +}; diff --git a/src/components/submit/JobSubmitter.tsx b/src/components/submit/JobSubmitter.tsx new file mode 100644 index 0000000..43d5033 --- /dev/null +++ b/src/components/submit/JobSubmitter.tsx @@ -0,0 +1,171 @@ +import React, { useState, useEffect, useRef, Dispatch } from "react"; +import { + TextField, + Button, + Box, + Checkbox, + FormControlLabel, + Accordion, + AccordionSummary, + AccordionDetails, +} from "@mui/material"; +import CircularProgress from "@mui/material/CircularProgress"; +import ExpandMoreIcon from "@mui/icons-material/ExpandMore"; +import { Element, Result } from "../../types"; +import { useAuth } from "../../contexts/AuthContext"; + +interface stateProps { + submittedURL: string; + setSubmittedURL: Dispatch>; + rows: Element[]; + setResults: Dispatch>; + setSnackbarMessage: Dispatch>; + setSnackbarOpen: Dispatch>; +} + +interface Props { + stateProps: stateProps; +} + +interface JobOptions { + multi_page_scrape: boolean; + custom_headers: null | Object; +} + +export const JobSubmitter = ({ stateProps }: Props) => { + const { user } = useAuth(); + + const { + submittedURL, + setSubmittedURL, + rows, + setResults, + setSnackbarMessage, + setSnackbarOpen, + } = stateProps; + + const [isValidURL, setIsValidUrl] = useState(true); + const [urlError, setUrlError] = useState(null); + const [loading, setLoading] = useState(false); + const [jobOptions, setJobOptions] = useState({ + multi_page_scrape: false, + custom_headers: null, + }); + + function validateURL(url: string): boolean { + try { + new URL(url); + return true; + } catch (_) { + return false; + } + } + + const handleSubmit = () => { + if (!validateURL(submittedURL)) { + setIsValidUrl(false); + setUrlError("Please enter a valid URL."); + return; + } + + setIsValidUrl(true); + setUrlError(null); + setLoading(true); + + fetch("/api/submit-scrape-job", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + url: submittedURL, + elements: rows, + user: user?.email, + time_created: new Date().toISOString(), + }), + }) + .then((response) => { + if (!response.ok) { + return response.json().then((error) => { + throw new Error(error.error); + }); + } + return response.json(); + }) + .then((data) => setResults(data)) + .catch((error) => { + setSnackbarMessage(error.message || "An error occurred."); + setSnackbarOpen(true); + }) + .finally(() => setLoading(false)); + }; + + return ( + <> +
+ setSubmittedURL(e.target.value)} + error={!isValidURL} + helperText={!isValidURL ? urlError : ""} + /> + +
+ + + setJobOptions((prevJobOptions) => ({ + ...prevJobOptions, + multi_page_scrape: !prevJobOptions.multi_page_scrape, + })) + } + /> + } + > + + } + aria-controls="panel1-content" + id="panel1-header" + > + Custom Headers (JSON) + + + + setJobOptions((prevJobOptions) => ({ + ...prevJobOptions, + custom_headers: e.target.value, + })) + } + style={{ maxHeight: "20vh", overflow: "auto" }} + className="mt-2" + /> + + + + + ); +}; diff --git a/src/components/submit/ResultsTable.tsx b/src/components/submit/ResultsTable.tsx new file mode 100644 index 0000000..53dafaf --- /dev/null +++ b/src/components/submit/ResultsTable.tsx @@ -0,0 +1,69 @@ +import React, { useState, useEffect, useRef } from "react"; +import { + Typography, + Table, + TableBody, + TableContainer, + TableCell, + TableHead, + TableRow, + Box, +} from "@mui/material"; +import { Result } from "../../types"; + +interface stateProps { + results: Result; +} + +interface Props { + stateProps: stateProps; + resultsRef: React.MutableRefObject; +} + +export const ResultsTable = ({ stateProps, resultsRef }: Props) => { + const { results } = stateProps; + + return ( + <> + {Object.keys(results).length ? ( + <> + Results + + + + + + Name + + + XPath + + + Text + + + + + {Object.keys(results).map((key, index) => ( + + {results[key].map((result, resultIndex) => ( + + {result.name} + {result.xpath} + {result.text} + + ))} + + ))} + +
+
+ + ) : null} + + ); +}; diff --git a/src/components/submit/index.ts b/src/components/submit/index.ts new file mode 100644 index 0000000..f0dc927 --- /dev/null +++ b/src/components/submit/index.ts @@ -0,0 +1,3 @@ +export * from "./ElementTable"; +export * from "./JobSubmitter"; +export * from "./ResultsTable"; diff --git a/src/pages/index.tsx b/src/pages/index.tsx index 07d9c59..77eae4f 100644 --- a/src/pages/index.tsx +++ b/src/pages/index.tsx @@ -1,90 +1,19 @@ import React, { useState, useEffect, useRef } from "react"; -import { - Typography, - FormControl, - InputLabel, - Select, - MenuItem, - TextField, - Button, - Table, - TableBody, - TableContainer, - TableCell, - TableHead, - TableRow, - Container, - Box, - IconButton, - Tooltip, - Snackbar, - Alert, - Checkbox, - FormControlLabel, - Accordion, - AccordionActions, - AccordionSummary, - AccordionDetails, -} from "@mui/material"; -import AddIcon from "@mui/icons-material/Add"; -import { useAuth } from "../contexts/AuthContext"; +import { Typography, Container, Box, Snackbar, Alert } from "@mui/material"; import { useRouter } from "next/router"; -import CircularProgress from "@mui/material/CircularProgress"; -import ExpandMoreIcon from "@mui/icons-material/ExpandMore"; - -interface Element { - name: string; - xpath: string; - url: string; -} - -interface ScrapeResult { - xpath: string; - text: string; - name: string; -} - -interface JobOptions { - multi_page_scrape: boolean; - custom_headers: null | Object; -} - -type Result = { - [key: string]: ScrapeResult[]; -}; - -function validateURL(url: string): boolean { - try { - new URL(url); - return true; - } catch (_) { - return false; - } -} +import { Element, Result } from "../types"; +import { ElementTable, JobSubmitter, ResultsTable } from "../components/submit"; const Home = () => { - const { user } = useAuth(); const router = useRouter(); const { elements, url } = router.query; const [submittedURL, setSubmittedURL] = useState(""); - const [isValidURL, setIsValidUrl] = useState(true); - const [urlError, setUrlError] = useState(null); const [rows, setRows] = useState([]); - const [results, setResults] = useState(null); - const [newRow, setNewRow] = useState({ - name: "", - xpath: "", - url: "", - }); - const [loading, setLoading] = useState(false); + const [results, setResults] = useState({}); const [snackbarOpen, setSnackbarOpen] = useState(false); const [snackbarMessage, setSnackbarMessage] = useState(""); - const [jobOptions, setJobOptions] = useState({ - multi_page_scrape: false, - custom_headers: null, - }); const resultsRef = useRef(null); @@ -103,57 +32,6 @@ const Home = () => { } }, [results]); - const handleAddRow = () => { - const updatedRow = { ...newRow, url: submittedURL }; - setRows([...rows, updatedRow]); - setNewRow({ name: "", xpath: "", url: "" }); - }; - - const handleDeleteRow = (elementName: string) => { - setRows( - rows.filter((r) => { - return elementName !== r.name; - }), - ); - }; - - const handleSubmit = () => { - if (!validateURL(submittedURL)) { - setIsValidUrl(false); - setUrlError("Please enter a valid URL."); - return; - } - - setIsValidUrl(true); - setUrlError(null); - setLoading(true); - - fetch("/api/submit-scrape-job", { - method: "POST", - headers: { "content-type": "application/json" }, - body: JSON.stringify({ - url: submittedURL, - elements: rows, - user: user?.email, - time_created: new Date().toISOString(), - }), - }) - .then((response) => { - if (!response.ok) { - return response.json().then((error) => { - throw new Error(error.error); - }); - } - return response.json(); - }) - .then((data) => setResults(data)) - .catch((error) => { - setSnackbarMessage(error.message || "An error occurred."); - setSnackbarOpen(true); - }) - .finally(() => setLoading(false)); - }; - const handleCloseSnackbar = () => { setSnackbarOpen(false); }; @@ -172,182 +50,22 @@ const Home = () => { Scraperr -
- setSubmittedURL(e.target.value)} - error={!isValidURL} - helperText={!isValidURL ? urlError : ""} - /> - -
- - - setJobOptions((prevJobOptions) => ({ - ...prevJobOptions, - multi_page_scrape: !prevJobOptions.multi_page_scrape, - })) - } - /> - } - > - - } - aria-controls="panel1-content" - id="panel1-header" - > - Custom Headers (JSON) - - - - setJobOptions((prevJobOptions) => ({ - ...prevJobOptions, - custom_headers: e.target.value, - })) - } - style={{ maxHeight: "20vh", overflow: "auto" }} - className="mt-2" - /> - - - - - setNewRow({ ...newRow, name: e.target.value })} - /> - setNewRow({ ...newRow, xpath: e.target.value })} - /> - 0 && newRow.name.length > 0 - ? "Add Element" - : "Fill out all fields to add an element" - } - placement="top" - > - - 0 && newRow.name.length > 0)} - > - - - - - - Elements - - - - - - Name - - - XPath - - - - - {rows.map((row, index) => ( - - - {row.name} - - - {row.xpath} - - - - - - ))} - -
-
- {results && ( - <> - Results - - - - - - Name - - - XPath - - - Text - - - - - {Object.keys(results).map((key, index) => ( - - {results[key].map((result, resultIndex) => ( - - {result.name} - {result.xpath} - {result.text} - - ))} - - ))} - -
-
- - )} + + +