diff --git a/README.md b/README.md index e3a70d6..3437f2b 100644 --- a/README.md +++ b/README.md @@ -13,12 +13,16 @@ From the table, users can download an excel sheet of the job's results, along wi ## Features +### Submitting URLs for Scraping + - Submit/Queue URLs for web scraping - Add and manage elements to scrape using XPath - Scrape all pages within same domain - Add custom json headers to send in requests to URLs - Display results of scraped data +### Managing Previous Jobs + ![main_page](https://github.com/jaypyles/www-scrape/blob/master/docs/main_page.png) - Download csv containing results @@ -28,14 +32,20 @@ From the table, users can download an excel sheet of the job's results, along wi ![job_page](https://github.com/jaypyles/www-scrape/blob/master/docs/job_page.png) +### User Management + - User login/signup to organize jobs ![login](https://github.com/jaypyles/www-scrape/blob/master/docs/login.png) +### Log Viewing + - View app logs inside of web ui ![logs](https://github.com/jaypyles/www-scrape/blob/master/docs/log_page.png) +### Statistics View + - View a small statistics view of jobs ran ![statistics](https://github.com/jaypyles/www-scrape/blob/master/docs/stats_page.png) diff --git a/api/backend/app.py b/api/backend/app.py index 1dbbdf8..95ac85b 100644 --- a/api/backend/app.py +++ b/api/backend/app.py @@ -57,6 +57,7 @@ app.add_middleware( ) app.mount("/_next/static", StaticFiles(directory="./dist/_next/static"), name="static") +app.mount("/images", StaticFiles(directory="./dist/images"), name="images") @app.get("/") diff --git a/docker-compose.yml b/docker-compose.yml index 62e127e..a4842fa 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,6 +5,8 @@ services: build: context: ./ container_name: scraperr + ports: + - 9000:8000 env_file: - ./.env volumes: diff --git a/docs/logo_picture.png b/docs/logo_picture.png index 29eda3f..3aad15e 100644 Binary files a/docs/logo_picture.png and b/docs/logo_picture.png differ diff --git a/next.config.mjs b/next.config.mjs index 2781a46..b21f9d2 100644 --- a/next.config.mjs +++ b/next.config.mjs @@ -2,6 +2,7 @@ const nextConfig = { output: "export", distDir: "./dist", + images: { unoptimized: true }, async rewrites() { return [ { diff --git a/public/favicon.ico b/public/favicon.ico index d92a3be..434a2ad 100644 Binary files a/public/favicon.ico and b/public/favicon.ico differ diff --git a/public/favicon.ico:Zone.Identifier b/public/favicon.ico:Zone.Identifier new file mode 100644 index 0000000..b65a7df --- /dev/null +++ b/public/favicon.ico:Zone.Identifier @@ -0,0 +1,4 @@ +[ZoneTransfer] +ZoneId=3 +ReferrerUrl=https://cloudconvert.com/ +HostUrl=https://us-east.storage.cloudconvert.com/tasks/ff0a6031-1745-4e41-871e-35f102bfe11b/scraperr_logo.ico?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=cloudconvert-production%2F20240722%2Fva%2Fs3%2Faws4_request&X-Amz-Date=20240722T165500Z&X-Amz-Expires=86400&X-Amz-Signature=df96cece92026dc08d5b1dbfc2391beffa5101137fade6587625c467f088feb7&X-Amz-SignedHeaders=host&response-content-disposition=attachment%3B%20filename%3D%22scraperr_logo.ico%22&response-content-type=image%2Fvnd.microsoft.icon&x-id=GetObject diff --git a/public/images/scraperr_logo.png b/public/images/scraperr_logo.png new file mode 100644 index 0000000..3fd7df0 Binary files /dev/null and b/public/images/scraperr_logo.png differ diff --git a/src/components/JobTable.tsx b/src/components/JobTable.tsx index e603e92..cbf4bfc 100644 --- a/src/components/JobTable.tsx +++ b/src/components/JobTable.tsx @@ -32,6 +32,7 @@ const COLOR_MAP: ColorMap = { Queued: "rgba(255,201,5,0.25)", Scraping: "rgba(3,104,255,0.25)", Completed: "rgba(5,255,51,0.25)", + Failed: "rgba(214,0,25,0.25)", }; const JobTable: React.FC = ({ jobs, fetchJobs }) => { diff --git a/src/components/NavDrawer.tsx b/src/components/NavDrawer.tsx index 5bccca6..6d8b0b7 100644 --- a/src/components/NavDrawer.tsx +++ b/src/components/NavDrawer.tsx @@ -24,6 +24,7 @@ import TerminalIcon from "@mui/icons-material/Terminal"; import BarChart from "@mui/icons-material/BarChart"; import { useRouter } from "next/router"; import { useTheme } from "@mui/material/styles"; +import Image from "next/image"; interface NavDrawerProps { toggleTheme: () => void; @@ -112,11 +113,9 @@ const NavDrawer: React.FC = ({ toggleTheme, isDarkMode }) => { )} - + } aria-controls="panel1a-content" id="panel1a-header" > - Settings + Quick Settings
diff --git a/src/components/jobs/JobQueue.tsx b/src/components/jobs/JobQueue.tsx index 7badba6..46dbc89 100644 --- a/src/components/jobs/JobQueue.tsx +++ b/src/components/jobs/JobQueue.tsx @@ -136,31 +136,37 @@ export const JobQueue = ({ {new Date(row.time_created).toLocaleString()} - + {row.status} - - - + + + + + ))} diff --git a/src/components/submit/ElementTable.tsx b/src/components/submit/ElementTable.tsx index 97ac4fa..9146cc2 100644 --- a/src/components/submit/ElementTable.tsx +++ b/src/components/submit/ElementTable.tsx @@ -33,7 +33,7 @@ export const ElementTable = ({ rows, setRows, submittedURL }: Props) => { const handleAddRow = () => { const updatedRow = { ...newRow, url: submittedURL }; - setRows([...rows, updatedRow]); + setRows([updatedRow, ...rows]); setNewRow({ name: "", xpath: "", url: "" }); }; @@ -44,87 +44,126 @@ export const ElementTable = ({ rows, setRows, submittedURL }: Props) => { }) ); }; + return ( - <> - - setNewRow({ ...newRow, name: e.target.value })} - /> - setNewRow({ ...newRow, xpath: e.target.value })} - /> - 0 && newRow.name.length > 0 - ? "Add Element" - : "Fill out all fields to add an element" - } - placement="top" + + + + Elements to Scrape + + - - + 0 && newRow.name.length > 0)} > - - - - + + + + Name + + + XPath + + + Actions + + + + + + + + setNewRow({ ...newRow, name: e.target.value }) + } + /> + + + + setNewRow({ ...newRow, xpath: e.target.value }) + } + /> + + + 0 && newRow.name.length > 0 + ? "Add Element" + : "Fill out all fields to add an element" + } + placement="top" + > + + 0 && newRow.name.length > 0) + } + > + + + + + + + {rows.map((row, index) => ( + + + {row.name} + + + {row.xpath} + + + + + + ))} + +
+
+ - Elements - - - - - - Name - - - XPath - - - - - {rows.map((row, index) => ( - - - {row.name} - - - {row.xpath} - - - - - - ))} - -
-
- + ); }; diff --git a/src/components/submit/JobSubmitter.tsx b/src/components/submit/JobSubmitter.tsx index 0f8e8de..0284544 100644 --- a/src/components/submit/JobSubmitter.tsx +++ b/src/components/submit/JobSubmitter.tsx @@ -15,7 +15,8 @@ interface StateProps { submittedURL: string; setSubmittedURL: Dispatch>; rows: Element[]; - setResults: Dispatch>; + isValidURL: boolean; + setIsValidUrl: Dispatch>; setSnackbarMessage: Dispatch>; setSnackbarOpen: Dispatch>; setSnackbarSeverity: Dispatch>; @@ -40,13 +41,13 @@ export const JobSubmitter = ({ stateProps }: Props) => { submittedURL, setSubmittedURL, rows, - setResults, + isValidURL, + setIsValidUrl, setSnackbarMessage, setSnackbarOpen, setSnackbarSeverity, } = stateProps; - const [isValidURL, setIsValidUrl] = useState(true); const [urlError, setUrlError] = useState(null); const [loading, setLoading] = useState(false); const [jobOptions, setJobOptions] = useState({ @@ -157,15 +158,17 @@ export const JobSubmitter = ({ stateProps }: Props) => { onChange={(e) => setSubmittedURL(e.target.value)} error={!isValidURL} helperText={!isValidURL ? urlError : ""} + className="rounded-md" /> diff --git a/src/pages/index.tsx b/src/pages/index.tsx index 74c36c5..a90a44d 100644 --- a/src/pages/index.tsx +++ b/src/pages/index.tsx @@ -14,6 +14,7 @@ const Home = () => { const [snackbarOpen, setSnackbarOpen] = useState(false); const [snackbarMessage, setSnackbarMessage] = useState(""); const [snackbarSeverity, setSnackbarSeverity] = useState("error"); + const [isValidURL, setIsValidUrl] = useState(true); const resultsRef = useRef(null); @@ -74,26 +75,26 @@ const Home = () => { minHeight="100vh" py={4} > - - - Scraperr - + - + {submittedURL.length ? ( + + ) : null} {snackbarSeverity === "info" ? : } diff --git a/src/styles/themes.ts b/src/styles/themes.ts index e158854..019ca78 100644 --- a/src/styles/themes.ts +++ b/src/styles/themes.ts @@ -70,7 +70,24 @@ const lightTheme = createTheme({ secondary: "#333333", }, }, + ...commonThemeOptions, + components: { + ...commonThemeOptions.components, + MuiButton: { + styleOverrides: { + root: { + color: "white", + "&.MuiButton-root": { + backgroundColor: "#034efc", + }, + "&:hover": { + backgroundColor: "#027be0", + }, + }, + }, + }, + }, }); const darkTheme = createTheme({ @@ -132,6 +149,12 @@ const darkTheme = createTheme({ styleOverrides: { root: { color: "white", + "&.MuiButton-root": { + backgroundColor: "#034efc", + }, + "&:hover": { + backgroundColor: "#027be0", + }, }, }, }, diff --git a/tailwind.config.js b/tailwind.config.js index a07246d..3b2d05c 100644 --- a/tailwind.config.js +++ b/tailwind.config.js @@ -1,6 +1,23 @@ /** @type {import('tailwindcss').Config} */ module.exports = { content: ["./src/**/*.{js,jsx,ts,tsx}"], - theme: {}, + theme: { + extend: { + animation: { + fadeIn: "fadeIn 0.5s ease-in-out", + fadeOut: "fadeOut 0.5s ease-in-out", + }, + keyframes: { + fadeIn: { + "0%": { opacity: 0 }, + "100%": { opacity: 1 }, + }, + fadeOut: { + "0%": { opacity: 1 }, + "100%": { opacity: 0 }, + }, + }, + }, + }, plugins: [], };