diff --git a/README.md b/README.md index 484938c..902b6ea 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,13 @@ -# Scraperr +![logo_picture](https://github.com/jaypyles/www-scrape/blob/master/docs/logo_picture.png) + +
+ MongoDB + FastAPI + Next JS + TailwindCSS +
+ +# Summary Scraperr is a self-hosted web application that allows users to scrape data from web pages by specifying elements via XPath. Users can submit URLs and the corresponding elements to be scraped, and the results will be displayed in a table. @@ -6,6 +15,8 @@ From the table, users can download an excel sheet of the job's results, along wi ## Features +### Submitting URLs for Scraping + - Submit/Queue URLs for web scraping - Add and manage elements to scrape using XPath - Scrape all pages within same domain @@ -14,6 +25,8 @@ From the table, users can download an excel sheet of the job's results, along wi ![main_page](https://github.com/jaypyles/www-scrape/blob/master/docs/main_page.png) +### Managing Previous Jobs + - Download csv containing results - Rerun jobs - View status of queued jobs @@ -21,14 +34,20 @@ From the table, users can download an excel sheet of the job's results, along wi ![job_page](https://github.com/jaypyles/www-scrape/blob/master/docs/job_page.png) +### User Management + - User login/signup to organize jobs ![login](https://github.com/jaypyles/www-scrape/blob/master/docs/login.png) +### Log Viewing + - View app logs inside of web ui ![logs](https://github.com/jaypyles/www-scrape/blob/master/docs/log_page.png) +### Statistics View + - View a small statistics view of jobs ran ![statistics](https://github.com/jaypyles/www-scrape/blob/master/docs/stats_page.png) @@ -73,38 +92,9 @@ The app provides its own `traefik` configuration to use independently, but can e ## API Endpoints -Use this service as an API for your own projects. +Use this service as an API for your own projects. Due to this using FastAPI, a docs page is available at `/docs` for the API. -- `/api/submit-scrape-job`: Endpoint to submit the scraping job. Accepts a POST request with the following payload: - - ```json - { - "url": "http://example.com", - "elements": [ - { - "name": "ElementName", - "xpath": "/div[@class='example']" - } - ], - "user": "user@example.com", - "time_created": "2024-07-07T12:34:56.789Z" - } - ``` - -- `/api/retrieve-scrape-jobs`: Endpoint to retrieve jobs made by specific accounts. - - ```json - { - "user": "user@example.com" - } - ``` - -- `/api/download`: Endpoint to download job in csv format. - ```json - { - "id": "85312b8b8b204aacab9631f2d76f1af0" - } - ``` +![docs](https://github.com/jaypyles/www-scrape/blob/master/docs/docs_page.png) ## Troubleshooting diff --git a/api/backend/app.py b/api/backend/app.py index 1dbbdf8..95ac85b 100644 --- a/api/backend/app.py +++ b/api/backend/app.py @@ -57,6 +57,7 @@ app.add_middleware( ) app.mount("/_next/static", StaticFiles(directory="./dist/_next/static"), name="static") +app.mount("/images", StaticFiles(directory="./dist/images"), name="images") @app.get("/") diff --git a/docker-compose.yml b/docker-compose.yml index 62e127e..a4842fa 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,6 +5,8 @@ services: build: context: ./ container_name: scraperr + ports: + - 9000:8000 env_file: - ./.env volumes: diff --git a/docs/docs_page.png b/docs/docs_page.png new file mode 100644 index 0000000..052c270 Binary files /dev/null and b/docs/docs_page.png differ diff --git a/docs/job_page.png b/docs/job_page.png index 732acba..02e8ba7 100644 Binary files a/docs/job_page.png and b/docs/job_page.png differ diff --git a/docs/logo_picture.png b/docs/logo_picture.png new file mode 100644 index 0000000..3aad15e Binary files /dev/null and b/docs/logo_picture.png differ diff --git a/docs/main_page.png b/docs/main_page.png index e34a137..d3d53e2 100644 Binary files a/docs/main_page.png and b/docs/main_page.png differ diff --git a/next.config.mjs b/next.config.mjs index 2781a46..b21f9d2 100644 --- a/next.config.mjs +++ b/next.config.mjs @@ -2,6 +2,7 @@ const nextConfig = { output: "export", distDir: "./dist", + images: { unoptimized: true }, async rewrites() { return [ { diff --git a/public/favicon.ico b/public/favicon.ico index d92a3be..434a2ad 100644 Binary files a/public/favicon.ico and b/public/favicon.ico differ diff --git a/public/favicon.ico:Zone.Identifier b/public/favicon.ico:Zone.Identifier new file mode 100644 index 0000000..b65a7df --- /dev/null +++ b/public/favicon.ico:Zone.Identifier @@ -0,0 +1,4 @@ +[ZoneTransfer] +ZoneId=3 +ReferrerUrl=https://cloudconvert.com/ +HostUrl=https://us-east.storage.cloudconvert.com/tasks/ff0a6031-1745-4e41-871e-35f102bfe11b/scraperr_logo.ico?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=cloudconvert-production%2F20240722%2Fva%2Fs3%2Faws4_request&X-Amz-Date=20240722T165500Z&X-Amz-Expires=86400&X-Amz-Signature=df96cece92026dc08d5b1dbfc2391beffa5101137fade6587625c467f088feb7&X-Amz-SignedHeaders=host&response-content-disposition=attachment%3B%20filename%3D%22scraperr_logo.ico%22&response-content-type=image%2Fvnd.microsoft.icon&x-id=GetObject diff --git a/public/images/scraperr_logo.png b/public/images/scraperr_logo.png new file mode 100644 index 0000000..3fd7df0 Binary files /dev/null and b/public/images/scraperr_logo.png differ diff --git a/src/components/JobTable.tsx b/src/components/JobTable.tsx index e603e92..cbf4bfc 100644 --- a/src/components/JobTable.tsx +++ b/src/components/JobTable.tsx @@ -32,6 +32,7 @@ const COLOR_MAP: ColorMap = { Queued: "rgba(255,201,5,0.25)", Scraping: "rgba(3,104,255,0.25)", Completed: "rgba(5,255,51,0.25)", + Failed: "rgba(214,0,25,0.25)", }; const JobTable: React.FC = ({ jobs, fetchJobs }) => { diff --git a/src/components/NavDrawer.tsx b/src/components/NavDrawer.tsx index 5bccca6..6d8b0b7 100644 --- a/src/components/NavDrawer.tsx +++ b/src/components/NavDrawer.tsx @@ -24,6 +24,7 @@ import TerminalIcon from "@mui/icons-material/Terminal"; import BarChart from "@mui/icons-material/BarChart"; import { useRouter } from "next/router"; import { useTheme } from "@mui/material/styles"; +import Image from "next/image"; interface NavDrawerProps { toggleTheme: () => void; @@ -112,11 +113,9 @@ const NavDrawer: React.FC = ({ toggleTheme, isDarkMode }) => { )} - + } aria-controls="panel1a-content" id="panel1a-header" > - Settings + Quick Settings
diff --git a/src/components/jobs/JobQueue.tsx b/src/components/jobs/JobQueue.tsx index 7badba6..46dbc89 100644 --- a/src/components/jobs/JobQueue.tsx +++ b/src/components/jobs/JobQueue.tsx @@ -136,31 +136,37 @@ export const JobQueue = ({ {new Date(row.time_created).toLocaleString()} - + {row.status} - - - + + + + + ))} diff --git a/src/components/submit/ElementTable.tsx b/src/components/submit/ElementTable.tsx index 97ac4fa..9146cc2 100644 --- a/src/components/submit/ElementTable.tsx +++ b/src/components/submit/ElementTable.tsx @@ -33,7 +33,7 @@ export const ElementTable = ({ rows, setRows, submittedURL }: Props) => { const handleAddRow = () => { const updatedRow = { ...newRow, url: submittedURL }; - setRows([...rows, updatedRow]); + setRows([updatedRow, ...rows]); setNewRow({ name: "", xpath: "", url: "" }); }; @@ -44,87 +44,126 @@ export const ElementTable = ({ rows, setRows, submittedURL }: Props) => { }) ); }; + return ( - <> - - setNewRow({ ...newRow, name: e.target.value })} - /> - setNewRow({ ...newRow, xpath: e.target.value })} - /> - 0 && newRow.name.length > 0 - ? "Add Element" - : "Fill out all fields to add an element" - } - placement="top" + + + + Elements to Scrape + + - - + 0 && newRow.name.length > 0)} > - - - - + + + + Name + + + XPath + + + Actions + + + + + + + + setNewRow({ ...newRow, name: e.target.value }) + } + /> + + + + setNewRow({ ...newRow, xpath: e.target.value }) + } + /> + + + 0 && newRow.name.length > 0 + ? "Add Element" + : "Fill out all fields to add an element" + } + placement="top" + > + + 0 && newRow.name.length > 0) + } + > + + + + + + + {rows.map((row, index) => ( + + + {row.name} + + + {row.xpath} + + + + + + ))} + +
+
+ - Elements - - - - - - Name - - - XPath - - - - - {rows.map((row, index) => ( - - - {row.name} - - - {row.xpath} - - - - - - ))} - -
- - + ); }; diff --git a/src/components/submit/JobSubmitter.tsx b/src/components/submit/JobSubmitter.tsx index 0f8e8de..0284544 100644 --- a/src/components/submit/JobSubmitter.tsx +++ b/src/components/submit/JobSubmitter.tsx @@ -15,7 +15,8 @@ interface StateProps { submittedURL: string; setSubmittedURL: Dispatch>; rows: Element[]; - setResults: Dispatch>; + isValidURL: boolean; + setIsValidUrl: Dispatch>; setSnackbarMessage: Dispatch>; setSnackbarOpen: Dispatch>; setSnackbarSeverity: Dispatch>; @@ -40,13 +41,13 @@ export const JobSubmitter = ({ stateProps }: Props) => { submittedURL, setSubmittedURL, rows, - setResults, + isValidURL, + setIsValidUrl, setSnackbarMessage, setSnackbarOpen, setSnackbarSeverity, } = stateProps; - const [isValidURL, setIsValidUrl] = useState(true); const [urlError, setUrlError] = useState(null); const [loading, setLoading] = useState(false); const [jobOptions, setJobOptions] = useState({ @@ -157,15 +158,17 @@ export const JobSubmitter = ({ stateProps }: Props) => { onChange={(e) => setSubmittedURL(e.target.value)} error={!isValidURL} helperText={!isValidURL ? urlError : ""} + className="rounded-md" /> diff --git a/src/pages/index.tsx b/src/pages/index.tsx index 74c36c5..a90a44d 100644 --- a/src/pages/index.tsx +++ b/src/pages/index.tsx @@ -14,6 +14,7 @@ const Home = () => { const [snackbarOpen, setSnackbarOpen] = useState(false); const [snackbarMessage, setSnackbarMessage] = useState(""); const [snackbarSeverity, setSnackbarSeverity] = useState("error"); + const [isValidURL, setIsValidUrl] = useState(true); const resultsRef = useRef(null); @@ -74,26 +75,26 @@ const Home = () => { minHeight="100vh" py={4} > - - - Scraperr - + - + {submittedURL.length ? ( + + ) : null} {snackbarSeverity === "info" ? : } diff --git a/src/styles/themes.ts b/src/styles/themes.ts index e158854..019ca78 100644 --- a/src/styles/themes.ts +++ b/src/styles/themes.ts @@ -70,7 +70,24 @@ const lightTheme = createTheme({ secondary: "#333333", }, }, + ...commonThemeOptions, + components: { + ...commonThemeOptions.components, + MuiButton: { + styleOverrides: { + root: { + color: "white", + "&.MuiButton-root": { + backgroundColor: "#034efc", + }, + "&:hover": { + backgroundColor: "#027be0", + }, + }, + }, + }, + }, }); const darkTheme = createTheme({ @@ -132,6 +149,12 @@ const darkTheme = createTheme({ styleOverrides: { root: { color: "white", + "&.MuiButton-root": { + backgroundColor: "#034efc", + }, + "&:hover": { + backgroundColor: "#027be0", + }, }, }, }, diff --git a/tailwind.config.js b/tailwind.config.js index a07246d..3b2d05c 100644 --- a/tailwind.config.js +++ b/tailwind.config.js @@ -1,6 +1,23 @@ /** @type {import('tailwindcss').Config} */ module.exports = { content: ["./src/**/*.{js,jsx,ts,tsx}"], - theme: {}, + theme: { + extend: { + animation: { + fadeIn: "fadeIn 0.5s ease-in-out", + fadeOut: "fadeOut 0.5s ease-in-out", + }, + keyframes: { + fadeIn: { + "0%": { opacity: 0 }, + "100%": { opacity: 1 }, + }, + fadeOut: { + "0%": { opacity: 1 }, + "100%": { opacity: 0 }, + }, + }, + }, + }, plugins: [], };