Merge pull request #19 from jaypyles/ui-changes

UI changes
2026-08-24 21:36:29 +00:00 · 2024-07-22 16:04:17 -05:00
parent 9b3859b7bd 50509a5915
commit 0ac79bdffb
19 changed files with 233 additions and 148 deletions
@@ -1,4 +1,13 @@
-# Scraperr
+![logo_picture](https://github.com/jaypyles/www-scrape/blob/master/docs/logo_picture.png)
+
+<div align="center">
+  <img src="https://img.shields.io/badge/MongoDB-%234ea94b.svg?style=for-the-badge&logo=mongodb&logoColor=white" alt="MongoDB" />
+  <img src="https://img.shields.io/badge/FastAPI-005571?style=for-the-badge&logo=fastapi" alt="FastAPI" />
+  <img src="https://img.shields.io/badge/Next-black?style=for-the-badge&logo=next.js&logoColor=white" alt="Next JS" />
+  <img src="https://img.shields.io/badge/tailwindcss-%2338B2AC.svg?style=for-the-badge&logo=tailwind-css&logoColor=white" alt="TailwindCSS" />
+</div>
+
+# Summary

 Scraperr is a self-hosted web application that allows users to scrape data from web pages by specifying elements via XPath. Users can submit URLs and the corresponding elements to be scraped, and the results will be displayed in a table.

@@ -6,6 +15,8 @@ From the table, users can download an excel sheet of the job's results, along wi

 ## Features

+### Submitting URLs for Scraping
+
 - Submit/Queue URLs for web scraping
 - Add and manage elements to scrape using XPath
 - Scrape all pages within same domain
@@ -14,6 +25,8 @@ From the table, users can download an excel sheet of the job's results, along wi

 ![main_page](https://github.com/jaypyles/www-scrape/blob/master/docs/main_page.png)

+### Managing Previous Jobs
+
 - Download csv containing results
 - Rerun jobs
 - View status of queued jobs
@@ -21,14 +34,20 @@ From the table, users can download an excel sheet of the job's results, along wi

 ![job_page](https://github.com/jaypyles/www-scrape/blob/master/docs/job_page.png)

+### User Management
+
 - User login/signup to organize jobs

 ![login](https://github.com/jaypyles/www-scrape/blob/master/docs/login.png)

+### Log Viewing
+
 - View app logs inside of web ui

 ![logs](https://github.com/jaypyles/www-scrape/blob/master/docs/log_page.png)

+### Statistics View
+
 - View a small statistics view of jobs ran

 ![statistics](https://github.com/jaypyles/www-scrape/blob/master/docs/stats_page.png)
@@ -73,38 +92,9 @@ The app provides its own `traefik` configuration to use independently, but can e

 ## API Endpoints

-Use this service as an API for your own projects.
+Use this service as an API for your own projects. Due to this using FastAPI, a docs page is available at `/docs` for the API.

- `/api/submit-scrape-job`: Endpoint to submit the scraping job. Accepts a POST request with the following payload:
-
-  ```json
-  {
-    "url": "http://example.com",
-    "elements": [
-      {
-        "name": "ElementName",
-        "xpath": "/div[@class='example']"
-      }
-    ],
-    "user": "user@example.com",
-    "time_created": "2024-07-07T12:34:56.789Z"
-  }
-  ```
-
- `/api/retrieve-scrape-jobs`: Endpoint to retrieve jobs made by specific accounts.
-
-  ```json
-  {
-    "user": "user@example.com"
-  }
-  ```
-
- `/api/download`: Endpoint to download job in csv format.
-  ```json
-  {
-    "id": "85312b8b8b204aacab9631f2d76f1af0"
-  }
-  ```
+![docs](https://github.com/jaypyles/www-scrape/blob/master/docs/docs_page.png)

 ## Troubleshooting

@@ -57,6 +57,7 @@ app.add_middleware(
 )

 app.mount("/_next/static", StaticFiles(directory="./dist/_next/static"), name="static")
+app.mount("/images", StaticFiles(directory="./dist/images"), name="images")


@app.get("/")
@@ -5,6 +5,8 @@ services:
    build:
      context: ./
    container_name: scraperr
+    ports:
+      - 9000:8000
    env_file:
      - ./.env
    volumes:
@@ -2,6 +2,7 @@
 const nextConfig = {
  output: "export",
  distDir: "./dist",
+  images: { unoptimized: true },
  async rewrites() {
    return [
      {
@@ -0,0 +1,4 @@
+[ZoneTransfer]
+ZoneId=3
+ReferrerUrl=https://cloudconvert.com/
+HostUrl=https://us-east.storage.cloudconvert.com/tasks/ff0a6031-1745-4e41-871e-35f102bfe11b/scraperr_logo.ico?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=cloudconvert-production%2F20240722%2Fva%2Fs3%2Faws4_request&X-Amz-Date=20240722T165500Z&X-Amz-Expires=86400&X-Amz-Signature=df96cece92026dc08d5b1dbfc2391beffa5101137fade6587625c467f088feb7&X-Amz-SignedHeaders=host&response-content-disposition=attachment%3B%20filename%3D%22scraperr_logo.ico%22&response-content-type=image%2Fvnd.microsoft.icon&x-id=GetObject
@@ -32,6 +32,7 @@ const COLOR_MAP: ColorMap = {
  Queued: "rgba(255,201,5,0.25)",
  Scraping: "rgba(3,104,255,0.25)",
  Completed: "rgba(5,255,51,0.25)",
+  Failed: "rgba(214,0,25,0.25)",
 };

 const JobTable: React.FC<JobTableProps> = ({ jobs, fetchJobs }) => {
@@ -24,6 +24,7 @@ import TerminalIcon from "@mui/icons-material/Terminal";
 import BarChart from "@mui/icons-material/BarChart";
 import { useRouter } from "next/router";
 import { useTheme } from "@mui/material/styles";
+import Image from "next/image";

 interface NavDrawerProps {
  toggleTheme: () => void;
@@ -112,11 +113,9 @@ const NavDrawer: React.FC<NavDrawerProps> = ({ toggleTheme, isDarkMode }) => {
              </Typography>
              <Button
                variant="contained"
-                color="primary"
                onClick={logout}
                sx={{
                  width: "100%",
-                  color: theme.palette.mode === "light" ? "#000000" : "#ffffff",
                }}
              >
                Logout
@@ -125,24 +124,22 @@ const NavDrawer: React.FC<NavDrawerProps> = ({ toggleTheme, isDarkMode }) => {
          ) : (
            <Button
              variant="contained"
-              color="primary"
              onClick={() => router.push("/login")}
              sx={{
                width: "100%",
-                color: theme.palette.mode === "light" ? "#000000" : "#ffffff",
              }}
            >
              Login
            </Button>
          )}
          <Divider sx={{ marginTop: 2, marginBottom: 2 }}></Divider>
-          <Accordion sx={{ padding: 0, width: "90%" }}>
+          <Accordion sx={{ padding: 0, width: "90%", marginBottom: 1 }}>
            <AccordionSummary
              expandIcon={<ExpandMoreIcon />}
              aria-controls="panel1a-content"
              id="panel1a-header"
            >
-              <Typography>Settings</Typography>
+              <Typography>Quick Settings</Typography>
            </AccordionSummary>
            <AccordionDetails>
              <div className="flex flex-row mr-1">
@@ -136,31 +136,37 @@ export const JobQueue = ({
                {new Date(row.time_created).toLocaleString()}
              </Box>
            </TableCell>
-            <TableCell sx={{ maxWidth: 150, overflow: "auto" }}>
+            <TableCell sx={{ maxWidth: 50, overflow: "auto" }}>
              <Box sx={{ maxHeight: 100, overflow: "auto" }}>
                <Box
                  className="rounded-md p-2 text-center"
-                  sx={{ bgcolor: colors[row.status], opactity: "50%" }}
+                  sx={{ bgcolor: colors[row.status] }}
                >
                  {row.status}
                </Box>
              </Box>
            </TableCell>
-            <TableCell sx={{ maxWidth: 100, overflow: "auto" }}>
-              <Button
-                onClick={() => {
-                  onDownload([row.id]);
-                }}
-              >
-                Download
-              </Button>
-              <Button
-                onClick={() =>
-                  onNavigate(row.elements, row.url, row.job_options)
-                }
-              >
-                Rerun
-              </Button>
+            <TableCell sx={{ maxWidth: 150, overflow: "auto" }}>
+              <Box sx={{ display: "flex", gap: 1 }}>
+                <Button
+                  onClick={() => {
+                    onDownload([row.id]);
+                  }}
+                  size="small"
+                  sx={{ minWidth: 0, padding: "4px 8px" }}
+                >
+                  Download
+                </Button>
+                <Button
+                  onClick={() =>
+                    onNavigate(row.elements, row.url, row.job_options)
+                  }
+                  size="small"
+                  sx={{ minWidth: 0, padding: "4px 8px" }}
+                >
+                  Rerun
+                </Button>
+              </Box>
            </TableCell>
          </TableRow>
        ))}
@@ -33,7 +33,7 @@ export const ElementTable = ({ rows, setRows, submittedURL }: Props) => {

  const handleAddRow = () => {
    const updatedRow = { ...newRow, url: submittedURL };
-    setRows([...rows, updatedRow]);
+    setRows([updatedRow, ...rows]);
    setNewRow({ name: "", xpath: "", url: "" });
  };

@@ -44,87 +44,126 @@ export const ElementTable = ({ rows, setRows, submittedURL }: Props) => {
      })
    );
  };
+
  return (
-    <>
-      <Box display="flex" gap={2} marginBottom={2} className="items-center">
-        <TextField
-          label="Name"
-          variant="outlined"
-          fullWidth
-          value={newRow.name}
-          onChange={(e) => setNewRow({ ...newRow, name: e.target.value })}
-        />
-        <TextField
-          label="XPath"
-          variant="outlined"
-          fullWidth
-          value={newRow.xpath}
-          onChange={(e) => setNewRow({ ...newRow, xpath: e.target.value })}
-        />
-        <Tooltip
-          title={
-            newRow.xpath.length > 0 && newRow.name.length > 0
-              ? "Add Element"
-              : "Fill out all fields to add an element"
-          }
-          placement="top"
+    <Box className="animate-fadeIn p-2" bgcolor="background.paper">
+      <Box className="text-center mb-4">
+        <Typography variant="h4" sx={{ marginBottom: 1 }}>
+          Elements to Scrape
+        </Typography>
+        <TableContainer
+          component={Box}
+          sx={{ maxHeight: "50%", overflow: "auto" }}
        >
-          <span>
-            <IconButton
-              aria-label="add"
-              size="small"
-              onClick={handleAddRow}
+          <div className="rounded-lg shadow-md border border-gray-300 overflow-hidden">
+            <Table
+              stickyHeader
+              className="mb-4"
              sx={{
-                height: "40px",
-                width: "40px",
+                tableLayout: "fixed",
+                width: "100%",
+                "& .MuiTableCell-root": {
+                  borderBottom: "1px solid #e0e0e0",
+                },
              }}
-              disabled={!(newRow.xpath.length > 0 && newRow.name.length > 0)}
            >
-              <AddIcon
-                fontSize="inherit"
-                sx={{
-                  color: theme.palette.mode === "light" ? "#000000" : "#ffffff",
-                }}
-              />
-            </IconButton>
-          </span>
-        </Tooltip>
+              <TableHead>
+                <TableRow>
+                  <TableCell>
+                    <Typography sx={{ fontWeight: "bold" }}>Name</Typography>
+                  </TableCell>
+                  <TableCell>
+                    <Typography sx={{ fontWeight: "bold" }}>XPath</Typography>
+                  </TableCell>
+                  <TableCell>
+                    <Typography sx={{ fontWeight: "bold" }}>Actions</Typography>
+                  </TableCell>
+                </TableRow>
+              </TableHead>
+              <TableBody>
+                <TableRow>
+                  <TableCell>
+                    <TextField
+                      label="Name"
+                      variant="outlined"
+                      fullWidth
+                      value={newRow.name}
+                      onChange={(e) =>
+                        setNewRow({ ...newRow, name: e.target.value })
+                      }
+                    />
+                  </TableCell>
+                  <TableCell>
+                    <TextField
+                      label="XPath"
+                      variant="outlined"
+                      fullWidth
+                      value={newRow.xpath}
+                      onChange={(e) =>
+                        setNewRow({ ...newRow, xpath: e.target.value })
+                      }
+                    />
+                  </TableCell>
+                  <TableCell>
+                    <Tooltip
+                      title={
+                        newRow.xpath.length > 0 && newRow.name.length > 0
+                          ? "Add Element"
+                          : "Fill out all fields to add an element"
+                      }
+                      placement="top"
+                    >
+                      <span>
+                        <IconButton
+                          aria-label="add"
+                          size="small"
+                          onClick={handleAddRow}
+                          sx={{
+                            height: "40px",
+                            width: "40px",
+                          }}
+                          disabled={
+                            !(newRow.xpath.length > 0 && newRow.name.length > 0)
+                          }
+                        >
+                          <AddIcon
+                            fontSize="inherit"
+                            sx={{
+                              color:
+                                theme.palette.mode === "light"
+                                  ? "#000000"
+                                  : "#ffffff",
+                            }}
+                          />
+                        </IconButton>
+                      </span>
+                    </Tooltip>
+                  </TableCell>
+                </TableRow>
+                {rows.map((row, index) => (
+                  <TableRow key={index}>
+                    <TableCell>
+                      <Typography>{row.name}</Typography>
+                    </TableCell>
+                    <TableCell>
+                      <Typography>{row.xpath}</Typography>
+                    </TableCell>
+                    <TableCell>
+                      <Button
+                        onClick={() => handleDeleteRow(row.name)}
+                        className="!bg-red-500 bg-opacity-50 !text-white font-semibold rounded-md 
+                        transition-transform transform hover:scale-105 hover:bg-red-500"
+                      >
+                        Delete
+                      </Button>
+                    </TableCell>
+                  </TableRow>
+                ))}
+              </TableBody>
+            </Table>
+          </div>
+        </TableContainer>
      </Box>
-      <Typography variant="h4">Elements</Typography>
-      <TableContainer
-        component={Box}
-        sx={{ maxHeight: "50%", overflow: "auto" }}
-      >
-        <Table stickyHeader className="mb-4">
-          <TableHead>
-            <TableRow>
-              <TableCell>
-                <Typography sx={{ fontWeight: "bold" }}>Name</Typography>
-              </TableCell>
-              <TableCell>
-                <Typography sx={{ fontWeight: "bold" }}>XPath</Typography>
-              </TableCell>
-            </TableRow>
-          </TableHead>
-          <TableBody>
-            {rows.map((row, index) => (
-              <TableRow key={index}>
-                <TableCell>
-                  <Typography>{row.name}</Typography>
-                </TableCell>
-                <TableCell>
-                  <Typography>{row.xpath}</Typography>
-                </TableCell>
-                <TableCell>
-                  <Button onClick={() => handleDeleteRow(row.name)}>
-                    Delete
-                  </Button>
-                </TableCell>
-              </TableRow>
-            ))}
-          </TableBody>
-        </Table>
-      </TableContainer>
-    </>
+    </Box>
  );
 };
@@ -15,7 +15,8 @@ interface StateProps {
  submittedURL: string;
  setSubmittedURL: Dispatch<React.SetStateAction<string>>;
  rows: Element[];
-  setResults: Dispatch<React.SetStateAction<Result>>;
+  isValidURL: boolean;
+  setIsValidUrl: Dispatch<React.SetStateAction<boolean>>;
  setSnackbarMessage: Dispatch<React.SetStateAction<string>>;
  setSnackbarOpen: Dispatch<React.SetStateAction<boolean>>;
  setSnackbarSeverity: Dispatch<React.SetStateAction<string>>;
@@ -40,13 +41,13 @@ export const JobSubmitter = ({ stateProps }: Props) => {
    submittedURL,
    setSubmittedURL,
    rows,
-    setResults,
+    isValidURL,
+    setIsValidUrl,
    setSnackbarMessage,
    setSnackbarOpen,
    setSnackbarSeverity,
  } = stateProps;

-  const [isValidURL, setIsValidUrl] = useState<boolean>(true);
  const [urlError, setUrlError] = useState<string | null>(null);
  const [loading, setLoading] = useState<boolean>(false);
  const [jobOptions, setJobOptions] = useState<JobOptions>({
@@ -157,15 +158,17 @@ export const JobSubmitter = ({ stateProps }: Props) => {
          onChange={(e) => setSubmittedURL(e.target.value)}
          error={!isValidURL}
          helperText={!isValidURL ? urlError : ""}
+          className="rounded-md"
        />
        <Button
          variant="contained"
-          color="primary"
          size="small"
          onClick={handleSubmit}
          disabled={!(rows.length > 0) || loading}
+          className={`bg-gradient-to-r from-[#034efc] to-gray-500 text-white font-semibold rounded-md 
+                    transition-transform transform hover:scale-105 disabled:opacity-50`}
        >
-          {loading ? <CircularProgress size={24} /> : "Submit"}
+          {loading ? <CircularProgress size={24} color="inherit" /> : "Submit"}
        </Button>
      </div>
      <Box bgcolor="background.paper" className="flex flex-col mb-2 rounded-md">
@@ -14,6 +14,7 @@ const Home = () => {
  const [snackbarOpen, setSnackbarOpen] = useState<boolean>(false);
  const [snackbarMessage, setSnackbarMessage] = useState<string>("");
  const [snackbarSeverity, setSnackbarSeverity] = useState<string>("error");
+  const [isValidURL, setIsValidUrl] = useState<boolean>(true);

  const resultsRef = useRef<HTMLTableElement | null>(null);

@@ -74,26 +75,26 @@ const Home = () => {
      minHeight="100vh"
      py={4}
    >
-      <Container maxWidth="md">
-        <Typography variant="h1" gutterBottom textAlign="center">
-          Scraperr
-        </Typography>
+      <Container maxWidth="lg">
        <JobSubmitter
          stateProps={{
            submittedURL,
            setSubmittedURL,
            rows,
-            setResults,
+            isValidURL,
+            setIsValidUrl,
            setSnackbarMessage,
            setSnackbarOpen,
            setSnackbarSeverity,
          }}
        />
-        <ElementTable
-          rows={rows}
-          setRows={setRows}
-          submittedURL={submittedURL}
-        />
+        {submittedURL.length ? (
+          <ElementTable
+            rows={rows}
+            setRows={setRows}
+            submittedURL={submittedURL}
+          />
+        ) : null}
      </Container>
      {snackbarSeverity === "info" ? <NotifySnackbar /> : <ErrorSnackbar />}
    </Box>
@@ -70,7 +70,24 @@ const lightTheme = createTheme({
      secondary: "#333333",
    },
  },
+
  ...commonThemeOptions,
+  components: {
+    ...commonThemeOptions.components,
+    MuiButton: {
+      styleOverrides: {
+        root: {
+          color: "white",
+          "&.MuiButton-root": {
+            backgroundColor: "#034efc",
+          },
+          "&:hover": {
+            backgroundColor: "#027be0",
+          },
+        },
+      },
+    },
+  },
 });

 const darkTheme = createTheme({
@@ -132,6 +149,12 @@ const darkTheme = createTheme({
      styleOverrides: {
        root: {
          color: "white",
+          "&.MuiButton-root": {
+            backgroundColor: "#034efc",
+          },
+          "&:hover": {
+            backgroundColor: "#027be0",
+          },
        },
      },
    },
@@ -1,6 +1,23 @@
 /** @type {import('tailwindcss').Config} */
 module.exports = {
  content: ["./src/**/*.{js,jsx,ts,tsx}"],
-  theme: {},
+  theme: {
+    extend: {
+      animation: {
+        fadeIn: "fadeIn 0.5s ease-in-out",
+        fadeOut: "fadeOut 0.5s ease-in-out",
+      },
+      keyframes: {
+        fadeIn: {
+          "0%": { opacity: 0 },
+          "100%": { opacity: 1 },
+        },
+        fadeOut: {
+          "0%": { opacity: 1 },
+          "100%": { opacity: 0 },
+        },
+      },
+    },
+  },
  plugins: [],
 };