wip: update UI

2025-12-14 11:46:17 +00:00 · 2024-07-22 15:57:32 -05:00
parent cd3cd7eea2
commit 5b548060db
16 changed files with 221 additions and 116 deletions
--- a/README.md
+++ b/README.md
@@ -13,12 +13,16 @@ From the table, users can download an excel sheet of the job's results, along wi

 ## Features

+### Submitting URLs for Scraping
+
 - Submit/Queue URLs for web scraping
 - Add and manage elements to scrape using XPath
 - Scrape all pages within same domain
 - Add custom json headers to send in requests to URLs
 - Display results of scraped data

+### Managing Previous Jobs
+
 ![main_page](https://github.com/jaypyles/www-scrape/blob/master/docs/main_page.png)

 - Download csv containing results
@@ -28,14 +32,20 @@ From the table, users can download an excel sheet of the job's results, along wi

 ![job_page](https://github.com/jaypyles/www-scrape/blob/master/docs/job_page.png)

+### User Management
+
 - User login/signup to organize jobs

 ![login](https://github.com/jaypyles/www-scrape/blob/master/docs/login.png)

+### Log Viewing
+
 - View app logs inside of web ui

 ![logs](https://github.com/jaypyles/www-scrape/blob/master/docs/log_page.png)

+### Statistics View
+
 - View a small statistics view of jobs ran

 ![statistics](https://github.com/jaypyles/www-scrape/blob/master/docs/stats_page.png)
--- a/api/backend/app.py
+++ b/api/backend/app.py
@@ -57,6 +57,7 @@ app.add_middleware(
 )

 app.mount("/_next/static", StaticFiles(directory="./dist/_next/static"), name="static")
+app.mount("/images", StaticFiles(directory="./dist/images"), name="images")


@app.get("/")
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -5,6 +5,8 @@ services:
    build:
      context: ./
    container_name: scraperr
+    ports:
+      - 9000:8000
    env_file:
      - ./.env
    volumes:
--- a/docs/logo_picture.png
+++ b/docs/logo_picture.png
--- a/next.config.mjs
+++ b/next.config.mjs
@@ -2,6 +2,7 @@
 const nextConfig = {
  output: "export",
  distDir: "./dist",
+  images: { unoptimized: true },
  async rewrites() {
    return [
      {
--- a/public/favicon.ico
+++ b/public/favicon.ico
--- a/public/favicon.ico:Zone.Identifier
+++ b/public/favicon.ico:Zone.Identifier
@@ -0,0 +1,4 @@
+[ZoneTransfer]
+ZoneId=3
+ReferrerUrl=https://cloudconvert.com/
+HostUrl=https://us-east.storage.cloudconvert.com/tasks/ff0a6031-1745-4e41-871e-35f102bfe11b/scraperr_logo.ico?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=cloudconvert-production%2F20240722%2Fva%2Fs3%2Faws4_request&X-Amz-Date=20240722T165500Z&X-Amz-Expires=86400&X-Amz-Signature=df96cece92026dc08d5b1dbfc2391beffa5101137fade6587625c467f088feb7&X-Amz-SignedHeaders=host&response-content-disposition=attachment%3B%20filename%3D%22scraperr_logo.ico%22&response-content-type=image%2Fvnd.microsoft.icon&x-id=GetObject
--- a/public/images/scraperr_logo.png
+++ b/public/images/scraperr_logo.png
--- a/src/components/JobTable.tsx
+++ b/src/components/JobTable.tsx
@@ -32,6 +32,7 @@ const COLOR_MAP: ColorMap = {
  Queued: "rgba(255,201,5,0.25)",
  Scraping: "rgba(3,104,255,0.25)",
  Completed: "rgba(5,255,51,0.25)",
+  Failed: "rgba(214,0,25,0.25)",
 };

 const JobTable: React.FC<JobTableProps> = ({ jobs, fetchJobs }) => {
--- a/src/components/NavDrawer.tsx
+++ b/src/components/NavDrawer.tsx
@@ -24,6 +24,7 @@ import TerminalIcon from "@mui/icons-material/Terminal";
 import BarChart from "@mui/icons-material/BarChart";
 import { useRouter } from "next/router";
 import { useTheme } from "@mui/material/styles";
+import Image from "next/image";

 interface NavDrawerProps {
  toggleTheme: () => void;
@@ -112,11 +113,9 @@ const NavDrawer: React.FC<NavDrawerProps> = ({ toggleTheme, isDarkMode }) => {
              </Typography>
              <Button
                variant="contained"
-                color="primary"
                onClick={logout}
                sx={{
                  width: "100%",
-                  color: theme.palette.mode === "light" ? "#000000" : "#ffffff",
                }}
              >
                Logout
@@ -125,24 +124,22 @@ const NavDrawer: React.FC<NavDrawerProps> = ({ toggleTheme, isDarkMode }) => {
          ) : (
            <Button
              variant="contained"
-              color="primary"
              onClick={() => router.push("/login")}
              sx={{
                width: "100%",
-                color: theme.palette.mode === "light" ? "#000000" : "#ffffff",
              }}
            >
              Login
            </Button>
          )}
          <Divider sx={{ marginTop: 2, marginBottom: 2 }}></Divider>
-          <Accordion sx={{ padding: 0, width: "90%" }}>
+          <Accordion sx={{ padding: 0, width: "90%", marginBottom: 1 }}>
            <AccordionSummary
              expandIcon={<ExpandMoreIcon />}
              aria-controls="panel1a-content"
              id="panel1a-header"
            >
-              <Typography>Settings</Typography>
+              <Typography>Quick Settings</Typography>
            </AccordionSummary>
            <AccordionDetails>
              <div className="flex flex-row mr-1">
--- a/src/components/jobs/JobQueue.tsx
+++ b/src/components/jobs/JobQueue.tsx
@@ -136,21 +136,24 @@ export const JobQueue = ({
                {new Date(row.time_created).toLocaleString()}
              </Box>
            </TableCell>
-            <TableCell sx={{ maxWidth: 150, overflow: "auto" }}>
+            <TableCell sx={{ maxWidth: 50, overflow: "auto" }}>
              <Box sx={{ maxHeight: 100, overflow: "auto" }}>
                <Box
                  className="rounded-md p-2 text-center"
-                  sx={{ bgcolor: colors[row.status], opactity: "50%" }}
+                  sx={{ bgcolor: colors[row.status] }}
                >
                  {row.status}
                </Box>
              </Box>
            </TableCell>
-            <TableCell sx={{ maxWidth: 100, overflow: "auto" }}>
+            <TableCell sx={{ maxWidth: 150, overflow: "auto" }}>
+              <Box sx={{ display: "flex", gap: 1 }}>
                <Button
                  onClick={() => {
                    onDownload([row.id]);
                  }}
+                  size="small"
+                  sx={{ minWidth: 0, padding: "4px 8px" }}
                >
                  Download
                </Button>
@@ -158,9 +161,12 @@ export const JobQueue = ({
                  onClick={() =>
                    onNavigate(row.elements, row.url, row.job_options)
                  }
+                  size="small"
+                  sx={{ minWidth: 0, padding: "4px 8px" }}
                >
                  Rerun
                </Button>
+              </Box>
            </TableCell>
          </TableRow>
        ))}
--- a/src/components/submit/ElementTable.tsx
+++ b/src/components/submit/ElementTable.tsx
@@ -33,7 +33,7 @@ export const ElementTable = ({ rows, setRows, submittedURL }: Props) => {

  const handleAddRow = () => {
    const updatedRow = { ...newRow, url: submittedURL };
-    setRows([...rows, updatedRow]);
+    setRows([updatedRow, ...rows]);
    setNewRow({ name: "", xpath: "", url: "" });
  };

@@ -44,23 +44,67 @@ export const ElementTable = ({ rows, setRows, submittedURL }: Props) => {
      })
    );
  };
+
  return (
-    <>
-      <Box display="flex" gap={2} marginBottom={2} className="items-center">
+    <Box className="animate-fadeIn p-2" bgcolor="background.paper">
+      <Box className="text-center mb-4">
+        <Typography variant="h4" sx={{ marginBottom: 1 }}>
+          Elements to Scrape
+        </Typography>
+        <TableContainer
+          component={Box}
+          sx={{ maxHeight: "50%", overflow: "auto" }}
+        >
+          <div className="rounded-lg shadow-md border border-gray-300 overflow-hidden">
+            <Table
+              stickyHeader
+              className="mb-4"
+              sx={{
+                tableLayout: "fixed",
+                width: "100%",
+                "& .MuiTableCell-root": {
+                  borderBottom: "1px solid #e0e0e0",
+                },
+              }}
+            >
+              <TableHead>
+                <TableRow>
+                  <TableCell>
+                    <Typography sx={{ fontWeight: "bold" }}>Name</Typography>
+                  </TableCell>
+                  <TableCell>
+                    <Typography sx={{ fontWeight: "bold" }}>XPath</Typography>
+                  </TableCell>
+                  <TableCell>
+                    <Typography sx={{ fontWeight: "bold" }}>Actions</Typography>
+                  </TableCell>
+                </TableRow>
+              </TableHead>
+              <TableBody>
+                <TableRow>
+                  <TableCell>
                    <TextField
                      label="Name"
                      variant="outlined"
                      fullWidth
                      value={newRow.name}
-          onChange={(e) => setNewRow({ ...newRow, name: e.target.value })}
+                      onChange={(e) =>
+                        setNewRow({ ...newRow, name: e.target.value })
+                      }
                    />
+                  </TableCell>
+                  <TableCell>
                    <TextField
                      label="XPath"
                      variant="outlined"
                      fullWidth
                      value={newRow.xpath}
-          onChange={(e) => setNewRow({ ...newRow, xpath: e.target.value })}
+                      onChange={(e) =>
+                        setNewRow({ ...newRow, xpath: e.target.value })
+                      }
                    />
+                  </TableCell>
+                  <TableCell>
                    <Tooltip
                      title={
                        newRow.xpath.length > 0 && newRow.name.length > 0
@@ -78,35 +122,24 @@ export const ElementTable = ({ rows, setRows, submittedURL }: Props) => {
                            height: "40px",
                            width: "40px",
                          }}
-              disabled={!(newRow.xpath.length > 0 && newRow.name.length > 0)}
+                          disabled={
+                            !(newRow.xpath.length > 0 && newRow.name.length > 0)
+                          }
                        >
                          <AddIcon
                            fontSize="inherit"
                            sx={{
-                  color: theme.palette.mode === "light" ? "#000000" : "#ffffff",
+                              color:
+                                theme.palette.mode === "light"
+                                  ? "#000000"
+                                  : "#ffffff",
                            }}
                          />
                        </IconButton>
                      </span>
                    </Tooltip>
-      </Box>
-      <Typography variant="h4">Elements</Typography>
-      <TableContainer
-        component={Box}
-        sx={{ maxHeight: "50%", overflow: "auto" }}
-      >
-        <Table stickyHeader className="mb-4">
-          <TableHead>
-            <TableRow>
-              <TableCell>
-                <Typography sx={{ fontWeight: "bold" }}>Name</Typography>
-              </TableCell>
-              <TableCell>
-                <Typography sx={{ fontWeight: "bold" }}>XPath</Typography>
                  </TableCell>
                </TableRow>
-          </TableHead>
-          <TableBody>
                {rows.map((row, index) => (
                  <TableRow key={index}>
                    <TableCell>
@@ -116,7 +149,11 @@ export const ElementTable = ({ rows, setRows, submittedURL }: Props) => {
                      <Typography>{row.xpath}</Typography>
                    </TableCell>
                    <TableCell>
-                  <Button onClick={() => handleDeleteRow(row.name)}>
+                      <Button
+                        onClick={() => handleDeleteRow(row.name)}
+                        className="!bg-red-500 bg-opacity-50 !text-white font-semibold rounded-md 
+                        transition-transform transform hover:scale-105 hover:bg-red-500"
+                      >
                        Delete
                      </Button>
                    </TableCell>
@@ -124,7 +161,9 @@ export const ElementTable = ({ rows, setRows, submittedURL }: Props) => {
                ))}
              </TableBody>
            </Table>
+          </div>
        </TableContainer>
-    </>
+      </Box>
+    </Box>
  );
 };
--- a/src/components/submit/JobSubmitter.tsx
+++ b/src/components/submit/JobSubmitter.tsx
@@ -15,7 +15,8 @@ interface StateProps {
  submittedURL: string;
  setSubmittedURL: Dispatch<React.SetStateAction<string>>;
  rows: Element[];
-  setResults: Dispatch<React.SetStateAction<Result>>;
+  isValidURL: boolean;
+  setIsValidUrl: Dispatch<React.SetStateAction<boolean>>;
  setSnackbarMessage: Dispatch<React.SetStateAction<string>>;
  setSnackbarOpen: Dispatch<React.SetStateAction<boolean>>;
  setSnackbarSeverity: Dispatch<React.SetStateAction<string>>;
@@ -40,13 +41,13 @@ export const JobSubmitter = ({ stateProps }: Props) => {
    submittedURL,
    setSubmittedURL,
    rows,
-    setResults,
+    isValidURL,
+    setIsValidUrl,
    setSnackbarMessage,
    setSnackbarOpen,
    setSnackbarSeverity,
  } = stateProps;

-  const [isValidURL, setIsValidUrl] = useState<boolean>(true);
  const [urlError, setUrlError] = useState<string | null>(null);
  const [loading, setLoading] = useState<boolean>(false);
  const [jobOptions, setJobOptions] = useState<JobOptions>({
@@ -157,15 +158,17 @@ export const JobSubmitter = ({ stateProps }: Props) => {
          onChange={(e) => setSubmittedURL(e.target.value)}
          error={!isValidURL}
          helperText={!isValidURL ? urlError : ""}
+          className="rounded-md"
        />
        <Button
          variant="contained"
-          color="primary"
          size="small"
          onClick={handleSubmit}
          disabled={!(rows.length > 0) || loading}
+          className={`bg-gradient-to-r from-[#034efc] to-gray-500 text-white font-semibold rounded-md 
+                    transition-transform transform hover:scale-105 disabled:opacity-50`}
        >
-          {loading ? <CircularProgress size={24} /> : "Submit"}
+          {loading ? <CircularProgress size={24} color="inherit" /> : "Submit"}
        </Button>
      </div>
      <Box bgcolor="background.paper" className="flex flex-col mb-2 rounded-md">
--- a/src/pages/index.tsx
+++ b/src/pages/index.tsx
@@ -14,6 +14,7 @@ const Home = () => {
  const [snackbarOpen, setSnackbarOpen] = useState<boolean>(false);
  const [snackbarMessage, setSnackbarMessage] = useState<string>("");
  const [snackbarSeverity, setSnackbarSeverity] = useState<string>("error");
+  const [isValidURL, setIsValidUrl] = useState<boolean>(true);

  const resultsRef = useRef<HTMLTableElement | null>(null);

@@ -74,26 +75,26 @@ const Home = () => {
      minHeight="100vh"
      py={4}
    >
-      <Container maxWidth="md">
-        <Typography variant="h1" gutterBottom textAlign="center">
-          Scraperr
-        </Typography>
+      <Container maxWidth="lg">
        <JobSubmitter
          stateProps={{
            submittedURL,
            setSubmittedURL,
            rows,
-            setResults,
+            isValidURL,
+            setIsValidUrl,
            setSnackbarMessage,
            setSnackbarOpen,
            setSnackbarSeverity,
          }}
        />
+        {submittedURL.length ? (
          <ElementTable
            rows={rows}
            setRows={setRows}
            submittedURL={submittedURL}
          />
+        ) : null}
      </Container>
      {snackbarSeverity === "info" ? <NotifySnackbar /> : <ErrorSnackbar />}
    </Box>
--- a/src/styles/themes.ts
+++ b/src/styles/themes.ts
@@ -70,7 +70,24 @@ const lightTheme = createTheme({
      secondary: "#333333",
    },
  },
+
  ...commonThemeOptions,
+  components: {
+    ...commonThemeOptions.components,
+    MuiButton: {
+      styleOverrides: {
+        root: {
+          color: "white",
+          "&.MuiButton-root": {
+            backgroundColor: "#034efc",
+          },
+          "&:hover": {
+            backgroundColor: "#027be0",
+          },
+        },
+      },
+    },
+  },
 });

 const darkTheme = createTheme({
@@ -132,6 +149,12 @@ const darkTheme = createTheme({
      styleOverrides: {
        root: {
          color: "white",
+          "&.MuiButton-root": {
+            backgroundColor: "#034efc",
+          },
+          "&:hover": {
+            backgroundColor: "#027be0",
+          },
        },
      },
    },
--- a/tailwind.config.js
+++ b/tailwind.config.js
@@ -1,6 +1,23 @@
 /** @type {import('tailwindcss').Config} */
 module.exports = {
  content: ["./src/**/*.{js,jsx,ts,tsx}"],
-  theme: {},
+  theme: {
+    extend: {
+      animation: {
+        fadeIn: "fadeIn 0.5s ease-in-out",
+        fadeOut: "fadeOut 0.5s ease-in-out",
+      },
+      keyframes: {
+        fadeIn: {
+          "0%": { opacity: 0 },
+          "100%": { opacity: 1 },
+        },
+        fadeOut: {
+          "0%": { opacity: 1 },
+          "100%": { opacity: 0 },
+        },
+      },
+    },
+  },
  plugins: [],
 };