chore: refactor wip

cypress/e2e/advanced-job-options.cy.ts

@@ -0,0 +1,80 @@
+import { login } from "../utilities/authentication.utils";
+import {
+  addCustomHeaders,
+  addSiteMapAction,
+  cleanUpJobs,
+  mockSubmitJob,
+  submitBasicJob,
+  waitForJobCompletion,
+} from "../utilities/job.utilities";
+
+describe.only("Advanced Job Options", () => {
+  beforeEach(() => {
+    mockSubmitJob();
+    login();
+    cy.visit("/");
+  });
+
+  afterEach(() => {
+    cleanUpJobs();
+  });
+
+  it.only("should handle custom headers", () => {
+    const customHeaders = {
+      "User-Agent": "Test Agent",
+      "Accept-Language": "en-US",
+    };
+
+    addCustomHeaders(customHeaders);
+    submitBasicJob("https://httpbin.org/headers", "headers", "//pre");
+
+    cy.wait("@submitScrapeJob").then((interception) => {
+      expect(interception.response?.statusCode).to.eq(200);
+      expect(
+        interception.request?.body.data.job_options.custom_headers
+      ).to.deep.equal(customHeaders);
+    });
+
+    cy.get("li").contains("Jobs").click();
+    waitForJobCompletion("https://httpbin.org/headers");
+  });
+
+  it("should handle site map actions", () => {
+    addSiteMapAction("click", "//button[contains(text(), 'Load More')]");
+    addSiteMapAction("input", "//input[@type='search']", "test search");
+
+    submitBasicJob("https://example.com", "content", "//div[@class='content']");
+
+    cy.wait("@submitScrapeJob").then((interception) => {
+      expect(interception.response?.statusCode).to.eq(200);
+      const siteMap = interception.request?.body.data.job_options.site_map;
+      expect(siteMap.actions).to.have.length(2);
+      expect(siteMap.actions[0].type).to.equal("click");
+      expect(siteMap.actions[1].type).to.equal("input");
+    });
+
+    cy.get("li").contains("Jobs").click();
+    waitForJobCompletion("https://example.com");
+  });
+
+  it("should handle multiple elements", () => {
+    cy.get('[data-cy="url-input"]').type("https://books.toscrape.com");
+    cy.get('[data-cy="name-field"]').type("titles");
+    cy.get('[data-cy="xpath-field"]').type("//h3");
+    cy.get('[data-cy="add-button"]').click();
+
+    cy.get('[data-cy="name-field"]').type("prices");
+    cy.get('[data-cy="xpath-field"]').type("//p[@class='price_color']");
+    cy.get('[data-cy="add-button"]').click();
+
+    cy.contains("Submit").click();
+
+    cy.wait("@submitScrapeJob").then((interception) => {
+      expect(interception.response?.statusCode).to.eq(200);
+      expect(interception.request?.body.data.elements).to.have.length(2);
+    });
+
+    cy.get("li").contains("Jobs").click();
+    waitForJobCompletion("https://books.toscrape.com");
+  });
+});

cypress/utilities/job.utilities.ts

@@ -7,3 +7,54 @@ export const cleanUpJobs = () => {
 
   cy.get("[data-testid='DeleteIcon']").click();
 };
+
+export const submitBasicJob = (url: string, name: string, xpath: string) => {
+  cy.get('[data-cy="url-input"]').type(url);
+  cy.get('[data-cy="name-field"]').type(name);
+  cy.get('[data-cy="xpath-field"]').type(xpath);
+  cy.get('[data-cy="add-button"]').click();
+  cy.contains("Submit").click();
+};
+
+export const waitForJobCompletion = (url: string) => {
+  cy.contains("div", url, { timeout: 10000 }).should("exist");
+  cy.contains("div", "Completed", { timeout: 20000 }).should("exist");
+};
+
+export const enableMultiPageScraping = () => {
+  cy.get("button").contains("Advanced Job Options").click();
+  cy.get('[data-cy="multi-page-toggle"]').click();
+  cy.get("body").type("{esc}");
+};
+
+export const addCustomHeaders = (headers: Record<string, string>) => {
+  cy.get("button").contains("Advanced Job Options").click();
+  cy.get('[name="custom_headers"]').type(JSON.stringify(headers), {
+    parseSpecialCharSequences: false,
+  });
+  cy.get("body").type("{esc}");
+};
+
+export const addCustomCookies = (cookies: Record<string, string>) => {
+  cy.get("button").contains("Advanced Job Options").click();
+  cy.get('[name="custom_cookies"]').type(JSON.stringify(cookies));
+  cy.get("body").type("{esc}");
+};
+
+export const addSiteMapAction = (
+  type: "click" | "input",
+  xpath: string,
+  input?: string
+) => {
+  cy.get("button").contains("Create Site Map").click();
+  cy.get('[data-cy="site-map-select"]').select(type);
+  cy.get('[data-cy="site-map-xpath"]').type(xpath);
+  if (type === "input" && input) {
+    cy.get('[data-cy="site-map-input"]').type(input);
+  }
+  cy.get('[data-cy="add-site-map-action"]').click();
+};
+
+export const mockSubmitJob = () => {
+  cy.intercept("POST", "/api/submit-scrape-job").as("submitScrapeJob");
+};

package.json

@@ -36,6 +36,7 @@
     "react-router": "^6.14.1",
     "react-router-dom": "^6.14.1",
     "react-spinners": "^0.14.1",
+    "react-toastify": "^11.0.5",
     "redux-persist": "^6.0.0",
     "typescript": "^4.9.5",
     "web-vitals": "^2.1.4"

src/components/common/advanced-job-options/dialog/advanced-job-options-dialog.tsx

@@ -25,7 +25,7 @@ import {
   Typography,
   useTheme,
 } from "@mui/material";
-import { Dispatch, SetStateAction } from "react";
+import { Dispatch, SetStateAction, useEffect, useState } from "react";
 
 export type AdvancedJobOptionsDialogProps = {
   open: boolean;
@@ -43,31 +43,45 @@ export const AdvancedJobOptionsDialog = ({
   multiPageScrapeEnabled = true,
 }: AdvancedJobOptionsDialogProps) => {
   const theme = useTheme();
+  const [localJobOptions, setLocalJobOptions] =
+    useState<RawJobOptions>(jobOptions);
+
+  // Update local state when prop changes
+  useEffect(() => {
+    setLocalJobOptions(jobOptions);
+  }, [jobOptions]);
+
   const handleMultiPageScrapeChange = () => {
-    setJobOptions((prevJobOptions) => ({
+    setLocalJobOptions((prevJobOptions) => ({
       ...prevJobOptions,
       multi_page_scrape: !prevJobOptions.multi_page_scrape,
     }));
   };
 
   const handleProxiesChange = (e: React.ChangeEvent<HTMLInputElement>) => {
-    setJobOptions((prevJobOptions) => ({
+    setLocalJobOptions((prevJobOptions) => ({
       ...prevJobOptions,
       proxies: e.target.value,
     }));
   };
 
   const handleCollectMediaChange = () => {
-    setJobOptions((prevJobOptions) => ({
+    setLocalJobOptions((prevJobOptions) => ({
       ...prevJobOptions,
       collect_media: !prevJobOptions.collect_media,
     }));
   };
 
+  const handleClose = () => {
+    // Save the local state back to the parent before closing
+    setJobOptions(localJobOptions);
+    onClose();
+  };
+
   return (
     <Dialog
       open={open}
-      onClose={onClose}
+      onClose={handleClose}
       maxWidth="md"
       fullWidth
       PaperProps={{
@@ -122,7 +136,7 @@ export const AdvancedJobOptionsDialog = ({
               <FormControlLabel
                 control={
                   <Checkbox
-                    checked={jobOptions.multi_page_scrape}
+                    checked={localJobOptions.multi_page_scrape}
                     onChange={handleMultiPageScrapeChange}
                     disabled={!multiPageScrapeEnabled}
                   />
@@ -147,7 +161,7 @@ export const AdvancedJobOptionsDialog = ({
               <FormControlLabel
                 control={
                   <Checkbox
-                    checked={jobOptions.collect_media}
+                    checked={localJobOptions.collect_media}
                     onChange={handleCollectMediaChange}
                     data-cy="collect-media-checkbox"
                   />
@@ -233,7 +247,7 @@ export const AdvancedJobOptionsDialog = ({
                   fullWidth
                   variant="outlined"
                   size="small"
-                  value={jobOptions.proxies}
+                  value={localJobOptions.proxies}
                   onChange={handleProxiesChange}
                   InputProps={{
                     startAdornment: (
@@ -251,8 +265,9 @@ export const AdvancedJobOptionsDialog = ({
               label="Custom Headers"
               placeholder='{"User-Agent": "CustomAgent", "Accept": "*/*"}'
               urlParam="custom_headers"
+              name="custom_headers"
               onChange={(value) => {
-                setJobOptions((prevJobOptions) => ({
+                setLocalJobOptions((prevJobOptions) => ({
                   ...prevJobOptions,
                   custom_headers: value,
                 }));
@@ -264,8 +279,9 @@ export const AdvancedJobOptionsDialog = ({
               label="Custom Cookies"
               placeholder='[{"name": "value", "name2": "value2"}]'
               urlParam="custom_cookies"
+              name="custom_cookies"
               onChange={(value) => {
-                setJobOptions((prevJobOptions) => ({
+                setLocalJobOptions((prevJobOptions) => ({
                   ...prevJobOptions,
                   custom_cookies: value,
                 }));

src/components/common/expanded-table-input/expanded-table-input.tsx

@@ -23,6 +23,7 @@ export type ExpandedTableInputProps = {
   onChange: (value: any) => void;
   placeholder: string;
   urlParam: string;
+  name: string;
 };
 
 export const ExpandedTableInput = ({
@@ -30,6 +31,7 @@ export const ExpandedTableInput = ({
   onChange,
   placeholder,
   urlParam,
+  name,
 }: ExpandedTableInputProps) => {
   const theme = useTheme();
   const [value, setValue] = useState("");
@@ -150,6 +152,7 @@ export const ExpandedTableInput = ({
           size="small"
           error={jsonError !== null}
           helperText={jsonError ?? ""}
+          name={name}
         />
 
         {parsedHeaders && parsedHeaders.length > 0 && (

src/components/common/nav-drawer/user-control/logged-in-control/logged-in-control.module.css

@@ -1,7 +1,7 @@
 .welcome {
-  margin: 0.25rem;
+  margin: 0.5rem !important;
 }
 
 .userControlButton {
   width: 100%;
-}
+}

src/components/submit/job-submitter/job-submitter.tsx

@@ -1,15 +1,14 @@
 "use client";
 
 import { AdvancedJobOptions } from "@/components/common/advanced-job-options";
+import { useSubmitJob } from "@/hooks/use-submit-job";
 import { parseJobOptions } from "@/lib";
-import { initialJobOptions, RawJobOptions } from "@/types/job";
+import { useUser } from "@/store/hooks";
 import { useRouter } from "next/router";
-import { useEffect, useState } from "react";
+import { useEffect } from "react";
 import { JobSubmitterHeader } from "./job-submitter-header";
 import { JobSubmitterInput } from "./job-submitter-input";
 import { useJobSubmitterProvider } from "./provider";
-import { useUser } from "@/store/hooks";
-import { useSubmitJob } from "@/hooks/use-submit-job";
 
 export const JobSubmitter = () => {
   const router = useRouter();
@@ -17,10 +16,8 @@ export const JobSubmitter = () => {
   const { user } = useUser();
 
   const { submitJob, loading, error } = useSubmitJob();
-  const { submittedURL, rows, siteMap, setSiteMap } = useJobSubmitterProvider();
-
-  const [jobOptions, setJobOptions] =
-    useState<RawJobOptions>(initialJobOptions);
+  const { submittedURL, rows, siteMap, setSiteMap, jobOptions, setJobOptions } =
+    useJobSubmitterProvider();
 
   useEffect(() => {
     if (job_options) {
@@ -32,6 +29,11 @@ export const JobSubmitter = () => {
     await submitJob(submittedURL, rows, user, jobOptions, siteMap, false, null);
   };
 
+  console.log(jobOptions);
+  useEffect(() => {
+    console.log(jobOptions);
+  }, [jobOptions]);
+
   return (
     <div>
       <JobSubmitterHeader />

src/components/submit/job-submitter/provider.tsx

@@ -1,12 +1,13 @@
+import { Element, RawJobOptions, Result, SiteMap } from "@/types";
+import { initialJobOptions } from "@/types/job";
 import React, {
   createContext,
+  Dispatch,
   PropsWithChildren,
   useContext,
-  useState,
-  Dispatch,
   useMemo,
+  useState,
 } from "react";
-import { Element, Result, SiteMap } from "@/types";
 
 type JobSubmitterProviderType = {
   submittedURL: string;
@@ -25,6 +26,8 @@ type JobSubmitterProviderType = {
   setIsValidUrl: Dispatch<React.SetStateAction<boolean>>;
   siteMap: SiteMap | null;
   setSiteMap: Dispatch<React.SetStateAction<SiteMap | null>>;
+  jobOptions: RawJobOptions;
+  setJobOptions: Dispatch<React.SetStateAction<RawJobOptions>>;
   closeSnackbar: () => void;
 };
 
@@ -41,6 +44,8 @@ export const Provider = ({ children }: PropsWithChildren) => {
   const [snackbarSeverity, setSnackbarSeverity] = useState<string>("error");
   const [isValidURL, setIsValidUrl] = useState<boolean>(true);
   const [siteMap, setSiteMap] = useState<SiteMap | null>(null);
+  const [jobOptions, setJobOptions] =
+    useState<RawJobOptions>(initialJobOptions);
 
   const closeSnackbar = () => {
     setSnackbarOpen(false);
@@ -66,6 +71,8 @@ export const Provider = ({ children }: PropsWithChildren) => {
       setIsValidUrl,
       siteMap,
       setSiteMap,
+      jobOptions,
+      setJobOptions,
       closeSnackbar,
     }),
     [
@@ -77,6 +84,7 @@ export const Provider = ({ children }: PropsWithChildren) => {
       snackbarSeverity,
       isValidURL,
       siteMap,
+      jobOptions,
       closeSnackbar,
     ]
   );

src/pages/_app.tsx

@@ -1,15 +1,16 @@
-import "bootstrap/dist/css/bootstrap.min.css";
 import "@/styles/globals.css";
+import "bootstrap/dist/css/bootstrap.min.css";
 
-import React, { useState, useEffect } from "react";
+import { NavDrawer } from "@/components/common";
+import { persistor, store } from "@/store/store";
+import { darkTheme, lightTheme } from "@/styles/themes";
+import { Box, CssBaseline, ThemeProvider } from "@mui/material";
 import type { AppProps } from "next/app";
 import Head from "next/head";
-import { ThemeProvider, CssBaseline, Box } from "@mui/material";
-import { NavDrawer } from "@/components/common";
-import { darkTheme, lightTheme } from "@/styles/themes";
+import React, { useEffect, useState } from "react";
 import { Provider } from "react-redux";
+import { ToastContainer } from "react-toastify";
 import { PersistGate } from "redux-persist/integration/react";
-import { store, persistor } from "@/store/store";
 
 const App: React.FC<AppProps> = ({ Component, pageProps }) => {
   const [isDarkMode, setIsDarkMode] = useState(false);
@@ -56,6 +57,7 @@ const App: React.FC<AppProps> = ({ Component, pageProps }) => {
                 <Component {...pageProps} />
               </Box>
             </Box>
+            <ToastContainer theme={isDarkMode ? "dark" : "light"} />
           </ThemeProvider>
         </PersistGate>
       </Provider>

src/pages/login.tsx

@@ -1,13 +1,14 @@
 "use client";
 
+import { ApiService } from "@/services";
 import { useUser, useUserSettings } from "@/store/hooks";
 import { Box, Button, TextField, Typography } from "@mui/material";
 import { useTheme } from "@mui/material/styles";
 import axios from "axios";
 import { useRouter } from "next/router";
 import React, { useEffect, useState } from "react";
+import { toast } from "react-toastify";
 import { getUserSettings } from "../lib";
-import { ApiService } from "@/services";
 
 type Mode = "login" | "signup";
 
@@ -37,9 +38,6 @@ const AuthForm: React.FC = () => {
       if (mode === "login") {
         const user = await ApiService.login(email, password);
 
-        console.log(user);
-        alert("Login successful");
-
         const userSettings = await getUserSettings();
         setUserSettings(userSettings);
 
@@ -52,15 +50,16 @@ const AuthForm: React.FC = () => {
           error: null,
         });
 
+        toast.success("Login successful");
         router.push("/");
       } else {
         await ApiService.register(email, password, fullName);
-        alert("Signup successful");
-        router.push("/login");
+        setMode("login");
+        toast.success("Registration successful");
       }
     } catch (error) {
       console.error(error);
-      alert(`${mode.charAt(0).toUpperCase() + mode.slice(1)} failed`);
+      toast.error("There was an error logging or registering");
     }
   };
 

yarn.lock

@@ -4765,6 +4765,13 @@ react-style-singleton@^2.2.2, react-style-singleton@^2.2.3:
     get-nonce "^1.0.0"
     tslib "^2.0.0"
 
+react-toastify@^11.0.5:
+  version "11.0.5"
+  resolved "https://registry.yarnpkg.com/react-toastify/-/react-toastify-11.0.5.tgz#ce4c42d10eeb433988ab2264d3e445c4e9d13313"
+  integrity sha512-EpqHBGvnSTtHYhCPLxML05NLY2ZX0JURbAdNYa6BUkk+amz4wbKBQvoKQAB0ardvSarUBuY4Q4s1sluAzZwkmA==
+  dependencies:
+    clsx "^2.1.1"
+
 react-transition-group@^4.4.5:
   version "4.4.5"
   resolved "https://registry.yarnpkg.com/react-transition-group/-/react-transition-group-4.4.5.tgz#e53d4e3f3344da8521489fbef8f2581d42becdd1"