lee/Scraperr
1
0
Fork 0
mirror of https://github.com/jaypyles/Scraperr.git synced 2025-10-30 05:57:12 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

chore: wip add upload/import

Browse Source
This commit is contained in:
Jayden Pyles
2025-06-08 21:23:36 -05:00
parent 9381ba9232
commit 87cdd809b6
10 changed files with 186 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
src/components/common/advanced-job-options/dialog/advanced-job-options-dialog.tsx
View File

@@ -1,4 +1,6 @@
import { ExpandedTableInput } from "@/components/common/expanded-table-input";
import { UploadFile } from "@/components/common/upload-file";
import { useImportJobConfig } from "@/hooks/use-import-job-config";
import { RawJobOptions } from "@/types";
import {
Code as CodeIcon,
@@ -26,6 +28,7 @@ import {
useTheme,
} from "@mui/material";
import { Dispatch, SetStateAction, useEffect, useState } from "react";
import { toast } from "react-toastify";
export type AdvancedJobOptionsDialogProps = {
open: boolean;
@@ -43,6 +46,7 @@ export const AdvancedJobOptionsDialog = ({
multiPageScrapeEnabled = true,
}: AdvancedJobOptionsDialogProps) => {
const theme = useTheme();
const { handleUploadFile } = useImportJobConfig();
const [localJobOptions, setLocalJobOptions] =
useState<RawJobOptions>(jobOptions);
@@ -69,6 +73,12 @@ export const AdvancedJobOptionsDialog = ({
onClose();
};
const onUploadFile = (file: File) => {
handleUploadFile(file);
handleClose();
toast.success("Job config uploaded successfully");
};
return (
<Dialog
open={open}
@@ -99,11 +109,14 @@ export const AdvancedJobOptionsDialog = ({
<Typography variant="h6" component="div">
Advanced Job Options
</Typography>
<Settings
sx={{
color: theme.palette.primary.contrastText,
}}
/>
<Box sx={{ display: "flex", alignItems: "center", gap: 1 }}>
<UploadFile message="Upload Job Config" onUploadFile={onUploadFile} />
<Settings
sx={{
color: theme.palette.primary.contrastText,
}}
/>
</Box>
</DialogTitle>
<DialogContent

1
src/components/common/upload-file/index.ts Normal file
View File

@@ -0,0 +1 @@
export * from "./upload-file";

24
src/components/common/upload-file/upload-file.tsx Normal file
View File

@@ -0,0 +1,24 @@
import { Box, Button, Typography } from "@mui/material";
export type UploadFileProps = {
message: string;
onUploadFile: (file: File) => void;
};
export const UploadFile = ({ message, onUploadFile }: UploadFileProps) => {
const handleUploadFile = (event: React.ChangeEvent<HTMLInputElement>) => {
const file = event.target.files?.[0];
if (file) {
onUploadFile(file);
}
};
return (
<Box>
<Button variant="contained" component="label">
<Typography>{message}</Typography>
<input type="file" hidden onChange={handleUploadFile} />
</Button>
</Box>
);
};

45
src/components/jobs/favorites.tsx
View File

@@ -1,18 +1,18 @@
import React from "react";
import StarIcon from "@mui/icons-material/Star";
import {
Tooltip,
Box,
Button,
Checkbox,
IconButton,
Table,
TableBody,
TableCell,
TableHead,
TableRow,
Box,
Checkbox,
Button,
Tooltip,
} from "@mui/material";
import router from "next/router";
import { Job } from "../../types";
import StarIcon from "@mui/icons-material/Star";
interface stateProps {
selectedJobs: Set<string>;
@@ -21,7 +21,12 @@ interface stateProps {
interface Props {
onSelectJob: (job: string) => void;
onNavigate: (elements: Object[], url: string, options: any) => void;
onNavigate: (
id: string,
elements: Object[],
url: string,
options: any
) => void;
onFavorite: (ids: string[], field: string, value: any) => void;
stateProps: stateProps;
}
@@ -87,11 +92,29 @@ export const Favorites = ({
</TableCell>
<TableCell sx={{ maxWidth: 100, overflow: "auto" }}>
<Button
onClick={() =>
onNavigate(row.elements, row.url, row.job_options)
}
onClick={() => {
if (row.agent_mode) {
router.push({
pathname: "/agent",
query: {
url: row.url,
prompt: row.prompt,
job_options: JSON.stringify(row.job_options),
id: row.id,
},
});
} else {
onNavigate(row.id, row.elements, row.url, row.job_options);
}
}}
size="small"
sx={{
minWidth: 0,
padding: "4px 8px",
fontSize: "0.625rem",
}}
>
Run
Rerun
</Button>
</TableCell>
</TableRow>

35
src/components/jobs/job-queue.tsx
View File

@@ -1,5 +1,11 @@
"use client";
import { AutoAwesome, Image, VideoCameraBack } from "@mui/icons-material";
import { useExportJobConfig } from "@/hooks/use-export-job-config";
import {
AutoAwesome,
Image,
Settings,
VideoCameraBack,
} from "@mui/icons-material";
import StarIcon from "@mui/icons-material/Star";
import {
Box,
@@ -30,7 +36,12 @@ interface Props {
colors: stringMap;
onSelectJob: (job: string) => void;
onDownload: (job: string[]) => void;
onNavigate: (elements: Object[], url: string, options: any) => void;
onNavigate: (
id: string,
elements: Object[],
url: string,
options: any
) => void;
onFavorite: (ids: string[], field: string, value: any) => void;
onJobClick: (job: Job) => void;
stateProps: stateProps;
@@ -46,6 +57,7 @@ export const JobQueue = ({
onJobClick,
}: Props) => {
const { selectedJobs, filteredJobs } = stateProps;
const { exportJobConfig } = useExportJobConfig();
const router = useRouter();
return (
@@ -116,6 +128,17 @@ export const JobQueue = ({
</IconButton>
</span>
</Tooltip>
<Tooltip title="Export Job Configuration">
<span>
<IconButton
onClick={() => {
exportJobConfig(row);
}}
>
<Settings />
</IconButton>
</span>
</Tooltip>
{row.job_options.collect_media && (
<Tooltip title="View Media">
<span>
@@ -214,10 +237,16 @@ export const JobQueue = ({
url: row.url,
prompt: row.prompt,
job_options: JSON.stringify(row.job_options),
id: row.id,
},
});
} else {
onNavigate(row.elements, row.url, row.job_options);
onNavigate(
row.id,
row.elements,
row.url,
row.job_options
);
}
}}
size="small"

8
src/components/jobs/job-table.tsx
View File

@@ -47,10 +47,16 @@ export const JobTable: React.FC<JobTableProps> = ({ jobs, setJobs }) => {
setJobDownloadDialogOpen(true);
};
const handleNavigate = (elements: Object[], url: string, options: any) => {
const handleNavigate = (
id: string,
elements: Object[],
url: string,
options: any
) => {
router.push({
pathname: "/",
query: {
id,
elements: JSON.stringify(elements),
url: url,
job_options: JSON.stringify(options),

27
src/hooks/use-export-job-config.ts Normal file
View File

@@ -0,0 +1,27 @@
import { Job } from "@/types";
export const useExportJobConfig = () => {
const exportJobConfig = async (job: Job) => {
const jobConfig = {
url: job.url,
prompt: job.prompt,
job_options: job.job_options,
elements: job.elements,
agent_mode: job.agent_mode,
};
const jobConfigString = JSON.stringify(jobConfig);
const blob = new Blob([jobConfigString], { type: "application/json" });
const url = window.URL.createObjectURL(blob);
const a = document.createElement("a");
a.style.display = "none";
a.href = url;
a.download = `job_${job.id}.json`;
document.body.appendChild(a);
a.click();
window.URL.revokeObjectURL(url);
document.body.removeChild(a);
};
return { exportJobConfig };
};

36
src/hooks/use-import-job-config.ts Normal file
View File

@@ -0,0 +1,36 @@
import { useJobSubmitterProvider } from "@/components/submit/job-submitter/provider";
import { useRouter } from "next/router";
export const useImportJobConfig = () => {
const router = useRouter();
const { setJobOptions, setSiteMap, setSubmittedURL, setRows } =
useJobSubmitterProvider();
const handleUploadFile = (file: File) => {
const reader = new FileReader();
reader.onload = (e) => {
const jobConfig = JSON.parse(e.target?.result as string);
if (jobConfig.agent_mode) {
router.push({
pathname: "/agent",
query: {
url: jobConfig.url,
prompt: jobConfig.prompt,
job_options: JSON.stringify(jobConfig.job_options),
},
});
}
setJobOptions(jobConfig.job_options);
setSiteMap(jobConfig.site_map);
setSubmittedURL(jobConfig.url);
setRows(jobConfig.elements);
};
reader.readAsText(file);
};
return { handleUploadFile };
};

6
src/hooks/use-submit-job.ts
View File

@@ -25,7 +25,8 @@ export const useSubmitJob = () => {
jobOptions: RawJobOptions,
siteMap: SiteMap | null,
agentMode: boolean,
prompt: string | null
prompt: string | null,
id?: string
) => {
if (!validateURL(submittedURL)) {
setIsValidUrl(false);
@@ -61,7 +62,8 @@ export const useSubmitJob = () => {
customCookies,
siteMap,
agentMode,
prompt || undefined
prompt || undefined,
id
)
.then(async (response) => {
if (!response.ok) {

5
src/services/api-service/functions/submit-job.ts
View File

@@ -9,14 +9,15 @@ export const submitJob = async (
customCookies: any,
siteMap: SiteMap | null,
agentMode: boolean = false,
prompt?: string
prompt?: string,
id?: string
) => {
console.log(user);
return await fetch(`/api/submit-scrape-job`, {
method: "POST",
headers: { "content-type": "application/json" },
body: JSON.stringify({
data: {
id,
url: submittedURL,
elements: rows,
user: user?.email,