mirror of
https://github.com/jaypyles/Scraperr.git
synced 2026-10-03 16:47:44 +00:00
feat: convert to self-hosted version
This commit is contained in:
@@ -1,6 +1,5 @@
|
||||
import React, { useState } from "react";
|
||||
import { useAuth } from "../useAuth";
|
||||
import { LogoutOptions, RedirectLoginOptions } from "@auth0/auth0-react";
|
||||
import { useAuth } from "../hooks/useAuth";
|
||||
import {
|
||||
Box,
|
||||
Drawer,
|
||||
@@ -22,20 +21,10 @@ import { useRouter } from "next/router";
|
||||
|
||||
const NavDrawer: React.FC = () => {
|
||||
const router = useRouter();
|
||||
const { loginWithRedirect, logout, user, isAuthenticated } = useAuth();
|
||||
const { login, logout, user, isAuthenticated } = useAuth();
|
||||
const [open, setOpen] = useState<boolean>(false);
|
||||
|
||||
const handleLogout = () => {
|
||||
const logoutOptions: LogoutOptions = {};
|
||||
logout(logoutOptions);
|
||||
};
|
||||
|
||||
const handleLogin = () => {
|
||||
const loginOptions: RedirectLoginOptions = {
|
||||
authorizationParams: { redirect_uri: "http://localhost" },
|
||||
};
|
||||
loginWithRedirect(loginOptions);
|
||||
};
|
||||
console.log(user);
|
||||
|
||||
const toggleDrawer =
|
||||
(open: boolean) => (event: React.KeyboardEvent | React.MouseEvent) => {
|
||||
@@ -80,7 +69,7 @@ const NavDrawer: React.FC = () => {
|
||||
return (
|
||||
<>
|
||||
<AppBar position="static">
|
||||
<Toolbar>
|
||||
<Toolbar className="flex flex-row justify-between items-center">
|
||||
<IconButton
|
||||
edge="start"
|
||||
color="inherit"
|
||||
@@ -90,16 +79,16 @@ const NavDrawer: React.FC = () => {
|
||||
<MenuIcon />
|
||||
</IconButton>
|
||||
{isAuthenticated ? (
|
||||
<>
|
||||
<div className="flex flex-row items-center">
|
||||
<Typography variant="body1" sx={{ marginRight: 2 }}>
|
||||
Welcome, {user?.name}
|
||||
Welcome, {user?.full_name}
|
||||
</Typography>
|
||||
<Button color="inherit" onClick={handleLogout}>
|
||||
<Button color="inherit" onClick={logout}>
|
||||
Logout
|
||||
</Button>
|
||||
</>
|
||||
</div>
|
||||
) : (
|
||||
<Button color="inherit" onClick={handleLogin}>
|
||||
<Button color="inherit" onClick={() => router.push("/login")}>
|
||||
Login
|
||||
</Button>
|
||||
)}
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
import { useState, useEffect } from "react";
|
||||
import axios from "axios";
|
||||
|
||||
type User = {
|
||||
full_name: string;
|
||||
email: string;
|
||||
disabled: null | boolean;
|
||||
};
|
||||
|
||||
export const useAuth = () => {
|
||||
const [isAuthenticated, setIsAuthenticated] = useState(false);
|
||||
const [user, setUser] = useState<User | null>(null);
|
||||
|
||||
useEffect(() => {
|
||||
const fetchUserInfo = async () => {
|
||||
try {
|
||||
const response = await axios.get("/api/auth/users/me", {
|
||||
headers: {
|
||||
Authorization: `Bearer ${localStorage.getItem("token")}`,
|
||||
},
|
||||
});
|
||||
setUser(response.data);
|
||||
setIsAuthenticated(true);
|
||||
} catch (error) {
|
||||
setIsAuthenticated(false);
|
||||
setUser(null);
|
||||
}
|
||||
};
|
||||
|
||||
const token = localStorage.getItem("token");
|
||||
if (token) {
|
||||
fetchUserInfo();
|
||||
}
|
||||
}, []);
|
||||
|
||||
const login = () => {
|
||||
window.location.href = "http://localhost:8000/api/auth/login";
|
||||
};
|
||||
|
||||
const logout = () => {
|
||||
localStorage.removeItem("token");
|
||||
setIsAuthenticated(false);
|
||||
setUser(null);
|
||||
};
|
||||
|
||||
return {
|
||||
isAuthenticated,
|
||||
user,
|
||||
login,
|
||||
logout,
|
||||
};
|
||||
};
|
||||
@@ -1,19 +0,0 @@
|
||||
import { initAuth0 } from "@auth0/nextjs-auth0";
|
||||
|
||||
const secret = process.env.AUTH0_SECRET;
|
||||
|
||||
if (!secret) {
|
||||
throw new Error("Secret not found");
|
||||
}
|
||||
|
||||
export default initAuth0({
|
||||
secret: process.env.AUTH0_SECRET as string,
|
||||
baseURL: process.env.AUTH0_BASE_URL as string,
|
||||
issuerBaseURL: process.env.AUTH0_ISSUER_BASE_URL as string,
|
||||
clientID: process.env.AUTH0_CLIENT_ID as string,
|
||||
clientSecret: process.env.AUTH0_CLIENT_SECRET as string,
|
||||
routes: {
|
||||
callback: "/auth/callback",
|
||||
postLogoutRedirect: "/",
|
||||
},
|
||||
});
|
||||
+4
-17
@@ -4,32 +4,19 @@ import "../styles/globals.css";
|
||||
import React from "react";
|
||||
import type { AppProps } from "next/app";
|
||||
import Head from "next/head";
|
||||
import { Auth0Provider } from "@auth0/auth0-react";
|
||||
import { SessionProvider } from "next-auth/react";
|
||||
import NavDrawer from "../components/NavDrawer";
|
||||
|
||||
const domain = process.env.NEXT_PUBLIC_AUTH0_ISSUER_BASE_URL || "";
|
||||
const clientId = process.env.NEXT_PUBLIC_AUTH0_CLIENT_ID || "";
|
||||
|
||||
console.log(domain);
|
||||
|
||||
const App: React.FC<AppProps> = ({ Component, pageProps }) => {
|
||||
return (
|
||||
<>
|
||||
<Head>
|
||||
<title>Webapp Template</title>
|
||||
</Head>
|
||||
<Auth0Provider
|
||||
domain={domain}
|
||||
clientId={clientId}
|
||||
authorizationParams={{
|
||||
redirect_uri: "http://localhost",
|
||||
}}
|
||||
cacheLocation="localstorage"
|
||||
useRefreshTokens={true}
|
||||
>
|
||||
<NavDrawer></NavDrawer>
|
||||
<SessionProvider session={pageProps.session}>
|
||||
<NavDrawer />
|
||||
<Component {...pageProps} />
|
||||
</Auth0Provider>
|
||||
</SessionProvider>
|
||||
</>
|
||||
);
|
||||
};
|
||||
|
||||
+124
-48
@@ -1,4 +1,4 @@
|
||||
import React, { useState, useEffect } from "react";
|
||||
import React, { useState, useEffect, useRef } from "react";
|
||||
import {
|
||||
Typography,
|
||||
TextField,
|
||||
@@ -10,9 +10,11 @@ import {
|
||||
TableRow,
|
||||
Container,
|
||||
Box,
|
||||
IconButton,
|
||||
Tooltip,
|
||||
} from "@mui/material";
|
||||
import AddIcon from "@mui/icons-material/Add";
|
||||
import { useAuth0 } from "@auth0/auth0-react";
|
||||
import { useAuth } from "../hooks/useAuth";
|
||||
import { useRouter } from "next/router";
|
||||
|
||||
interface Element {
|
||||
@@ -31,13 +33,24 @@ type Result = {
|
||||
[key: string]: ScrapeResult[];
|
||||
};
|
||||
|
||||
function validateURL(url: string): boolean {
|
||||
try {
|
||||
new URL(url);
|
||||
return true;
|
||||
} catch (_) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
const Home = () => {
|
||||
const { user } = useAuth0();
|
||||
const { user } = useAuth();
|
||||
const router = useRouter();
|
||||
|
||||
const { elements, url } = router.query;
|
||||
|
||||
const [submittedURL, setUrl] = useState("");
|
||||
const [submittedURL, setSubmittedURL] = useState("");
|
||||
const [isValidURL, setIsValidUrl] = useState<boolean>(true);
|
||||
const [urlError, setUrlError] = useState<string | null>(null);
|
||||
const [rows, setRows] = useState<Element[]>([]);
|
||||
const [results, setResults] = useState<null | Result>(null);
|
||||
const [newRow, setNewRow] = useState<Element>({
|
||||
@@ -46,29 +59,54 @@ const Home = () => {
|
||||
url: "",
|
||||
});
|
||||
|
||||
const resultsRef = useRef<HTMLTableElement | null>(null);
|
||||
|
||||
useEffect(() => {
|
||||
if (elements) {
|
||||
setRows(JSON.parse(elements as string));
|
||||
}
|
||||
if (url) {
|
||||
setUrl(url as string);
|
||||
setSubmittedURL(url as string);
|
||||
}
|
||||
}, [elements, url]);
|
||||
|
||||
useEffect(() => {
|
||||
if (results && resultsRef.current) {
|
||||
resultsRef.current.scrollIntoView({ behavior: "smooth" });
|
||||
}
|
||||
}, [results]);
|
||||
|
||||
const handleAddRow = () => {
|
||||
newRow.url = submittedURL;
|
||||
setRows([...rows, newRow]);
|
||||
const updatedRow = { ...newRow, url: submittedURL };
|
||||
setRows([...rows, updatedRow]);
|
||||
setNewRow({ name: "", xpath: "", url: "" });
|
||||
};
|
||||
|
||||
const handleDeleteRow = (elementName: string) => {
|
||||
setRows(
|
||||
rows.filter((r) => {
|
||||
return elementName !== r.name;
|
||||
}),
|
||||
);
|
||||
};
|
||||
|
||||
const handleSubmit = () => {
|
||||
if (!validateURL(submittedURL)) {
|
||||
setIsValidUrl(false);
|
||||
setUrlError("Please enter a valid URL.");
|
||||
return;
|
||||
}
|
||||
|
||||
setIsValidUrl(true);
|
||||
setUrlError(null);
|
||||
|
||||
fetch("/api/submit-scrape-job", {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
url: url,
|
||||
url: submittedURL,
|
||||
elements: rows,
|
||||
user: user?.name,
|
||||
user: user?.email,
|
||||
time_created: new Date().toISOString(),
|
||||
}),
|
||||
})
|
||||
@@ -77,25 +115,43 @@ const Home = () => {
|
||||
};
|
||||
|
||||
return (
|
||||
<>
|
||||
<Box
|
||||
display="flex"
|
||||
flexDirection="column"
|
||||
justifyContent="center"
|
||||
alignItems="center"
|
||||
minHeight="100vh"
|
||||
py={4}
|
||||
>
|
||||
<Container maxWidth="md">
|
||||
<Typography variant="h1" gutterBottom>
|
||||
<Typography variant="h1" gutterBottom textAlign="center">
|
||||
Web Scraper
|
||||
</Typography>
|
||||
<div style={{ marginBottom: "20px" }}>
|
||||
<div
|
||||
style={{ marginBottom: "20px" }}
|
||||
className="flex flex-row space-x-4 items-center"
|
||||
>
|
||||
<TextField
|
||||
label="URL"
|
||||
variant="outlined"
|
||||
fullWidth
|
||||
value={url}
|
||||
onChange={(e) => setUrl(e.target.value)}
|
||||
style={{ marginBottom: "10px" }}
|
||||
value={submittedURL}
|
||||
onChange={(e) => setSubmittedURL(e.target.value)}
|
||||
error={!isValidURL}
|
||||
helperText={!isValidURL ? urlError : ""}
|
||||
/>
|
||||
<Button variant="contained" color="primary" onClick={handleSubmit}>
|
||||
<Button
|
||||
className="!text-black"
|
||||
variant="contained"
|
||||
color="primary"
|
||||
size="small"
|
||||
onClick={handleSubmit}
|
||||
disabled={!(rows.length > 0)}
|
||||
>
|
||||
Submit
|
||||
</Button>
|
||||
</div>
|
||||
<Box display="flex" gap={2} marginBottom={2}>
|
||||
<Box display="flex" gap={2} marginBottom={2} className="items-center">
|
||||
<TextField
|
||||
label="Name"
|
||||
variant="outlined"
|
||||
@@ -110,17 +166,29 @@ const Home = () => {
|
||||
value={newRow.xpath}
|
||||
onChange={(e) => setNewRow({ ...newRow, xpath: e.target.value })}
|
||||
/>
|
||||
<Button
|
||||
variant="contained"
|
||||
color="secondary"
|
||||
startIcon={<AddIcon />}
|
||||
onClick={handleAddRow}
|
||||
<Tooltip
|
||||
title={
|
||||
newRow.xpath.length > 0 && newRow.name.length > 0
|
||||
? "Add Element"
|
||||
: "Fill out all fields to add an element"
|
||||
}
|
||||
placement="top"
|
||||
>
|
||||
Add Elements
|
||||
</Button>
|
||||
<span>
|
||||
<IconButton
|
||||
aria-label="add"
|
||||
size="small"
|
||||
onClick={handleAddRow}
|
||||
sx={{ height: "40px", width: "40px" }}
|
||||
disabled={!(newRow.xpath.length > 0 && newRow.name.length > 0)}
|
||||
>
|
||||
<AddIcon fontSize="inherit" sx={{ color: "black" }} />
|
||||
</IconButton>
|
||||
</span>
|
||||
</Tooltip>
|
||||
</Box>
|
||||
<Typography variant="h4">Elements</Typography>
|
||||
<Table>
|
||||
<Table className="mb-4">
|
||||
<TableHead>
|
||||
<TableRow>
|
||||
<TableCell>Name</TableCell>
|
||||
@@ -136,36 +204,44 @@ const Home = () => {
|
||||
<TableCell>
|
||||
<TextField variant="outlined" fullWidth value={row.xpath} />
|
||||
</TableCell>
|
||||
<TableCell>
|
||||
<Button onClick={() => handleDeleteRow(row.name)}>
|
||||
Delete
|
||||
</Button>
|
||||
</TableCell>
|
||||
</TableRow>
|
||||
))}
|
||||
</TableBody>
|
||||
</Table>
|
||||
{results && (
|
||||
<Table style={{ marginTop: "20px" }}>
|
||||
<TableHead>
|
||||
<TableRow>
|
||||
<TableCell>Name</TableCell>
|
||||
<TableCell>XPath</TableCell>
|
||||
<TableCell>Text</TableCell>
|
||||
</TableRow>
|
||||
</TableHead>
|
||||
<TableBody>
|
||||
{Object.keys(results).map((key, index) => (
|
||||
<React.Fragment key={index}>
|
||||
{results[key].map((result, resultIndex) => (
|
||||
<TableRow key={resultIndex}>
|
||||
<TableCell>{result.name}</TableCell>
|
||||
<TableCell>{result.xpath}</TableCell>
|
||||
<TableCell>{result.text}</TableCell>
|
||||
</TableRow>
|
||||
))}
|
||||
</React.Fragment>
|
||||
))}
|
||||
</TableBody>
|
||||
</Table>
|
||||
<>
|
||||
<Typography variant="h4">Results</Typography>
|
||||
<Table ref={resultsRef} style={{ marginTop: "20px" }}>
|
||||
<TableHead>
|
||||
<TableRow>
|
||||
<TableCell>Name</TableCell>
|
||||
<TableCell>XPath</TableCell>
|
||||
<TableCell>Text</TableCell>
|
||||
</TableRow>
|
||||
</TableHead>
|
||||
<TableBody>
|
||||
{Object.keys(results).map((key, index) => (
|
||||
<React.Fragment key={index}>
|
||||
{results[key].map((result, resultIndex) => (
|
||||
<TableRow key={resultIndex}>
|
||||
<TableCell>{result.name}</TableCell>
|
||||
<TableCell>{result.xpath}</TableCell>
|
||||
<TableCell>{result.text}</TableCell>
|
||||
</TableRow>
|
||||
))}
|
||||
</React.Fragment>
|
||||
))}
|
||||
</TableBody>
|
||||
</Table>
|
||||
</>
|
||||
)}
|
||||
</Container>
|
||||
</>
|
||||
</Box>
|
||||
);
|
||||
};
|
||||
|
||||
|
||||
+13
-15
@@ -1,26 +1,24 @@
|
||||
import { useAuth0 } from "@auth0/auth0-react";
|
||||
import React, { useEffect, useState } from "react";
|
||||
import JobTable from "../components/JobTable";
|
||||
import { useAuth } from "../hooks/useAuth";
|
||||
|
||||
const Jobs = () => {
|
||||
const { user } = useAuth0();
|
||||
const { user } = useAuth();
|
||||
const [jobs, setJobs] = useState([]);
|
||||
|
||||
useEffect(() => {
|
||||
fetch("/api/retrieve-scrape-jobs", {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({ user: user?.name }),
|
||||
})
|
||||
.then((response) => response.json())
|
||||
.then((data) => setJobs(data));
|
||||
}, []);
|
||||
if (user) {
|
||||
fetch("/api/retrieve-scrape-jobs", {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({ user: user?.email }),
|
||||
})
|
||||
.then((response) => response.json())
|
||||
.then((data) => setJobs(data));
|
||||
}
|
||||
}, [user]);
|
||||
|
||||
return (
|
||||
<>
|
||||
<JobTable jobs={jobs} />
|
||||
</>
|
||||
);
|
||||
return <JobTable jobs={jobs} />;
|
||||
};
|
||||
|
||||
export default Jobs;
|
||||
|
||||
@@ -0,0 +1,122 @@
|
||||
import React, { useState } from "react";
|
||||
import axios from "axios";
|
||||
import { Button, TextField, Typography, Container, Box } from "@mui/material";
|
||||
|
||||
type Mode = "login" | "signup";
|
||||
|
||||
const AuthForm: React.FC = () => {
|
||||
const [mode, setMode] = useState<Mode>("login");
|
||||
const [email, setEmail] = useState<string>("");
|
||||
const [password, setPassword] = useState<string>("");
|
||||
const [fullName, setFullName] = useState<string>("");
|
||||
|
||||
const handleSubmit = async (event: React.FormEvent) => {
|
||||
event.preventDefault();
|
||||
try {
|
||||
if (mode === "login") {
|
||||
const params = new URLSearchParams();
|
||||
params.append("username", email);
|
||||
params.append("password", password);
|
||||
const response = await axios.post(
|
||||
"http://localhost:8000/api/auth/token",
|
||||
params,
|
||||
);
|
||||
localStorage.setItem("token", response.data.access_token);
|
||||
alert("Login successful");
|
||||
} else {
|
||||
await axios.post("http://localhost:8000/api/auth/signup", {
|
||||
email: email,
|
||||
password: password,
|
||||
full_name: fullName,
|
||||
});
|
||||
alert("Signup successful");
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(error);
|
||||
alert(`${mode.charAt(0).toUpperCase() + mode.slice(1)} failed`);
|
||||
}
|
||||
};
|
||||
|
||||
const toggleMode = () => {
|
||||
setMode((prevMode) => (prevMode === "login" ? "signup" : "login"));
|
||||
};
|
||||
|
||||
return (
|
||||
<Container component="main" maxWidth="xs">
|
||||
<Box
|
||||
sx={{
|
||||
marginTop: 8,
|
||||
display: "flex",
|
||||
flexDirection: "column",
|
||||
alignItems: "center",
|
||||
}}
|
||||
>
|
||||
<Typography component="h1" variant="h5">
|
||||
{mode.charAt(0).toUpperCase() + mode.slice(1)}
|
||||
</Typography>
|
||||
<Box component="form" onSubmit={handleSubmit} sx={{ mt: 3 }}>
|
||||
<TextField
|
||||
variant="outlined"
|
||||
margin="normal"
|
||||
required
|
||||
fullWidth
|
||||
id="email"
|
||||
label="Email Address"
|
||||
name="email"
|
||||
autoComplete="email"
|
||||
autoFocus
|
||||
value={email}
|
||||
onChange={(e) => setEmail(e.target.value)}
|
||||
/>
|
||||
<TextField
|
||||
variant="outlined"
|
||||
margin="normal"
|
||||
required
|
||||
fullWidth
|
||||
name="password"
|
||||
label="Password"
|
||||
type="password"
|
||||
id="password"
|
||||
autoComplete="current-password"
|
||||
value={password}
|
||||
onChange={(e) => setPassword(e.target.value)}
|
||||
/>
|
||||
{mode === "signup" && (
|
||||
<TextField
|
||||
variant="outlined"
|
||||
margin="normal"
|
||||
required
|
||||
fullWidth
|
||||
name="fullName"
|
||||
label="Full Name"
|
||||
type="text"
|
||||
id="fullName"
|
||||
autoComplete="name"
|
||||
value={fullName}
|
||||
onChange={(e) => setFullName(e.target.value)}
|
||||
/>
|
||||
)}
|
||||
<Button
|
||||
type="submit"
|
||||
fullWidth
|
||||
variant="contained"
|
||||
color="primary"
|
||||
sx={{ mt: 3, mb: 2 }}
|
||||
>
|
||||
{mode.charAt(0).toUpperCase() + mode.slice(1)}
|
||||
</Button>
|
||||
<Button
|
||||
onClick={toggleMode}
|
||||
fullWidth
|
||||
variant="outlined"
|
||||
color="secondary"
|
||||
>
|
||||
Switch to {mode === "login" ? "Signup" : "Login"}
|
||||
</Button>
|
||||
</Box>
|
||||
</Box>
|
||||
</Container>
|
||||
);
|
||||
};
|
||||
|
||||
export default AuthForm;
|
||||
@@ -1,13 +0,0 @@
|
||||
import { useAuth0 } from "@auth0/auth0-react";
|
||||
|
||||
export const useAuth = () => {
|
||||
const { loginWithRedirect, logout, user, isAuthenticated, isLoading } =
|
||||
useAuth0();
|
||||
return {
|
||||
loginWithRedirect,
|
||||
logout,
|
||||
user,
|
||||
isAuthenticated,
|
||||
isLoading,
|
||||
};
|
||||
};
|
||||
Reference in New Issue
Block a user