lee/Scraperr
1
0
Fork 0
mirror of https://github.com/jaypyles/Scraperr.git synced 2026-05-03 07:50:41 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

wip: use ssr

Browse Source
This commit is contained in:
Jayden Pyles
2024-07-23 14:25:53 -05:00
parent c396746587
commit ffced1ebf2
19 changed files with 148 additions and 131 deletions
Download Patch File Download Diff File Expand all files Collapse all files
api/backend/app.py
+8 -11
View File
@@ -3,9 +3,6 @@ import uuid
import logging
from io import BytesIO
from openpyxl import Workbook
from typing import Any
from datetime import datetime
from bson import ObjectId
# PDM
from fastapi import BackgroundTasks, FastAPI, HTTPException
@@ -56,18 +53,18 @@ app.add_middleware(
allow_headers=["*"],
)
app.mount("/_next/static", StaticFiles(directory="./dist/_next/static"), name="static")
app.mount("/images", StaticFiles(directory="./dist/images"), name="images")
# app.mount("/_next/static", StaticFiles(directory="./dist/_next/static"), name="static")
# app.mount("/images", StaticFiles(directory="./dist/images"), name="images")
@app.get("/")
def read_root():
return FileResponse("./dist/index.html")
# @app.get("/")
# def read_root():
# return FileResponse("./dist/index.html")
@app.get("/favicon.ico")
def read_favicon():
return FileResponse("dist/favicon.ico")
# @app.get("/favicon.ico")
# def read_favicon():
# return FileResponse("dist/favicon.ico")
@app.post("/api/update")
next.config.mjs
+2 -7
View File
@@ -3,13 +3,8 @@ const nextConfig = {
output: "export",
distDir: "./dist",
images: { unoptimized: true },
async rewrites() {
return [
{
source: "/auth/:path*",
destination: "/api/auth/:path*",
},
];
env: {
DOMAIN: "http://localhost:8000",
},
};
package-lock.json
Generated
+24 -11
View File
@@ -22,6 +22,7 @@
"axios": "^1.7.2",
"bootstrap": "^5.3.0",
"chart.js": "^4.4.3",
"cookie": "^0.6.0",
"framer-motion": "^4.1.17",
"next": "^14.2.4",
"next-auth": "^4.24.7",
@@ -115,14 +116,6 @@
"next": ">=10"
}
},
"node_modules/@auth0/nextjs-auth0/node_modules/cookie": {
"version": "0.6.0",
"resolved": "https://registry.npmjs.org/cookie/-/cookie-0.6.0.tgz",
"integrity": "sha512-U71cyTamuh1CRNCfpGY6to28lxvNwPG4Guz/EVjgf3Jmzv0vlDp1atT9eS5dDjMYHucpHbWns6Lwf3BKz6svdw==",
"engines": {
"node": ">= 0.6"
}
},
"node_modules/@babel/code-frame": {
"version": "7.23.5",
"resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.23.5.tgz",
@@ -8022,6 +8015,7 @@
"version": "4.4.3",
"resolved": "https://registry.npmjs.org/chart.js/-/chart.js-4.4.3.tgz",
"integrity": "sha512-qK1gkGSRYcJzqrrzdR6a+I0vQ4/R+SoODXyAjscQ/4mzuNzySaMCd+hyVxitSY1+L2fjPD1Gbn+ibNqRmwQeLw==",
"license": "MIT",
"dependencies": {
"@kurkle/color": "^0.3.0"
},
@@ -8340,9 +8334,10 @@
"integrity": "sha512-ASFBup0Mz1uyiIjANan1jzLQami9z1PoYSZCiiYW2FczPbenXc45FZdBZLzOT+r6+iciuEModtmCti+hjaAk0A=="
},
"node_modules/cookie": {
"version": "0.5.0",
"resolved": "https://registry.npmjs.org/cookie/-/cookie-0.5.0.tgz",
"integrity": "sha512-YZ3GUyn/o8gfKJlnlX7g7xq4gyO6OSuhGPKaaGssGB2qgDUS0gPgtTvoyZLTt9Ab6dC4hfc9dV5arkvc/OCmrw==",
"version": "0.6.0",
"resolved": "https://registry.npmjs.org/cookie/-/cookie-0.6.0.tgz",
"integrity": "sha512-U71cyTamuh1CRNCfpGY6to28lxvNwPG4Guz/EVjgf3Jmzv0vlDp1atT9eS5dDjMYHucpHbWns6Lwf3BKz6svdw==",
"license": "MIT",
"engines": {
"node": ">= 0.6"
}
@@ -10274,6 +10269,15 @@
"resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz",
"integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg=="
},
"node_modules/express/node_modules/cookie": {
"version": "0.5.0",
"resolved": "https://registry.npmjs.org/cookie/-/cookie-0.5.0.tgz",
"integrity": "sha512-YZ3GUyn/o8gfKJlnlX7g7xq4gyO6OSuhGPKaaGssGB2qgDUS0gPgtTvoyZLTt9Ab6dC4hfc9dV5arkvc/OCmrw==",
"license": "MIT",
"engines": {
"node": ">= 0.6"
}
},
"node_modules/express/node_modules/debug": {
"version": "2.6.9",
"resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
@@ -15360,6 +15364,15 @@
}
}
},
"node_modules/next-auth/node_modules/cookie": {
"version": "0.5.0",
"resolved": "https://registry.npmjs.org/cookie/-/cookie-0.5.0.tgz",
"integrity": "sha512-YZ3GUyn/o8gfKJlnlX7g7xq4gyO6OSuhGPKaaGssGB2qgDUS0gPgtTvoyZLTt9Ab6dC4hfc9dV5arkvc/OCmrw==",
"license": "MIT",
"engines": {
"node": ">= 0.6"
}
},
"node_modules/no-case": {
"version": "3.0.4",
"resolved": "https://registry.npmjs.org/no-case/-/no-case-3.0.4.tgz",
package.json
+1
View File
@@ -17,6 +17,7 @@
"axios": "^1.7.2",
"bootstrap": "^5.3.0",
"chart.js": "^4.4.3",
"cookie": "^0.6.0",
"framer-motion": "^4.1.17",
"next": "^14.2.4",
"next-auth": "^4.24.7",
src/components/JobTable.tsx
+4 -3
View File
@@ -18,6 +18,7 @@ import StarIcon from "@mui/icons-material/Star";
import { useRouter } from "next/router";
import { Favorites, JobQueue } from "./jobs";
import { Job } from "../types";
import { Constants } from "../lib";
interface JobTableProps {
jobs: Job[];
@@ -45,7 +46,7 @@ const JobTable: React.FC<JobTableProps> = ({ jobs, fetchJobs }) => {
const router = useRouter();
const handleDownload = async (ids: string[]) => {
const response = await fetch("/api/download", {
const response = await fetch(`${Constants.DOMAIN}/api/download`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ ids: ids }),
@@ -100,7 +101,7 @@ const JobTable: React.FC<JobTableProps> = ({ jobs, fetchJobs }) => {
};
const handleDeleteSelected = async () => {
const response = await fetch("/api/delete-scrape-jobs", {
const response = await fetch(`${Constants.DOMAIN}/api/delete-scrape-jobs`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ ids: Array.from(selectedJobs) }),
@@ -130,7 +131,7 @@ const JobTable: React.FC<JobTableProps> = ({ jobs, fetchJobs }) => {
value: value,
};
await fetch("/api/update", {
await fetch(`${Constants.DOMAIN}/api/update`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify(postBody),
src/components/NavDrawer.tsx
+4 -5
View File
@@ -1,3 +1,5 @@
"use client";
import React from "react";
import { useAuth } from "../contexts/AuthContext";
import {
@@ -10,7 +12,6 @@ import {
Typography,
Button,
Switch,
Tooltip,
Drawer,
Divider,
Accordion,
@@ -23,8 +24,6 @@ import ExpandMoreIcon from "@mui/icons-material/ExpandMore";
import TerminalIcon from "@mui/icons-material/Terminal";
import BarChart from "@mui/icons-material/BarChart";
import { useRouter } from "next/router";
import { useTheme } from "@mui/material/styles";
import Image from "next/image";
interface NavDrawerProps {
toggleTheme: () => void;
@@ -35,7 +34,7 @@ const drawerWidth = 240;
const NavDrawer: React.FC<NavDrawerProps> = ({ toggleTheme, isDarkMode }) => {
const router = useRouter();
const theme = useTheme();
const { logout, user, isAuthenticated } = useAuth();
return (
@@ -143,7 +142,7 @@ const NavDrawer: React.FC<NavDrawerProps> = ({ toggleTheme, isDarkMode }) => {
</AccordionSummary>
<AccordionDetails>
<div className="flex flex-row mr-1">
<Typography className="mr-2">
<Typography className="mr-2" component="span">
<p className="text-sm">Dark Theme Toggle</p>
</Typography>
<Switch checked={isDarkMode} onChange={toggleTheme} />
src/components/submit/ElementTable.tsx
+2
View File
@@ -1,3 +1,5 @@
"use client";
import React, { useState, Dispatch, SetStateAction } from "react";
import {
Typography,
src/components/submit/JobSubmitter.tsx
+7 -4
View File
@@ -1,3 +1,5 @@
"use client";
import React, { useEffect, useState, Dispatch } from "react";
import {
TextField,
@@ -7,9 +9,10 @@ import {
FormControlLabel,
CircularProgress,
} from "@mui/material";
import { Element, Result } from "../../types";
import { Element } from "../../types";
import { useAuth } from "../../contexts/AuthContext";
import { useRouter } from "next/router";
import { Constants } from "../../lib";
interface StateProps {
submittedURL: string;
@@ -89,7 +92,7 @@ export const JobSubmitter = ({ stateProps }: Props) => {
return;
}
fetch("/api/submit-scrape-job", {
fetch(`${Constants.DOMAIN}/api/submit-scrape-job`, {
method: "POST",
headers: { "content-type": "application/json" },
body: JSON.stringify({
@@ -112,12 +115,12 @@ export const JobSubmitter = ({ stateProps }: Props) => {
return response.json();
})
.then((data) => {
setSnackbarMessage(data);
setSnackbarMessage(data || "Job submitted successfully.");
setSnackbarSeverity("info");
setSnackbarOpen(true);
})
.catch((error) => {
setSnackbarMessage(error.message || "An error occurred.");
setSnackbarMessage(error || "An error occurred.");
setSnackbarSeverity("error");
setSnackbarOpen(true);
})
src/components/submit/ResultsTable.tsx
-69
View File
@@ -1,69 +0,0 @@
import React, { useState, useEffect, useRef } from "react";
import {
Typography,
Table,
TableBody,
TableContainer,
TableCell,
TableHead,
TableRow,
Box,
} from "@mui/material";
import { Result } from "../../types";
interface stateProps {
results: Result;
}
interface Props {
stateProps: stateProps;
resultsRef: React.MutableRefObject<HTMLElement | null>;
}
export const ResultsTable = ({ stateProps, resultsRef }: Props) => {
const { results } = stateProps;
return (
<>
{Object.keys(results).length ? (
<>
<Typography variant="h4">Results</Typography>
<TableContainer
component={Box}
ref={resultsRef}
sx={{ maxHeight: "50%", overflow: "auto", marginTop: "20px" }}
>
<Table stickyHeader>
<TableHead>
<TableRow>
<TableCell>
<Typography sx={{ fontWeight: "bold" }}>Name</Typography>
</TableCell>
<TableCell>
<Typography sx={{ fontWeight: "bold" }}>XPath</Typography>
</TableCell>
<TableCell>
<Typography sx={{ fontWeight: "bold" }}>Text</Typography>
</TableCell>
</TableRow>
</TableHead>
<TableBody>
{Object.keys(results).map((key, index) => (
<React.Fragment key={index}>
{results[key].map((result, resultIndex) => (
<TableRow key={resultIndex}>
<TableCell>{result.name}</TableCell>
<TableCell>{result.xpath}</TableCell>
<TableCell>{result.text}</TableCell>
</TableRow>
))}
</React.Fragment>
))}
</TableBody>
</Table>
</TableContainer>
</>
) : null}
</>
);
};
src/components/submit/index.ts
-1
View File
@@ -1,3 +1,2 @@
export * from "./ElementTable";
export * from "./JobSubmitter";
export * from "./ResultsTable";
src/constants.ts
-3
View File
@@ -1,3 +0,0 @@
const Constants = {};
export default Constants;
src/contexts/AuthContext.tsx
+16 -6
View File
@@ -1,11 +1,13 @@
import React, { createContext, useContext, useState, useEffect } from "react";
import axios from "axios";
import { Constants } from "../lib";
interface AuthContextProps {
user: any;
isAuthenticated: boolean;
login: (email: string, password: string) => Promise<void>;
logout: () => void;
setUser: (user: any) => void;
}
const AuthContext = createContext<AuthContextProps | undefined>(undefined);
@@ -22,7 +24,7 @@ export const AuthProvider: React.FC<AuthProps> = ({ children }) => {
const token = localStorage.getItem("token");
if (token) {
axios
.get("/api/auth/users/me", {
.get(`${Constants.DOMAIN}/api/auth/users/me`, {
headers: { Authorization: `Bearer ${token}` },
})
.then((response) => {
@@ -39,11 +41,17 @@ export const AuthProvider: React.FC<AuthProps> = ({ children }) => {
const params = new URLSearchParams();
params.append("username", email);
params.append("password", password);
const response = await axios.post("/api/auth/token", params);
const response = await axios.post(
`${Constants.DOMAIN}/api/auth/token`,
params
);
localStorage.setItem("token", response.data.access_token);
const userResponse = await axios.get("/api/auth/users/me", {
headers: { Authorization: `Bearer ${response.data.access_token}` },
});
const userResponse = await axios.get(
`${Constants.DOMAIN}/api/auth/users/me`,
{
headers: { Authorization: `Bearer ${response.data.access_token}` },
}
);
setUser(userResponse.data);
setIsAuthenticated(true);
};
@@ -55,7 +63,9 @@ export const AuthProvider: React.FC<AuthProps> = ({ children }) => {
};
return (
<AuthContext.Provider value={{ user, isAuthenticated, login, logout }}>
<AuthContext.Provider
value={{ user, isAuthenticated, login, logout, setUser }}
>
{children}
</AuthContext.Provider>
);
src/lib/constants.ts
+3
View File
@@ -0,0 +1,3 @@
export const Constants = {
DOMAIN: process.env.DOMAIN,
};
src/lib/index.ts
+1
View File
@@ -0,0 +1 @@
export * from "./constants";
src/pages/_app.tsx
-2
View File
@@ -9,8 +9,6 @@ import NavDrawer from "../components/NavDrawer";
import { darkTheme, lightTheme } from "../styles/themes";
import { AuthProvider } from "../contexts/AuthContext";
const drawerWidth = 240;
const App: React.FC<AppProps> = ({ Component, pageProps }) => {
const [isDarkMode, setIsDarkMode] = useState(false);
src/pages/index.tsx
+4 -2
View File
@@ -1,8 +1,10 @@
"use client";
import React, { useState, useEffect, useRef } from "react";
import { Typography, Container, Box, Snackbar, Alert } from "@mui/material";
import { Container, Box, Snackbar, Alert } from "@mui/material";
import { useRouter } from "next/router";
import { Element, Result } from "../types";
import { ElementTable, JobSubmitter, ResultsTable } from "../components/submit";
import { ElementTable, JobSubmitter } from "../components/submit";
const Home = () => {
const router = useRouter();
src/pages/jobs.tsx
+65 -4
View File
@@ -2,13 +2,29 @@ import React, { useEffect, useState } from "react";
import JobTable from "../components/JobTable";
import { useAuth } from "../contexts/AuthContext";
import { Box } from "@mui/system";
import { Constants } from "../lib";
import { Job } from "../types";
import { GetServerSideProps } from "next";
import axios from "axios";
import { cookies } from "next/headers";
const Jobs = () => {
const { user } = useAuth();
const [jobs, setJobs] = useState([]);
interface JobsProps {
initialJobs: Job[];
initialUser: any;
}
const Jobs: React.FC<JobsProps> = ({ initialJobs, initialUser }) => {
const { user, setUser } = useAuth();
const [jobs, setJobs] = useState<Job[]>(initialJobs || []);
useEffect(() => {
if (!user && initialUser) {
setUser(initialUser);
}
}, [user, initialUser, setUser]);
const fetchJobs = async () => {
await fetch("/api/retrieve-scrape-jobs", {
await fetch(`${Constants.DOMAIN}/api/retrieve-scrape-jobs`, {
method: "POST",
headers: { "content-type": "application/json" },
body: JSON.stringify({ user: user?.email }),
@@ -62,4 +78,49 @@ const Jobs = () => {
);
};
export const getServerSideProps: GetServerSideProps = async (context) => {
const { req, res } = context;
const cookies = cookies.parse(req.headers.cookie || "");
const token = cookies.token;
let user = null;
let initialJobs: Job[] = [];
if (token) {
try {
const userResponse = await axios.get(
`${Constants.DOMAIN}/api/auth/users/me`,
{
headers: { Authorization: `Bearer ${token}` },
}
);
user = userResponse.data;
const jobsResponse = await axios.post(
`${Constants.DOMAIN}/api/retrieve-scrape-jobs`,
{ user: user.email },
{ headers: { "content-type": "application/json" } }
);
initialJobs = jobsResponse.data;
} catch (error) {
console.error("Error fetching user or jobs:", error);
res.setHeader(
"Set-Cookie",
cookies.serialize("token", "", {
maxAge: -1,
path: "/",
})
);
}
}
return {
props: {
initialJobs,
initialUser: user,
},
};
};
export default Jobs;
src/pages/login.tsx
+4 -1
View File
@@ -1,9 +1,12 @@
"use client";
import React, { useState } from "react";
import axios from "axios";
import { Button, TextField, Typography, Box } from "@mui/material";
import { useTheme } from "@mui/material/styles";
import { useRouter } from "next/router";
import { useAuth } from "../contexts/AuthContext";
import { Constants } from "../lib";
type Mode = "login" | "signup";
@@ -24,7 +27,7 @@ const AuthForm: React.FC = () => {
alert("Login successful");
router.push("/");
} else {
await axios.post("/api/auth/signup", {
await axios.post(`${Constants.DOMAIN}/api/auth/signup`, {
email: email,
password: password,
full_name: fullName,
src/pages/statistics.tsx
+3 -2
View File
@@ -2,6 +2,7 @@ import React, { useEffect, useRef, useState } from "react";
import { Chart, registerables } from "chart.js";
import { Box, Typography, useTheme } from "@mui/material";
import { useAuth } from "../contexts/AuthContext";
import { Constants } from "../lib";
Chart.register(...registerables);
@@ -17,7 +18,7 @@ const Statistics: React.FC = () => {
const fetchElementsData = async () => {
try {
const response = await fetch(
"/api/statistics/get-average-element-per-link",
`${Constants.DOMAIN}/api/statistics/get-average-element-per-link`,
{
method: "POST",
headers: { "Content-Type": "application/json" },
@@ -91,7 +92,7 @@ const Statistics: React.FC = () => {
const fetchJobsData = async () => {
try {
const response = await fetch(
"/api/statistics/get-average-jobs-per-day",
`${Constants.DOMAIN}/api/statistics/get-average-jobs-per-day`,
{
method: "POST",
headers: { "Content-Type": "application/json" },