3 Commits

Author SHA1 Message Date
Jayden Pyles
8703f706a1 feat: add in optional registration (#65)
* feat: add in optional registration

* fix: issue with registration var

* fix: issue with registration var

* fix: issue with registration var
2025-05-11 11:11:19 -05:00
Jayden Pyles
b40d378bbf fix: chat jobs not loading (#64)
Some checks failed
Unit Tests / unit-tests (push) Has been cancelled
Unit Tests / cypress-tests (push) Has been cancelled
Unit Tests / success-message (push) Has been cancelled
2025-05-10 18:34:42 -05:00
Jayden Pyles
8123e1f149 docs: update README [skip ci] 2025-05-10 15:24:16 -05:00
10 changed files with 444 additions and 361 deletions

View File

@@ -24,6 +24,7 @@ View the [docs](https://scraperr-docs.pages.dev) for a quickstart guide and more
- Scrape all pages within same domain
- Add custom json headers to send in requests to URLs
- Display results of scraped data
- Download media found on the page (images, videos, etc.)
![main_page](https://github.com/jaypyles/www-scrape/blob/master/docs/main_page.png)

View File

@@ -67,4 +67,4 @@ async def ai(c: AI):
@ai_router.get("/ai/check")
async def check():
return JSONResponse(content=bool(open_ai_key or llama_model))
return JSONResponse(content={"ai_enabled": bool(open_ai_key or llama_model)})

View File

@@ -1,5 +1,6 @@
# STL
from datetime import timedelta
import os
# PDM
from fastapi import Depends, APIRouter, HTTPException, status
@@ -61,3 +62,8 @@ async def create_user(user: UserCreate):
@auth_router.get("/auth/users/me", response_model=User)
async def read_users_me(current_user: User = Depends(get_current_user)):
return current_user
@auth_router.get("/auth/check")
async def check_auth():
return {"registration": os.environ.get("REGISTRATION_ENABLED", "True") == "True"}

View File

@@ -1,6 +1,9 @@
import os
from api.backend.database.common import connect, QUERIES
import logging
from api.backend.auth.auth_utils import get_password_hash
LOG = logging.getLogger(__name__)
@@ -12,4 +15,29 @@ def init_database():
LOG.info(f"Executing query: {query}")
_ = cursor.execute(query)
if os.environ.get("REGISTRATION_ENABLED", "True") == "False":
default_user_email = os.environ.get("DEFAULT_USER_EMAIL")
default_user_password = os.environ.get("DEFAULT_USER_PASSWORD")
default_user_full_name = os.environ.get("DEFAULT_USER_FULL_NAME")
if (
not default_user_email
or not default_user_password
or not default_user_full_name
):
LOG.error(
"DEFAULT_USER_EMAIL, DEFAULT_USER_PASSWORD, or DEFAULT_USER_FULL_NAME is not set!"
)
exit(1)
query = "INSERT INTO users (email, hashed_password, full_name) VALUES (?, ?, ?)"
_ = cursor.execute(
query,
(
default_user_email,
get_password_hash(default_user_password),
default_user_full_name,
),
)
cursor.close()

View File

@@ -28,10 +28,6 @@ export const JobSelector = ({
const [popoverJob, setPopoverJob] = useState<Job | null>(null);
const theme = useTheme();
useEffect(() => {
fetchJobs(setJobs, { chat: true });
}, []);
const handlePopoverOpen = (
event: React.MouseEvent<HTMLElement>,
job: Job
@@ -124,7 +120,9 @@ export const JobSelector = ({
fontStyle: "italic",
}}
>
{new Date(popoverJob.time_created).toLocaleString()}
{popoverJob.time_created
? new Date(popoverJob.time_created).toLocaleString()
: "Unknown"}
</Typography>
</div>
</Box>

View File

@@ -0,0 +1,351 @@
import React, { useEffect, useRef, useState } from "react";
import {
Box,
TextField,
Typography,
Paper,
useTheme,
IconButton,
Tooltip,
} from "@mui/material";
import { JobSelector } from "../../ai";
import { Job, Message } from "../../../types";
import { useSearchParams } from "next/navigation";
import { checkAI, fetchJob, fetchJobs, updateJob } from "../../../lib";
import SendIcon from "@mui/icons-material/Send";
import EditNoteIcon from "@mui/icons-material/EditNote";
export const AI: React.FC = () => {
const theme = useTheme();
const [currentMessage, setCurrentMessage] = useState<string>("");
const [selectedJob, setSelectedJob] = useState<Job | null>(null);
const [messages, setMessages] = useState<Message[]>([]);
const [aiEnabled, setAiEnabled] = useState<boolean>(false);
const [jobs, setJobs] = useState<Job[]>([]);
const [thinking, setThinking] = useState<boolean>(false);
const searchParams = useSearchParams();
const getJobFromParam = async () => {
const jobId = searchParams.get("job");
if (jobId) {
const job = await fetchJob(jobId);
if (job.length) {
setSelectedJob(job[0]);
if (job[0].chat) {
setMessages(job[0].chat);
}
}
}
};
useEffect(() => {
checkAI(setAiEnabled);
getJobFromParam();
}, []);
useEffect(() => {
if (selectedJob?.chat) {
setMessages(selectedJob?.chat);
return;
}
setMessages([]);
}, [selectedJob]);
const handleMessageSend = async (msg: string) => {
if (!selectedJob) {
throw Error("Job is not currently selected, but should be.");
}
const updatedMessages = await sendMessage(msg);
await updateJob([selectedJob?.id], "chat", updatedMessages);
};
const sendMessage = async (msg: string) => {
const newMessage = {
content: msg,
role: "user",
};
setMessages((prevMessages) => [...prevMessages, newMessage]);
setCurrentMessage("");
setThinking(true);
const jobMessage = {
role: "system",
content: `Here is the content return from a scraping job: ${JSON.stringify(
selectedJob?.result
)} for the url: ${
selectedJob?.url
}. The following messages will pertain to the content of the scraped job.`,
};
const response = await fetch("/api/ai", {
method: "POST",
headers: {
"Content-Type": "application/json",
},
body: JSON.stringify({
data: { messages: [jobMessage, ...messages, newMessage] },
}),
});
const updatedMessages = [...messages, newMessage];
const reader = response.body?.getReader();
const decoder = new TextDecoder("utf-8");
let aiResponse = "";
if (reader) {
setThinking(false);
while (true) {
const { done, value } = await reader.read();
if (done) break;
const chunk = decoder.decode(value, { stream: true });
aiResponse += chunk;
setMessages((prevMessages) => {
const lastMessage = prevMessages[prevMessages.length - 1];
if (lastMessage && lastMessage.role === "assistant") {
return [
...prevMessages.slice(0, -1),
{ ...lastMessage, content: aiResponse },
];
} else {
return [
...prevMessages,
{
content: aiResponse,
role: "assistant",
},
];
}
});
}
}
return [...updatedMessages, { role: "assistant", content: aiResponse }];
};
const handleNewChat = (selectedJob: Job) => {
updateJob([selectedJob.id], "chat", []);
setMessages([]);
};
useEffect(() => {
fetchJobs(setJobs);
}, []);
return (
<Box
sx={{
display: "flex",
flexDirection: "column",
height: "95vh",
maxWidth: "100%",
paddingLeft: 0,
paddingRight: 0,
borderRadius: "8px",
border:
theme.palette.mode === "light" ? "solid white" : "solid #4b5057",
boxShadow: "0 4px 8px rgba(0, 0, 0, 0.1)",
overflow: "hidden",
}}
>
{aiEnabled ? (
<>
<Paper
elevation={3}
sx={{
p: 2,
textAlign: "center",
fontSize: "1.2em",
position: "relative",
borderRadius: "8px 8px 0 0",
borderBottom: `2px solid ${theme.palette.divider}`,
}}
>
<Box
sx={{
display: "flex",
justifyContent: "center",
alignItems: "center",
position: "relative",
padding: theme.spacing(1),
}}
>
<Typography
sx={{
flex: 1,
textAlign: "center",
}}
>
Chat with AI
</Typography>
<JobSelector
selectedJob={selectedJob}
setSelectedJob={setSelectedJob}
setJobs={setJobs}
jobs={jobs}
sxProps={{
position: "absolute",
right: theme.spacing(2),
width: "25%",
}}
/>
</Box>
</Paper>
<Box
sx={{
position: "relative",
flex: 1,
p: 2,
overflowY: "auto",
maxHeight: "100%",
}}
>
{!selectedJob ? (
<Box
sx={{
position: "absolute",
top: 0,
left: "50%",
transform: "translateX(-50%)",
padding: 2,
bgcolor: "rgba(128,128,128,0.1)",
mt: 1,
borderRadius: "8px",
}}
className="rounded-md"
>
<Typography variant="body1">
Select a Job to Begin Chatting
</Typography>
</Box>
) : (
<>
{messages &&
messages.map((message, index) => (
<Box
key={index}
sx={{
my: 2,
p: 1,
borderRadius: "8px",
boxShadow: "0 2px 4px rgba(0, 0, 0, 0.1)",
bgcolor:
message.role === "user"
? theme.palette.UserMessage.main
: theme.palette.AIMessage.main,
marginLeft: message.role === "user" ? "auto" : "",
maxWidth: "40%",
}}
>
<Typography variant="body1" sx={{ color: "white" }}>
{message.content}
</Typography>
</Box>
))}
{thinking && (
<Box
sx={{
width: "full",
display: "flex",
flexDirection: "column",
justifyContent: "start",
}}
>
<Typography
sx={{
bgcolor: "rgba(128,128,128,0.1)",
maxWidth: "20%",
my: 2,
p: 1,
borderRadius: "8px",
boxShadow: "0 2px 4px rgba(0, 0, 0, 0.1)",
}}
variant="body1"
>
AI is thinking...
</Typography>
</Box>
)}
</>
)}
</Box>
<Box
sx={{
display: "flex",
p: 2,
borderTop: `1px solid ${theme.palette.divider}`,
}}
>
<Tooltip title="New Chat" placement="top">
<IconButton
disabled={!(messages.length > 0)}
sx={{ marginRight: 2 }}
size="medium"
onClick={() => {
if (!selectedJob) {
throw new Error("Selected job must be present but isn't.");
}
handleNewChat(selectedJob);
}}
>
<EditNoteIcon fontSize="medium" />
</IconButton>
</Tooltip>
<TextField
fullWidth
placeholder="Type your message here..."
disabled={!selectedJob}
value={currentMessage}
onChange={(e) => setCurrentMessage(e.target.value)}
onKeyDown={(e) => {
if (e.key === "Enter") {
handleMessageSend(currentMessage);
}
}}
sx={{ borderRadius: "8px" }}
/>
<Tooltip title="Send" placement="top">
<IconButton
color="primary"
sx={{ ml: 2 }}
disabled={!selectedJob}
onClick={() => {
handleMessageSend(currentMessage);
}}
>
<SendIcon />
</IconButton>
</Tooltip>
</Box>
</>
) : (
<Box
bgcolor="background.default"
minHeight="100vh"
display="flex"
justifyContent="center"
alignItems="center"
>
<h4
style={{
color: "#fff",
padding: "20px",
borderRadius: "8px",
background: "rgba(0, 0, 0, 0.6)",
boxShadow: "0 4px 8px rgba(0, 0, 0, 0.2)",
}}
>
Must set either OPENAI_KEY or OLLAMA_MODEL to use AI features.
</h4>
</Box>
)}
</Box>
);
};

View File

@@ -48,14 +48,14 @@ export const checkAI = async (
) => {
const token = Cookies.get("token");
try {
const response = await fetch("/api/ai/check", {
const response = await fetch("/api/check", {
headers: {
"content-type": "application/json",
Authorization: `Bearer ${token}`,
},
});
const data = await response.json();
setAiEnabled(data);
setAiEnabled(data.ai_enabled);
} catch (error) {
console.error("Error fetching jobs:", error);
throw error;

View File

@@ -17,12 +17,21 @@ export default async function handler(
}
);
const checksResponse = await fetch(
`${global.process.env.NEXT_PUBLIC_API_URL}/api/auth/check`,
{
method: "GET",
headers,
}
);
if (!response.ok) {
throw new Error(`Error: ${response.statusText}`);
}
const result = await response.json();
res.status(200).json(result);
const checksResult = await checksResponse.json();
res.status(200).json({ ...result, ...checksResult });
} catch (error) {
console.error("Error submitting scrape job:", error);
res.status(500).json({ error: "Internal Server Error" });

View File

@@ -1,348 +1 @@
import React, { useEffect, useRef, useState } from "react";
import {
Box,
TextField,
Typography,
Paper,
useTheme,
IconButton,
Tooltip,
} from "@mui/material";
import { JobSelector } from "../components/ai";
import { Job, Message } from "../types";
import { useSearchParams } from "next/navigation";
import { checkAI, fetchJob, fetchJobs, updateJob } from "../lib";
import SendIcon from "@mui/icons-material/Send";
import EditNoteIcon from "@mui/icons-material/EditNote";
const AI: React.FC = () => {
const theme = useTheme();
const [currentMessage, setCurrentMessage] = useState<string>("");
const [selectedJob, setSelectedJob] = useState<Job | null>(null);
const [messages, setMessages] = useState<Message[]>([]);
const [aiEnabled, setAiEnabled] = useState<boolean>(false);
const [jobs, setJobs] = useState<Job[]>([]);
const [thinking, setThinking] = useState<boolean>(false);
const searchParams = useSearchParams();
const getJobFromParam = async () => {
const jobId = searchParams.get("job");
if (jobId) {
const job = await fetchJob(jobId);
if (job.length) {
setSelectedJob(job[0]);
if (job[0].chat) {
setMessages(job[0].chat);
}
}
}
};
useEffect(() => {
checkAI(setAiEnabled);
getJobFromParam();
}, []);
useEffect(() => {
if (selectedJob?.chat) {
setMessages(selectedJob?.chat);
return;
}
setMessages([]);
}, [selectedJob]);
const handleMessageSend = async (msg: string) => {
if (!selectedJob) {
throw Error("Job is not currently selected, but should be.");
}
const updatedMessages = await sendMessage(msg);
await updateJob([selectedJob?.id], "chat", updatedMessages);
};
const sendMessage = async (msg: string) => {
const newMessage = {
content: msg,
role: "user",
};
setMessages((prevMessages) => [...prevMessages, newMessage]);
setCurrentMessage("");
setThinking(true);
const jobMessage = {
role: "system",
content: `Here is the content return from a scraping job: ${JSON.stringify(
selectedJob?.result
)} for the url: ${
selectedJob?.url
}. The following messages will pertain to the content of the scraped job.`,
};
const response = await fetch("/api/ai", {
method: "POST",
headers: {
"Content-Type": "application/json",
},
body: JSON.stringify({
data: { messages: [jobMessage, ...messages, newMessage] },
}),
});
const updatedMessages = [...messages, newMessage];
const reader = response.body?.getReader();
const decoder = new TextDecoder("utf-8");
let aiResponse = "";
if (reader) {
setThinking(false);
while (true) {
const { done, value } = await reader.read();
if (done) break;
const chunk = decoder.decode(value, { stream: true });
aiResponse += chunk;
setMessages((prevMessages) => {
const lastMessage = prevMessages[prevMessages.length - 1];
if (lastMessage && lastMessage.role === "assistant") {
return [
...prevMessages.slice(0, -1),
{ ...lastMessage, content: aiResponse },
];
} else {
return [
...prevMessages,
{
content: aiResponse,
role: "assistant",
},
];
}
});
}
}
return [...updatedMessages, { role: "assistant", content: aiResponse }];
};
const handleNewChat = (selectedJob: Job) => {
updateJob([selectedJob.id], "chat", []);
setMessages([]);
fetchJobs(setJobs, { chat: true });
};
return (
<Box
sx={{
display: "flex",
flexDirection: "column",
height: "95vh",
maxWidth: "100%",
paddingLeft: 0,
paddingRight: 0,
borderRadius: "8px",
border:
theme.palette.mode === "light" ? "solid white" : "solid #4b5057",
boxShadow: "0 4px 8px rgba(0, 0, 0, 0.1)",
overflow: "hidden",
}}
>
{aiEnabled ? (
<>
<Paper
elevation={3}
sx={{
p: 2,
textAlign: "center",
fontSize: "1.2em",
position: "relative",
borderRadius: "8px 8px 0 0",
borderBottom: `2px solid ${theme.palette.divider}`,
}}
>
<Box
sx={{
display: "flex",
justifyContent: "center",
alignItems: "center",
position: "relative",
padding: theme.spacing(1),
}}
>
<Typography
sx={{
flex: 1,
textAlign: "center",
}}
>
Chat with AI
</Typography>
<JobSelector
selectedJob={selectedJob}
setSelectedJob={setSelectedJob}
setJobs={setJobs}
jobs={jobs}
sxProps={{
position: "absolute",
right: theme.spacing(2),
width: "25%",
}}
/>
</Box>
</Paper>
<Box
sx={{
position: "relative",
flex: 1,
p: 2,
overflowY: "auto",
maxHeight: "100%",
}}
>
{!selectedJob ? (
<Box
sx={{
position: "absolute",
top: 0,
left: "50%",
transform: "translateX(-50%)",
padding: 2,
bgcolor: "rgba(128,128,128,0.1)",
mt: 1,
borderRadius: "8px",
}}
className="rounded-md"
>
<Typography variant="body1">
Select a Job to Begin Chatting
</Typography>
</Box>
) : (
<>
{messages &&
messages.map((message, index) => (
<Box
key={index}
sx={{
my: 2,
p: 1,
borderRadius: "8px",
boxShadow: "0 2px 4px rgba(0, 0, 0, 0.1)",
bgcolor:
message.role === "user"
? theme.palette.UserMessage.main
: theme.palette.AIMessage.main,
marginLeft: message.role === "user" ? "auto" : "",
maxWidth: "40%",
}}
>
<Typography variant="body1" sx={{ color: "white" }}>
{message.content}
</Typography>
</Box>
))}
{thinking && (
<Box
sx={{
width: "full",
display: "flex",
flexDirection: "column",
justifyContent: "start",
}}
>
<Typography
sx={{
bgcolor: "rgba(128,128,128,0.1)",
maxWidth: "20%",
my: 2,
p: 1,
borderRadius: "8px",
boxShadow: "0 2px 4px rgba(0, 0, 0, 0.1)",
}}
variant="body1"
>
AI is thinking...
</Typography>
</Box>
)}
</>
)}
</Box>
<Box
sx={{
display: "flex",
p: 2,
borderTop: `1px solid ${theme.palette.divider}`,
}}
>
<Tooltip title="New Chat" placement="top">
<IconButton
disabled={!(messages.length > 0)}
sx={{ marginRight: 2 }}
size="medium"
onClick={() => {
if (!selectedJob) {
throw new Error("Selected job must be present but isn't.");
}
handleNewChat(selectedJob);
}}
>
<EditNoteIcon fontSize="medium" />
</IconButton>
</Tooltip>
<TextField
fullWidth
placeholder="Type your message here..."
disabled={!selectedJob}
value={currentMessage}
onChange={(e) => setCurrentMessage(e.target.value)}
onKeyDown={(e) => {
if (e.key === "Enter") {
handleMessageSend(currentMessage);
}
}}
sx={{ borderRadius: "8px" }}
/>
<Tooltip title="Send" placement="top">
<IconButton
color="primary"
sx={{ ml: 2 }}
disabled={!selectedJob}
onClick={() => {
handleMessageSend(currentMessage);
}}
>
<SendIcon />
</IconButton>
</Tooltip>
</Box>
</>
) : (
<Box
bgcolor="background.default"
minHeight="100vh"
display="flex"
justifyContent="center"
alignItems="center"
>
<h4
style={{
color: "#fff",
padding: "20px",
borderRadius: "8px",
background: "rgba(0, 0, 0, 0.6)",
boxShadow: "0 4px 8px rgba(0, 0, 0, 0.2)",
}}
>
Must set either OPENAI_KEY or OLLAMA_MODEL to use AI features.
</h4>
</Box>
)}
</Box>
);
};
export default AI;
export { AI as default } from "../components/pages/chat/chat";

View File

@@ -1,6 +1,6 @@
"use client";
import React, { useState } from "react";
import React, { useEffect, useState } from "react";
import axios from "axios";
import { Button, TextField, Typography, Box } from "@mui/material";
import { useTheme } from "@mui/material/styles";
@@ -18,7 +18,16 @@ const AuthForm: React.FC = () => {
const theme = useTheme();
const router = useRouter();
const { login } = useAuth();
const [registrationEnabled, setRegistrationEnabled] = useState<boolean>(true);
const checkRegistrationEnabled = async () => {
const response = await axios.get(`/api/check`);
setRegistrationEnabled(response.data.registration);
};
useEffect(() => {
checkRegistrationEnabled();
}, []);
const handleSubmit = async (event: React.FormEvent) => {
event.preventDefault();
try {
@@ -124,9 +133,37 @@ const AuthForm: React.FC = () => {
>
{mode.charAt(0).toUpperCase() + mode.slice(1)}
</Button>
<Button onClick={toggleMode} fullWidth variant="text" color="primary">
{mode === "login" ? "No Account? Sign up" : "Login"}
</Button>
{registrationEnabled && (
<Button
onClick={toggleMode}
fullWidth
variant="text"
color="primary"
>
{mode === "login" ? "No Account? Sign up" : "Login"}
</Button>
)}
{!registrationEnabled && (
<div
style={{
marginTop: 10,
width: "100%",
textAlign: "center",
border: "1px solid #ccc",
backgroundColor: "#f8f8f8",
padding: 8,
borderRadius: 4,
display: "flex",
justifyContent: "center",
alignItems: "center",
}}
>
<Typography variant="body2" color="text">
Registration has been disabled
</Typography>
</div>
)}
</Box>
</Box>
</Box>