mirror of
https://github.com/jaypyles/Scraperr.git
synced 2025-12-12 10:45:58 +00:00
fix: small fixes to accompany the docs implementation (#35)
This commit is contained in:
@@ -4,6 +4,7 @@ from gc import disable
|
|||||||
from queue import Empty
|
from queue import Empty
|
||||||
from typing import Any, Optional
|
from typing import Any, Optional
|
||||||
from datetime import datetime, timedelta
|
from datetime import datetime, timedelta
|
||||||
|
import logging
|
||||||
|
|
||||||
# PDM
|
# PDM
|
||||||
from jose import JWTError, jwt
|
from jose import JWTError, jwt
|
||||||
@@ -16,6 +17,8 @@ from fastapi.security import OAuth2PasswordBearer
|
|||||||
from api.backend.schemas import User, UserInDB, TokenData
|
from api.backend.schemas import User, UserInDB, TokenData
|
||||||
from api.backend.database import get_user_collection
|
from api.backend.database import get_user_collection
|
||||||
|
|
||||||
|
LOG = logging.getLogger(__name__)
|
||||||
|
|
||||||
_ = load_dotenv()
|
_ = load_dotenv()
|
||||||
|
|
||||||
SECRET_KEY = os.getenv("SECRET_KEY") or ""
|
SECRET_KEY = os.getenv("SECRET_KEY") or ""
|
||||||
@@ -74,10 +77,16 @@ def create_access_token(
|
|||||||
|
|
||||||
|
|
||||||
async def get_current_user(token: str = Depends(oauth2_scheme)):
|
async def get_current_user(token: str = Depends(oauth2_scheme)):
|
||||||
|
LOG.info(f"Getting current user with token: {token}")
|
||||||
|
|
||||||
|
if not token:
|
||||||
|
return EMPTY_USER
|
||||||
|
|
||||||
try:
|
try:
|
||||||
payload: Optional[dict[str, Any]] = jwt.decode(
|
payload: Optional[dict[str, Any]] = jwt.decode(
|
||||||
token, SECRET_KEY, algorithms=[ALGORITHM]
|
token, SECRET_KEY, algorithms=[ALGORITHM]
|
||||||
)
|
)
|
||||||
|
|
||||||
if not payload:
|
if not payload:
|
||||||
return EMPTY_USER
|
return EMPTY_USER
|
||||||
|
|
||||||
@@ -91,6 +100,10 @@ async def get_current_user(token: str = Depends(oauth2_scheme)):
|
|||||||
except JWTError:
|
except JWTError:
|
||||||
return EMPTY_USER
|
return EMPTY_USER
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
LOG.error(f"Exception occurred: {e}")
|
||||||
|
return EMPTY_USER
|
||||||
|
|
||||||
user = await get_user(email=token_data.email)
|
user = await get_user(email=token_data.email)
|
||||||
|
|
||||||
if user is None:
|
if user is None:
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ from io import StringIO
|
|||||||
import csv
|
import csv
|
||||||
import logging
|
import logging
|
||||||
import random
|
import random
|
||||||
|
from typing import Optional
|
||||||
|
|
||||||
# PDM
|
# PDM
|
||||||
from fastapi import Depends, APIRouter
|
from fastapi import Depends, APIRouter
|
||||||
@@ -26,7 +27,7 @@ from api.backend.models import (
|
|||||||
Job,
|
Job,
|
||||||
)
|
)
|
||||||
from api.backend.schemas import User
|
from api.backend.schemas import User
|
||||||
from api.backend.auth.auth_utils import get_current_user
|
from api.backend.auth.auth_utils import get_current_user, EMPTY_USER
|
||||||
from api.backend.utils import clean_text
|
from api.backend.utils import clean_text
|
||||||
|
|
||||||
LOG = logging.getLogger(__name__)
|
LOG = logging.getLogger(__name__)
|
||||||
@@ -49,7 +50,7 @@ async def submit_scrape_job(job: Job):
|
|||||||
job_dict = job.model_dump()
|
job_dict = job.model_dump()
|
||||||
await insert(job_dict)
|
await insert(job_dict)
|
||||||
|
|
||||||
return JSONResponse(content=f"Job queued for scraping: {job.id}")
|
return JSONResponse(content={"id": job.id})
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
return JSONResponse(content={"error": str(e)}, status_code=500)
|
return JSONResponse(content={"error": str(e)}, status_code=500)
|
||||||
|
|
||||||
@@ -70,6 +71,7 @@ async def retrieve_scrape_jobs(
|
|||||||
@job_router.get("/job/{id}")
|
@job_router.get("/job/{id}")
|
||||||
async def job(id: str, user: User = Depends(get_current_user)):
|
async def job(id: str, user: User = Depends(get_current_user)):
|
||||||
LOG.info(f"Retrieving jobs for account: {user.email}")
|
LOG.info(f"Retrieving jobs for account: {user.email}")
|
||||||
|
|
||||||
try:
|
try:
|
||||||
filter = {"user": user.email, "id": id}
|
filter = {"user": user.email, "id": id}
|
||||||
results = await query(filter)
|
results = await query(filter)
|
||||||
|
|||||||
@@ -7,12 +7,10 @@ from lxml import etree
|
|||||||
from seleniumwire import webdriver
|
from seleniumwire import webdriver
|
||||||
from lxml.etree import _Element # type: ignore [reportPrivateImport]
|
from lxml.etree import _Element # type: ignore [reportPrivateImport]
|
||||||
from fake_useragent import UserAgent
|
from fake_useragent import UserAgent
|
||||||
from webdriver_manager.chrome import ChromeDriverManager
|
|
||||||
from selenium.webdriver.support import expected_conditions as EC
|
from selenium.webdriver.support import expected_conditions as EC
|
||||||
from selenium.webdriver.common.by import By
|
from selenium.webdriver.common.by import By
|
||||||
from selenium.webdriver.support.ui import WebDriverWait
|
from selenium.webdriver.support.ui import WebDriverWait
|
||||||
from selenium.webdriver.chrome.options import Options as ChromeOptions
|
from selenium.webdriver.chrome.options import Options as ChromeOptions
|
||||||
from selenium.webdriver.chrome.service import Service
|
|
||||||
from urllib.parse import urlparse, urljoin
|
from urllib.parse import urlparse, urljoin
|
||||||
|
|
||||||
from api.backend.models import Element, CapturedElement
|
from api.backend.models import Element, CapturedElement
|
||||||
|
|||||||
@@ -111,7 +111,10 @@ export const JobSubmitter = ({ stateProps }: Props) => {
|
|||||||
return response.json();
|
return response.json();
|
||||||
})
|
})
|
||||||
.then((data) => {
|
.then((data) => {
|
||||||
setSnackbarMessage(data || "Job submitted successfully.");
|
setSnackbarMessage(
|
||||||
|
`Job: ${data.id} submitted successfully.` ||
|
||||||
|
"Job submitted successfully."
|
||||||
|
);
|
||||||
setSnackbarSeverity("info");
|
setSnackbarSeverity("info");
|
||||||
setSnackbarOpen(true);
|
setSnackbarOpen(true);
|
||||||
})
|
})
|
||||||
|
|||||||
@@ -219,27 +219,28 @@ const AI: React.FC = () => {
|
|||||||
</Box>
|
</Box>
|
||||||
) : (
|
) : (
|
||||||
<>
|
<>
|
||||||
{messages.map((message, index) => (
|
{messages &&
|
||||||
<Box
|
messages.map((message, index) => (
|
||||||
key={index}
|
<Box
|
||||||
sx={{
|
key={index}
|
||||||
my: 2,
|
sx={{
|
||||||
p: 1,
|
my: 2,
|
||||||
borderRadius: "8px",
|
p: 1,
|
||||||
boxShadow: "0 2px 4px rgba(0, 0, 0, 0.1)",
|
borderRadius: "8px",
|
||||||
bgcolor:
|
boxShadow: "0 2px 4px rgba(0, 0, 0, 0.1)",
|
||||||
message.role === "user"
|
bgcolor:
|
||||||
? theme.palette.UserMessage.main
|
message.role === "user"
|
||||||
: theme.palette.AIMessage.main,
|
? theme.palette.UserMessage.main
|
||||||
marginLeft: message.role === "user" ? "auto" : "",
|
: theme.palette.AIMessage.main,
|
||||||
maxWidth: "40%",
|
marginLeft: message.role === "user" ? "auto" : "",
|
||||||
}}
|
maxWidth: "40%",
|
||||||
>
|
}}
|
||||||
<Typography variant="body1" sx={{ color: "white" }}>
|
>
|
||||||
{message.content}
|
<Typography variant="body1" sx={{ color: "white" }}>
|
||||||
</Typography>
|
{message.content}
|
||||||
</Box>
|
</Typography>
|
||||||
))}
|
</Box>
|
||||||
|
))}
|
||||||
{thinking && (
|
{thinking && (
|
||||||
<Box
|
<Box
|
||||||
sx={{
|
sx={{
|
||||||
|
|||||||
Reference in New Issue
Block a user