diff --git a/api/backend/__init__.py b/api/backend/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/api/backend/app.py b/api/backend/app.py index 82e46a6..f86eb7e 100644 --- a/api/backend/app.py +++ b/api/backend/app.py @@ -3,12 +3,12 @@ import os import uuid import logging import traceback -from io import BytesIO +from io import StringIO from typing import Optional +import csv # PDM -from fastapi import Depends, FastAPI, HTTPException, BackgroundTasks -from openpyxl import Workbook +from fastapi import Depends, FastAPI, HTTPException from fastapi.encoders import jsonable_encoder from fastapi.responses import JSONResponse, StreamingResponse from fastapi.middleware.cors import CORSMiddleware @@ -27,8 +27,8 @@ from api.backend.models import ( UpdateJobs, DownloadJob, FetchOptions, - SubmitScrapeJob, DeleteScrapeJobs, + Job, ) from api.backend.schemas import User from api.backend.ai.ai_router import ai_router @@ -79,14 +79,13 @@ async def update(update_jobs: UpdateJobs, user: User = Depends(get_current_user) @app.post("/submit-scrape-job") -async def submit_scrape_job(job: SubmitScrapeJob, background_tasks: BackgroundTasks): +async def submit_scrape_job(job: Job): LOG.info(f"Recieved job: {job}") try: job.id = uuid.uuid4().hex - if job.user: - job_dict = job.model_dump() - await insert(job_dict) + job_dict = job.model_dump() + await insert(job_dict) return JSONResponse(content=f"Job queued for scraping: {job.id}") except Exception as e: @@ -103,7 +102,7 @@ async def retrieve_scrape_jobs( return JSONResponse(content=jsonable_encoder(results[::-1])) except Exception as e: LOG.error(f"Exception occurred: {e}") - return JSONResponse(content={"error": str(e)}, status_code=500) + return JSONResponse(content=[], status_code=500) @app.get("/job/{id}") @@ -128,63 +127,40 @@ def clean_text(text: str): @app.post("/download") async def download(download_job: DownloadJob): LOG.info(f"Downloading job with ids: {download_job.ids}") + try: results = await query({"id": {"$in": download_job.ids}}) - flattened_results = [] + csv_buffer = StringIO() + csv_writer = csv.writer(csv_buffer) + + headers = ["id", "url", "element_name", "xpath", "text", "user", "time_created"] + csv_writer.writerow(headers) + for result in results: for res in result["result"]: for url, elements in res.items(): for element_name, values in elements.items(): for value in values: text = clean_text(value.get("text", "")) - flattened_results.append( - { - "id": result.get("id", None), - "url": url, - "element_name": element_name, - "xpath": value.get("xpath", ""), - "text": text, - "user": result.get("user", ""), - "time_created": result.get("time_created", ""), - } + csv_writer.writerow( + [ + result.get("id", ""), + url, + element_name, + value.get("xpath", ""), + text, + result.get("user", ""), + result.get("time_created", ""), + ] ) - # Create an Excel workbook and sheet - workbook = Workbook() - sheet = workbook.active - assert sheet - sheet.title = "Results" - - # Write the header - headers = ["id", "url", "element_name", "xpath", "text", "user", "time_created"] - sheet.append(headers) - - # Write the rows - for row in flattened_results: - sheet.append( - [ - row["id"], - row["url"], - row["element_name"], - row["xpath"], - row["text"], - row["user"], - row["time_created"], - ] - ) - - # Save the workbook to a BytesIO buffer - excel_buffer = BytesIO() - workbook.save(excel_buffer) - _ = excel_buffer.seek(0) - - # Create the response + _ = csv_buffer.seek(0) response = StreamingResponse( - excel_buffer, - media_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + csv_buffer, + media_type="text/csv", ) - response.headers["Content-Disposition"] = "attachment; filename=export.xlsx" + response.headers["Content-Disposition"] = "attachment; filename=export.csv" return response except Exception as e: diff --git a/api/backend/auth/auth_utils.py b/api/backend/auth/auth_utils.py index 7f7e11c..945e752 100644 --- a/api/backend/auth/auth_utils.py +++ b/api/backend/auth/auth_utils.py @@ -1,5 +1,7 @@ # STL import os +from gc import disable +from queue import Empty from typing import Any, Optional from datetime import datetime, timedelta @@ -23,6 +25,8 @@ ACCESS_TOKEN_EXPIRE_MINUTES = os.getenv("ACCESS_TOKEN_EXPIRE_MINUTES") pwd_context = CryptContext(schemes=["bcrypt"], deprecated="auto") oauth2_scheme = OAuth2PasswordBearer(tokenUrl="auth/token") +EMPTY_USER = User(email="", full_name="", disabled=False) + def verify_password(plain_password: str, hashed_password: str): return pwd_context.verify(plain_password, hashed_password) @@ -70,6 +74,32 @@ def create_access_token( async def get_current_user(token: str = Depends(oauth2_scheme)): + try: + payload: Optional[dict[str, Any]] = jwt.decode( + token, SECRET_KEY, algorithms=[ALGORITHM] + ) + if not payload: + return EMPTY_USER + + email = payload.get("sub") + + if email is None: + return EMPTY_USER + + token_data = TokenData(email=email) + + except JWTError: + return EMPTY_USER + + user = await get_user(email=token_data.email) + + if user is None: + return EMPTY_USER + + return user + + +async def require_user(token: str = Depends(oauth2_scheme)): credentials_exception = HTTPException( status_code=status.HTTP_401_UNAUTHORIZED, detail="Could not validate credentials", diff --git a/api/backend/models.py b/api/backend/models.py index 1d71768..8bc7066 100644 --- a/api/backend/models.py +++ b/api/backend/models.py @@ -1,5 +1,5 @@ # STL -from typing import Any, Optional +from typing import Any, Optional, Union from datetime import datetime # PDM @@ -27,18 +27,6 @@ class JobOptions(pydantic.BaseModel): custom_headers: Optional[dict[str, Any]] -class SubmitScrapeJob(pydantic.BaseModel): - id: Optional[str] = None - url: str - elements: list[Element] - user: Optional[str] = None - time_created: Optional[datetime] = None - result: Optional[dict[str, Any]] = None - job_options: JobOptions - status: str = "Queued" - chat: Optional[str] = None - - class RetrieveScrapeJobs(pydantic.BaseModel): user: str @@ -63,3 +51,15 @@ class UpdateJobs(pydantic.BaseModel): class AI(pydantic.BaseModel): messages: list[Any] + + +class Job(pydantic.BaseModel): + id: Optional[str] = None + url: str + elements: list[Element] + user: str = "" + time_created: Optional[Union[datetime, str]] = None + result: list[dict[str, dict[str, list[CapturedElement]]]] = [] + job_options: JobOptions + status: str = "Queued" + chat: Optional[str] = None diff --git a/api/backend/schemas.py b/api/backend/schemas.py index 27c0870..8099f98 100644 --- a/api/backend/schemas.py +++ b/api/backend/schemas.py @@ -1,5 +1,5 @@ # STL -from typing import Optional +from typing import Union, Literal, Optional # PDM from pydantic import EmailStr, BaseModel @@ -15,7 +15,7 @@ class TokenData(BaseModel): class User(BaseModel): - email: EmailStr + email: Union[EmailStr, Literal[""]] full_name: Optional[str] = None disabled: Optional[bool] = None diff --git a/api/backend/tests/factories/job_factory.py b/api/backend/tests/factories/job_factory.py new file mode 100644 index 0000000..88a28b4 --- /dev/null +++ b/api/backend/tests/factories/job_factory.py @@ -0,0 +1,42 @@ +from api.backend.models import Element, Job, JobOptions, CapturedElement +import uuid +from faker import Faker + +fake = Faker() + + +def create_job(): + return Job( + id=uuid.uuid4().hex, + url="https://example.com", + elements=[Element(name="test", xpath="xpath")], + job_options=JobOptions(multi_page_scrape=False, custom_headers={}), + ) + + +def create_completed_job() -> Job: + return Job( + id=uuid.uuid4().hex, + url="http://example.com", + elements=[ + Element( + name="element_name", + xpath="//div", + url="https://example.com", + ) + ], + job_options=JobOptions(multi_page_scrape=False, custom_headers={}), + user=fake.name(), + time_created=fake.date(), + result=[ + { + "https://example.com": { + "element_name": [ + CapturedElement( + xpath="//div", text="example", name="element_name" + ) + ] + } + } + ], + ) diff --git a/api/backend/tests/job/__init__.py b/api/backend/tests/job/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/api/backend/tests/job/test_download_job.py b/api/backend/tests/job/test_download_job.py new file mode 100644 index 0000000..5259c11 --- /dev/null +++ b/api/backend/tests/job/test_download_job.py @@ -0,0 +1,30 @@ +import pytest +from fastapi.testclient import TestClient +from unittest.mock import AsyncMock, patch +from api.backend.app import app +from api.backend.models import DownloadJob +from api.backend.tests.factories.job_factory import create_completed_job + +client = TestClient(app) + +mocked_job = create_completed_job().model_dump() +mock_results = [mocked_job] + + +@pytest.mark.asyncio +@patch("api.backend.app.query") +async def test_download(mock_query: AsyncMock): + mock_query.return_value = mock_results + download_job = DownloadJob(ids=[mocked_job["id"]]) + + # Make a POST request to the /download endpoint + response = client.post("/download", json=download_job.model_dump()) + + # Assertions + assert response.status_code == 200 + assert response.headers["Content-Disposition"] == "attachment; filename=export.csv" + + # Check the content of the CSV + csv_content = response.content.decode("utf-8") + expected_csv = f"id,url,element_name,xpath,text,user,time_created\r\n{mocked_job['id']},https://example.com,element_name,//div,example,{mocked_job['user']},{mocked_job['time_created']}\r\n" + assert csv_content == expected_csv diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 1864bdc..0c47476 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -18,3 +18,5 @@ services: scraperr_api: ports: - "8000:8000" + volumes: + - "$PWD/api:/project/api" diff --git a/pdm.lock b/pdm.lock index 92720e6..fe37490 100644 --- a/pdm.lock +++ b/pdm.lock @@ -5,10 +5,13 @@ groups = ["default", "dev"] strategy = [] lock_version = "4.5.0" -content_hash = "sha256:f2534e6409e8e8977bc777a05481c5549f8c26b6137c4d4d19cad7672fc7599f" +content_hash = "sha256:50e60db0b9c7d55c330310b64871570dd5494665b67a06138d8a78c2df377932" [[metadata.targets]] -requires_python = ">=3.10" +requires_python = "==3.10.12" +platform = "manylinux_2_35_x86_64" +implementation = "cpython" +gil_disabled = false [[package]] name = "aiohttp" @@ -509,16 +512,6 @@ files = [ {file = "click-8.1.7.tar.gz", hash = "sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de"}, ] -[[package]] -name = "colorama" -version = "0.4.6" -requires_python = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" -summary = "Cross-platform colored terminal text." -files = [ - {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, - {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, -] - [[package]] name = "cryptography" version = "42.0.8" @@ -664,12 +657,12 @@ files = [ [[package]] name = "exceptiongroup" -version = "1.2.1" +version = "1.2.2" requires_python = ">=3.7" summary = "Backport of PEP 654 (exception groups)" files = [ - {file = "exceptiongroup-1.2.1-py3-none-any.whl", hash = "sha256:5258b9ed329c5bbdd31a309f53cbfb0b155341807f6ff7606a1e801a891b29ad"}, - {file = "exceptiongroup-1.2.1.tar.gz", hash = "sha256:a4785e48b045528f5bfe627b6ad554ff32def154f42372786903b7abcfe1aa16"}, + {file = "exceptiongroup-1.2.2-py3-none-any.whl", hash = "sha256:3111b9d131c238bec2f8f516e123e14ba243563fb135d3fe885990585aa7795b"}, + {file = "exceptiongroup-1.2.2.tar.gz", hash = "sha256:47c2edf7c6738fafb49fd34290706d1a1a2f4d1c6df275526b62cbb4aa5393cc"}, ] [[package]] @@ -695,6 +688,20 @@ files = [ {file = "fake_useragent-1.5.1-py3-none-any.whl", hash = "sha256:57415096557c8a4e23b62a375c21c55af5fd4ba30549227f562d2c4f5b60e3b3"}, ] +[[package]] +name = "faker" +version = "30.6.0" +requires_python = ">=3.8" +summary = "Faker is a Python package that generates fake data for you." +dependencies = [ + "python-dateutil>=2.4", + "typing-extensions", +] +files = [ + {file = "Faker-30.6.0-py3-none-any.whl", hash = "sha256:37b5ab951f7367ea93edb865120e9717a7a649d6a4b223f1e4a47a8a20d9e85f"}, + {file = "faker-30.6.0.tar.gz", hash = "sha256:be0e548352c1be6f6d9c982003848a0d305868f160bb1fb7f945acffc347e676"}, +] + [[package]] name = "fastapi" version = "0.111.0" @@ -939,6 +946,16 @@ files = [ {file = "idna-3.7.tar.gz", hash = "sha256:028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc"}, ] +[[package]] +name = "iniconfig" +version = "2.0.0" +requires_python = ">=3.7" +summary = "brain-dead simple config-ini parsing" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + [[package]] name = "ipython" version = "8.26.0" @@ -1701,6 +1718,16 @@ files = [ {file = "pexpect-4.9.0.tar.gz", hash = "sha256:ee7d41123f3c9911050ea2c2dac107568dc43b2d3b0c7557a33212c398ead30f"}, ] +[[package]] +name = "pluggy" +version = "1.5.0" +requires_python = ">=3.8" +summary = "plugin and hook calling mechanisms for python" +files = [ + {file = "pluggy-1.5.0-py3-none-any.whl", hash = "sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669"}, + {file = "pluggy-1.5.0.tar.gz", hash = "sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1"}, +] + [[package]] name = "prompt-toolkit" version = "3.0.47" @@ -1836,19 +1863,6 @@ files = [ {file = "pydantic_core-2.20.1.tar.gz", hash = "sha256:26ca695eeee5f9f1aeeb211ffc12f10bcb6f71e2989988fda61dabd65db878d4"}, ] -[[package]] -name = "pydivert" -version = "2.1.0" -summary = "Python binding to windivert driver" -dependencies = [ - "enum34>=1.1.6; python_version == \"2.7\" or python_version == \"3.3\"", - "win-inet-pton>=1.0.1; python_version == \"2.7\" or python_version == \"3.3\"", -] -files = [ - {file = "pydivert-2.1.0-py2.py3-none-any.whl", hash = "sha256:382db488e3c37c03ec9ec94e061a0b24334d78dbaeebb7d4e4d32ce4355d9da1"}, - {file = "pydivert-2.1.0.tar.gz", hash = "sha256:f0e150f4ff591b78e35f514e319561dadff7f24a82186a171dd4d465483de5b4"}, -] - [[package]] name = "pyee" version = "11.1.0" @@ -1973,6 +1987,37 @@ files = [ {file = "PySocks-1.7.1.tar.gz", hash = "sha256:3f8804571ebe159c380ac6de37643bb4685970655d3bba243530d6558b799aa0"}, ] +[[package]] +name = "pytest" +version = "8.3.3" +requires_python = ">=3.8" +summary = "pytest: simple powerful testing with Python" +dependencies = [ + "colorama; sys_platform == \"win32\"", + "exceptiongroup>=1.0.0rc8; python_version < \"3.11\"", + "iniconfig", + "packaging", + "pluggy<2,>=1.5", + "tomli>=1; python_version < \"3.11\"", +] +files = [ + {file = "pytest-8.3.3-py3-none-any.whl", hash = "sha256:a6853c7375b2663155079443d2e45de913a911a11d669df02a50814944db57b2"}, + {file = "pytest-8.3.3.tar.gz", hash = "sha256:70b98107bd648308a7952b06e6ca9a50bc660be218d53c257cc1fc94fda10181"}, +] + +[[package]] +name = "pytest-asyncio" +version = "0.24.0" +requires_python = ">=3.8" +summary = "Pytest support for asyncio" +dependencies = [ + "pytest<9,>=8.2", +] +files = [ + {file = "pytest_asyncio-0.24.0-py3-none-any.whl", hash = "sha256:a811296ed596b69bf0b6f3dc40f83bcaf341b155a269052d82efa2b25ac7037b"}, + {file = "pytest_asyncio-0.24.0.tar.gz", hash = "sha256:d081d828e576d85f875399194281e92bf8a68d60d72d1a2faf2feddb6c46b276"}, +] + [[package]] name = "python-dateutil" version = "2.9.0.post0" @@ -2061,21 +2106,6 @@ files = [ {file = "pytz-2024.1.tar.gz", hash = "sha256:2a29735ea9c18baf14b448846bde5a48030ed267578472d8955cd0e7443a9812"}, ] -[[package]] -name = "pywin32" -version = "306" -summary = "Python for Window Extensions" -files = [ - {file = "pywin32-306-cp310-cp310-win32.whl", hash = "sha256:06d3420a5155ba65f0b72f2699b5bacf3109f36acbe8923765c22938a69dfc8d"}, - {file = "pywin32-306-cp310-cp310-win_amd64.whl", hash = "sha256:84f4471dbca1887ea3803d8848a1616429ac94a4a8d05f4bc9c5dcfd42ca99c8"}, - {file = "pywin32-306-cp311-cp311-win32.whl", hash = "sha256:e65028133d15b64d2ed8f06dd9fbc268352478d4f9289e69c190ecd6818b6407"}, - {file = "pywin32-306-cp311-cp311-win_amd64.whl", hash = "sha256:a7639f51c184c0272e93f244eb24dafca9b1855707d94c192d4a0b4c01e1100e"}, - {file = "pywin32-306-cp311-cp311-win_arm64.whl", hash = "sha256:70dba0c913d19f942a2db25217d9a1b726c278f483a919f1abfed79c9cf64d3a"}, - {file = "pywin32-306-cp312-cp312-win32.whl", hash = "sha256:383229d515657f4e3ed1343da8be101000562bf514591ff383ae940cad65458b"}, - {file = "pywin32-306-cp312-cp312-win_amd64.whl", hash = "sha256:37257794c1ad39ee9be652da0462dc2e394c8159dfd913a8a4e8eb6fd346da0e"}, - {file = "pywin32-306-cp312-cp312-win_arm64.whl", hash = "sha256:5821ec52f6d321aa59e2db7e0a35b997de60c201943557d108af9d4ae1ec7040"}, -] - [[package]] name = "pyyaml" version = "6.0.1" @@ -2329,6 +2359,16 @@ files = [ {file = "starlette-0.37.2.tar.gz", hash = "sha256:9af890290133b79fc3db55474ade20f6220a364a0402e0b556e7cd5e1e093823"}, ] +[[package]] +name = "tomli" +version = "2.0.2" +requires_python = ">=3.8" +summary = "A lil' TOML parser" +files = [ + {file = "tomli-2.0.2-py3-none-any.whl", hash = "sha256:2ebe24485c53d303f690b0ec092806a085f07af5a5aa1464f3931eec36caaa38"}, + {file = "tomli-2.0.2.tar.gz", hash = "sha256:d46d457a85337051c36524bc5349dd91b1877838e2979ac5ced3e710ed8a60ed"}, +] + [[package]] name = "tqdm" version = "4.66.4" @@ -2806,7 +2846,7 @@ version = "0.23.0" requires_python = ">=3.8" summary = "Zstandard bindings for Python" dependencies = [ - "cffi>=1.11; platform_python_implementation == \"PyPy\"", + "cffi>=1.17; platform_python_implementation == \"PyPy\"", ] files = [ {file = "zstandard-0.23.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bf0a05b6059c0528477fba9054d09179beb63744355cab9f38059548fedd46a9"}, diff --git a/pyproject.toml b/pyproject.toml index e52263b..f80985c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,6 +36,9 @@ dependencies = [ "docker>=7.1.0", "ollama>=0.3.0", "openai>=1.37.1", + "exceptiongroup>=1.2.2", + "Faker>=30.6.0", + "pytest-asyncio>=0.24.0", ] requires-python = ">=3.10" readme = "README.md" @@ -47,6 +50,7 @@ distribution = true [tool.pdm.dev-dependencies] dev = [ "ipython>=8.26.0", + "pytest>=8.3.3", ] [tool.pyright] include = ["./api/backend/"] diff --git a/src/components/common/NavDrawer.tsx b/src/components/common/NavDrawer.tsx deleted file mode 100644 index 4210987..0000000 --- a/src/components/common/NavDrawer.tsx +++ /dev/null @@ -1,169 +0,0 @@ -"use client"; - -import React from "react"; -import { useAuth } from "../../contexts/AuthContext"; -import { - Box, - List, - ListItem, - ListItemIcon, - ListItemButton, - ListItemText, - Typography, - Button, - Switch, - Drawer, - Divider, - Accordion, - AccordionSummary, - AccordionDetails, -} from "@mui/material"; -import HomeIcon from "@mui/icons-material/Home"; -import HttpIcon from "@mui/icons-material/Http"; -import ExpandMoreIcon from "@mui/icons-material/ExpandMore"; -import TerminalIcon from "@mui/icons-material/Terminal"; -import BarChart from "@mui/icons-material/BarChart"; -import AutoAwesomeIcon from "@mui/icons-material/AutoAwesome"; -import { useRouter } from "next/router"; - -interface NavDrawerProps { - toggleTheme: () => void; - isDarkMode: boolean; -} - -const drawerWidth = 240; - -export const NavDrawer: React.FC = ({ - toggleTheme, - isDarkMode, -}) => { - const router = useRouter(); - - const { logout, user, isAuthenticated } = useAuth(); - - return ( - - -
- - - router.push("/")}> - - - - - - - - - router.push("/jobs")}> - - - - - - - - - router.push("/chat")}> - - - - - - - - - router.push("/statistics")}> - - - - - - - - - router.push("/logs")}> - - - - - - - - -
- - {isAuthenticated ? ( - <> - - Welcome, {user?.full_name} - - - - ) : ( - - )} - - - } - aria-controls="panel1a-content" - id="panel1a-header" - > - Quick Settings - - -
- -

Dark Theme Toggle

-
- -
-
-
-
-
-
- ); -}; diff --git a/src/components/common/index.ts b/src/components/common/index.ts index ebd1fe0..f15de13 100644 --- a/src/components/common/index.ts +++ b/src/components/common/index.ts @@ -1 +1 @@ -export * from "./NavDrawer"; +export * from "./nav-drawer"; diff --git a/src/components/common/nav-drawer/index.ts b/src/components/common/nav-drawer/index.ts new file mode 100644 index 0000000..5f95cd2 --- /dev/null +++ b/src/components/common/nav-drawer/index.ts @@ -0,0 +1 @@ +export { NavDrawer } from "./nav-drawer"; diff --git a/src/components/common/nav-drawer/nav-drawer.module.css b/src/components/common/nav-drawer/nav-drawer.module.css new file mode 100644 index 0000000..b16e5a8 --- /dev/null +++ b/src/components/common/nav-drawer/nav-drawer.module.css @@ -0,0 +1,3 @@ +.userControl { + margin-bottom: 1rem; +} diff --git a/src/components/common/nav-drawer/nav-drawer.tsx b/src/components/common/nav-drawer/nav-drawer.tsx new file mode 100644 index 0000000..d23e107 --- /dev/null +++ b/src/components/common/nav-drawer/nav-drawer.tsx @@ -0,0 +1,62 @@ +"use client"; + +import React from "react"; +import { useAuth } from "../../../contexts/AuthContext"; +import { Box, Drawer, Divider } from "@mui/material"; + +import { QuickSettings } from "../../nav/quick-settings"; +import { NavItems } from "./nav-items/nav-items"; +import { UserControl } from "./user-control"; + +import classes from "./nav-drawer.module.css"; + +interface NavDrawerProps { + toggleTheme: () => void; + isDarkMode: boolean; +} + +const drawerWidth = 240; + +export const NavDrawer: React.FC = ({ + toggleTheme, + isDarkMode, +}) => { + const { logout, user, isAuthenticated } = useAuth(); + + return ( + + +
+ +
+
+ + +
+
+
+ ); +}; diff --git a/src/components/common/nav-drawer/nav-item/index.ts b/src/components/common/nav-drawer/nav-item/index.ts new file mode 100644 index 0000000..46009ed --- /dev/null +++ b/src/components/common/nav-drawer/nav-item/index.ts @@ -0,0 +1 @@ +export { default as NavItem } from "./nav-item"; diff --git a/src/components/common/nav-drawer/nav-item/nav-item.tsx b/src/components/common/nav-drawer/nav-item/nav-item.tsx new file mode 100644 index 0000000..f33fb87 --- /dev/null +++ b/src/components/common/nav-drawer/nav-item/nav-item.tsx @@ -0,0 +1,33 @@ +import { + ListItem, + ListItemButton, + ListItemIcon, + ListItemText, +} from "@mui/material"; +import { useRouter } from "next/router"; +import React from "react"; + +export type NavItemProps = { + icon: React.ReactNode; + text: string; + href: string; +}; + +const NavItem: React.FC = ({ icon, text, href }) => { + const router = useRouter(); + + const handleClick = () => { + router.push(href); + }; + + return ( + + + {icon} + + + + ); +}; + +export default NavItem; diff --git a/src/components/common/nav-drawer/nav-items/index.ts b/src/components/common/nav-drawer/nav-items/index.ts new file mode 100644 index 0000000..92bc374 --- /dev/null +++ b/src/components/common/nav-drawer/nav-items/index.ts @@ -0,0 +1 @@ +export { NavItems } from "./nav-items"; diff --git a/src/components/common/nav-drawer/nav-items/nav-items.tsx b/src/components/common/nav-drawer/nav-items/nav-items.tsx new file mode 100644 index 0000000..5b3355f --- /dev/null +++ b/src/components/common/nav-drawer/nav-items/nav-items.tsx @@ -0,0 +1,47 @@ +import React from "react"; +import { NavItem } from "../nav-item"; + +import HomeIcon from "@mui/icons-material/Home"; +import HttpIcon from "@mui/icons-material/Http"; +import TerminalIcon from "@mui/icons-material/Terminal"; +import BarChart from "@mui/icons-material/BarChart"; +import AutoAwesomeIcon from "@mui/icons-material/AutoAwesome"; +import { List } from "@mui/material"; + +const items = [ + { + icon: , + text: "Home", + href: "/", + }, + { + icon: , + text: "Previous Jobs", + href: "/jobs", + }, + { + icon: , + text: "Chat", + href: "/chat", + }, + { + icon: , + text: "Statistics", + href: "/statistics", + }, + { + icon: , + text: "View App Logs", + href: "/logs", + }, +]; + +export const NavItems = () => { + return ( + + {items.map((item) => ( + + ))} + + ); +}; diff --git a/src/components/common/nav-drawer/user-control/index.ts b/src/components/common/nav-drawer/user-control/index.ts new file mode 100644 index 0000000..f7f2d41 --- /dev/null +++ b/src/components/common/nav-drawer/user-control/index.ts @@ -0,0 +1 @@ +export * from "./user-control"; diff --git a/src/components/common/nav-drawer/user-control/logged-in-control/index.ts b/src/components/common/nav-drawer/user-control/logged-in-control/index.ts new file mode 100644 index 0000000..b2d8646 --- /dev/null +++ b/src/components/common/nav-drawer/user-control/logged-in-control/index.ts @@ -0,0 +1 @@ +export * from "./logged-in-control"; \ No newline at end of file diff --git a/src/components/common/nav-drawer/user-control/logged-in-control/logged-in-control.module.css b/src/components/common/nav-drawer/user-control/logged-in-control/logged-in-control.module.css new file mode 100644 index 0000000..68f3159 --- /dev/null +++ b/src/components/common/nav-drawer/user-control/logged-in-control/logged-in-control.module.css @@ -0,0 +1,7 @@ +.welcome { + margin: 0.25rem; +} + +.userControlButton { + width: 100%; +} \ No newline at end of file diff --git a/src/components/common/nav-drawer/user-control/logged-in-control/logged-in-control.tsx b/src/components/common/nav-drawer/user-control/logged-in-control/logged-in-control.tsx new file mode 100644 index 0000000..2c51473 --- /dev/null +++ b/src/components/common/nav-drawer/user-control/logged-in-control/logged-in-control.tsx @@ -0,0 +1,34 @@ +import React from "react"; +import { Typography, Button } from "@mui/material"; +import classes from "./logged-in-control.module.css"; + +type LoggedInControlProps = { + user: any; + logout: () => void; + children?: React.ReactNode; +}; + +export const LoggedInControl = ({ + user, + logout, + children, +}: LoggedInControlProps) => { + if (children) { + return <>{children}; + } + + return ( + <> + + Welcome, {user?.full_name} + + + + ); +}; diff --git a/src/components/common/nav-drawer/user-control/logged-out-control/index.ts b/src/components/common/nav-drawer/user-control/logged-out-control/index.ts new file mode 100644 index 0000000..f1d5201 --- /dev/null +++ b/src/components/common/nav-drawer/user-control/logged-out-control/index.ts @@ -0,0 +1 @@ +export * from "./logged-out-control"; diff --git a/src/components/common/nav-drawer/user-control/logged-out-control/logged-out-control.module.css b/src/components/common/nav-drawer/user-control/logged-out-control/logged-out-control.module.css new file mode 100644 index 0000000..48097a9 --- /dev/null +++ b/src/components/common/nav-drawer/user-control/logged-out-control/logged-out-control.module.css @@ -0,0 +1,3 @@ +.userControlButton { + width: 100%; +} diff --git a/src/components/common/nav-drawer/user-control/logged-out-control/logged-out-control.tsx b/src/components/common/nav-drawer/user-control/logged-out-control/logged-out-control.tsx new file mode 100644 index 0000000..e394af3 --- /dev/null +++ b/src/components/common/nav-drawer/user-control/logged-out-control/logged-out-control.tsx @@ -0,0 +1,29 @@ +import React from "react"; +import { Button } from "@mui/material"; +import classes from "./logged-out-control.module.css"; +import { useRouter } from "next/navigation"; + +export type LoggedOutControlProps = { + children?: React.ReactNode; +}; + +export const LoggedOutControl: React.FC = ({ + children, +}) => { + const router = useRouter(); + const login = () => router.push("/login"); + + if (children) { + return <>{children}; + } + + return ( + + ); +}; diff --git a/src/components/common/nav-drawer/user-control/user-control.module.css b/src/components/common/nav-drawer/user-control/user-control.module.css new file mode 100644 index 0000000..b9dd880 --- /dev/null +++ b/src/components/common/nav-drawer/user-control/user-control.module.css @@ -0,0 +1,13 @@ +.userControl { + display: flex; + flex-direction: column; + align-items: center; +} + +.welcome { + margin: 0.25rem; +} + +.userControlButton { + width: 100%; +} diff --git a/src/components/common/nav-drawer/user-control/user-control.tsx b/src/components/common/nav-drawer/user-control/user-control.tsx new file mode 100644 index 0000000..67d5b42 --- /dev/null +++ b/src/components/common/nav-drawer/user-control/user-control.tsx @@ -0,0 +1,37 @@ +import React from "react"; +import { Box } from "@mui/material"; +import clsx from "clsx"; + +import classes from "./user-control.module.css"; +import { LoggedInControl } from "./logged-in-control"; +import { LoggedOutControl } from "./logged-out-control"; + +export type UserControlProps = { + isAuthenticated: boolean; + user: any; + logout: () => void; + loggedInChildren?: React.ReactNode; + loggedOutChildren?: React.ReactNode; + className?: string; +}; + +export const UserControl = ({ + isAuthenticated, + user, + logout, + loggedInChildren, + loggedOutChildren, + className, +}: UserControlProps) => { + return ( + + {isAuthenticated ? ( + + {loggedInChildren} + + ) : ( + {loggedOutChildren} + )} + + ); +}; diff --git a/src/components/jobs/JobTable.tsx b/src/components/jobs/JobTable.tsx index 1b21b8f..fecea28 100644 --- a/src/components/jobs/JobTable.tsx +++ b/src/components/jobs/JobTable.tsx @@ -60,10 +60,11 @@ export const JobTable: React.FC = ({ jobs, setJobs }) => { const a = document.createElement("a"); a.style.display = "none"; a.href = url; - a.download = `job_${ids.splice(0, 1)}.xlsx`; + a.download = `job_${ids[0]}.csv`; document.body.appendChild(a); a.click(); window.URL.revokeObjectURL(url); + document.body.removeChild(a); } else { console.error("Failed to download the file."); } diff --git a/src/components/nav/quick-settings/index.ts b/src/components/nav/quick-settings/index.ts new file mode 100644 index 0000000..4f7e6be --- /dev/null +++ b/src/components/nav/quick-settings/index.ts @@ -0,0 +1 @@ +export * from "./quick-settings"; \ No newline at end of file diff --git a/src/components/nav/quick-settings/quick-settings.module.css b/src/components/nav/quick-settings/quick-settings.module.css new file mode 100644 index 0000000..4b1625a --- /dev/null +++ b/src/components/nav/quick-settings/quick-settings.module.css @@ -0,0 +1,13 @@ +.quickSettings { + padding: 0; + margin-bottom: 0.25rem; +} + +.details { + display: flex; + margin-right: 0.25rem; +} + +.detailsText p { + font-size: 1rem; +} diff --git a/src/components/nav/quick-settings/quick-settings.tsx b/src/components/nav/quick-settings/quick-settings.tsx new file mode 100644 index 0000000..dae1b17 --- /dev/null +++ b/src/components/nav/quick-settings/quick-settings.tsx @@ -0,0 +1,41 @@ +import React from "react"; + +import classes from "./quick-settings.module.css"; +import { + Accordion, + AccordionDetails, + AccordionSummary, + Switch, + Typography, +} from "@mui/material"; +import { ExpandMore } from "@mui/icons-material"; + +type QuickSettingsProps = { + toggleTheme: () => void; + isDarkMode: boolean; +}; + +export const QuickSettings: React.FC = ({ + toggleTheme, + isDarkMode, +}) => { + return ( + + } + aria-controls="panel1a-content" + id="panel1a-header" + > + Quick Settings + + +
+ +

Dark Theme Toggle

+
+ +
+
+
+ ); +}; diff --git a/src/components/submit/index.ts b/src/components/submit/index.ts index 7f96dc7..400e380 100644 --- a/src/components/submit/index.ts +++ b/src/components/submit/index.ts @@ -1,2 +1,2 @@ export * from "./ElementTable"; -export * from "./JobSubmitter"; +export * from "./job-submitter"; diff --git a/src/components/submit/job-submitter/index.ts b/src/components/submit/job-submitter/index.ts new file mode 100644 index 0000000..d1879c4 --- /dev/null +++ b/src/components/submit/job-submitter/index.ts @@ -0,0 +1 @@ +export { JobSubmitter } from "./job-submitter"; diff --git a/src/components/submit/job-submitter/job-submitter-header/index.ts b/src/components/submit/job-submitter/job-submitter-header/index.ts new file mode 100644 index 0000000..f5b8f04 --- /dev/null +++ b/src/components/submit/job-submitter/job-submitter-header/index.ts @@ -0,0 +1 @@ +export { JobSubmitterHeader } from "./job-submitter-header"; diff --git a/src/components/submit/job-submitter/job-submitter-header/job-submitter-header.module.css b/src/components/submit/job-submitter/job-submitter-header/job-submitter-header.module.css new file mode 100644 index 0000000..89762a2 --- /dev/null +++ b/src/components/submit/job-submitter/job-submitter-header/job-submitter-header.module.css @@ -0,0 +1,4 @@ +.jobSubmitterHeader { + margin-bottom: 1rem; + text-align: center; +} diff --git a/src/components/submit/job-submitter/job-submitter-header/job-submitter-header.tsx b/src/components/submit/job-submitter/job-submitter-header/job-submitter-header.tsx new file mode 100644 index 0000000..7a2fc87 --- /dev/null +++ b/src/components/submit/job-submitter/job-submitter-header/job-submitter-header.tsx @@ -0,0 +1,20 @@ +import React, { ReactNode } from "react"; +import { Typography } from "@mui/material"; +import classes from "./job-submitter-header.module.css"; + +interface JobSubmitterHeaderProps { + title?: string; + children?: ReactNode; +} + +export const JobSubmitterHeader: React.FC = ({ + title = "Scraping Made Easy", + children, +}) => { + return ( +
+ {title} + {children} +
+ ); +}; diff --git a/src/components/submit/job-submitter/job-submitter-input/index.ts b/src/components/submit/job-submitter/job-submitter-input/index.ts new file mode 100644 index 0000000..d820261 --- /dev/null +++ b/src/components/submit/job-submitter/job-submitter-input/index.ts @@ -0,0 +1 @@ +export { JobSubmitterInput } from "./job-submitter-input"; diff --git a/src/components/submit/job-submitter/job-submitter-input/job-submitter-input.module.css b/src/components/submit/job-submitter/job-submitter-input/job-submitter-input.module.css new file mode 100644 index 0000000..e69de29 diff --git a/src/components/submit/job-submitter/job-submitter-input/job-submitter-input.tsx b/src/components/submit/job-submitter/job-submitter-input/job-submitter-input.tsx new file mode 100644 index 0000000..cb6081c --- /dev/null +++ b/src/components/submit/job-submitter/job-submitter-input/job-submitter-input.tsx @@ -0,0 +1,48 @@ +import React, { Dispatch } from "react"; +import { TextField, Button, CircularProgress } from "@mui/material"; +import { Element } from "@/types"; + +export type JobSubmitterInputProps = { + submittedURL: string; + setSubmittedURL: Dispatch>; + isValidURL: boolean; + urlError: string | null; + handleSubmit: () => void; + loading: boolean; + rows: Element[]; +}; + +export const JobSubmitterInput = ({ + submittedURL, + setSubmittedURL, + isValidURL, + urlError, + handleSubmit, + loading, + rows, +}: JobSubmitterInputProps) => { + return ( +
+ setSubmittedURL(e.target.value)} + error={!isValidURL} + helperText={!isValidURL ? urlError : ""} + className="rounded-md" + /> + +
+ ); +}; diff --git a/src/components/submit/job-submitter/job-submitter-options/index.ts b/src/components/submit/job-submitter/job-submitter-options/index.ts new file mode 100644 index 0000000..7a04f8d --- /dev/null +++ b/src/components/submit/job-submitter/job-submitter-options/index.ts @@ -0,0 +1 @@ +export { JobSubmitterOptions } from "./job-submitter-options"; diff --git a/src/components/submit/job-submitter/job-submitter-options/job-submitter-options.tsx b/src/components/submit/job-submitter/job-submitter-options/job-submitter-options.tsx new file mode 100644 index 0000000..f29e5df --- /dev/null +++ b/src/components/submit/job-submitter/job-submitter-options/job-submitter-options.tsx @@ -0,0 +1,74 @@ +import { Box, FormControlLabel, Checkbox, TextField } from "@mui/material"; +import { Dispatch, SetStateAction } from "react"; + +import { JobOptions } from "@/types/job"; + +export type JobSubmitterOptionsProps = { + jobOptions: JobOptions; + setJobOptions: Dispatch>; + customJSONSelected: boolean; + setCustomJSONSelected: Dispatch>; +}; + +export const JobSubmitterOptions = ({ + jobOptions, + setJobOptions, + customJSONSelected, + setCustomJSONSelected, +}: JobSubmitterOptionsProps) => { + return ( + +
+ + setJobOptions((prevJobOptions) => ({ + ...prevJobOptions, + multi_page_scrape: !prevJobOptions.multi_page_scrape, + })) + } + /> + } + > + { + setCustomJSONSelected(!customJSONSelected); + setJobOptions((prevJobOptions) => ({ + ...prevJobOptions, + custom_headers: "", + })); + }} + /> + } + > +
+ {customJSONSelected ? ( +
+ + setJobOptions((prevJobOptions) => ({ + ...prevJobOptions, + custom_headers: e.target.value, + })) + } + style={{ maxHeight: "20vh", overflow: "auto" }} + className="mt-2" + /> +
+ ) : null} +
+ ); +}; diff --git a/src/components/submit/JobSubmitter.tsx b/src/components/submit/job-submitter/job-submitter.tsx similarity index 55% rename from src/components/submit/JobSubmitter.tsx rename to src/components/submit/job-submitter/job-submitter.tsx index bf464b7..5589323 100644 --- a/src/components/submit/JobSubmitter.tsx +++ b/src/components/submit/job-submitter/job-submitter.tsx @@ -1,19 +1,14 @@ "use client"; import React, { useEffect, useState, Dispatch } from "react"; -import { - TextField, - Button, - Box, - Checkbox, - FormControlLabel, - CircularProgress, - Typography, -} from "@mui/material"; -import { Element } from "../../types"; -import { useAuth } from "../../contexts/AuthContext"; +import { Element } from "@/types"; +import { useAuth } from "@/contexts/AuthContext"; import { useRouter } from "next/router"; -import { Constants } from "../../lib"; +import { Constants } from "@/lib"; + +import { JobSubmitterHeader } from "./job-submitter-header"; +import { JobSubmitterInput } from "./job-submitter-input"; +import { JobSubmitterOptions } from "./job-submitter-options"; interface StateProps { submittedURL: string; @@ -153,84 +148,22 @@ export const JobSubmitter = ({ stateProps }: Props) => { return ( <> - - Scraping Made Easy - -
- setSubmittedURL(e.target.value)} - error={!isValidURL} - helperText={!isValidURL ? urlError : ""} - className="rounded-md" +
+ + + -
- -
- - setJobOptions((prevJobOptions) => ({ - ...prevJobOptions, - multi_page_scrape: !prevJobOptions.multi_page_scrape, - })) - } - /> - } - > - { - setCustomJSONSelected(!customJSONSelected); - setJobOptions((prevJobOptions) => ({ - ...prevJobOptions, - custom_headers: "", - })); - }} - /> - } - > -
- {customJSONSelected ? ( -
- - setJobOptions((prevJobOptions) => ({ - ...prevJobOptions, - custom_headers: e.target.value, - })) - } - style={{ maxHeight: "20vh", overflow: "auto" }} - className="mt-2" - /> -
- ) : null} -
); }; diff --git a/src/pages/index.tsx b/src/pages/index.tsx index 7023350..34ad12f 100644 --- a/src/pages/index.tsx +++ b/src/pages/index.tsx @@ -1,10 +1,11 @@ "use client"; import React, { useState, useEffect, useRef } from "react"; -import { Container, Box, Snackbar, Alert } from "@mui/material"; +import { Button, Container, Box, Snackbar, Alert } from "@mui/material"; import { useRouter } from "next/router"; -import { Element, Result } from "../types"; -import { ElementTable, JobSubmitter } from "../components/submit"; +import { Element, Result } from "@/types"; +import { ElementTable } from "@/components/submit"; +import { JobSubmitter } from "@/components/submit/job-submitter"; const Home = () => { const router = useRouter(); @@ -54,13 +55,23 @@ const Home = () => { }; const NotifySnackbar = () => { + const goTo = () => { + router.push("/jobs"); + }; + + const action = ( + + ); + return ( - + {snackbarMessage} diff --git a/src/pages/jobs.tsx b/src/pages/jobs.tsx index 6e0492e..9a8a008 100644 --- a/src/pages/jobs.tsx +++ b/src/pages/jobs.tsx @@ -1,12 +1,10 @@ import React, { useEffect, useState } from "react"; import { JobTable } from "../components/jobs"; import { useAuth } from "../contexts/AuthContext"; -import { Box } from "@mui/system"; import { Job } from "../types"; import { GetServerSideProps } from "next/types"; import axios from "axios"; import { parseCookies } from "nookies"; -import Cookies from "js-cookie"; import { fetchJobs } from "../lib"; interface JobsProps { @@ -67,11 +65,7 @@ const Jobs: React.FC = ({ initialJobs, initialUser }) => { }, [user, initialUser, setUser]); useEffect(() => { - if (user) { - fetchJobs(setJobs); - } else { - setJobs([]); - } + fetchJobs(setJobs); }, [user]); useEffect(() => { @@ -81,33 +75,7 @@ const Jobs: React.FC = ({ initialJobs, initialUser }) => { return () => clearInterval(intervalId); }, []); - return ( - <> - {user ? ( - - ) : ( - -

- Previous jobs not viewable unless logged in. -

-
- )} - - ); + return ; }; export default Jobs; diff --git a/src/types/element.ts b/src/types/element.ts index 661925c..7053a47 100644 --- a/src/types/element.ts +++ b/src/types/element.ts @@ -1,5 +1,5 @@ -export interface Element { +export type Element = { name: string; xpath: string; url: string; -} +}; diff --git a/src/types/job.tsx b/src/types/job.ts similarity index 71% rename from src/types/job.tsx rename to src/types/job.ts index 722bd18..062bf32 100644 --- a/src/types/job.tsx +++ b/src/types/job.ts @@ -1,4 +1,5 @@ import { Message } from "./message"; + export interface Job { id: string; url: string; @@ -10,3 +11,8 @@ export interface Job { favorite: boolean; chat?: Message[]; } + +export type JobOptions = { + multi_page_scrape: boolean; + custom_headers: null | string; +}; diff --git a/tsconfig.json b/tsconfig.json index a2e2ef5..f92966a 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -26,8 +26,10 @@ "module": "commonjs" /* Specify what module code is generated. */, // "rootDir": "./", /* Specify the root folder within your source files. */ // "moduleResolution": "node10", /* Specify how TypeScript looks up a file from a given module specifier. */ - // "baseUrl": "./", /* Specify the base directory to resolve non-relative module names. */ - // "paths": {}, /* Specify a set of entries that re-map imports to additional lookup locations. */ + "baseUrl": "src" /* Specify the base directory to resolve non-relative module names. */, + "paths": { + "@/*": ["*"] + } /* Specify a set of entries that re-map imports to additional lookup locations. */, // "rootDirs": [], /* Allow multiple folders to be treated as one when resolving modules. */ // "typeRoots": [], /* Specify multiple folders that act like './node_modules/@types'. */ // "types": [], /* Specify type package names to be included without being referenced in a source file. */