feat: general rework (#32)

* feat: general rework [skip ci]

* feat: general rework [skip ci]

* feat: general rework [skip ci]

* feat: use csv [skip ci]

* feat: add testing [skip ci]

* fix: remove logging [skip ci]
This commit is contained in:
Jayden Pyles
2024-10-20 17:52:58 -05:00
committed by GitHub
parent 14cf2e9dbc
commit d3c6a3f6a3
49 changed files with 769 additions and 414 deletions

View File

@@ -1,5 +1,5 @@
# STL
from typing import Any, Optional
from typing import Any, Optional, Union
from datetime import datetime
# PDM
@@ -27,18 +27,6 @@ class JobOptions(pydantic.BaseModel):
custom_headers: Optional[dict[str, Any]]
class SubmitScrapeJob(pydantic.BaseModel):
id: Optional[str] = None
url: str
elements: list[Element]
user: Optional[str] = None
time_created: Optional[datetime] = None
result: Optional[dict[str, Any]] = None
job_options: JobOptions
status: str = "Queued"
chat: Optional[str] = None
class RetrieveScrapeJobs(pydantic.BaseModel):
user: str
@@ -63,3 +51,15 @@ class UpdateJobs(pydantic.BaseModel):
class AI(pydantic.BaseModel):
messages: list[Any]
class Job(pydantic.BaseModel):
id: Optional[str] = None
url: str
elements: list[Element]
user: str = ""
time_created: Optional[Union[datetime, str]] = None
result: list[dict[str, dict[str, list[CapturedElement]]]] = []
job_options: JobOptions
status: str = "Queued"
chat: Optional[str] = None