lee/Scraperr
1
0
Fork 0
mirror of https://github.com/jaypyles/Scraperr.git synced 2025-12-14 11:46:17 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

feat: add agent mode (#81)

Browse Source 
* chore: wip agent mode

* wip: add agent mode frontend

* wip: add agent mode frontend

* chore: cleanup code

* chore: cleanup code

* chore: cleanup code
This commit is contained in:
Jayden Pyles
2025-05-19 20:44:41 -05:00
committed by GitHub
parent d602d3330a
commit 5ebd96b62b
33 changed files with 869 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
api/backend/scraping.py
View File

@@ -9,7 +9,10 @@ from playwright.async_api import Page
from urllib.parse import urlparse, urljoin
from api.backend.models import Element, CapturedElement
from api.backend.job.scraping.scraping_utils import scrape_content
from api.backend.job.scraping.scraping_utils import (
clean_format_characters,
scrape_content,
)
from api.backend.job.site_mapping.site_mapping import handle_site_mapping
from api.backend.job.scraping.add_custom import add_custom_items
@@ -142,14 +145,7 @@ async def collect_scraped_elements(page: tuple[str, str], xpaths: list[Element])
else str(e) # type: ignore
)
text = text.strip()
text = text.replace("\n", " ")
text = text.replace("\t", " ")
text = text.replace("\r", " ")
text = text.replace("\f", " ")
text = text.replace("\v", " ")
text = text.replace("\b", " ")
text = text.replace("\a", " ")
text = clean_format_characters(text)
captured_element = CapturedElement(
xpath=elem.xpath, text=text, name=elem.name