lee/Scraperr
1
0
Fork 0
mirror of https://github.com/jaypyles/Scraperr.git synced 2025-12-14 11:46:17 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

feat: add recording viewer and vnc (#78)

Browse Source 
* feat: add recording viewer and vnc

* feat: add recording viewer and vnc

* feat: add recording viewer and vnc

* feat: add recording viewer and vnc

* chore: update gitignore [skip ci]

* chore: update dev compose [skip ci]

* fix: only run manually
This commit is contained in:
Jayden Pyles
2025-05-16 21:37:09 -05:00
committed by GitHub
parent 6b33723cac
commit 28de0f362c
34 changed files with 6355 additions and 11440 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
api/backend/scraping.py
View File

@@ -1,4 +1,5 @@
import logging
from pickle import FALSE
import random
from typing import Any, Optional, cast
@@ -14,6 +15,8 @@ from api.backend.job.site_mapping.site_mapping import handle_site_mapping
from api.backend.job.scraping.add_custom import add_custom_items
from api.backend.constants import RECORDINGS_ENABLED
LOG = logging.getLogger(__name__)
@@ -57,8 +60,9 @@ async def make_site_request(
proxy = random.choice(proxies)
LOG.info(f"Using proxy: {proxy}")
async with AsyncCamoufox(headless=True, proxy=proxy) as browser:
async with AsyncCamoufox(headless=not RECORDINGS_ENABLED, proxy=proxy) as browser:
page: Page = await browser.new_page()
await page.set_viewport_size({"width": 1920, "height": 1080})
# Add cookies and headers
await add_custom_items(url, page, custom_cookies, headers)