lee/Scraperr
1
0
Fork 0
mirror of https://github.com/jaypyles/Scraperr.git synced 2025-12-12 10:45:58 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

fix: better scrolling

Browse Source
This commit is contained in:
Jayden Pyles
2024-11-06 21:05:15 -06:00
parent fe5bc8859c
commit 86a1d32990
1 changed files with 3 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
api/backend/scraping.py
View File

@@ -106,7 +106,7 @@ async def make_site_request(
while True: while True:
driver.execute_script("window.scrollTo(0, document.body.scrollHeight);") driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
time.sleep(2) # Wait for the page to load time.sleep(3) # Wait for the page to load
new_height = driver.execute_script("return document.body.scrollHeight") new_height = driver.execute_script("return document.body.scrollHeight")
if new_height == last_height: if new_height == last_height:
@@ -114,9 +114,9 @@ async def make_site_request(
last_height = new_height last_height = new_height
driver.execute_script("return document.body.scrollHeight") final_height = driver.execute_script("return document.body.scrollHeight")
page_source = driver.page_source
page_source = driver.page_source
LOG.debug(f"Page source for url: {url}\n{page_source}") LOG.debug(f"Page source for url: {url}\n{page_source}")
pages.add((page_source, final_url)) pages.add((page_source, final_url))
finally: finally: