mirror of
https://github.com/jaypyles/Scraperr.git
synced 2025-12-14 19:55:56 +00:00
wip: add in job rerunning
This commit is contained in:
38
ipython.py
38
ipython.py
@@ -1,17 +1,37 @@
|
||||
# STL
|
||||
import asyncio
|
||||
import os
|
||||
|
||||
# LOCAL
|
||||
from api.backend.scraping import scrape
|
||||
# PDM
|
||||
import boto3
|
||||
from dotenv import load_dotenv
|
||||
|
||||
# Load environment variables from .env file
|
||||
load_dotenv()
|
||||
|
||||
|
||||
async def main():
|
||||
url = "https://darksouls3.wiki.fextralife.com/Dark+Souls+3"
|
||||
xpaths = [".//h3[@class='bonfire']", ".//div[@class='comment']"]
|
||||
scraped = await scrape(url, xpaths)
|
||||
def test_insert_and_delete():
|
||||
# Get environment variables
|
||||
region_name = os.getenv("AWS_REGION")
|
||||
# Initialize DynamoDB resource
|
||||
dynamodb = boto3.resource("dynamodb", region_name=region_name)
|
||||
table = dynamodb.Table("scrape")
|
||||
|
||||
print(scraped)
|
||||
# Item to insert
|
||||
item = {
|
||||
"id": "123", # Replace with the appropriate id value
|
||||
"attribute1": "value1",
|
||||
"attribute2": "value2",
|
||||
# Add more attributes as needed
|
||||
}
|
||||
|
||||
# Insert the item
|
||||
table.put_item(Item=item)
|
||||
print(f"Inserted item: {item}")
|
||||
|
||||
# Delete the item
|
||||
table.delete_item(Key={"id": "123"}) # Replace with the appropriate id value
|
||||
print(f"Deleted item with id: {item['id']}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
test_insert_and_delete()
|
||||
|
||||
Reference in New Issue
Block a user