wip: add in job rerunning

This commit is contained in:
Jayden Pyles
2024-07-06 16:56:56 -05:00
parent 70bdd01d9d
commit 8808b493e6
13 changed files with 607 additions and 70 deletions

View File

@@ -1,17 +1,37 @@
# STL
import asyncio
import os
# LOCAL
from api.backend.scraping import scrape
# PDM
import boto3
from dotenv import load_dotenv
# Load environment variables from .env file
load_dotenv()
async def main():
url = "https://darksouls3.wiki.fextralife.com/Dark+Souls+3"
xpaths = [".//h3[@class='bonfire']", ".//div[@class='comment']"]
scraped = await scrape(url, xpaths)
def test_insert_and_delete():
# Get environment variables
region_name = os.getenv("AWS_REGION")
# Initialize DynamoDB resource
dynamodb = boto3.resource("dynamodb", region_name=region_name)
table = dynamodb.Table("scrape")
print(scraped)
# Item to insert
item = {
"id": "123", # Replace with the appropriate id value
"attribute1": "value1",
"attribute2": "value2",
# Add more attributes as needed
}
# Insert the item
table.put_item(Item=item)
print(f"Inserted item: {item}")
# Delete the item
table.delete_item(Key={"id": "123"}) # Replace with the appropriate id value
print(f"Deleted item with id: {item['id']}")
if __name__ == "__main__":
asyncio.run(main())
test_insert_and_delete()