wip: add in job rerunning

This commit is contained in:
Jayden Pyles
2024-07-06 16:56:56 -05:00
parent 70bdd01d9d
commit 8808b493e6
13 changed files with 607 additions and 70 deletions

View File

@@ -1,8 +1,49 @@
# STL
import os
import logging
from typing import Any
# PDM
import boto3
from mypy_boto3_dynamodb.service_resource import Table, DynamoDBServiceResource
LOG = logging.getLogger(__name__)
def test_dyanmo():
dynamodb = boto3.resource("dynamodb", region_name="us-west-2")
table = dynamodb.Table("scrape")
print(table)
def connect_to_dynamo() -> Table:
region_name = os.getenv("AWS_REGION")
dynamodb: DynamoDBServiceResource = boto3.resource(
"dynamodb", region_name=region_name
)
return dynamodb.Table("scrape")
def insert(table: Table, item: dict[str, Any]) -> None:
i = table.put_item(Item=item)
LOG.info(f"Inserted item: {i}")
def query(table: Table, index_name: str, key_condition: Any) -> list[Any]:
try:
response = table.query(
IndexName=index_name, KeyConditionExpression=key_condition
)
items = response.get("Items", [])
for item in items:
LOG.info(f"Queried item: {item}")
return items
except Exception as e:
LOG.error(f"Failed to query table: {e}")
raise
def query_by_id(table: Table, key_condition: Any) -> list[Any]:
try:
response = table.query(KeyConditionExpression=key_condition)
items = response.get("Items", [])
for item in items:
LOG.info(f"Queried item: {item}")
return items
except Exception as e:
LOG.error(f"Failed to query table: {e}")
raise