From 8a933b88a725cf2d965d528a57ebfdd25927488d Mon Sep 17 00:00:00 2001 From: Jayden Pyles <111098627+jaypyles@users.noreply.github.com> Date: Sun, 11 May 2025 13:13:42 -0500 Subject: [PATCH] feat: add notification channels (#66) --- api/backend/worker/job_worker.py | 42 +++++++- api/backend/worker/logger.py | 5 + .../post_job_complete/discord_notification.py | 56 +++++++++++ .../post_job_complete/email_notifcation.py | 97 +++++++++++++++++++ .../worker/post_job_complete/models.py | 22 +++++ .../post_job_complete/post_job_complete.py | 21 ++++ docker-compose.dev.yml | 1 - src/components/jobs/JobTable.tsx | 12 ++- src/lib/helpers/parse-job-options.ts | 2 +- 9 files changed, 248 insertions(+), 10 deletions(-) create mode 100644 api/backend/worker/logger.py create mode 100644 api/backend/worker/post_job_complete/discord_notification.py create mode 100644 api/backend/worker/post_job_complete/email_notifcation.py create mode 100644 api/backend/worker/post_job_complete/models.py create mode 100644 api/backend/worker/post_job_complete/post_job_complete.py diff --git a/api/backend/worker/job_worker.py b/api/backend/worker/job_worker.py index 1eeaac9..15c43a5 100644 --- a/api/backend/worker/job_worker.py +++ b/api/backend/worker/job_worker.py @@ -1,21 +1,35 @@ +import os + from api.backend.job import get_queued_job, update_job from api.backend.scraping import scrape from api.backend.models import Element from fastapi.encoders import jsonable_encoder import asyncio -import logging -import sys import traceback from api.backend.database.startup import init_database -logging.basicConfig(stream=sys.stdout, level=logging.INFO) -LOG = logging.getLogger(__name__) +from api.backend.worker.post_job_complete.post_job_complete import post_job_complete +from api.backend.worker.logger import LOG + + +NOTIFICATION_CHANNEL = os.getenv("NOTIFICATION_CHANNEL", "") +NOTIFICATION_WEBHOOK_URL = os.getenv("NOTIFICATION_WEBHOOK_URL", "") +SCRAPERR_FRONTEND_URL = os.getenv("SCRAPERR_FRONTEND_URL", "") +EMAIL = os.getenv("EMAIL", "") +TO = os.getenv("TO", "") +SMTP_HOST = os.getenv("SMTP_HOST", "") +SMTP_PORT = int(os.getenv("SMTP_PORT", 587)) +SMTP_USER = os.getenv("SMTP_USER", "") +SMTP_PASSWORD = os.getenv("SMTP_PASSWORD", "") +USE_TLS = os.getenv("USE_TLS", "false").lower() == "true" async def process_job(): job = await get_queued_job() + status = "Queued" + if job: LOG.info(f"Beginning processing job: {job}.") try: @@ -36,10 +50,30 @@ async def process_job(): [job["id"]], field="result", value=jsonable_encoder(scraped) ) _ = await update_job([job["id"]], field="status", value="Completed") + status = "Completed" + except Exception as e: _ = await update_job([job["id"]], field="status", value="Failed") _ = await update_job([job["id"]], field="result", value=e) LOG.error(f"Exception as occured: {e}\n{traceback.print_exc()}") + status = "Failed" + finally: + job["status"] = status + await post_job_complete( + job, + { + "channel": NOTIFICATION_CHANNEL, + "webhook_url": NOTIFICATION_WEBHOOK_URL, + "scraperr_frontend_url": SCRAPERR_FRONTEND_URL, + "email": EMAIL, + "to": TO, + "smtp_host": SMTP_HOST, + "smtp_port": SMTP_PORT, + "smtp_user": SMTP_USER, + "smtp_password": SMTP_PASSWORD, + "use_tls": USE_TLS, + }, + ) async def main(): diff --git a/api/backend/worker/logger.py b/api/backend/worker/logger.py new file mode 100644 index 0000000..5b7cea4 --- /dev/null +++ b/api/backend/worker/logger.py @@ -0,0 +1,5 @@ +import logging +import sys + +logging.basicConfig(stream=sys.stdout, level=logging.INFO) +LOG = logging.getLogger(__name__) diff --git a/api/backend/worker/post_job_complete/discord_notification.py b/api/backend/worker/post_job_complete/discord_notification.py new file mode 100644 index 0000000..c54d3c1 --- /dev/null +++ b/api/backend/worker/post_job_complete/discord_notification.py @@ -0,0 +1,56 @@ +import json +from typing import Any + +import requests + +from api.backend.worker.logger import LOG +from api.backend.worker.post_job_complete.models import ( + PostJobCompleteOptions, + JOB_COLOR_MAP, +) + + +def discord_notification(job: dict[str, Any], options: PostJobCompleteOptions): + webhook_url = options["webhook_url"] + scraperr_frontend_url = options["scraperr_frontend_url"] + + LOG.info(f"Sending discord notification to {webhook_url}") + + embed = { + "title": "Job Completed", + "description": "Scraping job has been completed.", + "color": JOB_COLOR_MAP[job["status"]], + "url": f"{scraperr_frontend_url}/jobs?search={job['id']}&type=id", + "image": { + "url": "https://github.com/jaypyles/Scraperr/raw/master/docs/logo_picture.png", + }, + "author": { + "name": "Scraperr", + "url": "https://github.com/jaypyles/Scraperr", + }, + "fields": [ + { + "name": "Status", + "value": "Completed", + "inline": True, + }, + { + "name": "URL", + "value": job["url"], + "inline": True, + }, + { + "name": "ID", + "value": job["id"], + "inline": False, + }, + { + "name": "Options", + "value": f"```json\n{json.dumps(job['job_options'], indent=4)}\n```", + "inline": False, + }, + ], + } + + payload = {"embeds": [embed]} + requests.post(webhook_url, json=payload) diff --git a/api/backend/worker/post_job_complete/email_notifcation.py b/api/backend/worker/post_job_complete/email_notifcation.py new file mode 100644 index 0000000..8bf7faa --- /dev/null +++ b/api/backend/worker/post_job_complete/email_notifcation.py @@ -0,0 +1,97 @@ +import smtplib +import ssl +from email.mime.text import MIMEText +from email.mime.multipart import MIMEMultipart +import json +from typing import Any + +from api.backend.worker.logger import LOG + +from api.backend.worker.post_job_complete.models import ( + JOB_COLOR_MAP, + PostJobCompleteOptions, +) + + +def send_job_complete_email( + job: dict[str, Any], + options: PostJobCompleteOptions, +): + status = job["status"] + status_color = JOB_COLOR_MAP.get(status, 0x808080) + job_url = job["url"] + job_id = job["id"] + job_options_json = json.dumps(job["job_options"], indent=4) + frontend_url = options["scraperr_frontend_url"] + + subject = "📦 Job Completed - Scraperr Notification" + + html = f""" + + +

✅ Job Completed

+

Scraping job has been completed successfully.

+ + + Scraperr Logo + + +

Job Info:

+ + +

Options:

+ 
+{job_options_json}
+
+ +

View your job here:

+ Scraperr Job + +

+ Sent by Scraperr +

+ + + """ + + # Create email + message = MIMEMultipart("alternative") + message["From"] = options["email"] + message["To"] = options["to"] + message["Subject"] = subject + message.attach( + MIMEText( + "Job completed. View this email in HTML format for full details.", "plain" + ) + ) + message.attach(MIMEText(html, "html")) + + context = ssl.create_default_context() + + try: + if options["use_tls"]: + with smtplib.SMTP(options["smtp_host"], options["smtp_port"]) as server: + server.starttls(context=context) + server.login(options["smtp_user"], options["smtp_password"]) + server.sendmail( + from_addr=options["email"], + to_addrs=options["to"], + msg=message.as_string(), + ) + else: + with smtplib.SMTP_SSL( + options["smtp_host"], options["smtp_port"], context=context + ) as server: + server.login(options["smtp_user"], options["smtp_password"]) + server.sendmail( + from_addr=options["email"], + to_addrs=options["to"], + msg=message.as_string(), + ) + LOG.info("✅ Email sent successfully!") + except Exception as e: + LOG.error(f"❌ Failed to send email: {e}") diff --git a/api/backend/worker/post_job_complete/models.py b/api/backend/worker/post_job_complete/models.py new file mode 100644 index 0000000..1b57f82 --- /dev/null +++ b/api/backend/worker/post_job_complete/models.py @@ -0,0 +1,22 @@ +from typing import TypedDict + + +class PostJobCompleteOptions(TypedDict): + channel: str + webhook_url: str + scraperr_frontend_url: str + email: str + to: str + smtp_host: str + smtp_port: int + smtp_user: str + smtp_password: str + use_tls: bool + + +JOB_COLOR_MAP = { + "Queued": 0x0000FF, + "Scraping": 0x0000FF, + "Completed": 0x00FF00, + "Failed": 0xFF0000, +} diff --git a/api/backend/worker/post_job_complete/post_job_complete.py b/api/backend/worker/post_job_complete/post_job_complete.py new file mode 100644 index 0000000..09d5329 --- /dev/null +++ b/api/backend/worker/post_job_complete/post_job_complete.py @@ -0,0 +1,21 @@ +from typing import Any + +from api.backend.worker.post_job_complete.models import PostJobCompleteOptions +from api.backend.worker.post_job_complete.email_notifcation import ( + send_job_complete_email, +) +from api.backend.worker.post_job_complete.discord_notification import ( + discord_notification, +) + + +async def post_job_complete(job: dict[str, Any], options: PostJobCompleteOptions): + if not options.values(): + return + + if options["channel"] == "discord": + discord_notification(job, options) + elif options["channel"] == "email": + send_job_complete_email(job, options) + else: + raise ValueError(f"Invalid channel: {options['channel']}") diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 97cce17..678472d 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -14,4 +14,3 @@ services: - LOG_LEVEL=INFO volumes: - "$PWD/api:/project/api" - - "$PWD/scraping:/project/scraping" diff --git a/src/components/jobs/JobTable.tsx b/src/components/jobs/JobTable.tsx index c230696..d21f9de 100644 --- a/src/components/jobs/JobTable.tsx +++ b/src/components/jobs/JobTable.tsx @@ -1,4 +1,4 @@ -import React, { Dispatch, SetStateAction, useState } from "react"; +import React, { SetStateAction, useState } from "react"; import { IconButton, Box, @@ -18,8 +18,8 @@ import StarIcon from "@mui/icons-material/Star"; import { useRouter } from "next/router"; import { Favorites, JobQueue } from "."; import { Job } from "../../types"; -import { Constants } from "../../lib"; import Cookies from "js-cookie"; +import { useSearchParams } from "next/navigation"; interface JobTableProps { jobs: Job[]; @@ -38,10 +38,14 @@ const COLOR_MAP: ColorMap = { }; export const JobTable: React.FC = ({ jobs, setJobs }) => { + const searchParams = useSearchParams(); + const search = searchParams.get("search"); + const type = searchParams.get("type"); + const [selectedJobs, setSelectedJobs] = useState>(new Set()); const [allSelected, setAllSelected] = useState(false); - const [searchQuery, setSearchQuery] = useState(""); - const [searchMode, setSearchMode] = useState("url"); + const [searchQuery, setSearchQuery] = useState(search || ""); + const [searchMode, setSearchMode] = useState(type || "url"); const [favoriteView, setFavoriteView] = useState(false); const token = Cookies.get("token"); diff --git a/src/lib/helpers/parse-job-options.ts b/src/lib/helpers/parse-job-options.ts index 211f19a..106b780 100644 --- a/src/lib/helpers/parse-job-options.ts +++ b/src/lib/helpers/parse-job-options.ts @@ -28,7 +28,7 @@ export const parseJobOptions = ( newJobOptions.multi_page_scrape = jsonOptions.multi_page_scrape; - if (jsonOptions.proxies) { + if (jsonOptions.proxies.length > 0) { setProxiesSelected(true); newJobOptions.proxies = jsonOptions.proxies.join(","); }