lee/Scraperr
1
0
Fork 0
mirror of https://github.com/jaypyles/Scraperr.git synced 2026-08-25 05:46:28 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

feat: add notification channels (#66)

Browse Source
This commit is contained in:
Jayden Pyles
2025-05-11 13:13:42 -05:00
committed by GitHub
parent 863dbcd044
commit 8a933b88a7
9 changed files with 248 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
api/backend/worker/job_worker.py
+38 -4
View File
@@ -1,21 +1,35 @@
import os
from api.backend.job import get_queued_job, update_job
from api.backend.scraping import scrape
from api.backend.models import Element
from fastapi.encoders import jsonable_encoder
import asyncio
import logging
import sys
import traceback
from api.backend.database.startup import init_database
logging.basicConfig(stream=sys.stdout, level=logging.INFO)
LOG = logging.getLogger(__name__)
from api.backend.worker.post_job_complete.post_job_complete import post_job_complete
from api.backend.worker.logger import LOG
NOTIFICATION_CHANNEL = os.getenv("NOTIFICATION_CHANNEL", "")
NOTIFICATION_WEBHOOK_URL = os.getenv("NOTIFICATION_WEBHOOK_URL", "")
SCRAPERR_FRONTEND_URL = os.getenv("SCRAPERR_FRONTEND_URL", "")
EMAIL = os.getenv("EMAIL", "")
TO = os.getenv("TO", "")
SMTP_HOST = os.getenv("SMTP_HOST", "")
SMTP_PORT = int(os.getenv("SMTP_PORT", 587))
SMTP_USER = os.getenv("SMTP_USER", "")
SMTP_PASSWORD = os.getenv("SMTP_PASSWORD", "")
USE_TLS = os.getenv("USE_TLS", "false").lower() == "true"
async def process_job():
job = await get_queued_job()
status = "Queued"
if job:
LOG.info(f"Beginning processing job: {job}.")
try:
@@ -36,10 +50,30 @@ async def process_job():
[job["id"]], field="result", value=jsonable_encoder(scraped)
)
_ = await update_job([job["id"]], field="status", value="Completed")
status = "Completed"
except Exception as e:
_ = await update_job([job["id"]], field="status", value="Failed")
_ = await update_job([job["id"]], field="result", value=e)
LOG.error(f"Exception as occured: {e}\n{traceback.print_exc()}")
status = "Failed"
finally:
job["status"] = status
await post_job_complete(
job,
{
"channel": NOTIFICATION_CHANNEL,
"webhook_url": NOTIFICATION_WEBHOOK_URL,
"scraperr_frontend_url": SCRAPERR_FRONTEND_URL,
"email": EMAIL,
"to": TO,
"smtp_host": SMTP_HOST,
"smtp_port": SMTP_PORT,
"smtp_user": SMTP_USER,
"smtp_password": SMTP_PASSWORD,
"use_tls": USE_TLS,
},
)
async def main():
api/backend/worker/logger.py
+5
View File
@@ -0,0 +1,5 @@
import logging
import sys
logging.basicConfig(stream=sys.stdout, level=logging.INFO)
LOG = logging.getLogger(__name__)
api/backend/worker/post_job_complete/discord_notification.py
+56
View File
@@ -0,0 +1,56 @@
import json
from typing import Any
import requests
from api.backend.worker.logger import LOG
from api.backend.worker.post_job_complete.models import (
PostJobCompleteOptions,
JOB_COLOR_MAP,
)
def discord_notification(job: dict[str, Any], options: PostJobCompleteOptions):
webhook_url = options["webhook_url"]
scraperr_frontend_url = options["scraperr_frontend_url"]
LOG.info(f"Sending discord notification to {webhook_url}")
embed = {
"title": "Job Completed",
"description": "Scraping job has been completed.",
"color": JOB_COLOR_MAP[job["status"]],
"url": f"{scraperr_frontend_url}/jobs?search={job['id']}&type=id",
"image": {
"url": "https://github.com/jaypyles/Scraperr/raw/master/docs/logo_picture.png",
},
"author": {
"name": "Scraperr",
"url": "https://github.com/jaypyles/Scraperr",
},
"fields": [
{
"name": "Status",
"value": "Completed",
"inline": True,
},
{
"name": "URL",
"value": job["url"],
"inline": True,
},
{
"name": "ID",
"value": job["id"],
"inline": False,
},
{
"name": "Options",
"value": f"```json\n{json.dumps(job['job_options'], indent=4)}\n```",
"inline": False,
},
],
}
payload = {"embeds": [embed]}
requests.post(webhook_url, json=payload)
api/backend/worker/post_job_complete/email_notifcation.py
+97
View File
@@ -0,0 +1,97 @@
import smtplib
import ssl
from email.mime.text import MIMEText
from email.mime.multipart import MIMEMultipart
import json
from typing import Any
from api.backend.worker.logger import LOG
from api.backend.worker.post_job_complete.models import (
JOB_COLOR_MAP,
PostJobCompleteOptions,
)
def send_job_complete_email(
job: dict[str, Any],
options: PostJobCompleteOptions,
):
status = job["status"]
status_color = JOB_COLOR_MAP.get(status, 0x808080)
job_url = job["url"]
job_id = job["id"]
job_options_json = json.dumps(job["job_options"], indent=4)
frontend_url = options["scraperr_frontend_url"]
subject = "📦 Job Completed - Scraperr Notification"
html = f"""
<html>
<body style="font-family: Arial, sans-serif;">
<h2 style="color: #{status_color:06x};">✅ Job Completed</h2>
<p>Scraping job has been completed successfully.</p>
<a href="{frontend_url}/jobs?search={job_id}&type=id" target="_blank">
<img src="https://github.com/jaypyles/Scraperr/raw/master/docs/logo_picture.png" alt="Scraperr Logo" width="200">
</a>
<h3>Job Info:</h3>
<ul>
<li><strong>Status:</strong> {status}</li>
<li><strong>Job URL:</strong> <a href="{job_url}">{job_url}</a></li>
<li><strong>Job ID:</strong> {job_id}</li>
</ul>
<h3>Options:</h3>
<pre style="background-color:#f4f4f4; padding:10px; border-radius:5px;">
{job_options_json}
</pre>
<h3>View your job here:</h3>
<a href="{options['scraperr_frontend_url']}/jobs?search={job_id}&type=id">Scraperr Job</a>
<p style="font-size: 12px; color: gray;">
Sent by <a href="https://github.com/jaypyles/Scraperr" target="_blank">Scraperr</a>
</p>
</body>
</html>
"""
# Create email
message = MIMEMultipart("alternative")
message["From"] = options["email"]
message["To"] = options["to"]
message["Subject"] = subject
message.attach(
MIMEText(
"Job completed. View this email in HTML format for full details.", "plain"
)
)
message.attach(MIMEText(html, "html"))
context = ssl.create_default_context()
try:
if options["use_tls"]:
with smtplib.SMTP(options["smtp_host"], options["smtp_port"]) as server:
server.starttls(context=context)
server.login(options["smtp_user"], options["smtp_password"])
server.sendmail(
from_addr=options["email"],
to_addrs=options["to"],
msg=message.as_string(),
)
else:
with smtplib.SMTP_SSL(
options["smtp_host"], options["smtp_port"], context=context
) as server:
server.login(options["smtp_user"], options["smtp_password"])
server.sendmail(
from_addr=options["email"],
to_addrs=options["to"],
msg=message.as_string(),
)
LOG.info("✅ Email sent successfully!")
except Exception as e:
LOG.error(f"❌ Failed to send email: {e}")
api/backend/worker/post_job_complete/models.py
+22
View File
@@ -0,0 +1,22 @@
from typing import TypedDict
class PostJobCompleteOptions(TypedDict):
channel: str
webhook_url: str
scraperr_frontend_url: str
email: str
to: str
smtp_host: str
smtp_port: int
smtp_user: str
smtp_password: str
use_tls: bool
JOB_COLOR_MAP = {
"Queued": 0x0000FF,
"Scraping": 0x0000FF,
"Completed": 0x00FF00,
"Failed": 0xFF0000,
}
api/backend/worker/post_job_complete/post_job_complete.py
+21
View File
@@ -0,0 +1,21 @@
from typing import Any
from api.backend.worker.post_job_complete.models import PostJobCompleteOptions
from api.backend.worker.post_job_complete.email_notifcation import (
send_job_complete_email,
)
from api.backend.worker.post_job_complete.discord_notification import (
discord_notification,
)
async def post_job_complete(job: dict[str, Any], options: PostJobCompleteOptions):
if not options.values():
return
if options["channel"] == "discord":
discord_notification(job, options)
elif options["channel"] == "email":
send_job_complete_email(job, options)
else:
raise ValueError(f"Invalid channel: {options['channel']}")
docker-compose.dev.yml
-1
View File
@@ -14,4 +14,3 @@ services:
- LOG_LEVEL=INFO
volumes:
- "$PWD/api:/project/api"
- "$PWD/scraping:/project/scraping"
src/components/jobs/JobTable.tsx
+8 -4
View File
@@ -1,4 +1,4 @@
import React, { Dispatch, SetStateAction, useState } from "react";
import React, { SetStateAction, useState } from "react";
import {
IconButton,
Box,
@@ -18,8 +18,8 @@ import StarIcon from "@mui/icons-material/Star";
import { useRouter } from "next/router";
import { Favorites, JobQueue } from ".";
import { Job } from "../../types";
import { Constants } from "../../lib";
import Cookies from "js-cookie";
import { useSearchParams } from "next/navigation";
interface JobTableProps {
jobs: Job[];
@@ -38,10 +38,14 @@ const COLOR_MAP: ColorMap = {
};
export const JobTable: React.FC<JobTableProps> = ({ jobs, setJobs }) => {
const searchParams = useSearchParams();
const search = searchParams.get("search");
const type = searchParams.get("type");
const [selectedJobs, setSelectedJobs] = useState<Set<string>>(new Set());
const [allSelected, setAllSelected] = useState(false);
const [searchQuery, setSearchQuery] = useState<string>("");
const [searchMode, setSearchMode] = useState<string>("url");
const [searchQuery, setSearchQuery] = useState<string>(search || "");
const [searchMode, setSearchMode] = useState<string>(type || "url");
const [favoriteView, setFavoriteView] = useState<boolean>(false);
const token = Cookies.get("token");
src/lib/helpers/parse-job-options.ts
+1 -1
View File
@@ -28,7 +28,7 @@ export const parseJobOptions = (
newJobOptions.multi_page_scrape = jsonOptions.multi_page_scrape;
if (jsonOptions.proxies) {
if (jsonOptions.proxies.length > 0) {
setProxiesSelected(true);
newJobOptions.proxies = jsonOptions.proxies.join(",");
}