Fetchers - Define raw_content on the base fetcher, give a clear error when a fetcher can't supply the PDF

Fetchers that never set raw_content (puppeteer, playwright, selenium) crashed with
"'fetcher' object has no attribute 'raw_content'" when the response turned out to be a PDF.
Browsers render PDFs in their built-in viewer, so all they have is the viewer HTML; now the
processor reports that plainly instead of crashing or feeding HTML to pdftohtml.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
dgtlmoonandClaude Opus 5.5 committed 2026-10-13 10:52:40 +02:00
1 parent bbe02bb58d
commit 79dcdafa5a
2 files changed
+14 -2

No files matched your search

+3 -2
View File
@@ -87,6 +87,8 @@ class Fetcher():
favicon_blob = None
instock_data = None
instock_data_js = ""
# Original response bytes (needed for PDF etc), fetchers that can't provide them leave this as None
raw_content = None
screenshot_format = None
status_code = None
webdriver_js_execute_code = None
@@ -151,8 +153,7 @@ class Fetcher():
Call this after content has been saved to disk.
"""
self.content = None
if hasattr(self, 'raw_content'):
self.raw_content = None
self.raw_content = None
self.screenshot = None
self.xpath_data = None
# Keep headers and status_code as they're small
@@ -43,6 +43,11 @@ class PDFToHTMLToolNotFound(ValueError):
ValueError.__init__(self, msg)
class PDFNotAvailableFromFetcher(ValueError):
def __init__(self, msg):
ValueError.__init__(self, msg)
class FilterConfig:
"""Consolidates all filter and rule configurations from watch, tags, and global settings."""
@@ -481,6 +486,12 @@ class perform_site_check(difference_detection_processor):
# PDF preprocessing
if watch.is_pdf or stream_content_type.is_pdf:
# Browsers show PDFs in their built-in viewer, so a fetcher may only have the viewer's HTML
if not self.fetcher.raw_content or b'%PDF' not in self.fetcher.raw_content[:1024]:
raise PDFNotAvailableFromFetcher(
"Content is a PDF but this fetcher could not provide the original PDF file, "
"try the 'Basic fast Plaintext/HTTP Client' fetcher instead."
)
content = content_processor.preprocess_pdf(raw_content=self.fetcher.raw_content)
stream_content_type.is_html = True