mirror of
https://github.com/dgtlmoon/changedetection.io.git
synced 2026-11-03 12:34:58 +00:00
Fetchers - Define raw_content on the base fetcher, give a clear error when a fetcher can't supply the PDF
Fetchers that never set raw_content (puppeteer, playwright, selenium) crashed with "'fetcher' object has no attribute 'raw_content'" when the response turned out to be a PDF. Browsers render PDFs in their built-in viewer, so all they have is the viewer HTML; now the processor reports that plainly instead of crashing or feeding HTML to pdftohtml. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
bbe02bb58d
commit
79dcdafa5a
2 files changed
+14
-2
No files matched your search
@@ -87,6 +87,8 @@ class Fetcher():
|
||||
favicon_blob = None
|
||||
instock_data = None
|
||||
instock_data_js = ""
|
||||
# Original response bytes (needed for PDF etc), fetchers that can't provide them leave this as None
|
||||
raw_content = None
|
||||
screenshot_format = None
|
||||
status_code = None
|
||||
webdriver_js_execute_code = None
|
||||
@@ -151,8 +153,7 @@ class Fetcher():
|
||||
Call this after content has been saved to disk.
|
||||
"""
|
||||
self.content = None
|
||||
if hasattr(self, 'raw_content'):
|
||||
self.raw_content = None
|
||||
self.raw_content = None
|
||||
self.screenshot = None
|
||||
self.xpath_data = None
|
||||
# Keep headers and status_code as they're small
|
||||
|
||||
@@ -43,6 +43,11 @@ class PDFToHTMLToolNotFound(ValueError):
|
||||
ValueError.__init__(self, msg)
|
||||
|
||||
|
||||
class PDFNotAvailableFromFetcher(ValueError):
|
||||
def __init__(self, msg):
|
||||
ValueError.__init__(self, msg)
|
||||
|
||||
|
||||
class FilterConfig:
|
||||
"""Consolidates all filter and rule configurations from watch, tags, and global settings."""
|
||||
|
||||
@@ -481,6 +486,12 @@ class perform_site_check(difference_detection_processor):
|
||||
|
||||
# PDF preprocessing
|
||||
if watch.is_pdf or stream_content_type.is_pdf:
|
||||
# Browsers show PDFs in their built-in viewer, so a fetcher may only have the viewer's HTML
|
||||
if not self.fetcher.raw_content or b'%PDF' not in self.fetcher.raw_content[:1024]:
|
||||
raise PDFNotAvailableFromFetcher(
|
||||
"Content is a PDF but this fetcher could not provide the original PDF file, "
|
||||
"try the 'Basic fast Plaintext/HTTP Client' fetcher instead."
|
||||
)
|
||||
content = content_processor.preprocess_pdf(raw_content=self.fetcher.raw_content)
|
||||
stream_content_type.is_html = True
|
||||
|
||||
|
||||
Reference in new issue
Block a user