diff --git a/changedetectionio/processors/text_json_diff/processor.py b/changedetectionio/processors/text_json_diff/processor.py index 30e41b0f5..121a9c59d 100644 --- a/changedetectionio/processors/text_json_diff/processor.py +++ b/changedetectionio/processors/text_json_diff/processor.py @@ -24,7 +24,7 @@ json_filter_prefixes = ['json:', 'jq:', 'jqraw:'] DEFAULT_WHEN_NO_CONTENT_TYPE_HEADER = 'text/html' # When to apply the 'cdata to real HTML' hack -# @todo Some heuristic check instead? first and last bytes? +# @todo Some heuristic check instead? first and last bytes? maybe some new def that gets header+first 200 bytes? then we can unittest RSS_XML_CONTENT_TYPES = [ "application/rss+xml", "application/rdf+xml", @@ -37,7 +37,7 @@ RSS_XML_CONTENT_TYPES = [ ] # JSON Content-types -# @todo Some heuristic check instead? first and last bytes? +# @todo Some heuristic check instead? first and last bytes? maybe some new def that gets header+first 200 bytes? then we can unittest JSON_CONTENT_TYPES = [ "application/activity+json", "application/feed+json", @@ -73,6 +73,20 @@ class perform_site_check(difference_detection_processor): ctype_header = self.fetcher.get_all_headers().get('content-type', DEFAULT_WHEN_NO_CONTENT_TYPE_HEADER).lower() + # Try to detect better mime types if its a download or not announced as HTML + #@todo also goes into our heurestic detcet class + if 'attachment' in self.fetcher.get_all_headers().get('content-disposition','').lower() or 'octet-stream' in ctype_header: + logger.debug(f"Got a reply that may be a download or possibly a text attachment, checking..") + try: + import magic + mime = magic.from_buffer(self.fetcher.content, mime=True) + logger.debug(f"Guessing mime type, original content_type '{ctype_header}', mime type detected '{mime}'") + if mime and "/" in mime: # looks valid and is a valid mime type + ctype_header = mime + + except Exception as e: + logger.error(f"Error getting a more precise mime type from 'magic' library ({str(e)}") + # Unset any existing notification error update_obj = {'last_notification_error': False, 'last_error': False} @@ -180,20 +194,6 @@ class perform_site_check(difference_detection_processor): self.fetcher.content = html_tools.workarounds_for_obfuscations(self.fetcher.content) html_content = self.fetcher.content - is_attachment = 'attachment' in self.fetcher.get_all_headers().get('content-disposition', '').lower() or 'octet-stream' in ctype_header - - # Try to detect better mime types if its a download or not announced as HTML - if is_attachment: - logger.debug(f"Got a reply that may be a download or possibly a text attachment, checking..") - try: - import magic - mime = magic.from_buffer(html_content, mime=True) - logger.debug(f"Guessing mime type, original content_type '{ctype_header}', mime type detected '{mime}'") - if mime and "/" in mime: # looks valid and is a valid mime type - content_type = mime - except Exception as e: - logger.error(f"Error getting a more precise mime type from 'magic' library ({str(e)}") - # Some kind of "text" but definitely not RSS looking if (not is_rss and not 'html' in ctype_header diff --git a/changedetectionio/tests/test_backend.py b/changedetectionio/tests/test_backend.py index 0d25f6516..8a82b5a15 100644 --- a/changedetectionio/tests/test_backend.py +++ b/changedetectionio/tests/test_backend.py @@ -174,6 +174,8 @@ def test_non_text_mime_or_downloads(client, live_server, measure_memory_usage): but once the server sends content-type: application/octet-stream (which is usually done to force the browser to show the Download dialog), changedetection somehow ignores all line breaks and treats the document file as if everything is on one line. + WHAT THIS DOES - makes the system rely on 'magic' to determine what is it + :param client: :param live_server: :param measure_memory_usage: