diff --git a/changedetectionio/content_fetchers/base.py b/changedetectionio/content_fetchers/base.py index 66dd74036..2d29cfea5 100644 --- a/changedetectionio/content_fetchers/base.py +++ b/changedetectionio/content_fetchers/base.py @@ -51,6 +51,7 @@ class Fetcher(): instock_data = None instock_data_js = "" status_code = None + total_bytes = None webdriver_js_execute_code = None xpath_data = None xpath_element_js = "" diff --git a/changedetectionio/content_fetchers/requests.py b/changedetectionio/content_fetchers/requests.py index c39b26363..57342c2a5 100644 --- a/changedetectionio/content_fetchers/requests.py +++ b/changedetectionio/content_fetchers/requests.py @@ -12,6 +12,27 @@ from changedetectionio.content_fetchers.base import Fetcher class fetcher(Fetcher): fetcher_description = "Basic fast Plaintext/HTTP Client" + def get_total_bytes_received(self, response): + # Calculate the size of the response content + content_size = len(response.content) + # Calculate the size of the response headers + headers_size = sum(len(k) + len(v) for k, v in response.headers.items()) + len(response.headers) * 4 # adding 4 for ': ' and '\r\n' + + # Total bytes received + total_received = content_size + headers_size + return total_received + + def get_total_bytes_transferred(self, request): + # Calculate the size of the request headers + headers_size = sum(len(k) + len(v) for k, v in request.headers.items()) + len(request.headers) * 4 # adding 4 for ': ' and '\r\n' + + # Calculate the size of the request body, if any + body_size = len(request.body or '') + + # Total bytes transferred (request + response) + total_transferred = headers_size + body_size + return total_transferred + def __init__(self, proxy_override=None, custom_browser_connection_url=None): super().__init__() self.proxy_override = proxy_override @@ -60,6 +81,10 @@ class fetcher(Fetcher): proxies=proxies, verify=False) + total_received = self.get_total_bytes_received(response=r) + request_prepared = r.request + self.total_bytes = self.get_total_bytes_transferred(request_prepared) + total_received + # If the response did not tell us what encoding format to expect, Then use chardet to override what `requests` thinks. # For example - some sites don't tell us it's utf-8, but return utf-8 content # This seems to not occur when using webdriver/selenium, it seems to detect the text encoding more reliably.