From 912ab903c9bacdb6a5209271fa97c05cfede3397 Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Wed, 15 Oct 2025 12:33:39 +0200 Subject: [PATCH] Price currency and amount fixes --- .../processors/restock_diff/__init__.py | 84 ++++++++++++++++++- .../processors/restock_diff/processor.py | 24 ++++-- requirements.txt | 2 + 3 files changed, 97 insertions(+), 13 deletions(-) diff --git a/changedetectionio/processors/restock_diff/__init__.py b/changedetectionio/processors/restock_diff/__init__.py index 3d472beec..f2b13e2df 100644 --- a/changedetectionio/processors/restock_diff/__init__.py +++ b/changedetectionio/processors/restock_diff/__init__.py @@ -6,8 +6,76 @@ import re class Restock(dict): - def parse_currency(self, raw_value: str) -> Union[float, None]: - # Clean and standardize the value (ie 1,400.00 should be 1400.00), even better would be store the whole thing as an integer. + def _normalize_currency_code(self, currency: str) -> str: + """ + Normalize currency symbol or code to ISO 4217 code for consistency. + Uses iso4217parse for accurate conversion. + """ + if not currency: + return currency + + # If already a 3-letter code, likely already normalized + if len(currency) == 3 and currency.isupper(): + return currency + + try: + import iso4217parse + + # Parse the currency - returns list of possible matches + currencies = iso4217parse.parse(currency) + + if currencies: + # For ambiguous symbols, prefer common currencies + if currency == '$': + # Prefer USD for $ symbol + usd = [c for c in currencies if c.alpha3 == 'USD'] + if usd: + return 'USD' + elif currency == '£': + # Prefer GBP for £ symbol + gbp = [c for c in currencies if c.alpha3 == 'GBP'] + if gbp: + return 'GBP' + elif currency == '¥': + # Prefer JPY for ¥ symbol + jpy = [c for c in currencies if c.alpha3 == 'JPY'] + if jpy: + return 'JPY' + + # Return first match for unambiguous symbols + return currencies[0].alpha3 + except Exception: + pass + + # Fallback: return as-is if can't normalize + return currency + + def parse_currency(self, raw_value: str) -> Union[dict, None]: + """ + Parse price and currency from text, handling messy formats with extra text. + Returns dict with 'price' and 'currency' keys (ISO 4217 code), or None if parsing fails. + """ + try: + from price_parser import Price + # price-parser handles: + # - Extra text before/after ("Beginning at", "tax incl.") + # - Various number formats (1 099,00 or 1,099.00) + # - Currency symbols and codes + price_obj = Price.fromstring(raw_value) + + if price_obj.amount is not None: + result = {'price': float(price_obj.amount)} + if price_obj.currency: + # Normalize currency symbol to ISO 4217 code for consistency with metadata + normalized_currency = self._normalize_currency_code(price_obj.currency) + result['currency'] = normalized_currency + return result + + except Exception as e: + from loguru import logger + logger.trace(f"price-parser failed on '{raw_value}': {e}, falling back to manual parsing") + + # Fallback to existing manual parsing logic standardized_value = raw_value if ',' in standardized_value and '.' in standardized_value: @@ -24,7 +92,7 @@ class Restock(dict): if standardized_value: # Convert to float - return float(parse_decimal(standardized_value, locale='en')) + return {'price': float(parse_decimal(standardized_value, locale='en'))} return None @@ -51,7 +119,15 @@ class Restock(dict): # Custom logic to handle setting price and original_price if key == 'price' or key == 'original_price': if isinstance(value, str): - value = self.parse_currency(raw_value=value) + parsed = self.parse_currency(raw_value=value) + if parsed: + # Set the price value + value = parsed.get('price') + # Also set currency if found and not already set + if parsed.get('currency') and not self.get('currency'): + super().__setitem__('currency', parsed.get('currency')) + else: + value = None super().__setitem__(key, value) diff --git a/changedetectionio/processors/restock_diff/processor.py b/changedetectionio/processors/restock_diff/processor.py index 8e413973d..e755b19b3 100644 --- a/changedetectionio/processors/restock_diff/processor.py +++ b/changedetectionio/processors/restock_diff/processor.py @@ -180,18 +180,24 @@ def get_price_data_availability(html_content, price_change_custom_include_filter if filtered_content.strip(): # Convert HTML to text - price_text = html_tools.html_to_text( - html_content=filtered_content, - render_anchor_tag_content=False, - is_rss=False - ).strip() + import re + price_text = re.sub( + r'[\r\n\t]+', ' ', + html_tools.html_to_text( + html_content=filtered_content, + render_anchor_tag_content=False, + is_rss=False + ).strip() + ) # Parse the price from text try: - parsed_price = value.parse_currency(price_text) - if parsed_price is not None: - value['price'] = parsed_price - logger.debug(f"Extracted price from custom selector: {parsed_price} (from text: '{price_text}')") + parsed_result = value.parse_currency(price_text) + if parsed_result: + value['price'] = parsed_result.get('price') + if parsed_result.get('currency'): + value['currency'] = parsed_result.get('currency') + logger.debug(f"Extracted price from custom selector: {parsed_result.get('price')} {parsed_result.get('currency', '')} (from text: '{price_text}')") except Exception as e: logger.warning(f"Failed to parse price from '{price_text}': {e}") diff --git a/requirements.txt b/requirements.txt index d8eb4a02b..b6cfc780a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -105,6 +105,8 @@ extruct # For cleaning up unknown currency formats babel +# For normalizing currency symbols to ISO 4217 codes +iso4217parse levenshtein