Merge branch 'master' into browsersteps-requests

changedetectionio/html_tools.py

@@ -409,6 +409,23 @@ def has_ldjson_product_info(content):
     x=bool(pricing_data)
     return x
 
+
+def workarounds_for_obfuscations(content):
+    """
+    Some sites are using sneaky tactics to make prices and other information un-renderable by Inscriptis
+    This could go into its own Pip package in the future, for faster updates
+    """
+
+    # HomeDepot.com style <span>$<!-- -->90<!-- -->.<!-- -->74</span>
+    # https://github.com/weblyzard/inscriptis/issues/45
+    if not content:
+        return content
+
+    content = re.sub('<!--\s+-->', '', content)
+
+    return content
+
+
 def get_triggered_text(content, trigger_text):
     triggered_text = []
     result = strip_ignore_text(content=content,

changedetectionio/model/Watch.py

@@ -45,7 +45,6 @@ base_config = {
     'last_error': False,
     'last_viewed': 0,  # history key value of the last viewed via the [diff] link
     'method': 'GET',
-    'notification_alert_count': 0,
     # Custom notification content
     'notification_body': None,
     'notification_format': default_notification_format_for_watch,

changedetectionio/processors/text_json_diff.py

@@ -151,6 +151,7 @@ class perform_site_check(difference_detection_processor):
         if is_html or watch.is_source_type_url:
 
             # CSS Filter, extract the HTML that matches and feed that into the existing inscriptis::get_text
+            self.fetcher.content = html_tools.workarounds_for_obfuscations(self.fetcher.content)
             html_content = self.fetcher.content
 
             # If not JSON,  and if it's not text/plain..

changedetectionio/res/stock-not-in-stock.js

@@ -47,7 +47,6 @@ function isItemInStock() {
         'não estamos a aceitar encomendas',
         'out of stock',
         'out-of-stock',
-        'prodotto esaurito',
         'produkt niedostępny',
         'sold out',
         'sold-out',

changedetectionio/templates/edit.html

@@ -483,10 +483,6 @@ Unavailable") }}
                             <td>Last fetch time</td>
                             <td>{{ watch.fetch_time }}s</td>
                         </tr>
-                        <tr>
-                            <td>Notification alert count</td>
-                            <td>{{ watch.notification_alert_count }}</td>
-                        </tr>
                         </tbody>
                     </table>
                 </div>

changedetectionio/tests/test_obfuscations.py

@@ -2,7 +2,7 @@
 
 import time
 from flask import url_for
-from .util import live_server_setup, wait_for_all_checks
+from .util import live_server_setup
 
 
 def set_original_ignore_response():
@@ -21,7 +21,7 @@ def set_original_ignore_response():
 def test_obfuscations(client, live_server):
     set_original_ignore_response()
     live_server_setup(live_server)
-
+    time.sleep(1)
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
@@ -32,12 +32,12 @@ def test_obfuscations(client, live_server):
     assert b"1 Imported" in res.data
 
     # Give the thread time to pick it up
-    wait_for_all_checks(client)
+    time.sleep(3)
 
     # Check HTML conversion detected and workd
     res = client.get(
         url_for("preview_page", uuid="first"),
         follow_redirects=True
     )
-    # whitespace appears but it renders https://github.com/weblyzard/inscriptis/issues/45#issuecomment-1923339265
+
     assert b'$90.74' in res.data

changedetectionio/update_worker.py

@@ -150,10 +150,6 @@ class update_worker(threading.Thread):
         queued = False
         if n_object and n_object.get('notification_urls'):
             queued = True
-
-            count = watch.get('notification_alert_count', 0) + 1
-            self.datastore.update_watch(uuid=watch_uuid, update_obj={'notification_alert_count': count})
-
             self.queue_notification_for_watch(notification_q=self.notification_q, n_object=n_object, watch=watch)
 
         return queued

requirements.txt

@@ -8,7 +8,7 @@ flask_expects_json~=1.7
 flask_restful
 flask_wtf~=1.2
 flask~=2.3
-inscriptis~=2.4
+inscriptis~=2.2
 pytz
 timeago~=1.0
 validators~=0.21