Merge branch 'master' into extra-filters

changedetectionio/flask_app.py

@@ -1,6 +1,8 @@
 #!/usr/bin/env python3
 
 import datetime
+import importlib
+
 import flask_login
 import locale
 import os
@@ -10,7 +12,9 @@ import threading
 import time
 import timeago
 
+from .content_fetchers.exceptions import ReplyWithContentButNoText
 from .processors import find_processors, get_parent_module, get_custom_watch_obj_for_processor
+from .processors.text_json_diff.processor import FilterNotFoundInResponse
 from .safe_jinja import render as jinja_render
 from changedetectionio.strtobool import strtobool
 from copy import deepcopy
@@ -1396,6 +1400,57 @@ def changedetection_app(config=None, datastore_o=None):
         # Return a 500 error
         abort(500)
 
+    @app.route("/edit/<string:uuid>/preview-rendered", methods=['POST'])
+    @login_optionally_required
+    def watch_get_preview_rendered(uuid):
+        '''For when viewing the "preview" of the rendered text from inside of Edit'''
+        now = time.time()
+        import brotli
+        from . import forms
+
+        text_after_filter = ''
+        tmp_watch = deepcopy(datastore.data['watching'].get(uuid))
+
+        if tmp_watch and tmp_watch.history and os.path.isdir(tmp_watch.watch_data_dir):
+            # Splice in the temporary stuff from the form
+            form = forms.processor_text_json_diff_form(formdata=request.form if request.method == 'POST' else None,
+                                                       data=request.form
+                                                       )
+            # Only update vars that came in via the AJAX post
+            p = {k: v for k, v in form.data.items() if k in request.form.keys()}
+            tmp_watch.update(p)
+
+            latest_filename = next(reversed(tmp_watch.history))
+            html_fname = os.path.join(tmp_watch.watch_data_dir, f"{latest_filename}.html.br")
+            with open(html_fname, 'rb') as f:
+                decompressed_data = brotli.decompress(f.read()).decode('utf-8') if html_fname.endswith('.br') else f.read().decode('utf-8')
+
+                # Just like a normal change detection except provide a fake "watch" object and dont call .call_browser()
+                processor_module = importlib.import_module("changedetectionio.processors.text_json_diff.processor")
+                update_handler = processor_module.perform_site_check(datastore=datastore,
+                                                                     watch_uuid=uuid # probably not needed anymore anyway?
+                                                                     )
+                # Use the last loaded HTML as the input
+                update_handler.fetcher.content = decompressed_data
+                try:
+                    changed_detected, update_obj, contents, text_after_filter = update_handler.run_changedetection(
+                        watch=tmp_watch,
+                        skip_when_checksum_same=False,
+                    )
+                except FilterNotFoundInResponse as e:
+                    text_after_filter = f"Filter not found in HTML: {str(e)}"
+                except ReplyWithContentButNoText as e:
+                    text_after_filter = f"Filter found but no text (empty result)"
+                except Exception as e:
+                    text_after_filter = f"Error: {str(e)}"
+
+            if not text_after_filter.strip():
+                text_after_filter = 'Empty content'
+
+        logger.trace(f"Parsed in {time.time()-now:.3f}s")
+        return text_after_filter.strip()
+
+
     @app.route("/form/add/quickwatch", methods=['POST'])
     @login_optionally_required
     def form_quick_watch_add():

changedetectionio/forms.py

@@ -480,8 +480,10 @@ class processor_text_json_diff_form(commonSettingsForm):
     body = TextAreaField('Request body', [validators.Optional()])
     method = SelectField('Request method', choices=valid_method, default=default_method)
     ignore_status_codes = BooleanField('Ignore status codes (process non-2xx status codes as normal)', default=False)
-    check_unique_lines = BooleanField('Only trigger when unique lines appear', default=False)
+    check_unique_lines = BooleanField('Only trigger when unique lines appear in all history', default=False)
+    remove_duplicate_lines = BooleanField('Remove duplicate lines of text', default=False)
     sort_text_alphabetically =  BooleanField('Sort text alphabetically', default=False)
+    trim_text_whitespace = BooleanField('Trim whitespace before and after text', default=False)
 
     filter_text_added = BooleanField('Added lines', default=True)
     filter_text_replaced = BooleanField('Replaced/changed lines', default=True)

changedetectionio/model/__init__.py

@@ -59,6 +59,7 @@ class watch_base(dict):
             'time_between_check': {'weeks': None, 'days': None, 'hours': None, 'minutes': None, 'seconds': None},
             'time_between_check_use_default': True,
             'title': None,
+            'trim_text_whitespace': False,
             'track_ldjson_price_data': None,
             'trigger_text': [],  # List of text or regex to wait for until a change is detected
             'url': '',

changedetectionio/processors/__init__.py

@@ -1,4 +1,6 @@
 from abc import abstractmethod
+
+from changedetectionio.content_fetchers.base import Fetcher
 from changedetectionio.strtobool import strtobool
 
 from copy import deepcopy
@@ -23,10 +25,11 @@ class difference_detection_processor():
         super().__init__(*args, **kwargs)
         self.datastore = datastore
         self.watch = deepcopy(self.datastore.data['watching'].get(watch_uuid))
+        # Generic fetcher that should be extended (requests, playwright etc)
+        self.fetcher = Fetcher()
 
     def call_browser(self):
         from requests.structures import CaseInsensitiveDict
-        from changedetectionio.content_fetchers.exceptions import EmptyReply
 
         # Protect against file:// access
         if re.search(r'^file://', self.watch.get('url', '').strip(), re.IGNORECASE):
@@ -159,7 +162,7 @@ class difference_detection_processor():
         some_data = 'xxxxx'
         update_obj["previous_md5"] = hashlib.md5(some_data.encode('utf-8')).hexdigest()
         changed_detected = False
-        return changed_detected, update_obj, ''.encode('utf-8')
+        return changed_detected, update_obj, ''.encode('utf-8'), b''
 
 
 def find_sub_packages(package_name):

changedetectionio/processors/restock_diff/processor.py

@@ -290,4 +290,4 @@ class perform_site_check(difference_detection_processor):
         # Always record the new checksum
         update_obj["previous_md5"] = fetched_md5
 
-        return changed_detected, update_obj, snapshot_content.encode('utf-8').strip()
+        return changed_detected, update_obj, snapshot_content.encode('utf-8').strip(), b''

changedetectionio/processors/text_json_diff/processor.py

@@ -218,12 +218,6 @@ class perform_site_check(difference_detection_processor):
                             is_rss=is_rss)) #1874 activate the <title workaround hack
                         stripped_text_from_html = future.result()
 
-        if watch.get('sort_text_alphabetically') and stripped_text_from_html:
-            # Note: Because a <p>something</p> will add an extra line feed to signify the paragraph gap
-            # we end up with 'Some text\n\n', sorting will add all those extra \n at the start, so we remove them here.
-            stripped_text_from_html = stripped_text_from_html.replace('\n\n', '\n')
-            stripped_text_from_html = '\n'.join( sorted(stripped_text_from_html.splitlines(), key=lambda x: x.lower() ))
-
         # Re #340 - return the content before the 'ignore text' was applied
         text_content_before_ignored_filter = stripped_text_from_html.encode('utf-8')
 
@@ -250,7 +244,7 @@ class perform_site_check(difference_detection_processor):
                 # We had some content, but no differences were found
                 # Store our new file as the MD5 so it will trigger in the future
                 c = hashlib.md5(text_content_before_ignored_filter.translate(None, b'\r\n\t ')).hexdigest()
-                return False, {'previous_md5': c}, stripped_text_from_html.encode('utf-8')
+                return False, {'previous_md5': c}, stripped_text_from_html.encode('utf-8'), stripped_text_from_html.encode('utf-8')
             else:
                 stripped_text_from_html = rendered_diff
 
@@ -312,6 +306,20 @@ class perform_site_check(difference_detection_processor):
                 stripped_text_from_html = b''.join(regex_matched_output)
                 text_content_before_ignored_filter = stripped_text_from_html
 
+
+        if watch.get('sort_text_alphabetically') and stripped_text_from_html:
+            # Note: Because a <p>something</p> will add an extra line feed to signify the paragraph gap
+            # we end up with 'Some text\n\n', sorting will add all those extra \n at the start, so we remove them here.
+            stripped_text_from_html = stripped_text_from_html.replace(b'\n\n', b'\n')
+            stripped_text_from_html = '\n'.join(sorted(stripped_text_from_html.decode('utf-8').splitlines(), key=lambda x: x.lower())).encode('utf-8')
+
+        #
+        if watch.get('trim_text_whitespace') and stripped_text_from_html:
+            stripped_text_from_html = '\n'.join(line.strip() for line in stripped_text_from_html.decode('utf-8').splitlines()).encode('utf-8')
+#
+        if watch.get('remove_duplicate_lines') and stripped_text_from_html:
+            stripped_text_from_html = '\n'.join(dict.fromkeys(line for line in stripped_text_from_html.decode('utf-8').splitlines())).encode('utf-8')
+
         # Re #133 - if we should strip whitespaces from triggering the change detected comparison
         if self.datastore.data['settings']['application'].get('ignore_whitespace', False):
             fetched_md5 = hashlib.md5(stripped_text_from_html.translate(None, b'\r\n\t ')).hexdigest()
@@ -371,4 +379,4 @@ class perform_site_check(difference_detection_processor):
         if not watch.get('previous_md5'):
             watch['previous_md5'] = fetched_md5
 
-        return changed_detected, update_obj, text_content_before_ignored_filter
+        return changed_detected, update_obj, text_content_before_ignored_filter, stripped_text_from_html

changedetectionio/static/js/watch-settings.js

@@ -12,6 +12,50 @@ function toggleOpacity(checkboxSelector, fieldSelector, inverted) {
     checkbox.addEventListener('change', updateOpacity);
 }
 
+(function($) {
+    // Object to store ongoing requests by namespace
+    const requests = {};
+
+    $.abortiveSingularAjax = function(options) {
+        const namespace = options.namespace || 'default';
+
+        // Abort the current request in this namespace if it's still ongoing
+        if (requests[namespace]) {
+            requests[namespace].abort();
+        }
+
+        // Start a new AJAX request and store its reference in the correct namespace
+        requests[namespace] = $.ajax(options);
+
+        // Return the current request in case it's needed
+        return requests[namespace];
+    };
+})(jQuery);
+
+function request_textpreview_update() {
+    const data = {};
+    $('textarea:visible, input:visible').each(function () {
+        const $element = $(this); // Cache the jQuery object for the current element
+        const name = $element.attr('name'); // Get the name attribute of the element
+        data[name] = $element.is(':checkbox') ? ($element.is(':checked') ? $element.val() : undefined) : $element.val();
+    });
+
+    $.abortiveSingularAjax({
+        type: "POST",
+        url: preview_text_edit_filters_url,
+        data: data,
+        namespace: 'watchEdit'
+    }).done(function (data) {
+        $('#filters-and-triggers #text-preview-inner').text(data);
+    }).fail(function (error) {
+        if (error.statusText === 'abort') {
+            console.log('Request was aborted due to a new request being fired.');
+        } else {
+            $('#filters-and-triggers #text-preview-inner').text('There was an error communicating with the server.');
+        }
+    })
+}
+
 $(document).ready(function () {
     $('#notification-setting-reset-to-default').click(function (e) {
         $('#notification_title').val('');
@@ -27,5 +71,22 @@ $(document).ready(function () {
 
     toggleOpacity('#time_between_check_use_default', '#time_between_check', false);
 
+    const vh = Math.max(document.documentElement.clientHeight || 0, window.innerHeight || 0);
+    $("#text-preview-inner").css('max-height', (vh-300)+"px");
+
+    $("#activate-text-preview").click(function (e) {
+        $(this).fadeOut();
+        $('body').toggleClass('preview-text-enabled')
+
+        request_textpreview_update();
+        $("#text-preview-refresh").click(function (e) {
+            request_textpreview_update();
+        });
+        $('textarea:visible, input:visible').on('keyup keypress blur change click', function (e) {
+            request_textpreview_update();
+        });
+
+    });
+
 });
 

changedetectionio/static/styles/scss/parts/_preview_text_filter.scss

@@ -0,0 +1,45 @@
+body.preview-text-enabled {
+  #filters-and-triggers > div {
+    display: flex; /* Establishes Flexbox layout */
+    gap: 20px; /* Adds space between the columns */
+    position: relative; /* Ensures the sticky positioning is relative to this parent */
+  }
+
+  /* layout of the page */
+  #edit-text-filter, #text-preview {
+    flex: 1; /* Each column takes an equal amount of available space */
+    align-self: flex-start; /* Aligns the right column to the start, allowing it to maintain its content height */
+  }
+
+  #edit-text-filter {
+    #pro-tips {
+      display: none;
+    }
+  }
+
+  #text-preview {
+    position: sticky;
+    top: 25px;
+    display: block !important;
+  }
+
+  /* actual preview area */
+  #text-preview-inner {
+    background: var(--color-grey-900);
+    border: 1px solid var(--color-grey-600);
+    padding: 1rem;
+    color: #333;
+    font-family: "Courier New", Courier, monospace; /* Sets the font to a monospace type */
+    font-size: 12px;
+    overflow-x: scroll;
+    white-space: pre-wrap; /* Preserves whitespace and line breaks like <pre> */
+    overflow-wrap: break-word; /* Allows long words to break and wrap to the next line */
+  }
+}
+
+#activate-text-preview {
+  right: 0;
+  position: absolute;
+  z-index: 0;
+  box-shadow: 1px 1px 4px var(--color-shadow-jump);
+}

changedetectionio/static/styles/scss/styles.scss

@@ -12,6 +12,7 @@
 @import "parts/_darkmode";
 @import "parts/_menu";
 @import "parts/_love";
+@import "parts/preview_text_filter";
 
 body {
   color: var(--color-text);

changedetectionio/static/styles/styles.css

@@ -411,6 +411,47 @@ html[data-darkmode="true"] #toggle-light-mode .icon-dark {
     fill: #ff0000 !important;
     transition: all ease 0.3s !important; }
 
+body.preview-text-enabled {
+  /* layout of the page */
+  /* actual preview area */ }
+  body.preview-text-enabled #filters-and-triggers > div {
+    display: flex;
+    /* Establishes Flexbox layout */
+    gap: 20px;
+    /* Adds space between the columns */
+    position: relative;
+    /* Ensures the sticky positioning is relative to this parent */ }
+  body.preview-text-enabled #edit-text-filter, body.preview-text-enabled #text-preview {
+    flex: 1;
+    /* Each column takes an equal amount of available space */
+    align-self: flex-start;
+    /* Aligns the right column to the start, allowing it to maintain its content height */ }
+  body.preview-text-enabled #edit-text-filter #pro-tips {
+    display: none; }
+  body.preview-text-enabled #text-preview {
+    position: sticky;
+    top: 25px;
+    display: block !important; }
+  body.preview-text-enabled #text-preview-inner {
+    background: var(--color-grey-900);
+    border: 1px solid var(--color-grey-600);
+    padding: 1rem;
+    color: #333;
+    font-family: "Courier New", Courier, monospace;
+    /* Sets the font to a monospace type */
+    font-size: 12px;
+    overflow-x: scroll;
+    white-space: pre-wrap;
+    /* Preserves whitespace and line breaks like <pre> */
+    overflow-wrap: break-word;
+    /* Allows long words to break and wrap to the next line */ }
+
+#activate-text-preview {
+  right: 0;
+  position: absolute;
+  z-index: 0;
+  box-shadow: 1px 1px 4px var(--color-shadow-jump); }
+
 body {
   color: var(--color-text);
   background: var(--color-background-page);
@@ -1194,11 +1235,9 @@ ul {
   color: #fff;
   opacity: 0.7; }
 
-
 .restock-label svg {
   vertical-align: middle; }
 
-
 #chrome-extension-link {
   padding: 9px;
   border: 1px solid var(--color-grey-800);

changedetectionio/templates/base.html

@@ -33,7 +33,7 @@
     <script src="{{url_for('static_content', group='js', filename='csrf.js')}}" defer></script>
   </head>
 
-  <body>
+  <body class="">
     <div class="header">
       <div class="home-menu pure-menu pure-menu-horizontal pure-menu-fixed" id="nav-menu">
         {% if has_password and not current_user.is_authenticated %}

changedetectionio/templates/edit.html

@@ -254,7 +254,10 @@ User-Agent: wonderbra 1.0") }}
 
             {% if watch['processor'] == 'text_json_diff' %}
             <div class="tab-pane-inner" id="filters-and-triggers">
-                    <div class="pure-control-group">
+                <span id="activate-text-preview" class="pure-button pure-button-primary button-xsmall">Activate preview</span>
+              <div>
+              <div id="edit-text-filter">
+                    <div class="pure-control-group" id="pro-tips">
                             <strong>Pro-tips:</strong><br>
                             <ul>
                                 <li>
@@ -330,14 +333,21 @@ nav
                     <span class="pure-form-message-inline">So it's always better to select <strong>Added</strong>+<strong>Replaced</strong> when you're interested in new content.</span><br>
                     <span class="pure-form-message-inline">When content is merely moved in a list, it will also trigger an <strong>addition</strong>, consider enabling <code><strong>Only trigger when unique lines appear</strong></code></span>
                 </fieldset>
-
+                <fieldset class="pure-control-group">
+                    {{ render_checkbox_field(form.check_unique_lines) }}
+                    <span class="pure-form-message-inline">Good for websites that just move the content around, and you want to know when NEW content is added, compares new lines against all history for this watch.</span>
+                </fieldset>
+                <fieldset class="pure-control-group">
+                    {{ render_checkbox_field(form.remove_duplicate_lines) }}
+                    <span class="pure-form-message-inline">Remove duplicate lines of text</span>
+                </fieldset>
                 <fieldset class="pure-control-group">
                     {{ render_checkbox_field(form.sort_text_alphabetically) }}
                     <span class="pure-form-message-inline">Helps reduce changes detected caused by sites shuffling lines around, combine with <i>check unique lines</i> below.</span>
                 </fieldset>
                 <fieldset class="pure-control-group">
-                    {{ render_checkbox_field(form.check_unique_lines) }}
-                    <span class="pure-form-message-inline">Good for websites that just move the content around, and you want to know when NEW content is added, compares new lines against all history for this watch.</span>
+                    {{ render_checkbox_field(form.trim_text_whitespace) }}
+                    <span class="pure-form-message-inline">Remove any whitespace before and after each line of text</span>
                 </fieldset>
                 <fieldset>
                     <div class="pure-control-group">
@@ -387,7 +397,7 @@ Unavailable") }}
                 </fieldset>
                 <fieldset>
                     <div class="pure-control-group">
-                        {{ render_field(form.extract_text, rows=5, placeholder="\d+ online") }}
+                        {{ render_field(form.extract_text, rows=5, placeholder="Example: /\d+ online/") }}
                         <span class="pure-form-message-inline">
                     <ul>
                         <li>Extracts text in the final output (line by line) after other filters using regular expressions or string match;
@@ -407,7 +417,19 @@ Unavailable") }}
                 </fieldset>
                 </div>
             </div>
-            {% endif %}
+              <div id="text-preview" style="display: none;" >
+                    <script>
+                        const preview_text_edit_filters_url="{{url_for('watch_get_preview_rendered', uuid=uuid)}}";
+                    </script>
+                    <span><strong>Preview of the text that is used for changedetection after all filters run.</strong></span><br>
+                    {#<div id="text-preview-controls"><span id="text-preview-refresh" class="pure-button button-xsmall">Refresh</span></div>#}
+                <p>
+                    <div id="text-preview-inner"></div>
+                </p>
+            </div>
+          </div>
+        </div>
+        {% endif %}
         {# rendered sub Template #}
         {% if extra_form_content %}
             <div class="tab-pane-inner" id="extras_tab">

changedetectionio/tests/test_filter_failure_notification.py

@@ -116,9 +116,11 @@ def run_filter_test(client, live_server, content_filter):
         res = client.get(url_for("index"))
         assert b'Warning, no filters were found' in res.data
         assert not os.path.isfile("test-datastore/notification.txt")
-
+        time.sleep(1)
+        
     assert live_server.app.config['DATASTORE'].data['watching'][uuid]['consecutive_filter_failures'] == 5
 
+    time.sleep(2)
     # One more check should trigger the _FILTER_FAILURE_THRESHOLD_ATTEMPTS_DEFAULT threshold
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)

changedetectionio/tests/util.py

@@ -78,6 +78,7 @@ def set_more_modified_response():
 
 def wait_for_notification_endpoint_output():
     '''Apprise can take a few seconds to fire'''
+    #@todo - could check the apprise object directly instead of looking for this file
     from os.path import isfile
     for i in range(1, 20):
         time.sleep(1)

changedetectionio/update_worker.py

@@ -278,7 +278,7 @@ class update_worker(threading.Thread):
 
                         update_handler.call_browser()
 
-                        changed_detected, update_obj, contents = update_handler.run_changedetection(
+                        changed_detected, update_obj, contents, content_after_filters = update_handler.run_changedetection(
                             watch=watch,
                             skip_when_checksum_same=skip_when_same_checksum,
                         )