Merge branch 'master' into reverse-2626-memory

changedetectionio/flask_app.py

@@ -1,8 +1,6 @@
 #!/usr/bin/env python3
 
 import datetime
-import importlib
-
 import flask_login
 import locale
 import os
@@ -12,9 +10,7 @@ import threading
 import time
 import timeago
 
-from .content_fetchers.exceptions import ReplyWithContentButNoText
 from .processors import find_processors, get_parent_module, get_custom_watch_obj_for_processor
-from .processors.text_json_diff.processor import FilterNotFoundInResponse
 from .safe_jinja import render as jinja_render
 from changedetectionio.strtobool import strtobool
 from copy import deepcopy
@@ -1400,58 +1396,6 @@ def changedetection_app(config=None, datastore_o=None):
         # Return a 500 error
         abort(500)
 
-    @app.route("/edit/<string:uuid>/preview-rendered", methods=['POST'])
-    @login_optionally_required
-    def watch_get_preview_rendered(uuid):
-        '''For when viewing the "preview" of the rendered text from inside of Edit'''
-        now = time.time()
-        import brotli
-        from . import forms
-
-        text_after_filter = ''
-        tmp_watch = deepcopy(datastore.data['watching'].get(uuid))
-
-        if tmp_watch and tmp_watch.history and os.path.isdir(tmp_watch.watch_data_dir):
-            # Splice in the temporary stuff from the form
-            form = forms.processor_text_json_diff_form(formdata=request.form if request.method == 'POST' else None,
-                                                       data=request.form
-                                                       )
-            # Only update vars that came in via the AJAX post
-            p = {k: v for k, v in form.data.items() if k in request.form.keys()}
-            tmp_watch.update(p)
-
-            latest_filename = next(reversed(tmp_watch.history))
-            html_fname = os.path.join(tmp_watch.watch_data_dir, f"{latest_filename}.html.br")
-            with open(html_fname, 'rb') as f:
-                decompressed_data = brotli.decompress(f.read()).decode('utf-8') if html_fname.endswith('.br') else f.read().decode('utf-8')
-
-                # Just like a normal change detection except provide a fake "watch" object and dont call .call_browser()
-                processor_module = importlib.import_module("changedetectionio.processors.text_json_diff.processor")
-                update_handler = processor_module.perform_site_check(datastore=datastore,
-                                                                     watch_uuid=uuid # probably not needed anymore anyway?
-                                                                     )
-                # Use the last loaded HTML as the input
-                update_handler.fetcher.content = decompressed_data
-                update_handler.fetcher.headers['content-type'] = tmp_watch.get('content-type')
-                try:
-                    changed_detected, update_obj, contents, text_after_filter = update_handler.run_changedetection(
-                        watch=tmp_watch,
-                        skip_when_checksum_same=False,
-                    )
-                except FilterNotFoundInResponse as e:
-                    text_after_filter = f"Filter not found in HTML: {str(e)}"
-                except ReplyWithContentButNoText as e:
-                    text_after_filter = f"Filter found but no text (empty result)"
-                except Exception as e:
-                    text_after_filter = f"Error: {str(e)}"
-
-            if not text_after_filter.strip():
-                text_after_filter = 'Empty content'
-
-        logger.trace(f"Parsed in {time.time()-now:.3f}s")
-        return text_after_filter.strip()
-
-
     @app.route("/form/add/quickwatch", methods=['POST'])
     @login_optionally_required
     def form_quick_watch_add():

changedetectionio/model/__init__.py

@@ -18,7 +18,6 @@ class watch_base(dict):
             'check_count': 0,
             'check_unique_lines': False,  # On change-detected, compare against all history if its something new
             'consecutive_filter_failures': 0,  # Every time the CSS/xPath filter cannot be located, reset when all is fine.
-            'content-type': None,
             'date_created': None,
             'extract_text': [],  # Extract text by regex after filters
             'extract_title_as_title': False,

changedetectionio/processors/__init__.py

@@ -1,6 +1,4 @@
 from abc import abstractmethod
-
-from changedetectionio.content_fetchers.base import Fetcher
 from changedetectionio.strtobool import strtobool
 
 from copy import deepcopy
@@ -25,11 +23,10 @@ class difference_detection_processor():
         super().__init__(*args, **kwargs)
         self.datastore = datastore
         self.watch = deepcopy(self.datastore.data['watching'].get(watch_uuid))
-        # Generic fetcher that should be extended (requests, playwright etc)
-        self.fetcher = Fetcher()
 
     def call_browser(self):
         from requests.structures import CaseInsensitiveDict
+        from changedetectionio.content_fetchers.exceptions import EmptyReply
 
         # Protect against file:// access
         if re.search(r'^file://', self.watch.get('url', '').strip(), re.IGNORECASE):
@@ -162,7 +159,7 @@ class difference_detection_processor():
         some_data = 'xxxxx'
         update_obj["previous_md5"] = hashlib.md5(some_data.encode('utf-8')).hexdigest()
         changed_detected = False
-        return changed_detected, update_obj, ''.encode('utf-8'), b''
+        return changed_detected, update_obj, ''.encode('utf-8')
 
 
 def find_sub_packages(package_name):

changedetectionio/processors/restock_diff/processor.py

@@ -298,4 +298,4 @@ class perform_site_check(difference_detection_processor):
         # Always record the new checksum
         update_obj["previous_md5"] = fetched_md5
 
-        return changed_detected, update_obj, snapshot_content.encode('utf-8').strip(), b''
+        return changed_detected, update_obj, snapshot_content.encode('utf-8').strip()

changedetectionio/processors/text_json_diff/processor.py

@@ -207,7 +207,7 @@ class perform_site_check(difference_detection_processor):
             stripped_text_from_html = '\n'.join(line.strip() for line in stripped_text_from_html.replace("\n\n", "\n").splitlines())
 
         if watch.get('remove_duplicate_lines'):
-            stripped_text_from_html = '\n'.join(dict.fromkeys(line for line in stripped_text_from_html.replace("\n\n", "\n").splitlines()))
+            stripped_text_from_html = '\n'.join(dict.fromkeys(line.strip() for line in stripped_text_from_html.replace("\n\n", "\n").splitlines()))
 
         if watch.get('sort_text_alphabetically'):
             # Note: Because a <p>something</p> will add an extra line feed to signify the paragraph gap
@@ -242,7 +242,7 @@ class perform_site_check(difference_detection_processor):
                 # We had some content, but no differences were found
                 # Store our new file as the MD5 so it will trigger in the future
                 c = hashlib.md5(text_content_before_ignored_filter.translate(None, b'\r\n\t ')).hexdigest()
-                return False, {'previous_md5': c}, stripped_text_from_html.encode('utf-8'), stripped_text_from_html.encode('utf-8')
+                return False, {'previous_md5': c}, stripped_text_from_html.encode('utf-8')
             else:
                 stripped_text_from_html = rendered_diff
 
@@ -365,4 +365,4 @@ class perform_site_check(difference_detection_processor):
         if not watch.get('previous_md5'):
             watch['previous_md5'] = fetched_md5
 
-        return changed_detected, update_obj, text_content_before_ignored_filter, stripped_text_from_html
+        return changed_detected, update_obj, text_content_before_ignored_filter

changedetectionio/static/js/watch-settings.js

@@ -12,55 +12,6 @@ function toggleOpacity(checkboxSelector, fieldSelector, inverted) {
     checkbox.addEventListener('change', updateOpacity);
 }
 
-(function($) {
-    // Object to store ongoing requests by namespace
-    const requests = {};
-
-    $.abortiveSingularAjax = function(options) {
-        const namespace = options.namespace || 'default';
-
-        // Abort the current request in this namespace if it's still ongoing
-        if (requests[namespace]) {
-            requests[namespace].abort();
-        }
-
-        // Start a new AJAX request and store its reference in the correct namespace
-        requests[namespace] = $.ajax(options);
-
-        // Return the current request in case it's needed
-        return requests[namespace];
-    };
-})(jQuery);
-
-function request_textpreview_update() {
-    if (!$('body').hasClass('preview-text-enabled')) {
-        console.error("Preview text was requested but body tag was not setup")
-        return
-    }
-
-    const data = {};
-    $('textarea:visible, input:visible').each(function () {
-        const $element = $(this); // Cache the jQuery object for the current element
-        const name = $element.attr('name'); // Get the name attribute of the element
-        data[name] = $element.is(':checkbox') ? ($element.is(':checked') ? $element.val() : undefined) : $element.val();
-    });
-
-    $.abortiveSingularAjax({
-        type: "POST",
-        url: preview_text_edit_filters_url,
-        data: data,
-        namespace: 'watchEdit'
-    }).done(function (data) {
-        $('#filters-and-triggers #text-preview-inner').text(data);
-    }).fail(function (error) {
-        if (error.statusText === 'abort') {
-            console.log('Request was aborted due to a new request being fired.');
-        } else {
-            $('#filters-and-triggers #text-preview-inner').text('There was an error communicating with the server.');
-        }
-    })
-}
-
 $(document).ready(function () {
     $('#notification-setting-reset-to-default').click(function (e) {
         $('#notification_title').val('');
@@ -76,22 +27,5 @@ $(document).ready(function () {
 
     toggleOpacity('#time_between_check_use_default', '#time_between_check', false);
 
-    const vh = Math.max(document.documentElement.clientHeight || 0, window.innerHeight || 0);
-    $("#text-preview-inner").css('max-height', (vh-300)+"px");
-
-    // Realtime preview of 'Filters & Text' setup
-    var debounced_request_textpreview_update = request_textpreview_update.debounce(100);
-
-    $("#activate-text-preview").click(function (e) {
-        $('body').toggleClass('preview-text-enabled')
-        request_textpreview_update();
-
-        const method = $('body').hasClass('preview-text-enabled') ? 'on' : 'off';
-        $("#text-preview-refresh")[method]('click', debounced_request_textpreview_update);
-        $('textarea:visible')[method]('keyup blur', debounced_request_textpreview_update);
-        $('input:visible')[method]('keyup blur change', debounced_request_textpreview_update);
-        $("#filters-and-triggers-tab")[method]('click', debounced_request_textpreview_update);
-    });
-
 });
 

changedetectionio/static/styles/scss/parts/_browser-steps.scss

@@ -42,36 +42,26 @@
 
 
 #browser-steps .flex-wrapper {
+  display: flex;
+  flex-flow: row;
+  height: 70vh;
   font-size: 80%;
-
-  @media screen and (min-width: 800px) {
-    display: flex;
-    flex-flow: row;
-    height: 70vh;
-    #browser-steps-fieldlist {
-      flex-grow: 0; /* Don't allow it to grow */
-      flex-shrink: 0; /* Don't allow it to shrink */
-      flex-basis: auto; /* Base width is determined by the content */
-      max-width: 400px; /* Set a max width to prevent overflow */
-      padding-left: 1rem;
-      overflow-y: scroll;
-    }
-
-
-    #browser-steps-ui {
-      flex-grow: 1; /* Allow it to grow and fill the available space */
-      flex-shrink: 1; /* Allow it to shrink if needed */
-      flex-basis: 0; /* Start with 0 base width so it stretches as much as possible */
-    }
-  }
-
   #browser-steps-ui {
+    flex-grow: 1;      /* Allow it to grow and fill the available space */
+    flex-shrink: 1;    /* Allow it to shrink if needed */
+    flex-basis: 0;     /* Start with 0 base width so it stretches as much as possible */
     background-color: #eee;
     border-radius: 5px;
+
   }
 
-  #browser-steps-field-list {
-    text-align: center;
+  #browser-steps-fieldlist {
+    flex-grow: 0;      /* Don't allow it to grow */
+    flex-shrink: 0;    /* Don't allow it to shrink */
+    flex-basis: auto;  /* Base width is determined by the content */
+    max-width: 400px;  /* Set a max width to prevent overflow */
+    padding-left: 1rem;
+    overflow-y: scroll;
   }
 }
 

changedetectionio/static/styles/scss/parts/_preview_text_filter.scss

@@ -1,45 +0,0 @@
-body.preview-text-enabled {
-  #filters-and-triggers > div {
-    display: flex; /* Establishes Flexbox layout */
-    gap: 20px; /* Adds space between the columns */
-    position: relative; /* Ensures the sticky positioning is relative to this parent */
-  }
-
-  /* layout of the page */
-  #edit-text-filter, #text-preview {
-    flex: 1; /* Each column takes an equal amount of available space */
-    align-self: flex-start; /* Aligns the right column to the start, allowing it to maintain its content height */
-  }
-
-  #edit-text-filter {
-    #pro-tips {
-      display: none;
-    }
-  }
-
-  #text-preview {
-    position: sticky;
-    top: 25px;
-    display: block !important;
-  }
-
-  /* actual preview area */
-  #text-preview-inner {
-    background: var(--color-grey-900);
-    border: 1px solid var(--color-grey-600);
-    padding: 1rem;
-    color: #333;
-    font-family: "Courier New", Courier, monospace; /* Sets the font to a monospace type */
-    font-size: 12px;
-    overflow-x: scroll;
-    white-space: pre-wrap; /* Preserves whitespace and line breaks like <pre> */
-    overflow-wrap: break-word; /* Allows long words to break and wrap to the next line */
-  }
-}
-
-#activate-text-preview {
-  right: 0;
-  position: absolute;
-  z-index: 0;
-  box-shadow: 1px 1px 4px var(--color-shadow-jump);
-}

changedetectionio/static/styles/scss/styles.scss

@@ -12,7 +12,6 @@
 @import "parts/_darkmode";
 @import "parts/_menu";
 @import "parts/_love";
-@import "parts/preview_text_filter";
 
 body {
   color: var(--color-text);

changedetectionio/static/styles/styles.css

@@ -47,35 +47,30 @@
       display: none; }
 
 #browser-steps .flex-wrapper {
+  display: flex;
+  flex-flow: row;
+  height: 70vh;
   font-size: 80%; }
-  @media screen and (min-width: 800px) {
-    #browser-steps .flex-wrapper {
-      display: flex;
-      flex-flow: row;
-      height: 70vh; }
-      #browser-steps .flex-wrapper #browser-steps-fieldlist {
-        flex-grow: 0;
-        /* Don't allow it to grow */
-        flex-shrink: 0;
-        /* Don't allow it to shrink */
-        flex-basis: auto;
-        /* Base width is determined by the content */
-        max-width: 400px;
-        /* Set a max width to prevent overflow */
-        padding-left: 1rem;
-        overflow-y: scroll; }
-      #browser-steps .flex-wrapper #browser-steps-ui {
-        flex-grow: 1;
-        /* Allow it to grow and fill the available space */
-        flex-shrink: 1;
-        /* Allow it to shrink if needed */
-        flex-basis: 0;
-        /* Start with 0 base width so it stretches as much as possible */ } }
   #browser-steps .flex-wrapper #browser-steps-ui {
+    flex-grow: 1;
+    /* Allow it to grow and fill the available space */
+    flex-shrink: 1;
+    /* Allow it to shrink if needed */
+    flex-basis: 0;
+    /* Start with 0 base width so it stretches as much as possible */
     background-color: #eee;
     border-radius: 5px; }
-  #browser-steps .flex-wrapper #browser-steps-field-list {
-    text-align: center; }
+  #browser-steps .flex-wrapper #browser-steps-fieldlist {
+    flex-grow: 0;
+    /* Don't allow it to grow */
+    flex-shrink: 0;
+    /* Don't allow it to shrink */
+    flex-basis: auto;
+    /* Base width is determined by the content */
+    max-width: 400px;
+    /* Set a max width to prevent overflow */
+    padding-left: 1rem;
+    overflow-y: scroll; }
 
 /*  this is duplicate :( */
 #browsersteps-selector-wrapper {
@@ -433,47 +428,6 @@ html[data-darkmode="true"] #toggle-light-mode .icon-dark {
     fill: #ff0000 !important;
     transition: all ease 0.3s !important; }
 
-body.preview-text-enabled {
-  /* layout of the page */
-  /* actual preview area */ }
-  body.preview-text-enabled #filters-and-triggers > div {
-    display: flex;
-    /* Establishes Flexbox layout */
-    gap: 20px;
-    /* Adds space between the columns */
-    position: relative;
-    /* Ensures the sticky positioning is relative to this parent */ }
-  body.preview-text-enabled #edit-text-filter, body.preview-text-enabled #text-preview {
-    flex: 1;
-    /* Each column takes an equal amount of available space */
-    align-self: flex-start;
-    /* Aligns the right column to the start, allowing it to maintain its content height */ }
-  body.preview-text-enabled #edit-text-filter #pro-tips {
-    display: none; }
-  body.preview-text-enabled #text-preview {
-    position: sticky;
-    top: 25px;
-    display: block !important; }
-  body.preview-text-enabled #text-preview-inner {
-    background: var(--color-grey-900);
-    border: 1px solid var(--color-grey-600);
-    padding: 1rem;
-    color: #333;
-    font-family: "Courier New", Courier, monospace;
-    /* Sets the font to a monospace type */
-    font-size: 12px;
-    overflow-x: scroll;
-    white-space: pre-wrap;
-    /* Preserves whitespace and line breaks like <pre> */
-    overflow-wrap: break-word;
-    /* Allows long words to break and wrap to the next line */ }
-
-#activate-text-preview {
-  right: 0;
-  position: absolute;
-  z-index: 0;
-  box-shadow: 1px 1px 4px var(--color-shadow-jump); }
-
 body {
   color: var(--color-text);
   background: var(--color-background-page);

changedetectionio/templates/base.html

@@ -33,7 +33,7 @@
     <script src="{{url_for('static_content', group='js', filename='csrf.js')}}" defer></script>
   </head>
 
-  <body class="">
+  <body>
     <div class="header">
       <div class="home-menu pure-menu pure-menu-horizontal pure-menu-fixed" id="nav-menu">
         {% if has_password and not current_user.is_authenticated %}

changedetectionio/templates/edit.html

@@ -50,7 +50,7 @@
             {% endif %}
             {% if watch['processor'] == 'text_json_diff' %}
             <li class="tab"><a id="visualselector-tab" href="#visualselector">Visual Filter Selector</a></li>
-            <li class="tab" id="filters-and-triggers-tab"><a href="#filters-and-triggers">Filters &amp; Triggers</a></li>
+            <li class="tab"><a href="#filters-and-triggers">Filters &amp; Triggers</a></li>
             {% endif %}
             <li class="tab"><a href="#notifications">Notifications</a></li>
             <li class="tab"><a href="#stats">Stats</a></li>
@@ -254,10 +254,7 @@ User-Agent: wonderbra 1.0") }}
 
             {% if watch['processor'] == 'text_json_diff' %}
             <div class="tab-pane-inner" id="filters-and-triggers">
-                <span id="activate-text-preview" class="pure-button pure-button-primary button-xsmall">Activate preview</span>
-              <div>
-              <div id="edit-text-filter">
-                    <div class="pure-control-group" id="pro-tips">
+                    <div class="pure-control-group">
                             <strong>Pro-tips:</strong><br>
                             <ul>
                                 <li>
@@ -350,6 +347,10 @@ nav
                     {{ render_checkbox_field(form.trim_text_whitespace) }}
                     <span class="pure-form-message-inline">Remove any whitespace before and after each line of text</span>
                 </fieldset>
+                <fieldset class="pure-control-group">
+                    {{ render_checkbox_field(form.check_unique_lines) }}
+                    <span class="pure-form-message-inline">Good for websites that just move the content around, and you want to know when NEW content is added, compares new lines against all history for this watch.</span>
+                </fieldset>
                 <fieldset>
                     <div class="pure-control-group">
                         {{ render_field(form.trigger_text, rows=5, placeholder="Some text to wait for in a line
@@ -418,19 +419,7 @@ Unavailable") }}
                 </fieldset>
                 </div>
             </div>
-              <div id="text-preview" style="display: none;" >
-                    <script>
-                        const preview_text_edit_filters_url="{{url_for('watch_get_preview_rendered', uuid=uuid)}}";
-                    </script>
-                    <span><strong>Preview of the text that is used for changedetection after all filters run.</strong></span><br>
-                    {#<div id="text-preview-controls"><span id="text-preview-refresh" class="pure-button button-xsmall">Refresh</span></div>#}
-                <p>
-                    <div id="text-preview-inner"></div>
-                </p>
-            </div>
-          </div>
-        </div>
-        {% endif %}
+            {% endif %}
         {# rendered sub Template #}
         {% if extra_form_content %}
             <div class="tab-pane-inner" id="extras_tab">

changedetectionio/tests/test_encoding.py

@@ -3,7 +3,7 @@
 
 import time
 from flask import url_for
-from .util import live_server_setup, wait_for_all_checks, extract_UUID_from_client
+from .util import live_server_setup, wait_for_all_checks
 import pytest
 
 
@@ -38,11 +38,6 @@ def test_check_encoding_detection(client, live_server, measure_memory_usage):
     # Give the thread time to pick it up
     wait_for_all_checks(client)
 
-
-    # Content type recording worked
-    uuid = extract_UUID_from_client(client)
-    assert live_server.app.config['DATASTORE'].data['watching'][uuid]['content-type'] == "text/html"
-
     res = client.get(
         url_for("preview_page", uuid="first"),
         follow_redirects=True

changedetectionio/tests/test_filter_failure_notification.py

@@ -116,11 +116,9 @@ def run_filter_test(client, live_server, content_filter):
         res = client.get(url_for("index"))
         assert b'Warning, no filters were found' in res.data
         assert not os.path.isfile("test-datastore/notification.txt")
-        time.sleep(1)
-        
+
     assert live_server.app.config['DATASTORE'].data['watching'][uuid]['consecutive_filter_failures'] == 5
 
-    time.sleep(2)
     # One more check should trigger the _FILTER_FAILURE_THRESHOLD_ATTEMPTS_DEFAULT threshold
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)

changedetectionio/tests/util.py

@@ -78,7 +78,6 @@ def set_more_modified_response():
 
 def wait_for_notification_endpoint_output():
     '''Apprise can take a few seconds to fire'''
-    #@todo - could check the apprise object directly instead of looking for this file
     from os.path import isfile
     for i in range(1, 20):
         time.sleep(1)

changedetectionio/update_worker.py

@@ -278,7 +278,7 @@ class update_worker(threading.Thread):
 
                         update_handler.call_browser()
 
-                        changed_detected, update_obj, contents, content_after_filters = update_handler.run_changedetection(
+                        changed_detected, update_obj, contents = update_handler.run_changedetection(
                             watch=watch,
                             skip_when_checksum_same=skip_when_same_checksum,
                         )
@@ -491,8 +491,6 @@ class update_worker(threading.Thread):
                         if not self.datastore.data['watching'].get(uuid):
                             continue
 
-                        update_obj['content-type'] = update_handler.fetcher.get_all_headers().get('content-type', '').lower()
-
                         # Mark that we never had any failures
                         if not watch.get('ignore_status_codes'):
                             update_obj['consecutive_filter_failures'] = 0