revert

changedetectionio/content_fetchers/__init__.py

@@ -4,9 +4,7 @@ from loguru import logger
 from changedetectionio.content_fetchers.exceptions import BrowserStepsStepException
 import os
 
-# Visual Selector scraper - 'Button' is there because some sites have <button>OUT OF STOCK</button>.
-visualselector_xpath_selectors = 'div,span,form,table,tbody,tr,td,a,p,ul,li,h1,h2,h3,h4,header,footer,section,article,aside,details,main,nav,section,summary,button'
-
+visualselector_xpath_selectors = 'div,span,form,table,tbody,tr,td,a,p,ul,li,h1,h2,h3,h4,header,footer,section,article,aside,details,main,nav,section,summary'
 
 # available_fetchers() will scan this implementation looking for anything starting with html_
 # this information is used in the form selections

changedetectionio/flask_app.py

@@ -1,8 +1,6 @@
 #!/usr/bin/env python3
 
 import datetime
-import importlib
-
 import flask_login
 import locale
 import os
@@ -12,9 +10,7 @@ import threading
 import time
 import timeago
 
-from .content_fetchers.exceptions import ReplyWithContentButNoText
 from .processors import find_processors, get_parent_module, get_custom_watch_obj_for_processor
-from .processors.text_json_diff.processor import FilterNotFoundInResponse
 from .safe_jinja import render as jinja_render
 from changedetectionio.strtobool import strtobool
 from copy import deepcopy
@@ -1158,6 +1154,8 @@ def changedetection_app(config=None, datastore_o=None):
     @login_optionally_required
     def preview_page(uuid):
         content = []
+        ignored_line_numbers = []
+        trigger_line_numbers = []
         versions = []
         timestamp = None
 
@@ -1174,10 +1172,11 @@ def changedetection_app(config=None, datastore_o=None):
         system_uses_webdriver = datastore.data['settings']['application']['fetch_backend'] == 'html_webdriver'
         extra_stylesheets = [url_for('static_content', group='styles', filename='diff.css')]
 
+
         is_html_webdriver = False
         if (watch.get('fetch_backend') == 'system' and system_uses_webdriver) or watch.get('fetch_backend') == 'html_webdriver' or watch.get('fetch_backend', '').startswith('extra_browser_'):
             is_html_webdriver = True
-        triggered_line_numbers = []
+
         if datastore.data['watching'][uuid].history_n == 0 and (watch.get_error_text() or watch.get_error_snapshot()):
             flash("Preview unavailable - No fetch/check completed or triggers not reached", "error")
         else:
@@ -1190,12 +1189,31 @@ def changedetection_app(config=None, datastore_o=None):
 
             try:
                 versions = list(watch.history.keys())
-                content = watch.get_history_snapshot(timestamp)
+                tmp = watch.get_history_snapshot(timestamp).splitlines()
 
-                triggered_line_numbers = html_tools.strip_ignore_text(content=content,
-                                                                      wordlist=watch['trigger_text'],
-                                                                      mode='line numbers'
-                                                                      )
+                # Get what needs to be highlighted
+                ignore_rules = watch.get('ignore_text', []) + datastore.data['settings']['application']['global_ignore_text']
+
+                # .readlines will keep the \n, but we will parse it here again, in the future tidy this up
+                ignored_line_numbers = html_tools.strip_ignore_text(content="\n".join(tmp),
+                                                                    wordlist=ignore_rules,
+                                                                    mode='line numbers'
+                                                                    )
+
+                trigger_line_numbers = html_tools.strip_ignore_text(content="\n".join(tmp),
+                                                                    wordlist=watch['trigger_text'],
+                                                                    mode='line numbers'
+                                                                    )
+                # Prepare the classes and lines used in the template
+                i=0
+                for l in tmp:
+                    classes=[]
+                    i+=1
+                    if i in ignored_line_numbers:
+                        classes.append('ignored')
+                    if i in trigger_line_numbers:
+                        classes.append('triggered')
+                    content.append({'line': l, 'classes': ' '.join(classes)})
 
             except Exception as e:
                 content.append({'line': f"File doesnt exist or unable to read timestamp {timestamp}", 'classes': ''})
@@ -1206,7 +1224,8 @@ def changedetection_app(config=None, datastore_o=None):
                                  history_n=watch.history_n,
                                  extra_stylesheets=extra_stylesheets,
                                  extra_title=f" - Diff - {watch.label} @ {timestamp}",
-                                 triggered_line_numbers=triggered_line_numbers,
+                                 ignored_line_numbers=ignored_line_numbers,
+                                 triggered_line_numbers=trigger_line_numbers,
                                  current_diff_url=watch['url'],
                                  screenshot=watch.get_screenshot(),
                                  watch=watch,
@@ -1377,84 +1396,6 @@ def changedetection_app(config=None, datastore_o=None):
         # Return a 500 error
         abort(500)
 
-    # Ajax callback
-    @app.route("/edit/<string:uuid>/preview-rendered", methods=['POST'])
-    @login_optionally_required
-    def watch_get_preview_rendered(uuid):
-        from flask import jsonify
-        '''For when viewing the "preview" of the rendered text from inside of Edit'''
-        now = time.time()
-        import brotli
-        from . import forms
-
-        text_after_filter = ''
-        tmp_watch = deepcopy(datastore.data['watching'].get(uuid))
-
-        if tmp_watch and tmp_watch.history and os.path.isdir(tmp_watch.watch_data_dir):
-            # Splice in the temporary stuff from the form
-            form = forms.processor_text_json_diff_form(formdata=request.form if request.method == 'POST' else None,
-                                                       data=request.form
-                                                       )
-            # Only update vars that came in via the AJAX post
-            p = {k: v for k, v in form.data.items() if k in request.form.keys()}
-            tmp_watch.update(p)
-
-            latest_filename = next(reversed(tmp_watch.history))
-            html_fname = os.path.join(tmp_watch.watch_data_dir, f"{latest_filename}.html.br")
-            with open(html_fname, 'rb') as f:
-                decompressed_data = brotli.decompress(f.read()).decode('utf-8') if html_fname.endswith('.br') else f.read().decode('utf-8')
-
-                # Just like a normal change detection except provide a fake "watch" object and dont call .call_browser()
-                processor_module = importlib.import_module("changedetectionio.processors.text_json_diff.processor")
-                update_handler = processor_module.perform_site_check(datastore=datastore,
-                                                                     watch_uuid=uuid # probably not needed anymore anyway?
-                                                                     )
-                # Use the last loaded HTML as the input
-                update_handler.fetcher.content = decompressed_data
-                update_handler.fetcher.headers['content-type'] = tmp_watch.get('content-type')
-                try:
-                    changed_detected, update_obj, text_after_filter = update_handler.run_changedetection(
-                        watch=tmp_watch,
-                        skip_when_checksum_same=False,
-                    )
-                except FilterNotFoundInResponse as e:
-                    text_after_filter = f"Filter not found in HTML: {str(e)}"
-                except ReplyWithContentButNoText as e:
-                    text_after_filter = f"Filter found but no text (empty result)"
-                except Exception as e:
-                    text_after_filter = f"Error: {str(e)}"
-
-            if not text_after_filter.strip():
-                text_after_filter = 'Empty content'
-
-        # because run_changedetection always returns bytes due to saving the snapshots etc
-        text_after_filter = text_after_filter.decode('utf-8') if isinstance(text_after_filter, bytes) else text_after_filter
-
-        do_anchor = datastore.data["settings"]["application"].get("render_anchor_tag_content", False)
-
-        trigger_line_numbers = []
-        try:
-            text_before_filter = html_tools.html_to_text(html_content=decompressed_data,
-                                                         render_anchor_tag_content=do_anchor)
-
-            trigger_line_numbers = html_tools.strip_ignore_text(content=text_after_filter,
-                                                                wordlist=tmp_watch['trigger_text'],
-                                                                mode='line numbers'
-                                                                )
-        except Exception as e:
-            text_before_filter = f"Error: {str(e)}"
-
-        logger.trace(f"Parsed in {time.time() - now:.3f}s")
-
-        return jsonify(
-            {
-                'after_filter': text_after_filter,
-                'before_filter': text_before_filter.decode('utf-8') if isinstance(text_before_filter, bytes) else text_before_filter,
-                'trigger_line_numbers': trigger_line_numbers
-            }
-        )
-
-
     @app.route("/form/add/quickwatch", methods=['POST'])
     @login_optionally_required
     def form_quick_watch_add():

changedetectionio/forms.py

@@ -1,6 +1,5 @@
 import os
 import re
-from loguru import logger
 
 from changedetectionio.strtobool import strtobool
 
@@ -476,7 +475,7 @@ class processor_text_json_diff_form(commonSettingsForm):
 
     title = StringField('Title', default='')
 
-    ignore_text = StringListField('Remove lines containing', [ValidateListRegex()])
+    ignore_text = StringListField('Ignore text', [ValidateListRegex()])
     headers = StringDictKeyValue('Request headers')
     body = TextAreaField('Request body', [validators.Optional()])
     method = SelectField('Request method', choices=valid_method, default=default_method)
@@ -526,16 +525,9 @@ class processor_text_json_diff_form(commonSettingsForm):
         try:
             from changedetectionio.safe_jinja import render as jinja_render
             jinja_render(template_str=self.url.data)
-        except ModuleNotFoundError as e:
-            # incase jinja2_time or others is missing
-            logger.error(e)
-            self.url.errors.append(e)
-            result = False
         except Exception as e:
-            logger.error(e)
             self.url.errors.append('Invalid template syntax')
             result = False
-
         return result
 
 class SingleExtraProxy(Form):

changedetectionio/model/__init__.py

@@ -18,7 +18,6 @@ class watch_base(dict):
             'check_count': 0,
             'check_unique_lines': False,  # On change-detected, compare against all history if its something new
             'consecutive_filter_failures': 0,  # Every time the CSS/xPath filter cannot be located, reset when all is fine.
-            'content-type': None,
             'date_created': None,
             'extract_text': [],  # Extract text by regex after filters
             'extract_title_as_title': False,

changedetectionio/processors/__init__.py

@@ -1,14 +1,14 @@
 from abc import abstractmethod
-from changedetectionio.content_fetchers.base import Fetcher
 from changedetectionio.strtobool import strtobool
+
 from copy import deepcopy
 from loguru import logger
 import hashlib
-import importlib
-import inspect
 import os
-import pkgutil
 import re
+import importlib
+import pkgutil
+import inspect
 
 class difference_detection_processor():
 
@@ -23,11 +23,10 @@ class difference_detection_processor():
         super().__init__(*args, **kwargs)
         self.datastore = datastore
         self.watch = deepcopy(self.datastore.data['watching'].get(watch_uuid))
-        # Generic fetcher that should be extended (requests, playwright etc)
-        self.fetcher = Fetcher()
 
     def call_browser(self):
         from requests.structures import CaseInsensitiveDict
+        from changedetectionio.content_fetchers.exceptions import EmptyReply
 
         # Protect against file:// access
         if re.search(r'^file://', self.watch.get('url', '').strip(), re.IGNORECASE):
@@ -155,7 +154,7 @@ class difference_detection_processor():
         # After init, call run_changedetection() which will do the actual change-detection
 
     @abstractmethod
-    def run_changedetection(self, watch, skip_when_checksum_same: bool = True):
+    def run_changedetection(self, watch, skip_when_checksum_same=True):
         update_obj = {'last_notification_error': False, 'last_error': False}
         some_data = 'xxxxx'
         update_obj["previous_md5"] = hashlib.md5(some_data.encode('utf-8')).hexdigest()

changedetectionio/processors/restock_diff/processor.py

@@ -143,6 +143,8 @@ class perform_site_check(difference_detection_processor):
     def run_changedetection(self, watch, skip_when_checksum_same=True):
         import hashlib
 
+        from concurrent.futures import ProcessPoolExecutor
+        from functools import partial
         if not watch:
             raise Exception("Watch no longer exists.")
 
@@ -184,7 +186,11 @@ class perform_site_check(difference_detection_processor):
 
         itemprop_availability = {}
         try:
-            itemprop_availability = get_itemprop_availability(self.fetcher.content)
+            with ProcessPoolExecutor() as executor:
+                # Use functools.partial to create a callable with arguments
+                # anything using bs4/lxml etc is quite "leaky"
+                future = executor.submit(partial(get_itemprop_availability, self.fetcher.content))
+                itemprop_availability = future.result()
         except MoreThanOnePriceFound as e:
             # Add the real data
             raise ProcessorException(message="Cannot run, more than one price detected, this plugin is only for product pages with ONE product, try the content-change detection mode.",
@@ -229,13 +235,12 @@ class perform_site_check(difference_detection_processor):
                 xpath_data=self.fetcher.xpath_data
                 )
 
-        logger.debug(f"self.fetcher.instock_data is - '{self.fetcher.instock_data}' and itemprop_availability.get('availability') is {itemprop_availability.get('availability')}")
         # Nothing automatic in microdata found, revert to scraping the page
         if self.fetcher.instock_data and itemprop_availability.get('availability') is None:
             # 'Possibly in stock' comes from stock-not-in-stock.js when no string found above the fold.
             # Careful! this does not really come from chrome/js when the watch is set to plaintext
             update_obj['restock']["in_stock"] = True if self.fetcher.instock_data == 'Possibly in stock' else False
-            logger.debug(f"Watch UUID {watch.get('uuid')} restock check returned instock_data - '{self.fetcher.instock_data}' from JS scraper.")
+            logger.debug(f"Watch UUID {watch.get('uuid')} restock check returned '{self.fetcher.instock_data}' from JS scraper.")
 
         # What we store in the snapshot
         price = update_obj.get('restock').get('price') if update_obj.get('restock').get('price') else ""

changedetectionio/processors/text_json_diff/processor.py

@@ -36,6 +36,8 @@ class PDFToHTMLToolNotFound(ValueError):
 class perform_site_check(difference_detection_processor):
 
     def run_changedetection(self, watch, skip_when_checksum_same=True):
+        from concurrent.futures import ProcessPoolExecutor
+        from functools import partial
 
         changed_detected = False
         html_content = ""
@@ -172,20 +174,30 @@ class perform_site_check(difference_detection_processor):
                     for filter_rule in include_filters_rule:
                         # For HTML/XML we offer xpath as an option, just start a regular xPath "/.."
                         if filter_rule[0] == '/' or filter_rule.startswith('xpath:'):
-                            html_content += html_tools.xpath_filter(xpath_filter=filter_rule.replace('xpath:', ''),
+                            with ProcessPoolExecutor() as executor:
+                                # Use functools.partial to create a callable with arguments - anything using bs4/lxml etc is quite "leaky"
+                                future = executor.submit(partial(html_tools.xpath_filter, xpath_filter=filter_rule.replace('xpath:', ''),
                                                                     html_content=self.fetcher.content,
                                                                     append_pretty_line_formatting=not watch.is_source_type_url,
-                                                                    is_rss=is_rss)
+                                                                    is_rss=is_rss))
+                                html_content += future.result()
 
                         elif filter_rule.startswith('xpath1:'):
-                            html_content += html_tools.xpath1_filter(xpath_filter=filter_rule.replace('xpath1:', ''),
-                                                                     html_content=self.fetcher.content,
-                                                                     append_pretty_line_formatting=not watch.is_source_type_url,
-                                                                     is_rss=is_rss)
+                            with ProcessPoolExecutor() as executor:
+                                # Use functools.partial to create a callable with arguments - anything using bs4/lxml etc is quite "leaky"
+                                future = executor.submit(partial(html_tools.xpath1_filter, xpath_filter=filter_rule.replace('xpath1:', ''),
+                                                                    html_content=self.fetcher.content,
+                                                                    append_pretty_line_formatting=not watch.is_source_type_url,
+                                                                    is_rss=is_rss))
+                                html_content += future.result()
                         else:
-                            html_content += html_tools.include_filters(include_filters=filter_rule,
+                            with ProcessPoolExecutor() as executor:
+                                # Use functools.partial to create a callable with arguments - anything using bs4/lxml etc is quite "leaky"
+                                # CSS Filter, extract the HTML that matches and feed that into the existing inscriptis::get_text
+                                future = executor.submit(partial(html_tools.include_filters, include_filters=filter_rule,
                                                                        html_content=self.fetcher.content,
-                                                                       append_pretty_line_formatting=not watch.is_source_type_url)
+                                                                       append_pretty_line_formatting=not watch.is_source_type_url))
+                                html_content += future.result()
 
                     if not html_content.strip():
                         raise FilterNotFoundInResponse(msg=include_filters_rule, screenshot=self.fetcher.screenshot, xpath_data=self.fetcher.xpath_data)
@@ -198,15 +210,20 @@ class perform_site_check(difference_detection_processor):
                 else:
                     # extract text
                     do_anchor = self.datastore.data["settings"]["application"].get("render_anchor_tag_content", False)
-                    stripped_text_from_html = html_tools.html_to_text(html_content=html_content,
-                                                                      render_anchor_tag_content=do_anchor,
-                                                                      is_rss=is_rss)  # 1874 activate the <title workaround hack
+                    with ProcessPoolExecutor() as executor:
+                        # Use functools.partial to create a callable with arguments - anything using bs4/lxml etc is quite "leaky"
+                        # CSS Filter, extract the HTML that matches and feed that into the existing inscriptis::get_text
+                        future = executor.submit(partial(html_tools.html_to_text, html_content=html_content,
+                            render_anchor_tag_content=do_anchor,
+                            is_rss=is_rss)) #1874 activate the <title workaround hack
+                        stripped_text_from_html = future.result()
+
 
         if watch.get('trim_text_whitespace'):
             stripped_text_from_html = '\n'.join(line.strip() for line in stripped_text_from_html.replace("\n\n", "\n").splitlines())
 
         if watch.get('remove_duplicate_lines'):
-            stripped_text_from_html = '\n'.join(dict.fromkeys(line for line in stripped_text_from_html.replace("\n\n", "\n").splitlines()))
+            stripped_text_from_html = '\n'.join(dict.fromkeys(line.strip() for line in stripped_text_from_html.replace("\n\n", "\n").splitlines()))
 
         if watch.get('sort_text_alphabetically'):
             # Note: Because a <p>something</p> will add an extra line feed to signify the paragraph gap
@@ -214,8 +231,8 @@ class perform_site_check(difference_detection_processor):
             stripped_text_from_html = stripped_text_from_html.replace("\n\n", "\n")
             stripped_text_from_html = '\n'.join(sorted(stripped_text_from_html.splitlines(), key=lambda x: x.lower()))
 
+
         # Re #340 - return the content before the 'ignore text' was applied
-        # Also used to calculate/show what was removed
         text_content_before_ignored_filter = stripped_text_from_html.encode('utf-8')
 
         # @todo whitespace coming from missing rtrim()?
@@ -240,7 +257,7 @@ class perform_site_check(difference_detection_processor):
             if not rendered_diff and stripped_text_from_html:
                 # We had some content, but no differences were found
                 # Store our new file as the MD5 so it will trigger in the future
-                c = hashlib.md5(stripped_text_from_html.encode('utf-8').translate(None, b'\r\n\t ')).hexdigest()
+                c = hashlib.md5(text_content_before_ignored_filter.translate(None, b'\r\n\t ')).hexdigest()
                 return False, {'previous_md5': c}, stripped_text_from_html.encode('utf-8')
             else:
                 stripped_text_from_html = rendered_diff
@@ -364,5 +381,4 @@ class perform_site_check(difference_detection_processor):
         if not watch.get('previous_md5'):
             watch['previous_md5'] = fetched_md5
 
-        # stripped_text_from_html - Everything after filters and NO 'ignored' content
-        return changed_detected, update_obj, stripped_text_from_html
+        return changed_detected, update_obj, text_content_before_ignored_filter

changedetectionio/static/js/plugins.js

@@ -1,120 +0,0 @@
-(function($) {
-
-/*
-    $('#code-block').highlightLines([
-      {
-        'color': '#dd0000',
-        'lines': [10, 12]
-      },
-      {
-        'color': '#ee0000',
-        'lines': [15, 18]
-      }
-    ]);
-  });
-*/
-
-  $.fn.highlightLines = function(configurations) {
-    return this.each(function() {
-      const $pre = $(this);
-      const textContent = $pre.text();
-      const lines = textContent.split(/\r?\n/); // Handles both \n and \r\n line endings
-
-      // Build a map of line numbers to styles
-      const lineStyles = {};
-
-      configurations.forEach(config => {
-        const { color, lines: lineNumbers } = config;
-        lineNumbers.forEach(lineNumber => {
-          lineStyles[lineNumber] = color;
-        });
-      });
-
-      // Function to escape HTML characters
-      function escapeHtml(text) {
-        return text.replace(/[&<>"'`=\/]/g, function(s) {
-          return "&#" + s.charCodeAt(0) + ";";
-        });
-      }
-
-      // Process each line
-      const processedLines = lines.map((line, index) => {
-        const lineNumber = index + 1; // Line numbers start at 1
-        const escapedLine = escapeHtml(line);
-        const color = lineStyles[lineNumber];
-
-        if (color) {
-          // Wrap the line in a span with inline style
-          return `<span style="background-color: ${color}">${escapedLine}</span>`;
-        } else {
-          return escapedLine;
-        }
-      });
-
-      // Join the lines back together
-      const newContent = processedLines.join('\n');
-
-      // Set the new content as HTML
-      $pre.html(newContent);
-    });
-  };
-   $.fn.miniTabs = function(tabsConfig, options) {
-        const settings = {
-            tabClass: 'minitab',
-            tabsContainerClass: 'minitabs',
-            activeClass: 'active',
-            ...(options || {})
-        };
-
-        return this.each(function() {
-            const $wrapper = $(this);
-            const $contents = $wrapper.find('div[id]').hide();
-            const $tabsContainer = $('<div>', { class: settings.tabsContainerClass }).prependTo($wrapper);
-
-            // Generate tabs
-            Object.entries(tabsConfig).forEach(([tabTitle, contentSelector], index) => {
-                const $content = $wrapper.find(contentSelector);
-                if (index === 0) $content.show(); // Show first content by default
-
-                $('<a>', {
-                    class: `${settings.tabClass}${index === 0 ? ` ${settings.activeClass}` : ''}`,
-                    text: tabTitle,
-                    'data-target': contentSelector
-                }).appendTo($tabsContainer);
-            });
-
-            // Tab click event
-            $tabsContainer.on('click', `.${settings.tabClass}`, function(e) {
-                e.preventDefault();
-                const $tab = $(this);
-                const target = $tab.data('target');
-
-                // Update active tab
-                $tabsContainer.find(`.${settings.tabClass}`).removeClass(settings.activeClass);
-                $tab.addClass(settings.activeClass);
-
-                // Show/hide content
-                $contents.hide();
-                $wrapper.find(target).show();
-            });
-        });
-    };
-
-    // Object to store ongoing requests by namespace
-    const requests = {};
-
-    $.abortiveSingularAjax = function(options) {
-        const namespace = options.namespace || 'default';
-
-        // Abort the current request in this namespace if it's still ongoing
-        if (requests[namespace]) {
-            requests[namespace].abort();
-        }
-
-        // Start a new AJAX request and store its reference in the correct namespace
-        requests[namespace] = $.ajax(options);
-
-        // Return the current request in case it's needed
-        return requests[namespace];
-    };
-})(jQuery);

changedetectionio/static/js/preview.js

@@ -1,63 +1,53 @@
-function redirectToVersion(version) {
-    var currentUrl = window.location.href.split('?')[0]; // Base URL without query parameters
+function redirect_to_version(version) {
+    var currentUrl = window.location.href;
+    var baseUrl = currentUrl.split('?')[0]; // Base URL without query parameters
     var anchor = '';
 
     // Check if there is an anchor
-    if (currentUrl.indexOf('#') !== -1) {
-        anchor = currentUrl.substring(currentUrl.indexOf('#'));
-        currentUrl = currentUrl.substring(0, currentUrl.indexOf('#'));
+    if (baseUrl.indexOf('#') !== -1) {
+        anchor = baseUrl.substring(baseUrl.indexOf('#'));
+        baseUrl = baseUrl.substring(0, baseUrl.indexOf('#'));
     }
-
-    window.location.href = currentUrl + '?version=' + version + anchor;
+    window.location.href = baseUrl + '?version=' + version + anchor;
 }
 
-function setupDateWidget() {
-    $(document).on('keydown', function (event) {
-        var $selectElement = $('#preview-version');
-        var $selectedOption = $selectElement.find('option:selected');
-
-        if ($selectedOption.length) {
-            if (event.key === 'ArrowLeft' && $selectedOption.prev().length) {
-                redirectToVersion($selectedOption.prev().val());
-            } else if (event.key === 'ArrowRight' && $selectedOption.next().length) {
-                redirectToVersion($selectedOption.next().val());
+document.addEventListener('keydown', function (event) {
+    var selectElement = document.getElementById('preview-version');
+    if (selectElement) {
+        var selectedOption = selectElement.querySelector('option:checked');
+        if (selectedOption) {
+            if (event.key === 'ArrowLeft') {
+                if (selectedOption.previousElementSibling) {
+                    redirect_to_version(selectedOption.previousElementSibling.value);
+                }
+            } else if (event.key === 'ArrowRight') {
+                if (selectedOption.nextElementSibling) {
+                    redirect_to_version(selectedOption.nextElementSibling.value);
+                }
             }
         }
-    });
+    }
+});
 
-    $('#preview-version').on('change', function () {
-        redirectToVersion($(this).val());
-    });
 
-    var $selectedOption = $('#preview-version option:selected');
+document.getElementById('preview-version').addEventListener('change', function () {
+    redirect_to_version(this.value);
+});
 
-    if ($selectedOption.length) {
-        var $prevOption = $selectedOption.prev();
-        var $nextOption = $selectedOption.next();
-
-        if ($prevOption.length) {
-            $('#btn-previous').attr('href', '?version=' + $prevOption.val());
+var selectElement = document.getElementById('preview-version');
+if (selectElement) {
+    var selectedOption = selectElement.querySelector('option:checked');
+    if (selectedOption) {
+        if (selectedOption.previousElementSibling) {
+            document.getElementById('btn-previous').href = "?version=" + selectedOption.previousElementSibling.value;
         } else {
-            $('#btn-previous').remove();
+            document.getElementById('btn-previous').remove()
+        }
+        if (selectedOption.nextElementSibling) {
+            document.getElementById('btn-next').href = "?version=" + selectedOption.nextElementSibling.value;
+        } else {
+            document.getElementById('btn-next').remove()
         }
 
-        if ($nextOption.length) {
-            $('#btn-next').attr('href', '?version=' + $nextOption.val());
-        } else {
-            $('#btn-next').remove();
-        }
     }
 }
-
-$(document).ready(function () {
-    if ($('#preview-version').length) {
-        setupDateWidget();
-    }
-
-    $('#diff-col > pre').highlightLines([
-        {
-            'color': '#ee0000',
-            'lines': triggered_line_numbers
-        }
-    ]);
-});

changedetectionio/static/js/watch-settings.js

@@ -12,48 +12,6 @@ function toggleOpacity(checkboxSelector, fieldSelector, inverted) {
     checkbox.addEventListener('change', updateOpacity);
 }
 
-
-function request_textpreview_update() {
-    if (!$('body').hasClass('preview-text-enabled')) {
-        console.error("Preview text was requested but body tag was not setup")
-        return
-    }
-
-    const data = {};
-    $('textarea:visible, input:visible').each(function () {
-        const $element = $(this); // Cache the jQuery object for the current element
-        const name = $element.attr('name'); // Get the name attribute of the element
-        data[name] = $element.is(':checkbox') ? ($element.is(':checked') ? $element.val() : false) : $element.val();
-    });
-
-    $.abortiveSingularAjax({
-        type: "POST",
-        url: preview_text_edit_filters_url,
-        data: data,
-        namespace: 'watchEdit'
-    }).done(function (data) {
-        $('#filters-and-triggers #text-preview-before-inner').text(data['before_filter']);
-
-        $('#filters-and-triggers #text-preview-inner')
-            .text(data['after_filter'])
-            .highlightLines([
-                {
-                    'color': '#ee0000',
-                    'lines': data['trigger_line_numbers']
-                }
-            ]);
-
-
-
-    }).fail(function (error) {
-        if (error.statusText === 'abort') {
-            console.log('Request was aborted due to a new request being fired.');
-        } else {
-            $('#filters-and-triggers #text-preview-inner').text('There was an error communicating with the server.');
-        }
-    })
-}
-
 $(document).ready(function () {
     $('#notification-setting-reset-to-default').click(function (e) {
         $('#notification_title').val('');
@@ -69,26 +27,5 @@ $(document).ready(function () {
 
     toggleOpacity('#time_between_check_use_default', '#time_between_check', false);
 
-    const vh = Math.max(document.documentElement.clientHeight || 0, window.innerHeight || 0);
-    $("#text-preview-inner").css('max-height', (vh-300)+"px");
-    $("#text-preview-before-inner").css('max-height', (vh-300)+"px");
-
-    // Realtime preview of 'Filters & Text' setup
-    var debounced_request_textpreview_update = request_textpreview_update.debounce(100);
-
-    $("#activate-text-preview").click(function (e) {
-        $('body').toggleClass('preview-text-enabled')
-        request_textpreview_update();
-
-        const method = $('body').hasClass('preview-text-enabled') ? 'on' : 'off';
-        $("#text-preview-refresh")[method]('click', debounced_request_textpreview_update);
-        $('textarea:visible')[method]('keyup blur', debounced_request_textpreview_update);
-        $('input:visible')[method]('keyup blur change', debounced_request_textpreview_update);
-        $("#filters-and-triggers-tab")[method]('click', debounced_request_textpreview_update);
-    });
-    $('.minitabs-wrapper').miniTabs({
-        "Content after filters": "#text-preview-inner",
-        "Content raw/before filters": "#text-preview-before-inner"
-    });
 });
 

changedetectionio/static/styles/scss/parts/_minitabs.scss

@@ -1,37 +0,0 @@
-.minitabs-wrapper {
-  width: 100%;
-
-  > div[id] {
-    padding: 20px;
-    border: 1px solid #ccc;
-    border-top: none;
-  }
-
-  .minitabs {
-    display: flex;
-    border-bottom: 1px solid #ccc;
-  }
-
-  .minitab {
-    flex: 1;
-    text-align: center;
-    padding: 12px 0;
-    text-decoration: none;
-    color: #333;
-    background-color: #f1f1f1;
-    border: 1px solid #ccc;
-    border-bottom: none;
-    cursor: pointer;
-    transition: background-color 0.3s;
-  }
-
-  .minitab:hover {
-    background-color: #ddd;
-  }
-
-  .minitab.active {
-    background-color: #fff;
-    font-weight: bold;
-  }
-
-}

changedetectionio/static/styles/scss/parts/_preview_text_filter.scss

@@ -1,56 +0,0 @@
-@import "minitabs";
-
-body.preview-text-enabled {
-
-  @media (min-width: 800px) {
-    #filters-and-triggers > div {
-      display: flex; /* Establishes Flexbox layout */
-      gap: 20px; /* Adds space between the columns */
-      position: relative; /* Ensures the sticky positioning is relative to this parent */
-    }
-  }
-
-  /* layout of the page */
-  #edit-text-filter, #text-preview {
-    flex: 1; /* Each column takes an equal amount of available space */
-    align-self: flex-start; /* Aligns the right column to the start, allowing it to maintain its content height */
-  }
-
-  #edit-text-filter {
-    #pro-tips {
-      display: none;
-    }
-  }
-
-  #text-preview {
-    position: sticky;
-    top: 20px;
-    padding-top: 1rem;
-    padding-bottom: 1rem;
-    display: block !important;
-  }
-
-  #activate-text-preview {
-      background-color: var(--color-grey-500);
-  }
-
-  /* actual preview area */
-  .monospace-preview {
-    background: var(--color-background-input);
-    border: 1px solid var(--color-grey-600);
-    padding: 1rem;
-    color: var(--color-text-input);
-    font-family: "Courier New", Courier, monospace; /* Sets the font to a monospace type */
-    font-size: 70%;
-    overflow-x: scroll;
-    white-space: pre-wrap; /* Preserves whitespace and line breaks like <pre> */
-    overflow-wrap: break-word; /* Allows long words to break and wrap to the next line */
-  }
-}
-
-#activate-text-preview {
-  right: 0;
-  position: absolute;
-  z-index: 3;
-  box-shadow: 1px 1px 4px var(--color-shadow-jump);
-}

changedetectionio/static/styles/scss/styles.scss

@@ -12,7 +12,6 @@
 @import "parts/_darkmode";
 @import "parts/_menu";
 @import "parts/_love";
-@import "parts/preview_text_filter";
 
 body {
   color: var(--color-text);
@@ -321,6 +320,10 @@ a.pure-button-selected {
   background: var(--color-background-button-cancel);
 }
 
+#save_button {
+  margin-right: 1rem;
+}
+
 .messages {
   li {
     list-style: none;
@@ -617,9 +620,9 @@ footer {
       list-style: none;
 
       li {
-        display: flex;
-        align-items: center;
-        gap: 1em;
+        >* {
+          display: inline-block;
+        }
       }
     }
   }
@@ -679,12 +682,6 @@ footer {
       tr {
         th {
           display: inline-block;
-          // Hide the "Last" text for smaller screens
-          @media (max-width: 768px) {
-            .hide-on-mobile {
-              display: none; 
-            }
-          }
         }
       }
       .empty-cell {
@@ -700,24 +697,6 @@ footer {
       }
     }
 
-    tbody {
-      tr {
-        display: flex;
-        flex-wrap: wrap;
-
-        // The third child of each row will take up the remaining space
-        // This is useful for the URL column, which should expand to fill the remaining space
-        :nth-child(3) {
-          flex-grow: 1;
-        }
-        // The last three children (from the end) of each row will take up the full width
-        // This is useful for the "Last Checked", "Last Changed", and the action buttons columns, which should each take up the full width
-        :nth-last-child(-n+3) {
-          flex-basis: 100%;
-        }
-      }
-    }
-
     .last-checked {
       >span {
         vertical-align: middle;
@@ -836,11 +815,6 @@ textarea::placeholder {
 - We dont use 'size' with <input> because `size` is too unreliable to override, and will often push-out
 - Rely always on width in CSS
 */
-/** Set max width for input field */
-.m-d {
-  min-width: 100%;
-}
-
 @media only screen and (min-width: 761px) {
 
   /* m-d is medium-desktop */
@@ -956,13 +930,6 @@ body.full-width {
     background: var(--color-background);
   }
 
-  /* Make action buttons have consistent size and spacing */
-  #actions .pure-control-group {
-    display: flex;
-    gap: 0.625em;
-    flex-wrap: wrap;
-  }
-
   .pure-form-message-inline {
     padding-left: 0;
     color: var(--color-text-input-description);
@@ -1006,28 +973,6 @@ ul {
   }
 }
 
-@media only screen and (max-width: 760px) {
-  .time-check-widget {
-    tbody {
-      display: grid;
-      grid-template-columns: auto 1fr auto 1fr;
-      gap: 0.625em 0.3125em;
-      align-items: center;
-    }    
-    tr {
-      display: contents; 
-      th {
-        text-align: right;
-        padding-right: 5px;
-      }
-      input[type="number"] {
-        width: 100%;
-        max-width: 5em;
-      }
-    }
-  }
-}
-
 @import "parts/_visualselector";
 
 #webdriver_delay {

changedetectionio/static/styles/styles.css

@@ -428,78 +428,6 @@ html[data-darkmode="true"] #toggle-light-mode .icon-dark {
     fill: #ff0000 !important;
     transition: all ease 0.3s !important; }
 
-.minitabs-wrapper {
-  width: 100%; }
-  .minitabs-wrapper > div[id] {
-    padding: 20px;
-    border: 1px solid #ccc;
-    border-top: none; }
-  .minitabs-wrapper .minitabs {
-    display: flex;
-    border-bottom: 1px solid #ccc; }
-  .minitabs-wrapper .minitab {
-    flex: 1;
-    text-align: center;
-    padding: 12px 0;
-    text-decoration: none;
-    color: #333;
-    background-color: #f1f1f1;
-    border: 1px solid #ccc;
-    border-bottom: none;
-    cursor: pointer;
-    transition: background-color 0.3s; }
-  .minitabs-wrapper .minitab:hover {
-    background-color: #ddd; }
-  .minitabs-wrapper .minitab.active {
-    background-color: #fff;
-    font-weight: bold; }
-
-body.preview-text-enabled {
-  /* layout of the page */
-  /* actual preview area */ }
-  @media (min-width: 800px) {
-    body.preview-text-enabled #filters-and-triggers > div {
-      display: flex;
-      /* Establishes Flexbox layout */
-      gap: 20px;
-      /* Adds space between the columns */
-      position: relative;
-      /* Ensures the sticky positioning is relative to this parent */ } }
-  body.preview-text-enabled #edit-text-filter, body.preview-text-enabled #text-preview {
-    flex: 1;
-    /* Each column takes an equal amount of available space */
-    align-self: flex-start;
-    /* Aligns the right column to the start, allowing it to maintain its content height */ }
-  body.preview-text-enabled #edit-text-filter #pro-tips {
-    display: none; }
-  body.preview-text-enabled #text-preview {
-    position: sticky;
-    top: 20px;
-    padding-top: 1rem;
-    padding-bottom: 1rem;
-    display: block !important; }
-  body.preview-text-enabled #activate-text-preview {
-    background-color: var(--color-grey-500); }
-  body.preview-text-enabled .monospace-preview {
-    background: var(--color-background-input);
-    border: 1px solid var(--color-grey-600);
-    padding: 1rem;
-    color: var(--color-text-input);
-    font-family: "Courier New", Courier, monospace;
-    /* Sets the font to a monospace type */
-    font-size: 70%;
-    overflow-x: scroll;
-    white-space: pre-wrap;
-    /* Preserves whitespace and line breaks like <pre> */
-    overflow-wrap: break-word;
-    /* Allows long words to break and wrap to the next line */ }
-
-#activate-text-preview {
-  right: 0;
-  position: absolute;
-  z-index: 3;
-  box-shadow: 1px 1px 4px var(--color-shadow-jump); }
-
 body {
   color: var(--color-text);
   background: var(--color-background-page);
@@ -723,6 +651,9 @@ a.pure-button-selected {
 .button-cancel {
   background: var(--color-background-button-cancel); }
 
+#save_button {
+  margin-right: 1rem; }
+
 .messages li {
   list-style: none;
   padding: 1em;
@@ -921,10 +852,8 @@ footer {
   .pure-form .inline-radio ul {
     margin: 0px;
     list-style: none; }
-    .pure-form .inline-radio ul li {
-      display: flex;
-      align-items: center;
-      gap: 1em; }
+    .pure-form .inline-radio ul li > * {
+      display: inline-block; }
 
 @media only screen and (max-width: 760px), (min-device-width: 768px) and (max-device-width: 1024px) {
   .box {
@@ -960,24 +889,12 @@ footer {
     .watch-table thead {
       display: block; }
       .watch-table thead tr th {
-        display: inline-block; } }
-      @media only screen and (max-width: 760px) and (max-width: 768px), (min-device-width: 768px) and (max-device-width: 800px) and (max-width: 768px) {
-        .watch-table thead tr th .hide-on-mobile {
-          display: none; } }
-
-@media only screen and (max-width: 760px), (min-device-width: 768px) and (max-device-width: 800px) {
+        display: inline-block; }
       .watch-table thead .empty-cell {
         display: none; }
     .watch-table tbody td,
     .watch-table tbody tr {
       display: block; }
-    .watch-table tbody tr {
-      display: flex;
-      flex-wrap: wrap; }
-      .watch-table tbody tr :nth-child(3) {
-        flex-grow: 1; }
-      .watch-table tbody tr :nth-last-child(-n+3) {
-        flex-basis: 100%; }
     .watch-table .last-checked > span {
       vertical-align: middle; }
     .watch-table .last-checked::before {
@@ -1069,10 +986,6 @@ textarea::placeholder {
 - We dont use 'size' with <input> because `size` is too unreliable to override, and will often push-out
 - Rely always on width in CSS
 */
-/** Set max width for input field */
-.m-d {
-  min-width: 100%; }
-
 @media only screen and (min-width: 761px) {
   /* m-d is medium-desktop */
   .m-d {
@@ -1133,8 +1046,7 @@ body.full-width .edit-form {
 .edit-form {
   min-width: 70%;
   /* so it cant overflow */
-  max-width: 95%;
-  /* Make action buttons have consistent size and spacing */ }
+  max-width: 95%; }
   .edit-form .box-wrap {
     position: relative; }
   .edit-form .inner {
@@ -1143,10 +1055,6 @@ body.full-width .edit-form {
   .edit-form #actions {
     display: block;
     background: var(--color-background); }
-  .edit-form #actions .pure-control-group {
-    display: flex;
-    gap: 0.625em;
-    flex-wrap: wrap; }
   .edit-form .pure-form-message-inline {
     padding-left: 0;
     color: var(--color-text-input-description); }
@@ -1175,21 +1083,6 @@ ul {
   .time-check-widget tr input[type="number"] {
     width: 5em; }
 
-@media only screen and (max-width: 760px) {
-  .time-check-widget tbody {
-    display: grid;
-    grid-template-columns: auto 1fr auto 1fr;
-    gap: 0.625em 0.3125em;
-    align-items: center; }
-  .time-check-widget tr {
-    display: contents; }
-    .time-check-widget tr th {
-      text-align: right;
-      padding-right: 5px; }
-    .time-check-widget tr input[type="number"] {
-      width: 100%;
-      max-width: 5em; } }
-
 #selector-wrapper {
   height: 100%;
   text-align: center;

changedetectionio/templates/base.html

@@ -33,7 +33,7 @@
     <script src="{{url_for('static_content', group='js', filename='csrf.js')}}" defer></script>
   </head>
 
-  <body class="">
+  <body>
     <div class="header">
       <div class="home-menu pure-menu pure-menu-horizontal pure-menu-fixed" id="nav-menu">
         {% if has_password and not current_user.is_authenticated %}

changedetectionio/templates/edit.html

@@ -24,7 +24,7 @@
     const watch_visual_selector_data_url="{{url_for('static_content', group='visual_selector_data', filename=uuid)}}";
     const default_system_fetch_backend="{{ settings_application['fetch_backend'] }}";
 </script>
-<script src="{{url_for('static_content', group='js', filename='plugins.js')}}" defer></script>
+
 <script src="{{url_for('static_content', group='js', filename='watch-settings.js')}}" defer></script>
 <script src="{{url_for('static_content', group='js', filename='limit.js')}}" defer></script>
 <script src="{{url_for('static_content', group='js', filename='notifications.js')}}" defer></script>
@@ -50,7 +50,7 @@
             {% endif %}
             {% if watch['processor'] == 'text_json_diff' %}
             <li class="tab"><a id="visualselector-tab" href="#visualselector">Visual Filter Selector</a></li>
-            <li class="tab" id="filters-and-triggers-tab"><a href="#filters-and-triggers">Filters &amp; Triggers</a></li>
+            <li class="tab"><a href="#filters-and-triggers">Filters &amp; Triggers</a></li>
             {% endif %}
             <li class="tab"><a href="#notifications">Notifications</a></li>
             <li class="tab"><a href="#stats">Stats</a></li>
@@ -254,10 +254,7 @@ User-Agent: wonderbra 1.0") }}
 
             {% if watch['processor'] == 'text_json_diff' %}
             <div class="tab-pane-inner" id="filters-and-triggers">
-                <span id="activate-text-preview" class="pure-button pure-button-primary button-xsmall">Activate preview</span>
-              <div>
-              <div id="edit-text-filter">
-                    <div class="pure-control-group" id="pro-tips">
+                    <div class="pure-control-group">
                             <strong>Pro-tips:</strong><br>
                             <ul>
                                 <li>
@@ -350,6 +347,10 @@ nav
                     {{ render_checkbox_field(form.trim_text_whitespace) }}
                     <span class="pure-form-message-inline">Remove any whitespace before and after each line of text</span>
                 </fieldset>
+                <fieldset class="pure-control-group">
+                    {{ render_checkbox_field(form.check_unique_lines) }}
+                    <span class="pure-form-message-inline">Good for websites that just move the content around, and you want to know when NEW content is added, compares new lines against all history for this watch.</span>
+                </fieldset>
                 <fieldset>
                     <div class="pure-control-group">
                         {{ render_field(form.trigger_text, rows=5, placeholder="Some text to wait for in a line
@@ -371,10 +372,10 @@ nav
 ") }}
                     <span class="pure-form-message-inline">
                         <ul>
-                            <li>Matching text will be <strong>removed</strong> from the text snapshot</li>
                             <li>Each line processed separately, any line matching will be ignored (removed before creating the checksum)</li>
                             <li>Regular Expression support, wrap the entire line in forward slash <code>/regex/</code></li>
                             <li>Changing this will affect the comparison checksum which may trigger an alert</li>
+                            <li>Use the preview/show current tab to see ignores</li>
                         </ul>
                 </span>
 
@@ -418,26 +419,7 @@ Unavailable") }}
                 </fieldset>
                 </div>
             </div>
-              <div id="text-preview" style="display: none;" >
-                    <script>
-                        const preview_text_edit_filters_url="{{url_for('watch_get_preview_rendered', uuid=uuid)}}";
-                    </script>
-                    <br>
-                    {#<div id="text-preview-controls"><span id="text-preview-refresh" class="pure-button button-xsmall">Refresh</span></div>#}
-
-                    <div class="minitabs-wrapper">
-                        <div id="text-preview-inner" class="monospace-preview">
-                            <p>Loading...</p>
-                        </div>
-                        <div id="text-preview-before-inner" style="display: none;" class="monospace-preview">
-                            <p>Loading...</p>
-                        </div>
-                    </div>
-            </div>
-          </div>
-        </div>
-
-        {% endif %}
+            {% endif %}
         {# rendered sub Template #}
         {% if extra_form_content %}
             <div class="tab-pane-inner" id="extras_tab">

changedetectionio/templates/preview.html

@@ -3,13 +3,11 @@
 {% block content %}
     <script>
         const screenshot_url = "{{url_for('static_content', group='screenshot', filename=uuid)}}";
-        const triggered_line_numbers = {{ triggered_line_numbers|tojson }};
         {% if last_error_screenshot %}
             const error_screenshot_url = "{{url_for('static_content', group='screenshot', filename=uuid, error_screenshot=1) }}";
         {% endif %}
         const highlight_submit_ignore_url = "{{url_for('highlight_submit_ignore_url', uuid=uuid)}}";
     </script>
-    <script src="{{url_for('static_content', group='js', filename='plugins.js')}}"></script>
     <script src="{{ url_for('static_content', group='js', filename='diff-overview.js') }}" defer></script>
     <script src="{{ url_for('static_content', group='js', filename='preview.js') }}" defer></script>
     <script src="{{ url_for('static_content', group='js', filename='tabs.js') }}" defer></script>
@@ -69,15 +67,16 @@
 
         <div class="tab-pane-inner" id="text">
             <div class="snapshot-age">{{ current_version|format_timestamp_timeago }}</div>
+            <span class="ignored">Grey lines are ignored</span> <span class="triggered">Blue lines are triggers</span>
             <span class="tip"><strong>Pro-tip</strong>: Highlight text to add to ignore filters</span>
 
             <table>
                 <tbody>
                 <tr>
                     <td id="diff-col" class="highlightable-filter">
-                        <pre style="border-left: 2px solid #ddd;">
-{{ content }}
-                        </pre>
+                        {% for row in content %}
+                            <div class="{{ row.classes }}">{{ row.line }}</div>
+                        {% endfor %}
                     </td>
                 </tr>
                 </tbody>

changedetectionio/templates/settings.html

@@ -172,11 +172,11 @@ nav
                     <span class="pure-form-message-inline">Note: This is applied globally in addition to the per-watch rules.</span><br>
                     <span class="pure-form-message-inline">
                         <ul>
-                            <li>Matching text will be <strong>removed</strong> from the text snapshot</li>
                             <li>Note: This is applied globally in addition to the per-watch rules.</li>
                             <li>Each line processed separately, any line matching will be ignored (removed before creating the checksum)</li>
                             <li>Regular Expression support, wrap the entire line in forward slash <code>/regex/</code></li>
                             <li>Changing this will affect the comparison checksum which may trigger an alert</li>
+                            <li>Use the preview/show current tab to see ignores</li>
                         </ul>
                      </span>
                     </fieldset>

changedetectionio/templates/watch-overview.html

@@ -78,8 +78,8 @@
              {% if any_has_restock_price_processor %}
                 <th>Restock &amp; Price</th>
              {% endif %}
-                <th><a class="{{ 'active '+link_order if sort_attribute == 'last_checked' else 'inactive' }}" href="{{url_for('index', sort='last_checked', order=link_order, tag=active_tag_uuid)}}"><span class="hide-on-mobile">Last</span> Checked <span class='arrow {{link_order}}'></span></a></th>
-                <th><a class="{{ 'active '+link_order if sort_attribute == 'last_changed' else 'inactive' }}" href="{{url_for('index', sort='last_changed', order=link_order, tag=active_tag_uuid)}}"><span class="hide-on-mobile">Last</span> Changed <span class='arrow {{link_order}}'></span></a></th>
+                <th><a class="{{ 'active '+link_order if sort_attribute == 'last_checked' else 'inactive' }}" href="{{url_for('index', sort='last_checked', order=link_order, tag=active_tag_uuid)}}">Last Checked <span class='arrow {{link_order}}'></span></a></th>
+                <th><a class="{{ 'active '+link_order if sort_attribute == 'last_changed' else 'inactive' }}" href="{{url_for('index', sort='last_changed', order=link_order, tag=active_tag_uuid)}}">Last Changed <span class='arrow {{link_order}}'></span></a></th>
                 <th class="empty-cell"></th>
             </tr>
             </thead>
@@ -191,9 +191,9 @@
                     {% if watch.history_n >= 2 %}
 
                         {%  if is_unviewed %}
-                           <a href="{{ url_for('diff_history_page', uuid=watch.uuid, from_version=watch.get_next_snapshot_key_to_last_viewed) }}" target="{{watch.uuid}}" class="pure-button pure-button-primary diff-link">History</a>
+                           <a href="{{ url_for('diff_history_page', uuid=watch.uuid, from_version=watch.get_next_snapshot_key_to_last_viewed) }}" target="{{watch.uuid}}" class="pure-button pure-button-primary diff-link">Diff</a>
                         {% else %}
-                           <a href="{{ url_for('diff_history_page', uuid=watch.uuid)}}" target="{{watch.uuid}}" class="pure-button pure-button-primary diff-link">History</a>
+                           <a href="{{ url_for('diff_history_page', uuid=watch.uuid)}}" target="{{watch.uuid}}" class="pure-button pure-button-primary diff-link">Diff</a>
                         {% endif %}
 
                     {% else %}

changedetectionio/tests/proxy_list/test_select_custom_proxy.py

@@ -44,7 +44,7 @@ def test_select_custom(client, live_server, measure_memory_usage):
         follow_redirects=True
     )
     # We should see something via proxy
-    assert b' - 0.' in res.data
+    assert b'<div class=""> - 0.' in res.data
 
     #
     # Now we should see the request in the container logs for "squid-squid-custom" because it will be the only default

changedetectionio/tests/test_add_replace_remove_filter.py

@@ -39,8 +39,9 @@ def test_setup(client, live_server, measure_memory_usage):
     live_server_setup(live_server)
 
 def test_check_removed_line_contains_trigger(client, live_server, measure_memory_usage):
-    #live_server_setup(live_server)
+
     # Give the endpoint time to spin up
+    time.sleep(1)
     set_original()
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)
@@ -151,9 +152,7 @@ def test_check_add_line_contains_trigger(client, live_server, measure_memory_usa
 
     # A line thats not the trigger should not trigger anything
     res = client.get(url_for("form_watch_checknow"), follow_redirects=True)
-
     assert b'1 watches queued for rechecking.' in res.data
-
     wait_for_all_checks(client)
     res = client.get(url_for("index"))
     assert b'unviewed' not in res.data

changedetectionio/tests/test_encoding.py

@@ -3,7 +3,7 @@
 
 import time
 from flask import url_for
-from .util import live_server_setup, wait_for_all_checks, extract_UUID_from_client
+from .util import live_server_setup, wait_for_all_checks
 import pytest
 
 
@@ -38,11 +38,6 @@ def test_check_encoding_detection(client, live_server, measure_memory_usage):
     # Give the thread time to pick it up
     wait_for_all_checks(client)
 
-
-    # Content type recording worked
-    uuid = extract_UUID_from_client(client)
-    assert live_server.app.config['DATASTORE'].data['watching'][uuid]['content-type'] == "text/html"
-
     res = client.get(
         url_for("preview_page", uuid="first"),
         follow_redirects=True

changedetectionio/tests/test_extract_regex.py

@@ -115,9 +115,9 @@ def test_check_filter_multiline(client, live_server, measure_memory_usage):
     # Plaintext that doesnt look like a regex should match also
     assert b'and this should be' in res.data
 
-    assert b'Something' in res.data
-    assert b'across 6 billion multiple' in res.data
-    assert b'lines' in res.data
+    assert b'<div class="">Something' in res.data
+    assert b'<div class="">across 6 billion multiple' in res.data
+    assert b'<div class="">lines' in res.data
 
     # but the last one, which also says 'lines' shouldnt be here (non-greedy match checking)
     assert b'aaand something lines' not in res.data
@@ -183,19 +183,20 @@ def test_check_filter_and_regex_extract(client, live_server, measure_memory_usag
         follow_redirects=True
     )
 
-    assert b'1000 online' in res.data
+    # Class will be blank for now because the frontend didnt apply the diff
+    assert b'<div class="">1000 online' in res.data
 
     # All regex matching should be here
-    assert b'2000 online' in res.data
+    assert b'<div class="">2000 online' in res.data
 
     # Both regexs should be here
-    assert b'80 guests' in res.data
+    assert b'<div class="">80 guests' in res.data
 
     # Regex with flag handling should be here
-    assert b'SomeCase insensitive 3456' in res.data
+    assert b'<div class="">SomeCase insensitive 3456' in res.data
 
     # Singular group from /somecase insensitive (345\d)/i
-    assert b'3456' in res.data
+    assert b'<div class="">3456' in res.data
 
     # Regex with multiline flag handling should be here
 

changedetectionio/tests/test_filter_failure_notification.py

@@ -116,11 +116,9 @@ def run_filter_test(client, live_server, content_filter):
         res = client.get(url_for("index"))
         assert b'Warning, no filters were found' in res.data
         assert not os.path.isfile("test-datastore/notification.txt")
-        time.sleep(1)
-        
+
     assert live_server.app.config['DATASTORE'].data['watching'][uuid]['consecutive_filter_failures'] == 5
 
-    time.sleep(2)
     # One more check should trigger the _FILTER_FAILURE_THRESHOLD_ATTEMPTS_DEFAULT threshold
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)

changedetectionio/tests/test_ignore_text.py

@@ -79,14 +79,14 @@ def set_modified_ignore_response():
         f.write(test_return_data)
 
 
-# Ignore text now just removes it entirely, is a LOT more simpler code this way
-
 def test_check_ignore_text_functionality(client, live_server, measure_memory_usage):
 
     # Use a mix of case in ZzZ to prove it works case-insensitive.
     ignore_text = "XXXXX\r\nYYYYY\r\nzZzZZ\r\nnew ignore stuff"
     set_original_ignore_response()
 
+    # Give the endpoint time to spin up
+    time.sleep(1)
 
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)
@@ -151,10 +151,12 @@ def test_check_ignore_text_functionality(client, live_server, measure_memory_usa
     res = client.get(url_for("index"))
     assert b'unviewed' in res.data
 
+    # Check the preview/highlighter, we should be able to see what we ignored, but it should be highlighted
+    # We only introduce the "modified" content that includes what we ignore so we can prove the newest version also displays
+    # at /preview
     res = client.get(url_for("preview_page", uuid="first"))
-
-    # Should no longer be in the preview
-    assert b'new ignore stuff' not in res.data
+    # We should be able to see what we ignored
+    assert b'<div class="ignored">new ignore stuff' in res.data
 
     res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data

changedetectionio/tests/test_ignorehighlighter.py

@@ -23,7 +23,7 @@ def set_original_ignore_response():
         f.write(test_return_data)
 
 
-def test_ignore(client, live_server, measure_memory_usage):
+def test_highlight_ignore(client, live_server, measure_memory_usage):
     live_server_setup(live_server)
     set_original_ignore_response()
     test_url = url_for('test_endpoint', _external=True)
@@ -51,9 +51,9 @@ def test_ignore(client, live_server, measure_memory_usage):
     # Should return a link
     assert b'href' in res.data
 
-    # It should not be in the preview anymore
+    # And it should register in the preview page
     res = client.get(url_for("preview_page", uuid=uuid))
-    assert b'<div class="ignored">oh yeah 456' not in res.data
+    assert b'<div class="ignored">oh yeah 456' in res.data
 
     # Should be in base.html
     assert b'csrftoken' in res.data

changedetectionio/tests/test_jsonpath_jq_selector.py

@@ -499,7 +499,7 @@ def test_correct_header_detect(client, live_server, measure_memory_usage):
     )
 
     assert b'"hello": 123,' in res.data
-    assert b'"world": 123' in res.data
+    assert b'&#34;world&#34;: 123</div>' in res.data
 
     res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data

changedetectionio/tests/test_trigger.py

@@ -2,7 +2,7 @@
 
 import time
 from flask import url_for
-from .util import live_server_setup, wait_for_all_checks
+from . util import live_server_setup
 
 
 def set_original_ignore_response():
@@ -59,9 +59,12 @@ def test_trigger_functionality(client, live_server, measure_memory_usage):
 
     live_server_setup(live_server)
 
+    sleep_time_for_fetch_thread = 3
     trigger_text = "Add to cart"
     set_original_ignore_response()
 
+    # Give the endpoint time to spin up
+    time.sleep(1)
 
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)
@@ -86,14 +89,14 @@ def test_trigger_functionality(client, live_server, measure_memory_usage):
     )
     assert b"Updated watch." in res.data
 
-    wait_for_all_checks(client)
     # Check it saved
     res = client.get(
         url_for("edit_page", uuid="first"),
     )
     assert bytes(trigger_text.encode('utf-8')) in res.data
 
-
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
     
     # so that we set the state to 'unviewed' after all the edits
     client.get(url_for("diff_history_page", uuid="first"))
@@ -101,7 +104,8 @@ def test_trigger_functionality(client, live_server, measure_memory_usage):
     # Trigger a check
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
 
-    wait_for_all_checks(client)
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
 
     # It should report nothing found (no new 'unviewed' class)
     res = client.get(url_for("index"))
@@ -113,17 +117,19 @@ def test_trigger_functionality(client, live_server, measure_memory_usage):
 
     # Trigger a check
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    wait_for_all_checks(client)
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
 
     # It should report nothing found (no new 'unviewed' class)
     res = client.get(url_for("index"))
     assert b'unviewed' not in res.data
 
     # Now set the content which contains the trigger text
+    time.sleep(sleep_time_for_fetch_thread)
     set_modified_with_trigger_text_response()
 
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    wait_for_all_checks(client)
+    time.sleep(sleep_time_for_fetch_thread)
     res = client.get(url_for("index"))
     assert b'unviewed' in res.data
     
@@ -136,7 +142,4 @@ def test_trigger_functionality(client, live_server, measure_memory_usage):
     res = client.get(url_for("preview_page", uuid="first"))
 
     # We should be able to see what we triggered on
-    # The JS highlighter should tell us which lines (also used in the live-preview)
-    assert b'const triggered_line_numbers = [6]' in res.data
-    assert b'Add to cart' in res.data
-
+    assert b'<div class="triggered">Add to cart' in res.data

changedetectionio/tests/test_xpath_selector.py

@@ -161,8 +161,8 @@ def test_check_xpath_text_function_utf8(client, live_server, measure_memory_usag
         follow_redirects=True
     )
 
-    assert b'Stock Alert (UK): RPi CM4' in res.data
-    assert b'Stock Alert (UK): Big monitor' in res.data
+    assert b'<div class="">Stock Alert (UK): RPi CM4' in res.data
+    assert b'<div class="">Stock Alert (UK): Big monitor' in res.data
 
     res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data

changedetectionio/tests/util.py

@@ -78,7 +78,6 @@ def set_more_modified_response():
 
 def wait_for_notification_endpoint_output():
     '''Apprise can take a few seconds to fire'''
-    #@todo - could check the apprise object directly instead of looking for this file
     from os.path import isfile
     for i in range(1, 20):
         time.sleep(1)

changedetectionio/update_worker.py

@@ -491,8 +491,6 @@ class update_worker(threading.Thread):
                         if not self.datastore.data['watching'].get(uuid):
                             continue
 
-                        update_obj['content-type'] = update_handler.fetcher.get_all_headers().get('content-type', '').lower()
-
                         # Mark that we never had any failures
                         if not watch.get('ignore_status_codes'):
                             update_obj['consecutive_filter_failures'] = 0

docker-compose.yml

@@ -58,10 +58,6 @@ services:
   #
   #        Absolute minimum seconds to recheck, overrides any watch minimum, change to 0 to disable
   #      - MINIMUM_SECONDS_RECHECK_TIME=3
-  #
-  #        If you want to watch local files file:///path/to/file.txt (careful! security implications!)
-  #      - ALLOW_FILE_URI=False
-  
       # Comment out ports: when using behind a reverse proxy , enable networks: etc.
       ports:
         - 5000:5000

requirements.txt

@@ -93,5 +93,3 @@ babel
 
 # Needed for > 3.10, https://github.com/microsoft/playwright-python/issues/2096
 greenlet >= 3.0.3
-
-