Adding simple memory usage test

changedetectionio/blueprint/browser_steps/browser_steps.py

@@ -256,7 +256,8 @@ class browsersteps_live_ui(steppable_browser_interface):
     def get_current_state(self):
         """Return the screenshot and interactive elements mapping, generally always called after action_()"""
         import importlib.resources
-        xpath_element_js = importlib.resources.read_text("changedetectionio.content_fetchers.res", "xpath_element_scraper.js")
+        xpath_element_js = importlib.resources.files("changedetectionio.content_fetchers.res").joinpath('xpath_element_scraper.js').read_text()
+
         now = time.time()
         self.page.wait_for_timeout(1 * 1000)
 
@@ -289,7 +290,7 @@ class browsersteps_live_ui(steppable_browser_interface):
         """
         import importlib.resources
         self.page.evaluate("var include_filters=''")
-        xpath_element_js = importlib.resources.read_text("changedetectionio.content_fetchers.res", "xpath_element_scraper.js")
+        xpath_element_js = importlib.resources.files("changedetectionio.content_fetchers.res").joinpath('xpath_element_scraper.js').read_text()
         from changedetectionio.content_fetchers import visualselector_xpath_selectors
         xpath_element_js = xpath_element_js.replace('%ELEMENTS%', visualselector_xpath_selectors)
         xpath_data = self.page.evaluate("async () => {" + xpath_element_js + "}")

changedetectionio/content_fetchers/base.py

@@ -65,8 +65,8 @@ class Fetcher():
 
     def __init__(self):
         import importlib.resources
-        self.xpath_element_js = importlib.resources.read_text("changedetectionio.content_fetchers.res", 'xpath_element_scraper.js')
-        self.instock_data_js = importlib.resources.read_text("changedetectionio.content_fetchers.res", 'stock-not-in-stock.js')
+        self.xpath_element_js = importlib.resources.files("changedetectionio.content_fetchers.res").joinpath('xpath_element_scraper.js').read_text()
+        self.instock_data_js = importlib.resources.files("changedetectionio.content_fetchers.res").joinpath('stock-not-in-stock.js').read_text()
 
     @abstractmethod
     def get_error(self):

changedetectionio/content_fetchers/exceptions/__init__.py

@@ -87,11 +87,12 @@ class ScreenshotUnavailable(Exception):
 
 
 class ReplyWithContentButNoText(Exception):
-    def __init__(self, status_code, url, screenshot=None, has_filters=False, html_content=''):
+    def __init__(self, status_code, url, screenshot=None, has_filters=False, html_content='', xpath_data=None):
         # Set this so we can use it in other parts of the app
         self.status_code = status_code
         self.url = url
         self.screenshot = screenshot
         self.has_filters = has_filters
         self.html_content = html_content
+        self.xpath_data = xpath_data
         return

changedetectionio/content_fetchers/res/xpath_element_scraper.js

@@ -182,6 +182,7 @@ visibleElementsArray.forEach(function (element) {
 // Inject the current one set in the include_filters, which may be a CSS rule
 // used for displaying the current one in VisualSelector, where its not one we generated.
 if (include_filters.length) {
+    let results;
     // Foreach filter, go and find it on the page and add it to the results so we can visualise it again
     for (const f of include_filters) {
         bbox = false;
@@ -197,10 +198,15 @@ if (include_filters.length) {
             if (f.startsWith('/') || f.startsWith('xpath')) {
                 var qry_f = f.replace(/xpath(:|\d:)/, '')
                 console.log("[xpath] Scanning for included filter " + qry_f)
-                q = document.evaluate(qry_f, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null).singleNodeValue;
+                let xpathResult = document.evaluate(qry_f, document, null, XPathResult.ORDERED_NODE_SNAPSHOT_TYPE, null);
+                results = [];
+                for (let i = 0; i < xpathResult.snapshotLength; i++) {
+                    results.push(xpathResult.snapshotItem(i));
+                }
             } else {
                 console.log("[css] Scanning for included filter " + f)
-                q = document.querySelector(f);
+                console.log("[css] Scanning for included filter " + f);
+                results = document.querySelectorAll(f);
             }
         } catch (e) {
             // Maybe catch DOMException and alert?
@@ -208,44 +214,45 @@ if (include_filters.length) {
             console.log(e);
         }
 
-        if (q) {
-            // Try to resolve //something/text() back to its /something so we can atleast get the bounding box
-            try {
-                if (typeof q.nodeName == 'string' && q.nodeName === '#text') {
-                    q = q.parentElement
-                }
-            } catch (e) {
-                console.log(e)
-                console.log("xpath_element_scraper: #text resolver")
-            }
+        if (results.length) {
 
-            // #1231 - IN the case XPath attribute filter is applied, we will have to traverse up and find the element.
-            if (typeof q.getBoundingClientRect == 'function') {
-                bbox = q.getBoundingClientRect();
-                console.log("xpath_element_scraper: Got filter element, scroll from top was " + scroll_y)
-            } else {
+            // Iterate over the results
+            results.forEach(node => {
+                // Try to resolve //something/text() back to its /something so we can atleast get the bounding box
                 try {
-                    // Try and see we can find its ownerElement
-                    bbox = q.ownerElement.getBoundingClientRect();
-                    console.log("xpath_element_scraper: Got filter by ownerElement element, scroll from top was " + scroll_y)
+                    if (typeof node.nodeName == 'string' && node.nodeName === '#text') {
+                        node = node.parentElement
+                    }
                 } catch (e) {
                     console.log(e)
-                    console.log("xpath_element_scraper: error looking up q.ownerElement")
+                    console.log("xpath_element_scraper: #text resolver")
                 }
-            }
-        }
 
-        if (!q) {
-            console.log("xpath_element_scraper: filter element " + f + " was not found");
-        }
+                // #1231 - IN the case XPath attribute filter is applied, we will have to traverse up and find the element.
+                if (typeof node.getBoundingClientRect == 'function') {
+                    bbox = node.getBoundingClientRect();
+                    console.log("xpath_element_scraper: Got filter element, scroll from top was " + scroll_y)
+                } else {
+                    try {
+                        // Try and see we can find its ownerElement
+                        bbox = node.ownerElement.getBoundingClientRect();
+                        console.log("xpath_element_scraper: Got filter by ownerElement element, scroll from top was " + scroll_y)
+                    } catch (e) {
+                        console.log(e)
+                        console.log("xpath_element_scraper: error looking up q.ownerElement")
+                    }
+                }
 
-        if (bbox && bbox['width'] > 0 && bbox['height'] > 0) {
-            size_pos.push({
-                xpath: f,
-                width: parseInt(bbox['width']),
-                height: parseInt(bbox['height']),
-                left: parseInt(bbox['left']),
-                top: parseInt(bbox['top']) + scroll_y
+                if (bbox && bbox['width'] > 0 && bbox['height'] > 0) {
+                    size_pos.push({
+                        xpath: f,
+                        width: parseInt(bbox['width']),
+                        height: parseInt(bbox['height']),
+                        left: parseInt(bbox['left']),
+                        top: parseInt(bbox['top']) + scroll_y,
+                        highlight_as_custom_filter: true
+                    });
+                }
             });
         }
     }

changedetectionio/diff.py

@@ -1,62 +1,97 @@
-# used for the notifications, the front-end is using a JS library
-
 import difflib
+from typing import List, Iterator, Union
 
+def same_slicer(lst: List[str], start: int, end: int) -> List[str]:
+    """Return a slice of the list, or a single element if start == end."""
+    return lst[start:end] if start != end else [lst[start]]
 
-def same_slicer(l, a, b):
-    if a == b:
-        return [l[a]]
-    else:
-        return l[a:b]
-
-# like .compare but a little different output
-def customSequenceMatcher(before, after, include_equal=False, include_removed=True, include_added=True, include_replaced=True, include_change_type_prefix=True):
-    cruncher = difflib.SequenceMatcher(isjunk=lambda x: x in " \\t", a=before, b=after)
-
-    # @todo Line-by-line mode instead of buncghed, including `after` that is not in `before` (maybe unset?)
+def customSequenceMatcher(
+    before: List[str],
+    after: List[str],
+    include_equal: bool = False,
+    include_removed: bool = True,
+    include_added: bool = True,
+    include_replaced: bool = True,
+    include_change_type_prefix: bool = True
+) -> Iterator[List[str]]:
+    """
+    Compare two sequences and yield differences based on specified parameters.
+    
+    Args:
+        before (List[str]): Original sequence
+        after (List[str]): Modified sequence
+        include_equal (bool): Include unchanged parts
+        include_removed (bool): Include removed parts
+        include_added (bool): Include added parts
+        include_replaced (bool): Include replaced parts
+        include_change_type_prefix (bool): Add prefixes to indicate change types
+    
+    Yields:
+        List[str]: Differences between sequences
+    """
+    cruncher = difflib.SequenceMatcher(isjunk=lambda x: x in " \t", a=before, b=after)
+    
     for tag, alo, ahi, blo, bhi in cruncher.get_opcodes():
         if include_equal and tag == 'equal':
-            g = before[alo:ahi]
-            yield g
+            yield before[alo:ahi]
         elif include_removed and tag == 'delete':
-            row_prefix = "(removed) " if include_change_type_prefix else ''
-            g = [ row_prefix + i for i in same_slicer(before, alo, ahi)]
-            yield g
+            prefix = "(removed) " if include_change_type_prefix else ''
+            yield [f"{prefix}{line}" for line in same_slicer(before, alo, ahi)]
         elif include_replaced and tag == 'replace':
-            row_prefix = "(changed) " if include_change_type_prefix else ''
-            g = [row_prefix + i for i in same_slicer(before, alo, ahi)]
-            row_prefix = "(into) " if include_change_type_prefix else ''
-            g += [row_prefix + i for i in same_slicer(after, blo, bhi)]
-            yield g
+            prefix_changed = "(changed) " if include_change_type_prefix else ''
+            prefix_into = "(into) " if include_change_type_prefix else ''
+            yield [f"{prefix_changed}{line}" for line in same_slicer(before, alo, ahi)] + \
+                  [f"{prefix_into}{line}" for line in same_slicer(after, blo, bhi)]
         elif include_added and tag == 'insert':
-            row_prefix = "(added) " if include_change_type_prefix else ''
-            g = [row_prefix + i for i in same_slicer(after, blo, bhi)]
-            yield g
+            prefix = "(added) " if include_change_type_prefix else ''
+            yield [f"{prefix}{line}" for line in same_slicer(after, blo, bhi)]
 
-# only_differences - only return info about the differences, no context
-# line_feed_sep could be "<br>" or "<li>" or "\n" etc
-def render_diff(previous_version_file_contents, newest_version_file_contents, include_equal=False, include_removed=True, include_added=True, include_replaced=True, line_feed_sep="\n", include_change_type_prefix=True, patch_format=False):
-
-    newest_version_file_contents = [line.rstrip() for line in newest_version_file_contents.splitlines()]
-
-    if previous_version_file_contents:
-        previous_version_file_contents = [line.rstrip() for line in previous_version_file_contents.splitlines()]
-    else:
-        previous_version_file_contents = ""
+def render_diff(
+    previous_version_file_contents: str,
+    newest_version_file_contents: str,
+    include_equal: bool = False,
+    include_removed: bool = True,
+    include_added: bool = True,
+    include_replaced: bool = True,
+    line_feed_sep: str = "\n",
+    include_change_type_prefix: bool = True,
+    patch_format: bool = False
+) -> str:
+    """
+    Render the difference between two file contents.
+    
+    Args:
+        previous_version_file_contents (str): Original file contents
+        newest_version_file_contents (str): Modified file contents
+        include_equal (bool): Include unchanged parts
+        include_removed (bool): Include removed parts
+        include_added (bool): Include added parts
+        include_replaced (bool): Include replaced parts
+        line_feed_sep (str): Separator for lines in output
+        include_change_type_prefix (bool): Add prefixes to indicate change types
+        patch_format (bool): Use patch format for output
+    
+    Returns:
+        str: Rendered difference
+    """
+    newest_lines = [line.rstrip() for line in newest_version_file_contents.splitlines()]
+    previous_lines = [line.rstrip() for line in previous_version_file_contents.splitlines()] if previous_version_file_contents else []
 
     if patch_format:
-        patch = difflib.unified_diff(previous_version_file_contents, newest_version_file_contents)
+        patch = difflib.unified_diff(previous_lines, newest_lines)
         return line_feed_sep.join(patch)
 
-    rendered_diff = customSequenceMatcher(before=previous_version_file_contents,
-                                          after=newest_version_file_contents,
-                                          include_equal=include_equal,
-                                          include_removed=include_removed,
-                                          include_added=include_added,
-                                          include_replaced=include_replaced,
-                                          include_change_type_prefix=include_change_type_prefix)
+    rendered_diff = customSequenceMatcher(
+        before=previous_lines,
+        after=newest_lines,
+        include_equal=include_equal,
+        include_removed=include_removed,
+        include_added=include_added,
+        include_replaced=include_replaced,
+        include_change_type_prefix=include_change_type_prefix
+    )
 
-    # Recursively join lists
-    f = lambda L: line_feed_sep.join([f(x) if type(x) is list else x for x in L])
-    p= f(rendered_diff)
-    return p
+    def flatten(lst: List[Union[str, List[str]]]) -> str:
+        return line_feed_sep.join(flatten(x) if isinstance(x, list) else x for x in lst)
+
+    return flatten(rendered_diff)

changedetectionio/model/Watch.py

@@ -238,6 +238,8 @@ class model(dict):
 
         if len(tmp_history):
             self.__newest_history_key = list(tmp_history.keys())[-1]
+        else:
+            self.__newest_history_key = None
 
         self.__history_n = len(tmp_history)
 

changedetectionio/processors/text_json_diff.py

@@ -19,8 +19,9 @@ description = 'Detects all text changes where possible'
 json_filter_prefixes = ['json:', 'jq:', 'jqraw:']
 
 class FilterNotFoundInResponse(ValueError):
-    def __init__(self, msg, screenshot=None):
+    def __init__(self, msg, screenshot=None, xpath_data=None):
         self.screenshot = screenshot
+        self.xpath_data = xpath_data
         ValueError.__init__(self, msg)
 
 
@@ -185,7 +186,7 @@ class perform_site_check(difference_detection_processor):
                                                                        append_pretty_line_formatting=not watch.is_source_type_url)
 
                     if not html_content.strip():
-                        raise FilterNotFoundInResponse(msg=include_filters_rule, screenshot=self.fetcher.screenshot)
+                        raise FilterNotFoundInResponse(msg=include_filters_rule, screenshot=self.fetcher.screenshot, xpath_data=self.fetcher.xpath_data)
 
                 if has_subtractive_selectors:
                     html_content = html_tools.element_removal(subtractive_selectors, html_content)
@@ -243,9 +244,10 @@ class perform_site_check(difference_detection_processor):
         if not is_json and not empty_pages_are_a_change and len(stripped_text_from_html.strip()) == 0:
             raise content_fetchers.exceptions.ReplyWithContentButNoText(url=url,
                                                             status_code=self.fetcher.get_last_status_code(),
-                                                            screenshot=screenshot,
+                                                            screenshot=self.fetcher.screenshot,
                                                             has_filters=has_filter_rule,
-                                                            html_content=html_content
+                                                            html_content=html_content,
+                                                            xpath_data=self.fetcher.xpath_data
                                                             )
 
         # We rely on the actual text in the html output.. many sites have random script vars etc,

changedetectionio/static/js/visual-selector.js

@@ -2,250 +2,258 @@
 // All rights reserved.
 // yes - this is really a hack, if you are a front-ender and want to help, please get in touch!
 
-$(document).ready(function () {
+let runInClearMode = false;
 
-    var current_selected_i;
-    var state_clicked = false;
+$(document).ready(() => {
+    let currentSelections = [];
+    let currentSelection = null;
+    let appendToList = false;
+    let c, xctx, ctx;
+    let xScale = 1, yScale = 1;
+    let selectorImage, selectorImageRect, selectorData;
 
-    var c;
 
-    // greyed out fill context
-    var xctx;
-    // redline highlight context
-    var ctx;
+    // Global jQuery selectors with "Elem" appended
+    const $selectorCanvasElem = $('#selector-canvas');
+    const $includeFiltersElem = $("#include_filters");
+    const $selectorBackgroundElem = $("img#selector-background");
+    const $selectorCurrentXpathElem = $("#selector-current-xpath span");
+    const $fetchingUpdateNoticeElem = $('.fetching-update-notice');
+    const $selectorWrapperElem = $("#selector-wrapper");
 
-    var current_default_xpath = [];
-    var x_scale = 1;
-    var y_scale = 1;
-    var selector_image;
-    var selector_image_rect;
-    var selector_data;
+    // Color constants
+    const FILL_STYLE_HIGHLIGHT = 'rgba(205,0,0,0.35)';
+    const FILL_STYLE_GREYED_OUT = 'rgba(205,205,205,0.95)';
+    const STROKE_STYLE_HIGHLIGHT = 'rgba(255,0,0, 0.9)';
+    const FILL_STYLE_REDLINE = 'rgba(255,0,0, 0.1)';
+    const STROKE_STYLE_REDLINE = 'rgba(225,0,0,0.9)';
 
-    $('#visualselector-tab').click(function () {
-        $("img#selector-background").off('load');
-        state_clicked = false;
-        current_selected_i = false;
-        bootstrap_visualselector();
+    $('#visualselector-tab').click(() => {
+        $selectorBackgroundElem.off('load');
+        currentSelections = [];
+        bootstrapVisualSelector();
     });
 
-    $(document).on('keydown', function (event) {
-        if ($("img#selector-background").is(":visible")) {
-            if (event.key == "Escape") {
-                state_clicked = false;
-                ctx.clearRect(0, 0, c.width, c.height);
+    function clearReset() {
+        ctx.clearRect(0, 0, c.width, c.height);
+
+        if ($includeFiltersElem.val().length) {
+            alert("Existing filters under the 'Filters & Triggers' tab were cleared.");
+        }
+        $includeFiltersElem.val('');
+
+        currentSelections = [];
+
+        // Means we ignore the xpaths from the scraper marked as sel.highlight_as_custom_filter (it matched a previous selector)
+        runInClearMode = true;
+
+        highlightCurrentSelected();
+    }
+
+    function splitToList(v) {
+        return v.split('\n').map(line => line.trim()).filter(line => line.length > 0);
+    }
+
+    function sortScrapedElementsBySize() {
+        // Sort the currentSelections array by area (width * height) in descending order
+        selectorData['size_pos'].sort((a, b) => {
+            const areaA = a.width * a.height;
+            const areaB = b.width * b.height;
+            return areaB - areaA;
+        });
+    }
+
+    $(document).on('keydown keyup', (event) => {
+        if (event.code === 'ShiftLeft' || event.code === 'ShiftRight') {
+            appendToList = event.type === 'keydown';
+        }
+
+        if (event.type === 'keydown') {
+            if ($selectorBackgroundElem.is(":visible") && event.key === "Escape") {
+                clearReset();
             }
         }
     });
 
-    // For when the page loads
-    if (!window.location.hash || window.location.hash != '#visualselector') {
-        $("img#selector-background").attr('src', '');
+    $('#clear-selector').on('click', () => {
+        clearReset();
+    });
+    // So if they start switching between visualSelector and manual filters, stop it from rendering old filters
+    $('li.tab a').on('click', () => {
+        runInClearMode = true;
+    });
+
+    if (!window.location.hash || window.location.hash !== '#visualselector') {
+        $selectorBackgroundElem.attr('src', '');
         return;
     }
 
-    // Handle clearing button/link
-    $('#clear-selector').on('click', function (event) {
-        if (!state_clicked) {
-            alert('Oops, Nothing selected!');
-        }
-        state_clicked = false;
-        ctx.clearRect(0, 0, c.width, c.height);
-        xctx.clearRect(0, 0, c.width, c.height);
-        $("#include_filters").val('');
-    });
+    bootstrapVisualSelector();
 
-
-    bootstrap_visualselector();
-
-
-    function bootstrap_visualselector() {
-        if (1) {
-            // bootstrap it, this will trigger everything else
-            $("img#selector-background").on("error", function () {
-                $('.fetching-update-notice').html("<strong>Ooops!</strong> The VisualSelector tool needs atleast one fetched page, please unpause the watch and/or wait for the watch to complete fetching and then reload this page.");
-                $('.fetching-update-notice').css('color','#bb0000');
-                $('#selector-current-xpath').hide();
-                $('#clear-selector').hide();
-            }).bind('load', function () {
+    function bootstrapVisualSelector() {
+        $selectorBackgroundElem
+            .on("error", () => {
+                $fetchingUpdateNoticeElem.html("<strong>Ooops!</strong> The VisualSelector tool needs at least one fetched page, please unpause the watch and/or wait for the watch to complete fetching and then reload this page.")
+                    .css('color', '#bb0000');
+                $('#selector-current-xpath, #clear-selector').hide();
+            })
+            .on('load', () => {
                 console.log("Loaded background...");
                 c = document.getElementById("selector-canvas");
-                // greyed out fill context
                 xctx = c.getContext("2d");
-                // redline highlight context
                 ctx = c.getContext("2d");
-                if ($("#include_filters").val().trim().length) {
-                    current_default_xpath = $("#include_filters").val().split(/\r?\n/g);
-                } else {
-                    current_default_xpath = [];
-                }
-                fetch_data();
-                $('#selector-canvas').off("mousemove mousedown");
-                // screenshot_url defined in the edit.html template
-            }).attr("src", screenshot_url);
-        }
-        // Tell visualSelector that the image should update
-        var s = $("img#selector-background").attr('src') + "?" + new Date().getTime();
-        $("img#selector-background").attr('src', s)
+                fetchData();
+                $selectorCanvasElem.off("mousemove mousedown");
+            })
+            .attr("src", screenshot_url);
+
+        let s = `${$selectorBackgroundElem.attr('src')}?${new Date().getTime()}`;
+        $selectorBackgroundElem.attr('src', s);
     }
 
-    // This is fired once the img src is loaded in bootstrap_visualselector()
-    function fetch_data() {
-        // Image is ready
-        $('.fetching-update-notice').html("Fetching element data..");
+    function alertIfFilterNotFound() {
+        let existingFilters = splitToList($includeFiltersElem.val());
+        let sizePosXpaths = selectorData['size_pos'].map(sel => sel.xpath);
+
+        for (let filter of existingFilters) {
+            if (!sizePosXpaths.includes(filter)) {
+                alert(`One or more of your existing filters was not found and will be removed when a new filter is selected.`);
+                break;
+            }
+        }
+    }
+
+    function fetchData() {
+        $fetchingUpdateNoticeElem.html("Fetching element data..");
 
         $.ajax({
             url: watch_visual_selector_data_url,
             context: document.body
-        }).done(function (data) {
-            $('.fetching-update-notice').html("Rendering..");
-            selector_data = data;
-            console.log("Reported browser width from backend: " + data['browser_width']);
-            state_clicked = false;
-            set_scale();
-            reflow_selector();
-            $('.fetching-update-notice').fadeOut();
-        });
+        }).done((data) => {
+            $fetchingUpdateNoticeElem.html("Rendering..");
+            selectorData = data;
+            sortScrapedElementsBySize();
+            console.log(`Reported browser width from backend: ${data['browser_width']}`);
 
+            // Little sanity check for the user, alert them if something missing
+            alertIfFilterNotFound();
+
+            setScale();
+            reflowSelector();
+            $fetchingUpdateNoticeElem.fadeOut();
+        });
     }
 
+    function updateFiltersText() {
+        // Assuming currentSelections is already defined and contains the selections
+        let uniqueSelections = new Set(currentSelections.map(sel => (sel[0] === '/' ? `xpath:${sel.xpath}` : sel.xpath)));
 
-    function set_scale() {
-
-        // some things to check if the scaling doesnt work
-        // - that the widths/sizes really are about the actual screen size cat elements.json |grep -o width......|sort|uniq
-        $("#selector-wrapper").show();
-        selector_image = $("img#selector-background")[0];
-        selector_image_rect = selector_image.getBoundingClientRect();
-
-        // make the canvas the same size as the image
-        $('#selector-canvas').attr('height', selector_image_rect.height);
-        $('#selector-canvas').attr('width', selector_image_rect.width);
-        $('#selector-wrapper').attr('width', selector_image_rect.width);
-        x_scale = selector_image_rect.width / selector_data['browser_width'];
-        y_scale = selector_image_rect.height / selector_image.naturalHeight;
-        ctx.strokeStyle = 'rgba(255,0,0, 0.9)';
-        ctx.fillStyle = 'rgba(255,0,0, 0.1)';
-        ctx.lineWidth = 3;
-        console.log("scaling set  x: " + x_scale + " by y:" + y_scale);
-        $("#selector-current-xpath").css('max-width', selector_image_rect.width);
-    }
-
-    function reflow_selector() {
-        $(window).resize(function () {
-            set_scale();
-            highlight_current_selected_i();
-        });
-        var selector_currnt_xpath_text = $("#selector-current-xpath span");
-
-        set_scale();
-
-        console.log(selector_data['size_pos'].length + " selectors found");
-
-        // highlight the default one if we can find it in the xPath list
-        // or the xpath matches the default one
-        found = false;
-        if (current_default_xpath.length) {
-            // Find the first one that matches
-            // @todo In the future paint all that match
-            for (const c of current_default_xpath) {
-                for (var i = selector_data['size_pos'].length; i !== 0; i--) {
-                    if (selector_data['size_pos'][i - 1].xpath.trim() === c.trim()) {
-                        console.log("highlighting " + c);
-                        current_selected_i = i - 1;
-                        highlight_current_selected_i();
-                        found = true;
-                        break;
-                    }
-                }
-                if (found) {
-                    break;
-                }
-            }
-            if (!found) {
-                alert("Unfortunately your existing CSS/xPath Filter was no longer found!");
-            }
+        if (currentSelections.length > 0) {
+            // Convert the Set back to an array and join with newline characters
+            let textboxFilterText = Array.from(uniqueSelections).join("\n");
+            $includeFiltersElem.val(textboxFilterText);
         }
+    }
 
+    function setScale() {
+        $selectorWrapperElem.show();
+        selectorImage = $selectorBackgroundElem[0];
+        selectorImageRect = selectorImage.getBoundingClientRect();
 
-        $('#selector-canvas').bind('mousemove', function (e) {
-            if (state_clicked) {
-                return;
+        $selectorCanvasElem.attr({
+            'height': selectorImageRect.height,
+            'width': selectorImageRect.width
+        });
+        $selectorWrapperElem.attr('width', selectorImageRect.width);
+        $('#visual-selector-heading').css('max-width', selectorImageRect.width + "px")
+
+        xScale = selectorImageRect.width / selectorImage.naturalWidth;
+        yScale = selectorImageRect.height / selectorImage.naturalHeight;
+
+        ctx.strokeStyle = STROKE_STYLE_HIGHLIGHT;
+        ctx.fillStyle = FILL_STYLE_REDLINE;
+        ctx.lineWidth = 3;
+        console.log("Scaling set  x: " + xScale + " by y:" + yScale);
+        $("#selector-current-xpath").css('max-width', selectorImageRect.width);
+    }
+
+    function reflowSelector() {
+        $(window).resize(() => {
+            setScale();
+            highlightCurrentSelected();
+        });
+
+        setScale();
+
+        console.log(selectorData['size_pos'].length + " selectors found");
+
+        let existingFilters = splitToList($includeFiltersElem.val());
+
+        selectorData['size_pos'].forEach(sel => {
+            if ((!runInClearMode && sel.highlight_as_custom_filter) || existingFilters.includes(sel.xpath)) {
+                console.log("highlighting " + c);
+                currentSelections.push(sel);
             }
-            ctx.clearRect(0, 0, c.width, c.height);
-            current_selected_i = null;
+        });
 
-            // Add in offset
-            if ((typeof e.offsetX === "undefined" || typeof e.offsetY === "undefined") || (e.offsetX === 0 && e.offsetY === 0)) {
-                var targetOffset = $(e.target).offset();
+
+        highlightCurrentSelected();
+        updateFiltersText();
+
+        $selectorCanvasElem.bind('mousemove', handleMouseMove.debounce(5));
+        $selectorCanvasElem.bind('mousedown', handleMouseDown.debounce(5));
+        $selectorCanvasElem.bind('mouseleave', highlightCurrentSelected.debounce(5));
+
+        function handleMouseMove(e) {
+            if (!e.offsetX && !e.offsetY) {
+                const targetOffset = $(e.target).offset();
                 e.offsetX = e.pageX - targetOffset.left;
                 e.offsetY = e.pageY - targetOffset.top;
             }
 
-            // Reverse order - the most specific one should be deeper/"laster"
-            // Basically, find the most 'deepest'
-            var found = 0;
-            ctx.fillStyle = 'rgba(205,0,0,0.35)';
-            // Will be sorted by smallest width*height first
-            for (var i = 0; i <= selector_data['size_pos'].length; i++) {
-                // draw all of them? let them choose somehow?
-                var sel = selector_data['size_pos'][i];
-                // If we are in a bounding-box
-                if (e.offsetY > sel.top * y_scale && e.offsetY < sel.top * y_scale + sel.height * y_scale
-                    &&
-                    e.offsetX > sel.left * y_scale && e.offsetX < sel.left * y_scale + sel.width * y_scale
+            ctx.fillStyle = FILL_STYLE_HIGHLIGHT;
 
-                ) {
-
-                    // FOUND ONE
-                    set_current_selected_text(sel.xpath);
-                    ctx.strokeRect(sel.left * x_scale, sel.top * y_scale, sel.width * x_scale, sel.height * y_scale);
-                    ctx.fillRect(sel.left * x_scale, sel.top * y_scale, sel.width * x_scale, sel.height * y_scale);
-
-                    // no need to keep digging
-                    // @todo or, O to go out/up, I to go in
-                    // or double click to go up/out the selector?
-                    current_selected_i = i;
-                    found += 1;
-                    break;
+            selectorData['size_pos'].forEach(sel => {
+                if (e.offsetY > sel.top * yScale && e.offsetY < sel.top * yScale + sel.height * yScale &&
+                    e.offsetX > sel.left * yScale && e.offsetX < sel.left * yScale + sel.width * yScale) {
+                    setCurrentSelectedText(sel.xpath);
+                    drawHighlight(sel);
+                    currentSelections.push(sel);
+                    currentSelection = sel;
+                    highlightCurrentSelected();
+                    currentSelections.pop();
                 }
-            }
-
-        }.debounce(5));
-
-        function set_current_selected_text(s) {
-            selector_currnt_xpath_text[0].innerHTML = s;
-        }
-
-        function highlight_current_selected_i() {
-            if (state_clicked) {
-                state_clicked = false;
-                xctx.clearRect(0, 0, c.width, c.height);
-                return;
-            }
-
-            var sel = selector_data['size_pos'][current_selected_i];
-            if (sel[0] == '/') {
-                // @todo - not sure just checking / is right
-                $("#include_filters").val('xpath:' + sel.xpath);
-            } else {
-                $("#include_filters").val(sel.xpath);
-            }
-            xctx.fillStyle = 'rgba(205,205,205,0.95)';
-            xctx.strokeStyle = 'rgba(225,0,0,0.9)';
-            xctx.lineWidth = 3;
-            xctx.fillRect(0, 0, c.width, c.height);
-            // Clear out what only should be seen (make a clear/clean spot)
-            xctx.clearRect(sel.left * x_scale, sel.top * y_scale, sel.width * x_scale, sel.height * y_scale);
-            xctx.strokeRect(sel.left * x_scale, sel.top * y_scale, sel.width * x_scale, sel.height * y_scale);
-            state_clicked = true;
-            set_current_selected_text(sel.xpath);
-
+            })
         }
 
 
-        $('#selector-canvas').bind('mousedown', function (e) {
-            highlight_current_selected_i();
-        });
+        function setCurrentSelectedText(s) {
+            $selectorCurrentXpathElem[0].innerHTML = s;
+        }
+
+        function drawHighlight(sel) {
+            ctx.strokeRect(sel.left * xScale, sel.top * yScale, sel.width * xScale, sel.height * yScale);
+            ctx.fillRect(sel.left * xScale, sel.top * yScale, sel.width * xScale, sel.height * yScale);
+        }
+
+        function handleMouseDown() {
+            // If we are in 'appendToList' mode, grow the list, if not, just 1
+            currentSelections = appendToList ? [...currentSelections, currentSelection] : [currentSelection];
+            highlightCurrentSelected();
+            updateFiltersText();
+        }
+
     }
 
+    function highlightCurrentSelected() {
+        xctx.fillStyle = FILL_STYLE_GREYED_OUT;
+        xctx.strokeStyle = STROKE_STYLE_REDLINE;
+        xctx.lineWidth = 3;
+        xctx.clearRect(0, 0, c.width, c.height);
+
+        currentSelections.forEach(sel => {
+            //xctx.clearRect(sel.left * xScale, sel.top * yScale, sel.width * xScale, sel.height * yScale);
+            xctx.strokeRect(sel.left * xScale, sel.top * yScale, sel.width * xScale, sel.height * yScale);
+        });
+    }
 });

changedetectionio/static/styles/scss/parts/_visualselector.scss

@@ -1,6 +1,8 @@
 
 #selector-wrapper {
   height: 100%;
+  text-align: center;
+  
   max-height: 70vh;
   overflow-y: scroll;
   position: relative;

changedetectionio/static/styles/styles.css

@@ -1065,6 +1065,7 @@ ul {
 
 #selector-wrapper {
   height: 100%;
+  text-align: center;
   max-height: 70vh;
   overflow-y: scroll;
   position: relative; }

changedetectionio/store.py

@@ -242,6 +242,14 @@ class ChangeDetectionStore:
     def clear_watch_history(self, uuid):
         import pathlib
 
+        # JSON Data, Screenshots, Textfiles (history index and snapshots), HTML in the future etc
+        for item in pathlib.Path(os.path.join(self.datastore_path, uuid)).rglob("*.*"):
+            unlink(item)
+
+        # Force the attr to recalculate
+        bump = self.__data['watching'][uuid].history
+
+        # Do this last because it will trigger a recheck due to last_checked being zero
         self.__data['watching'][uuid].update({
                 'browser_steps_last_error_step' : None,
                 'check_count': 0,
@@ -258,13 +266,6 @@ class ChangeDetectionStore:
                 'track_ldjson_price_data': None,
             })
 
-        # JSON Data, Screenshots, Textfiles (history index and snapshots), HTML in the future etc
-        for item in pathlib.Path(os.path.join(self.datastore_path, uuid)).rglob("*.*"):
-            unlink(item)
-
-        # Force the attr to recalculate
-        bump = self.__data['watching'][uuid].history
-
         self.needs_write_urgent = True
 
     def add_watch(self, url, tag='', extras=None, tag_uuids=None, write_to_disk_now=True):
@@ -843,3 +844,8 @@ class ChangeDetectionStore:
                 # Something custom here
                 self.__data["watching"][uuid]['time_between_check_use_default'] = False
 
+    # Correctly set datatype for older installs where 'tag' was string and update_12 did not catch it
+    def update_16(self):
+        for uuid, watch in self.data['watching'].items():
+            if isinstance(watch.get('tags'), str):
+                self.data['watching'][uuid]['tags'] = []

changedetectionio/templates/edit.html

@@ -432,9 +432,8 @@ Unavailable") }}
                 <fieldset>
                     <div class="pure-control-group">
                         {% if visualselector_enabled %}
-                            <span class="pure-form-message-inline">
-                                The Visual Selector tool lets you select the <i>text</i> elements that will be used for the change detection &dash; after the <i>Browser Steps</i> has completed.<br>
-                                This tool is a helper to manage filters in the  "CSS/JSONPath/JQ/XPath Filters" box of the <a href="#filters-and-triggers">Filters & Triggers</a> tab.
+                            <span class="pure-form-message-inline" id="visual-selector-heading">
+                                The Visual Selector tool lets you select the <i>text</i> elements that will be used for the change detection. It automatically fills-in the filters in the "CSS/JSONPath/JQ/XPath Filters" box of the <a href="#filters-and-triggers">Filters & Triggers</a> tab. Use <strong>Shift+Click</strong> to select multiple items.
                             </span>
 
                             <div id="selector-header">

changedetectionio/tests/conftest.py

@@ -1,4 +1,7 @@
 #!/usr/bin/python3
+import resource
+import time
+from threading import Thread
 
 import pytest
 from changedetectionio import changedetection_app
@@ -23,6 +26,36 @@ def reportlog(pytestconfig):
     yield
     logger.remove(handler_id)
 
+
+def track_memory(memory_usage, ):
+    while not memory_usage["stop"]:
+        max_rss = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
+        memory_usage["peak"] = max(memory_usage["peak"], max_rss)
+        time.sleep(0.01)  # Adjust the sleep time as needed
+
+@pytest.fixture(scope='function')
+def measure_memory_usage(request):
+    memory_usage = {"peak": 0, "stop": False}
+    tracker_thread = Thread(target=track_memory, args=(memory_usage,))
+    tracker_thread.start()
+
+    yield
+
+    memory_usage["stop"] = True
+    tracker_thread.join()
+
+    # Note: ru_maxrss is in kilobytes on Unix-based systems
+    max_memory_used = memory_usage["peak"] / 1024  # Convert to MB
+    s = f"Peak memory used by the test {request.node.fspath} - '{request.node.name}': {max_memory_used:.2f} MB"
+    logger.debug(s)
+
+    with open("test-memory.log", 'a') as f:
+        f.write(f"{s}\n")
+
+    # Assert that the memory usage is less than 200MB
+    assert max_memory_used < 150, f"Memory usage exceeded 200MB: {max_memory_used:.2f} MB"
+
+
 def cleanup(datastore_path):
     import glob
     # Unlink test output files

changedetectionio/tests/custom_browser_url/test_custom_browser_url.py

@@ -77,13 +77,13 @@ def do_test(client, live_server, make_test_use_extra_browser=False):
 
 
 # Requires playwright to be installed
-def test_request_via_custom_browser_url(client, live_server):
+def test_request_via_custom_browser_url(client, live_server, measure_memory_usage):
     live_server_setup(live_server)
     # We do this so we can grep the logs of the custom container and see if the request actually went through that container
     do_test(client, live_server, make_test_use_extra_browser=True)
 
 
-def test_request_not_via_custom_browser_url(client, live_server):
+def test_request_not_via_custom_browser_url(client, live_server, measure_memory_usage):
     live_server_setup(live_server)
     # We do this so we can grep the logs of the custom container and see if the request actually went through that container
     do_test(client, live_server, make_test_use_extra_browser=False)

changedetectionio/tests/fetchers/test_content.py

@@ -6,7 +6,7 @@ from ..util import live_server_setup, wait_for_all_checks
 import logging
 
 # Requires playwright to be installed
-def test_fetch_webdriver_content(client, live_server):
+def test_fetch_webdriver_content(client, live_server, measure_memory_usage):
     live_server_setup(live_server)
 
     #####################

changedetectionio/tests/fetchers/test_custom_js_before_content.py

@@ -3,7 +3,7 @@ from flask import url_for
 from ..util import live_server_setup, wait_for_all_checks, extract_UUID_from_client
 
 
-def test_execute_custom_js(client, live_server):
+def test_execute_custom_js(client, live_server, measure_memory_usage):
 
     live_server_setup(live_server)
     assert os.getenv('PLAYWRIGHT_DRIVER_URL'), "Needs PLAYWRIGHT_DRIVER_URL set for this test"

changedetectionio/tests/proxy_list/test_multiple_proxy.py

@@ -5,7 +5,7 @@ from flask import url_for
 from ..util import live_server_setup, wait_for_all_checks
 
 
-def test_preferred_proxy(client, live_server):
+def test_preferred_proxy(client, live_server, measure_memory_usage):
     live_server_setup(live_server)
     url = "http://chosen.changedetection.io"
 

changedetectionio/tests/proxy_list/test_noproxy.py

@@ -5,7 +5,7 @@ from flask import url_for
 from ..util import live_server_setup, wait_for_all_checks, extract_UUID_from_client
 
 
-def test_noproxy_option(client, live_server):
+def test_noproxy_option(client, live_server, measure_memory_usage):
     live_server_setup(live_server)
     # Run by run_proxy_tests.sh
     # Call this URL then scan the containers that it never went through them

changedetectionio/tests/proxy_list/test_proxy.py

@@ -5,7 +5,7 @@ from flask import url_for
 from ..util import live_server_setup, wait_for_all_checks, extract_UUID_from_client
 
 # just make a request, we will grep in the docker logs to see it actually got called
-def test_check_basic_change_detection_functionality(client, live_server):
+def test_check_basic_change_detection_functionality(client, live_server, measure_memory_usage):
     live_server_setup(live_server)
     res = client.post(
         url_for("import_page"),

changedetectionio/tests/proxy_list/test_select_custom_proxy.py

@@ -6,7 +6,7 @@ from ..util import live_server_setup, wait_for_all_checks
 import os
 
 # just make a request, we will grep in the docker logs to see it actually got called
-def test_select_custom(client, live_server):
+def test_select_custom(client, live_server, measure_memory_usage):
     live_server_setup(live_server)
 
     # Goto settings, add our custom one

changedetectionio/tests/proxy_socks5/test_socks5_proxy.py

@@ -5,7 +5,7 @@ from flask import url_for
 from changedetectionio.tests.util import live_server_setup, wait_for_all_checks
 
 
-def test_socks5(client, live_server):
+def test_socks5(client, live_server, measure_memory_usage):
     live_server_setup(live_server)
 
     # Setup a proxy

changedetectionio/tests/proxy_socks5/test_socks5_proxy_sources.py

@@ -7,7 +7,7 @@ from changedetectionio.tests.util import live_server_setup, wait_for_all_checks
 
 # should be proxies.json mounted from run_proxy_tests.sh already
 # -v `pwd`/tests/proxy_socks5/proxies.json-example:/app/changedetectionio/test-datastore/proxies.json
-def test_socks5_from_proxiesjson_file(client, live_server):
+def test_socks5_from_proxiesjson_file(client, live_server, measure_memory_usage):
     live_server_setup(live_server)
 
     test_url = "https://changedetection.io/CHANGELOG.txt?socks-test-tag=" + os.getenv('SOCKSTEST', '')

changedetectionio/tests/restock/test_restock.py

@@ -48,7 +48,7 @@ def set_back_in_stock_response():
     return None
 
 # Add a site in paused mode, add an invalid filter, we should still have visual selector data ready
-def test_restock_detection(client, live_server):
+def test_restock_detection(client, live_server, measure_memory_usage):
 
     set_original_response()
     #assert os.getenv('PLAYWRIGHT_DRIVER_URL'), "Needs PLAYWRIGHT_DRIVER_URL set for this test"

changedetectionio/tests/smtp/test_notification_smtp.py

@@ -40,7 +40,7 @@ def get_last_message_from_smtp_server():
 
 # Requires running the test SMTP server
 
-def test_check_notification_email_formats_default_HTML(client, live_server):
+def test_check_notification_email_formats_default_HTML(client, live_server, measure_memory_usage):
     # live_server_setup(live_server)
     set_original_response()
 
@@ -92,7 +92,7 @@ def test_check_notification_email_formats_default_HTML(client, live_server):
     assert b'Deleted' in res.data
 
 
-def test_check_notification_email_formats_default_Text_override_HTML(client, live_server):
+def test_check_notification_email_formats_default_Text_override_HTML(client, live_server, measure_memory_usage):
     # live_server_setup(live_server)
 
     # HTML problems? see this

changedetectionio/tests/test_add_replace_remove_filter.py

@@ -35,10 +35,10 @@ def set_original(excluding=None, add_line=None):
     with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write(test_return_data)
 
-def test_setup(client, live_server):
+def test_setup(client, live_server, measure_memory_usage):
     live_server_setup(live_server)
 
-def test_check_removed_line_contains_trigger(client, live_server):
+def test_check_removed_line_contains_trigger(client, live_server, measure_memory_usage):
 
     # Give the endpoint time to spin up
     time.sleep(1)
@@ -103,7 +103,7 @@ def test_check_removed_line_contains_trigger(client, live_server):
     assert b'Deleted' in res.data
 
 
-def test_check_add_line_contains_trigger(client, live_server):
+def test_check_add_line_contains_trigger(client, live_server, measure_memory_usage):
     #live_server_setup(live_server)
 
     # Give the endpoint time to spin up

changedetectionio/tests/test_api.py

@@ -53,10 +53,10 @@ def is_valid_uuid(val):
         return False
 
 
-def test_setup(client, live_server):
+def test_setup(client, live_server, measure_memory_usage):
     live_server_setup(live_server)
 
-def test_api_simple(client, live_server):
+def test_api_simple(client, live_server, measure_memory_usage):
     #live_server_setup(live_server)
 
     api_key = extract_api_key_from_UI(client)
@@ -241,7 +241,7 @@ def test_api_simple(client, live_server):
     )
     assert len(res.json) == 0, "Watch list should be empty"
 
-def test_access_denied(client, live_server):
+def test_access_denied(client, live_server, measure_memory_usage):
     # `config_api_token_enabled` Should be On by default
     res = client.get(
         url_for("createwatch")
@@ -287,7 +287,7 @@ def test_access_denied(client, live_server):
     )
     assert b"Settings updated." in res.data
 
-def test_api_watch_PUT_update(client, live_server):
+def test_api_watch_PUT_update(client, live_server, measure_memory_usage):
 
     #live_server_setup(live_server)
     api_key = extract_api_key_from_UI(client)
@@ -369,7 +369,7 @@ def test_api_watch_PUT_update(client, live_server):
     assert b'Deleted' in res.data
 
 
-def test_api_import(client, live_server):
+def test_api_import(client, live_server, measure_memory_usage):
     api_key = extract_api_key_from_UI(client)
 
     res = client.post(

changedetectionio/tests/test_auth.py

@@ -5,7 +5,7 @@ from flask import url_for
 from .util import live_server_setup, wait_for_all_checks
 
 
-def test_basic_auth(client, live_server):
+def test_basic_auth(client, live_server, measure_memory_usage):
 
     live_server_setup(live_server)
 

changedetectionio/tests/test_automatic_follow_ldjson_price.py

@@ -76,11 +76,11 @@ def set_response_without_ldjson():
         f.write(test_return_data)
     return None
 
-def test_setup(client, live_server):
+def test_setup(client, live_server, measure_memory_usage):
     live_server_setup(live_server)
 
 # actually only really used by the distll.io importer, but could be handy too
-def test_check_ldjson_price_autodetect(client, live_server):
+def test_check_ldjson_price_autodetect(client, live_server, measure_memory_usage):
 
     set_response_with_ldjson()
 
@@ -167,7 +167,7 @@ def _test_runner_check_bad_format_ignored(live_server, client, has_ldjson_price_
     client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
 
 
-def test_bad_ldjson_is_correctly_ignored(client, live_server):
+def test_bad_ldjson_is_correctly_ignored(client, live_server, measure_memory_usage):
     #live_server_setup(live_server)
     test_return_data = """
             <html>

changedetectionio/tests/test_backend.py

@@ -3,7 +3,8 @@
 import time
 from flask import url_for
 from urllib.request import urlopen
-from .util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks, extract_rss_token_from_UI
+from .util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks, extract_rss_token_from_UI, \
+    extract_UUID_from_client
 
 sleep_time_for_fetch_thread = 3
 
@@ -16,7 +17,7 @@ def test_inscriptus():
     assert stripped_text_from_html == 'test!\nok man'
 
 
-def test_check_basic_change_detection_functionality(client, live_server):
+def test_check_basic_change_detection_functionality(client, live_server, measure_memory_usage):
     set_original_response()
     live_server_setup(live_server)
 
@@ -141,6 +142,14 @@ def test_check_basic_change_detection_functionality(client, live_server):
     assert b'Mark all viewed' not in res.data
     assert b'unviewed' not in res.data
 
+    # #2458 "clear history" should make the Watch object update its status correctly when the first snapshot lands again
+    uuid = extract_UUID_from_client(client)
+    client.get(url_for("clear_watch_history", uuid=uuid))
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    wait_for_all_checks(client)
+    res = client.get(url_for("index"))
+    assert b'preview/' in res.data
+
     #
     # Cleanup everything
     res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)

changedetectionio/tests/test_backup.py

@@ -8,7 +8,7 @@ import re
 import time
 
 
-def test_backup(client, live_server):
+def test_backup(client, live_server, measure_memory_usage):
     live_server_setup(live_server)
 
     set_original_response()

changedetectionio/tests/test_block_while_text_present.py

@@ -60,7 +60,7 @@ def set_modified_response_minus_block_text():
         f.write(test_return_data)
 
 
-def test_check_block_changedetection_text_NOT_present(client, live_server):
+def test_check_block_changedetection_text_NOT_present(client, live_server, measure_memory_usage):
 
     live_server_setup(live_server)
     # Use a mix of case in ZzZ to prove it works case-insensitive.

changedetectionio/tests/test_clone.py

@@ -6,7 +6,7 @@ from . util import live_server_setup
 
 
 
-def test_trigger_functionality(client, live_server):
+def test_trigger_functionality(client, live_server, measure_memory_usage):
 
     live_server_setup(live_server)
 

changedetectionio/tests/test_css_selector.py

@@ -70,7 +70,7 @@ def test_include_filters_output():
 
 
 # Tests the whole stack works with the CSS Filter
-def test_check_markup_include_filters_restriction(client, live_server):
+def test_check_markup_include_filters_restriction(client, live_server, measure_memory_usage):
     sleep_time_for_fetch_thread = 3
 
     include_filters = "#sametext"
@@ -124,7 +124,7 @@ def test_check_markup_include_filters_restriction(client, live_server):
 
 
 # Tests the whole stack works with the CSS Filter
-def test_check_multiple_filters(client, live_server):
+def test_check_multiple_filters(client, live_server, measure_memory_usage):
     sleep_time_for_fetch_thread = 3
 
     include_filters = "#blob-a\r\nxpath://*[contains(@id,'blob-b')]"
@@ -180,7 +180,7 @@ def test_check_multiple_filters(client, live_server):
 # The filter exists, but did not contain anything useful
 # Mainly used when the filter contains just an IMG, this can happen when someone selects an image in the visual-selector
 # Tests fetcher can throw a "ReplyWithContentButNoText" exception after applying filter and extracting text
-def test_filter_is_empty_help_suggestion(client, live_server):
+def test_filter_is_empty_help_suggestion(client, live_server, measure_memory_usage):
     #live_server_setup(live_server)
 
     include_filters = "#blob-a"

changedetectionio/tests/test_element_removal.py

@@ -106,7 +106,7 @@ across multiple lines
     )
 
 
-def test_element_removal_full(client, live_server):
+def test_element_removal_full(client, live_server, measure_memory_usage):
     sleep_time_for_fetch_thread = 3
 
     set_original_response()

changedetectionio/tests/test_encoding.py

@@ -24,7 +24,7 @@ def set_html_response():
 
 
 # In the case the server does not issue a charset= or doesnt have content_type header set
-def test_check_encoding_detection(client, live_server):
+def test_check_encoding_detection(client, live_server, measure_memory_usage):
     set_html_response()
 
     # Add our URL to the import page
@@ -50,7 +50,7 @@ def test_check_encoding_detection(client, live_server):
 
 
 # In the case the server does not issue a charset= or doesnt have content_type header set
-def test_check_encoding_detection_missing_content_type_header(client, live_server):
+def test_check_encoding_detection_missing_content_type_header(client, live_server, measure_memory_usage):
     set_html_response()
 
     # Add our URL to the import page

changedetectionio/tests/test_errorhandling.py

@@ -54,7 +54,7 @@ def _runner_test_http_errors(client, live_server, http_code, expected_text):
     assert b'Deleted' in res.data
 
 
-def test_http_error_handler(client, live_server):
+def test_http_error_handler(client, live_server, measure_memory_usage):
     _runner_test_http_errors(client, live_server, 403, 'Access denied')
     _runner_test_http_errors(client, live_server, 404, 'Page not found')
     _runner_test_http_errors(client, live_server, 500, '(Internal server error) received')
@@ -63,7 +63,7 @@ def test_http_error_handler(client, live_server):
     assert b'Deleted' in res.data
 
 # Just to be sure error text is properly handled
-def test_DNS_errors(client, live_server):
+def test_DNS_errors(client, live_server, measure_memory_usage):
     # Give the endpoint time to spin up
     time.sleep(1)
 
@@ -87,7 +87,7 @@ def test_DNS_errors(client, live_server):
     assert b'Deleted' in res.data
 
 # Re 1513
-def test_low_level_errors_clear_correctly(client, live_server):
+def test_low_level_errors_clear_correctly(client, live_server, measure_memory_usage):
     #live_server_setup(live_server)
     # Give the endpoint time to spin up
     time.sleep(1)

changedetectionio/tests/test_extract_csv.py

@@ -9,7 +9,7 @@ sleep_time_for_fetch_thread = 3
 
 
 
-def test_check_extract_text_from_diff(client, live_server):
+def test_check_extract_text_from_diff(client, live_server, measure_memory_usage):
     import time
     with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write("Now it's {} seconds since epoch, time flies!".format(str(time.time())))

changedetectionio/tests/test_extract_regex.py

@@ -67,10 +67,10 @@ def set_multiline_response():
     return None
 
 
-def test_setup(client, live_server):
+def test_setup(client, live_server, measure_memory_usage):
     live_server_setup(live_server)
 
-def test_check_filter_multiline(client, live_server):
+def test_check_filter_multiline(client, live_server, measure_memory_usage):
     #live_server_setup(live_server)
     set_multiline_response()
 
@@ -122,7 +122,7 @@ def test_check_filter_multiline(client, live_server):
     # but the last one, which also says 'lines' shouldnt be here (non-greedy match checking)
     assert b'aaand something lines' not in res.data
 
-def test_check_filter_and_regex_extract(client, live_server):
+def test_check_filter_and_regex_extract(client, live_server, measure_memory_usage):
     
     include_filters = ".changetext"
 
@@ -205,7 +205,7 @@ def test_check_filter_and_regex_extract(client, live_server):
 
 
 
-def test_regex_error_handling(client, live_server):
+def test_regex_error_handling(client, live_server, measure_memory_usage):
 
     #live_server_setup(live_server)
 

changedetectionio/tests/test_filter_exist_changes.py

@@ -41,7 +41,7 @@ def set_response_with_filter():
         f.write(test_return_data)
     return None
 
-def test_filter_doesnt_exist_then_exists_should_get_notification(client, live_server):
+def test_filter_doesnt_exist_then_exists_should_get_notification(client, live_server, measure_memory_usage):
 #  Filter knowingly doesn't exist, like someone setting up a known filter to see if some cinema tickets are on sale again
 #  And the page has that filter available
 #  Then I should get a notification

changedetectionio/tests/test_filter_failure_notification.py

@@ -151,10 +151,10 @@ def run_filter_test(client, live_server, content_filter):
 def test_setup(live_server):
     live_server_setup(live_server)
 
-def test_check_include_filters_failure_notification(client, live_server):
+def test_check_include_filters_failure_notification(client, live_server, measure_memory_usage):
     run_filter_test(client, live_server,'#nope-doesnt-exist')
 
-def test_check_xpath_filter_failure_notification(client, live_server):
+def test_check_xpath_filter_failure_notification(client, live_server, measure_memory_usage):
     run_filter_test(client, live_server, '//*[@id="nope-doesnt-exist"]')
 
 # Test that notification is never sent

changedetectionio/tests/test_group.py

@@ -6,7 +6,7 @@ from .util import live_server_setup, wait_for_all_checks, extract_rss_token_from
 import os
 
 
-def test_setup(client, live_server):
+def test_setup(client, live_server, measure_memory_usage):
     live_server_setup(live_server)
 
 def set_original_response():
@@ -39,7 +39,7 @@ def set_modified_response():
         f.write(test_return_data)
     return None
 
-def test_setup_group_tag(client, live_server):
+def test_setup_group_tag(client, live_server, measure_memory_usage):
     #live_server_setup(live_server)
     set_original_response()
 
@@ -130,7 +130,7 @@ def test_setup_group_tag(client, live_server):
     res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
 
-def test_tag_import_singular(client, live_server):
+def test_tag_import_singular(client, live_server, measure_memory_usage):
     #live_server_setup(live_server)
 
     test_url = url_for('test_endpoint', _external=True)
@@ -150,7 +150,7 @@ def test_tag_import_singular(client, live_server):
     res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
 
-def test_tag_add_in_ui(client, live_server):
+def test_tag_add_in_ui(client, live_server, measure_memory_usage):
     #live_server_setup(live_server)
 #
     res = client.post(
@@ -167,7 +167,7 @@ def test_tag_add_in_ui(client, live_server):
     res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
 
-def test_group_tag_notification(client, live_server):
+def test_group_tag_notification(client, live_server, measure_memory_usage):
     #live_server_setup(live_server)
     set_original_response()
 
@@ -235,7 +235,7 @@ def test_group_tag_notification(client, live_server):
     res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
 
-def test_limit_tag_ui(client, live_server):
+def test_limit_tag_ui(client, live_server, measure_memory_usage):
     #live_server_setup(live_server)
 
     test_url = url_for('test_endpoint', _external=True)
@@ -273,7 +273,7 @@ def test_limit_tag_ui(client, live_server):
     assert b'Deleted' in res.data
     res = client.get(url_for("tags.delete_all"), follow_redirects=True)
     assert b'All tags deleted' in res.data
-def test_clone_tag_on_import(client, live_server):
+def test_clone_tag_on_import(client, live_server, measure_memory_usage):
     #live_server_setup(live_server)
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
@@ -298,7 +298,7 @@ def test_clone_tag_on_import(client, live_server):
     res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
 
-def test_clone_tag_on_quickwatchform_add(client, live_server):
+def test_clone_tag_on_quickwatchform_add(client, live_server, measure_memory_usage):
     #live_server_setup(live_server)
 
     test_url = url_for('test_endpoint', _external=True)
@@ -328,7 +328,7 @@ def test_clone_tag_on_quickwatchform_add(client, live_server):
     res = client.get(url_for("tags.delete_all"), follow_redirects=True)
     assert b'All tags deleted' in res.data
 
-def test_order_of_filters_tag_filter_and_watch_filter(client, live_server):
+def test_order_of_filters_tag_filter_and_watch_filter(client, live_server, measure_memory_usage):
 
     # Add a tag with some config, import a tag and it should roughly work
     res = client.post(

changedetectionio/tests/test_history_consistency.py

@@ -8,7 +8,7 @@ from flask import url_for
 from .util import live_server_setup, wait_for_all_checks
 from urllib.parse import urlparse, parse_qs
 
-def test_consistent_history(client, live_server):
+def test_consistent_history(client, live_server, measure_memory_usage):
     live_server_setup(live_server)
 
     r = range(1, 30)

changedetectionio/tests/test_ignore_text.py

@@ -82,7 +82,7 @@ def set_modified_ignore_response():
         f.write(test_return_data)
 
 
-def test_check_ignore_text_functionality(client, live_server):
+def test_check_ignore_text_functionality(client, live_server, measure_memory_usage):
 
     # Use a mix of case in ZzZ to prove it works case-insensitive.
     ignore_text = "XXXXX\r\nYYYYY\r\nzZzZZ\r\nnew ignore stuff"
@@ -164,7 +164,7 @@ def test_check_ignore_text_functionality(client, live_server):
     res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
 
-def test_check_global_ignore_text_functionality(client, live_server):
+def test_check_global_ignore_text_functionality(client, live_server, measure_memory_usage):
 
     # Give the endpoint time to spin up
     time.sleep(1)

changedetectionio/tests/test_ignorehighlighter.py

@@ -23,7 +23,7 @@ def set_original_ignore_response():
         f.write(test_return_data)
 
 
-def test_highlight_ignore(client, live_server):
+def test_highlight_ignore(client, live_server, measure_memory_usage):
     live_server_setup(live_server)
     set_original_ignore_response()
     test_url = url_for('test_endpoint', _external=True)

changedetectionio/tests/test_ignorehyperlinks.py

@@ -40,7 +40,7 @@ def set_modified_ignore_response():
     with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write(test_return_data)
 
-def test_render_anchor_tag_content_true(client, live_server):
+def test_render_anchor_tag_content_true(client, live_server, measure_memory_usage):
     """Testing that the link changes are detected when
     render_anchor_tag_content setting is set to true"""
     sleep_time_for_fetch_thread = 3

changedetectionio/tests/test_ignorestatuscode.py

@@ -39,7 +39,7 @@ def set_some_changed_response():
         f.write(test_return_data)
 
 
-def test_normal_page_check_works_with_ignore_status_code(client, live_server):
+def test_normal_page_check_works_with_ignore_status_code(client, live_server, measure_memory_usage):
 
 
     # Give the endpoint time to spin up
@@ -85,7 +85,7 @@ def test_normal_page_check_works_with_ignore_status_code(client, live_server):
 
 
 # Tests the whole stack works with staus codes ignored
-def test_403_page_check_works_with_ignore_status_code(client, live_server):
+def test_403_page_check_works_with_ignore_status_code(client, live_server, measure_memory_usage):
     sleep_time_for_fetch_thread = 3
 
     set_original_response()

changedetectionio/tests/test_ignorewhitespace.py

@@ -49,7 +49,7 @@ def set_original_ignore_response():
 
 
 # If there was only a change in the whitespacing, then we shouldnt have a change detected
-def test_check_ignore_whitespace(client, live_server):
+def test_check_ignore_whitespace(client, live_server, measure_memory_usage):
     sleep_time_for_fetch_thread = 3
 
     # Give the endpoint time to spin up

changedetectionio/tests/test_import.py

@@ -8,10 +8,10 @@ from flask import url_for
 from .util import live_server_setup, wait_for_all_checks
 
 
-def test_setup(client, live_server):
+def test_setup(client, live_server, measure_memory_usage):
     live_server_setup(live_server)
 
-def test_import(client, live_server):
+def test_import(client, live_server, measure_memory_usage):
     # Give the endpoint time to spin up
     wait_for_all_checks(client)
 
@@ -34,7 +34,7 @@ https://example.com tag1, other tag"""
     res = client.get( url_for("index"))
     res = client.get( url_for("index"))
 
-def xtest_import_skip_url(client, live_server):
+def xtest_import_skip_url(client, live_server, measure_memory_usage):
 
 
     # Give the endpoint time to spin up
@@ -57,7 +57,7 @@ def xtest_import_skip_url(client, live_server):
     # Clear flask alerts
     res = client.get( url_for("index"))
 
-def test_import_distillio(client, live_server):
+def test_import_distillio(client, live_server, measure_memory_usage):
 
     distill_data='''
 {
@@ -123,7 +123,7 @@ def test_import_distillio(client, live_server):
     # Clear flask alerts
     res = client.get(url_for("index"))
 
-def test_import_custom_xlsx(client, live_server):
+def test_import_custom_xlsx(client, live_server, measure_memory_usage):
     """Test can upload a excel spreadsheet and the watches are created correctly"""
 
     #live_server_setup(live_server)
@@ -172,7 +172,7 @@ def test_import_custom_xlsx(client, live_server):
     res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
 
-def test_import_watchete_xlsx(client, live_server):
+def test_import_watchete_xlsx(client, live_server, measure_memory_usage):
     """Test can upload a excel spreadsheet and the watches are created correctly"""
 
     #live_server_setup(live_server)

changedetectionio/tests/test_jinja2.py

@@ -5,11 +5,11 @@ from flask import url_for
 from .util import live_server_setup, wait_for_all_checks
 
 
-def test_setup(client, live_server):
+def test_setup(client, live_server, measure_memory_usage):
     live_server_setup(live_server)
 
 # If there was only a change in the whitespacing, then we shouldnt have a change detected
-def test_jinja2_in_url_query(client, live_server):
+def test_jinja2_in_url_query(client, live_server, measure_memory_usage):
     #live_server_setup(live_server)
 
     # Add our URL to the import page
@@ -34,7 +34,7 @@ def test_jinja2_in_url_query(client, live_server):
     assert b'date=2' in res.data
 
 # https://techtonics.medium.com/secure-templating-with-jinja2-understanding-ssti-and-jinja2-sandbox-environment-b956edd60456
-def test_jinja2_security_url_query(client, live_server):
+def test_jinja2_security_url_query(client, live_server, measure_memory_usage):
     #live_server_setup(live_server)
 
     # Add our URL to the import page

changedetectionio/tests/test_jsonpath_jq_selector.py

@@ -201,7 +201,7 @@ def set_modified_response():
 
     return None
 
-def test_check_json_without_filter(client, live_server):
+def test_check_json_without_filter(client, live_server, measure_memory_usage):
     # Request a JSON document from a application/json source containing HTML
     # and be sure it doesn't get chewed up by instriptis
     set_json_response_with_html()
@@ -294,14 +294,14 @@ def check_json_filter(json_filter, client, live_server):
     res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
 
-def test_check_jsonpath_filter(client, live_server):
+def test_check_jsonpath_filter(client, live_server, measure_memory_usage):
     check_json_filter('json:boss.name', client, live_server)
 
-def test_check_jq_filter(client, live_server):
+def test_check_jq_filter(client, live_server, measure_memory_usage):
     if jq_support:
         check_json_filter('jq:.boss.name', client, live_server)
 
-def test_check_jqraw_filter(client, live_server):
+def test_check_jqraw_filter(client, live_server, measure_memory_usage):
     if jq_support:
         check_json_filter('jqraw:.boss.name', client, live_server)
 
@@ -352,14 +352,14 @@ def check_json_filter_bool_val(json_filter, client, live_server):
     res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
 
-def test_check_jsonpath_filter_bool_val(client, live_server):
+def test_check_jsonpath_filter_bool_val(client, live_server, measure_memory_usage):
     check_json_filter_bool_val("json:$['available']", client, live_server)
 
-def test_check_jq_filter_bool_val(client, live_server):
+def test_check_jq_filter_bool_val(client, live_server, measure_memory_usage):
     if jq_support:
         check_json_filter_bool_val("jq:.available", client, live_server)
 
-def test_check_jqraw_filter_bool_val(client, live_server):
+def test_check_jqraw_filter_bool_val(client, live_server, measure_memory_usage):
     if jq_support:
         check_json_filter_bool_val("jq:.available", client, live_server)
 
@@ -430,7 +430,7 @@ def check_json_ext_filter(json_filter, client, live_server):
     res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
 
-def test_ignore_json_order(client, live_server):
+def test_ignore_json_order(client, live_server, measure_memory_usage):
     # A change in order shouldn't trigger a notification
 
     with open("test-datastore/endpoint-content.txt", "w") as f:
@@ -472,7 +472,7 @@ def test_ignore_json_order(client, live_server):
     res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
 
-def test_correct_header_detect(client, live_server):
+def test_correct_header_detect(client, live_server, measure_memory_usage):
     # Like in https://github.com/dgtlmoon/changedetection.io/pull/1593
     # Specify extra html that JSON is sometimes wrapped in - when using SockpuppetBrowser / Puppeteer / Playwrightetc
     with open("test-datastore/endpoint-content.txt", "w") as f:
@@ -504,13 +504,13 @@ def test_correct_header_detect(client, live_server):
     res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
 
-def test_check_jsonpath_ext_filter(client, live_server):
+def test_check_jsonpath_ext_filter(client, live_server, measure_memory_usage):
     check_json_ext_filter('json:$[?(@.status==Sold)]', client, live_server)
 
-def test_check_jq_ext_filter(client, live_server):
+def test_check_jq_ext_filter(client, live_server, measure_memory_usage):
     if jq_support:
         check_json_ext_filter('jq:.[] | select(.status | contains("Sold"))', client, live_server)
 
-def test_check_jqraw_ext_filter(client, live_server):
+def test_check_jqraw_ext_filter(client, live_server, measure_memory_usage):
     if jq_support:
         check_json_ext_filter('jq:.[] | select(.status | contains("Sold"))', client, live_server)

changedetectionio/tests/test_nonrenderable_pages.py

@@ -22,7 +22,7 @@ def set_nonrenderable_response():
 
     return None
 
-def test_check_basic_change_detection_functionality(client, live_server):
+def test_check_basic_change_detection_functionality(client, live_server, measure_memory_usage):
     set_original_response()
     live_server_setup(live_server)
 

changedetectionio/tests/test_notification.py

@@ -21,7 +21,7 @@ def test_setup(live_server):
 
 # Hard to just add more live server URLs when one test is already running (I think)
 # So we add our test here (was in a different file)
-def test_check_notification(client, live_server):
+def test_check_notification(client, live_server, measure_memory_usage):
     #live_server_setup(live_server)
     set_original_response()
 
@@ -234,7 +234,7 @@ def test_check_notification(client, live_server):
         follow_redirects=True
     )
 
-def test_notification_validation(client, live_server):
+def test_notification_validation(client, live_server, measure_memory_usage):
 
     time.sleep(1)
 
@@ -273,7 +273,7 @@ def test_notification_validation(client, live_server):
 
 
 
-def test_notification_custom_endpoint_and_jinja2(client, live_server):
+def test_notification_custom_endpoint_and_jinja2(client, live_server, measure_memory_usage):
     #live_server_setup(live_server)
 
     # test_endpoint - that sends the contents of a file

changedetectionio/tests/test_notification_errors.py

@@ -4,7 +4,7 @@ from flask import url_for
 from .util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks
 import logging
 
-def test_check_notification_error_handling(client, live_server):
+def test_check_notification_error_handling(client, live_server, measure_memory_usage):
 
     live_server_setup(live_server)
     set_original_response()

changedetectionio/tests/test_obfuscations.py

@@ -18,7 +18,7 @@ def set_original_ignore_response():
         f.write(test_return_data)
 
 
-def test_obfuscations(client, live_server):
+def test_obfuscations(client, live_server, measure_memory_usage):
     set_original_ignore_response()
     live_server_setup(live_server)
     time.sleep(1)

changedetectionio/tests/test_pdf.py

@@ -6,7 +6,7 @@ from .util import set_original_response, set_modified_response, live_server_setu
 
 
 # `subtractive_selectors` should still work in `source:` type requests
-def test_fetch_pdf(client, live_server):
+def test_fetch_pdf(client, live_server, measure_memory_usage):
     import shutil
     shutil.copy("tests/test.pdf", "test-datastore/endpoint-test.pdf")
 

changedetectionio/tests/test_request.py

@@ -9,7 +9,7 @@ def test_setup(live_server):
 
 # Hard to just add more live server URLs when one test is already running (I think)
 # So we add our test here (was in a different file)
-def test_headers_in_request(client, live_server):
+def test_headers_in_request(client, live_server, measure_memory_usage):
     #ve_server_setup(live_server)
     # Add our URL to the import page
     test_url = url_for('test_headers', _external=True)
@@ -84,7 +84,7 @@ def test_headers_in_request(client, live_server):
     res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
 
-def test_body_in_request(client, live_server):
+def test_body_in_request(client, live_server, measure_memory_usage):
 
     # Add our URL to the import page
     test_url = url_for('test_body', _external=True)
@@ -177,7 +177,7 @@ def test_body_in_request(client, live_server):
     res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
 
-def test_method_in_request(client, live_server):
+def test_method_in_request(client, live_server, measure_memory_usage):
     # Add our URL to the import page
     test_url = url_for('test_method', _external=True)
     if os.getenv('PLAYWRIGHT_DRIVER_URL'):
@@ -254,7 +254,7 @@ def test_method_in_request(client, live_server):
     assert b'Deleted' in res.data
 
 # Re #2408 - user-agent override test, also should handle case-insensitive header deduplication
-def test_ua_global_override(client, live_server):
+def test_ua_global_override(client, live_server, measure_memory_usage):
     # live_server_setup(live_server)
     test_url = url_for('test_headers', _external=True)
 
@@ -309,7 +309,7 @@ def test_ua_global_override(client, live_server):
     res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
 
-def test_headers_textfile_in_request(client, live_server):
+def test_headers_textfile_in_request(client, live_server, measure_memory_usage):
     #live_server_setup(live_server)
     # Add our URL to the import page
 

changedetectionio/tests/test_rss.py

@@ -49,10 +49,10 @@ def set_original_cdata_xml():
         f.write(test_return_data)
 
 
-def test_setup(client, live_server):
+def test_setup(client, live_server, measure_memory_usage):
     live_server_setup(live_server)
 
-def test_rss_and_token(client, live_server):
+def test_rss_and_token(client, live_server, measure_memory_usage):
     #    live_server_setup(live_server)
 
     set_original_response()
@@ -90,7 +90,7 @@ def test_rss_and_token(client, live_server):
 
     client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
 
-def test_basic_cdata_rss_markup(client, live_server):
+def test_basic_cdata_rss_markup(client, live_server, measure_memory_usage):
     #live_server_setup(live_server)
 
     set_original_cdata_xml()
@@ -118,7 +118,7 @@ def test_basic_cdata_rss_markup(client, live_server):
     assert b'The days of Terminator' in res.data
     res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
 
-def test_rss_xpath_filtering(client, live_server):
+def test_rss_xpath_filtering(client, live_server, measure_memory_usage):
     #live_server_setup(live_server)
 
     set_original_cdata_xml()

changedetectionio/tests/test_search.py

@@ -5,7 +5,7 @@ import time
 def test_setup(live_server):
     live_server_setup(live_server)
 
-def test_basic_search(client, live_server):
+def test_basic_search(client, live_server, measure_memory_usage):
     #live_server_setup(live_server)
 
     urls = ['https://localhost:12300?first-result=1',
@@ -38,7 +38,7 @@ def test_basic_search(client, live_server):
     assert urls[1].encode('utf-8') not in res.data
 
 
-def test_search_in_tag_limit(client, live_server):
+def test_search_in_tag_limit(client, live_server, measure_memory_usage):
     #live_server_setup(live_server)
 
     urls = ['https://localhost:12300?first-result=1 tag-one',

changedetectionio/tests/test_security.py

@@ -2,10 +2,10 @@ from flask import url_for
 from .util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks
 import time
 
-def test_setup(client, live_server):
+def test_setup(client, live_server, measure_memory_usage):
     live_server_setup(live_server)
 
-def test_bad_access(client, live_server):
+def test_bad_access(client, live_server, measure_memory_usage):
     #live_server_setup(live_server)
     res = client.post(
         url_for("import_page"),
@@ -67,7 +67,7 @@ def test_bad_access(client, live_server):
 
     assert b'file:// type access is denied for security reasons.' in res.data
 
-def test_xss(client, live_server):
+def test_xss(client, live_server, measure_memory_usage):
     #live_server_setup(live_server)
     from changedetectionio.notification import (
         default_notification_format

changedetectionio/tests/test_share_watch.py

@@ -9,7 +9,7 @@ import re
 sleep_time_for_fetch_thread = 3
 
 
-def test_share_watch(client, live_server):
+def test_share_watch(client, live_server, measure_memory_usage):
     set_original_response()
     live_server_setup(live_server)
 

changedetectionio/tests/test_source.py

@@ -10,7 +10,7 @@ sleep_time_for_fetch_thread = 3
 def test_setup(live_server):
     live_server_setup(live_server)
 
-def test_check_basic_change_detection_functionality_source(client, live_server):
+def test_check_basic_change_detection_functionality_source(client, live_server, measure_memory_usage):
     set_original_response()
     test_url = 'source:'+url_for('test_endpoint', _external=True)
     # Add our URL to the import page
@@ -58,7 +58,7 @@ def test_check_basic_change_detection_functionality_source(client, live_server):
 
 
 # `subtractive_selectors` should still work in `source:` type requests
-def test_check_ignore_elements(client, live_server):
+def test_check_ignore_elements(client, live_server, measure_memory_usage):
     set_original_response()
     time.sleep(1)
     test_url = 'source:'+url_for('test_endpoint', _external=True)

changedetectionio/tests/test_trigger.py

@@ -55,7 +55,7 @@ def set_modified_with_trigger_text_response():
         f.write(test_return_data)
 
 
-def test_trigger_functionality(client, live_server):
+def test_trigger_functionality(client, live_server, measure_memory_usage):
 
     live_server_setup(live_server)
 

changedetectionio/tests/test_trigger_regex.py

@@ -22,7 +22,7 @@ def set_original_ignore_response():
 
 
 
-def test_trigger_regex_functionality(client, live_server):
+def test_trigger_regex_functionality(client, live_server, measure_memory_usage):
 
     live_server_setup(live_server)
 

changedetectionio/tests/test_trigger_regex_with_filter.py

@@ -22,7 +22,7 @@ def set_original_ignore_response():
 
 
 
-def test_trigger_regex_functionality_with_filter(client, live_server):
+def test_trigger_regex_functionality_with_filter(client, live_server, measure_memory_usage):
 
     live_server_setup(live_server)
     sleep_time_for_fetch_thread = 3

changedetectionio/tests/test_unique_lines.py

@@ -66,10 +66,10 @@ def set_modified_with_trigger_text_response():
     with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write(test_return_data)
 
-def test_setup(client, live_server):
+def test_setup(client, live_server, measure_memory_usage):
     live_server_setup(live_server)
 
-def test_unique_lines_functionality(client, live_server):
+def test_unique_lines_functionality(client, live_server, measure_memory_usage):
     #live_server_setup(live_server)
 
 
@@ -118,7 +118,7 @@ def test_unique_lines_functionality(client, live_server):
     res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
 
-def test_sort_lines_functionality(client, live_server):
+def test_sort_lines_functionality(client, live_server, measure_memory_usage):
     #live_server_setup(live_server)
 
     set_modified_swapped_lines_with_extra_text_for_sorting()

changedetectionio/tests/test_watch_fields_storage.py

@@ -4,7 +4,7 @@ from urllib.request import urlopen
 from . util import set_original_response, set_modified_response, live_server_setup
 
 
-def test_check_watch_field_storage(client, live_server):
+def test_check_watch_field_storage(client, live_server, measure_memory_usage):
     set_original_response()
     live_server_setup(live_server)
 

changedetectionio/tests/test_xpath_selector.py

@@ -49,7 +49,7 @@ def set_modified_response():
 
 
 # Handle utf-8 charset replies https://github.com/dgtlmoon/changedetection.io/pull/613
-def test_check_xpath_filter_utf8(client, live_server):
+def test_check_xpath_filter_utf8(client, live_server, measure_memory_usage):
     filter = '//item/*[self::description]'
 
     d = '''<?xml version="1.0" encoding="UTF-8"?>
@@ -105,7 +105,7 @@ def test_check_xpath_filter_utf8(client, live_server):
 
 
 # Handle utf-8 charset replies https://github.com/dgtlmoon/changedetection.io/pull/613
-def test_check_xpath_text_function_utf8(client, live_server):
+def test_check_xpath_text_function_utf8(client, live_server, measure_memory_usage):
     filter = '//item/title/text()'
 
     d = '''<?xml version="1.0" encoding="UTF-8"?>
@@ -168,7 +168,7 @@ def test_check_xpath_text_function_utf8(client, live_server):
     assert b'Deleted' in res.data
 
 
-def test_check_markup_xpath_filter_restriction(client, live_server):
+def test_check_markup_xpath_filter_restriction(client, live_server, measure_memory_usage):
     xpath_filter = "//*[contains(@class, 'sametext')]"
 
     set_original_response()
@@ -214,7 +214,7 @@ def test_check_markup_xpath_filter_restriction(client, live_server):
     assert b'Deleted' in res.data
 
 
-def test_xpath_validation(client, live_server):
+def test_xpath_validation(client, live_server, measure_memory_usage):
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
@@ -235,7 +235,7 @@ def test_xpath_validation(client, live_server):
     assert b'Deleted' in res.data
 
 
-def test_xpath23_prefix_validation(client, live_server):
+def test_xpath23_prefix_validation(client, live_server, measure_memory_usage):
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
@@ -255,7 +255,7 @@ def test_xpath23_prefix_validation(client, live_server):
     res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
 
-def test_xpath1_lxml(client, live_server):
+def test_xpath1_lxml(client, live_server, measure_memory_usage):
     #live_server_setup(live_server)
 
     d = '''<?xml version="1.0" encoding="UTF-8"?>
@@ -319,7 +319,7 @@ def test_xpath1_lxml(client, live_server):
     #####
 
 
-def test_xpath1_validation(client, live_server):
+def test_xpath1_validation(client, live_server, measure_memory_usage):
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
@@ -341,7 +341,7 @@ def test_xpath1_validation(client, live_server):
 
 
 # actually only really used by the distll.io importer, but could be handy too
-def test_check_with_prefix_include_filters(client, live_server):
+def test_check_with_prefix_include_filters(client, live_server, measure_memory_usage):
     res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
 
@@ -378,7 +378,7 @@ def test_check_with_prefix_include_filters(client, live_server):
     client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
 
 
-def test_various_rules(client, live_server):
+def test_various_rules(client, live_server, measure_memory_usage):
     # Just check these don't error
     # live_server_setup(live_server)
     with open("test-datastore/endpoint-content.txt", "w") as f:
@@ -426,7 +426,7 @@ def test_various_rules(client, live_server):
     assert b'Deleted' in res.data
 
 
-def test_xpath_20(client, live_server):
+def test_xpath_20(client, live_server, measure_memory_usage):
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
         url_for("import_page"),
@@ -463,7 +463,7 @@ def test_xpath_20(client, live_server):
     client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
 
 
-def test_xpath_20_function_count(client, live_server):
+def test_xpath_20_function_count(client, live_server, measure_memory_usage):
     set_original_response()
 
     # Add our URL to the import page
@@ -499,7 +499,7 @@ def test_xpath_20_function_count(client, live_server):
     client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
 
 
-def test_xpath_20_function_count2(client, live_server):
+def test_xpath_20_function_count2(client, live_server, measure_memory_usage):
     set_original_response()
 
     # Add our URL to the import page
@@ -535,7 +535,7 @@ def test_xpath_20_function_count2(client, live_server):
     client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
 
 
-def test_xpath_20_function_string_join_matches(client, live_server):
+def test_xpath_20_function_string_join_matches(client, live_server, measure_memory_usage):
     set_original_response()
 
     # Add our URL to the import page

changedetectionio/tests/visualselector/test_fetch_data.py

@@ -4,12 +4,12 @@ import os
 from flask import url_for
 from ..util import live_server_setup, wait_for_all_checks, extract_UUID_from_client
 
-def test_setup(client, live_server):
+def test_setup(client, live_server, measure_memory_usage):
     live_server_setup(live_server)
 
 
 # Add a site in paused mode, add an invalid filter, we should still have visual selector data ready
-def test_visual_selector_content_ready(client, live_server):
+def test_visual_selector_content_ready(client, live_server, measure_memory_usage):
 
     import os
     import json
@@ -79,7 +79,7 @@ def test_visual_selector_content_ready(client, live_server):
         follow_redirects=True
     )
 
-def test_basic_browserstep(client, live_server):
+def test_basic_browserstep(client, live_server, measure_memory_usage):
 
     #live_server_setup(live_server)
     assert os.getenv('PLAYWRIGHT_DRIVER_URL'), "Needs PLAYWRIGHT_DRIVER_URL set for this test"

changedetectionio/update_worker.py

@@ -250,8 +250,7 @@ class update_worker(threading.Thread):
                     # Clear last errors (move to preflight func?)
                     self.datastore.data['watching'][uuid]['browser_steps_last_error_step'] = None
 
-                    # DeepCopy so we can be sure we don't accidently change anything by reference
-                    watch = deepcopy(self.datastore.data['watching'].get(uuid))
+                    watch = self.datastore.data['watching'].get(uuid)
 
                     logger.info(f"Processing watch UUID {uuid} Priority {queued_item_data.priority} URL {watch['url']}")
                     now = time.time()
@@ -315,6 +314,9 @@ class update_worker(threading.Thread):
                         if e.screenshot:
                             watch.save_screenshot(screenshot=e.screenshot, as_error=True)
 
+                        if e.xpath_data:
+                            watch.save_xpath_data(data=e.xpath_data)
+                            
                         process_changedetection_results = False
 
                     except content_fetchers.exceptions.Non200ErrorCodeReceived as e:
@@ -345,8 +347,13 @@ class update_worker(threading.Thread):
 
                         err_text = "Warning, no filters were found, no change detection ran - Did the page change layout? update your Visual Filter if necessary."
                         self.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text})
+
+                        # Filter wasnt found, but we should still update the visual selector so that they can have a chance to set it up again
                         if e.screenshot:
-                            watch.save_screenshot(screenshot=e.screenshot, as_error=True)
+                            watch.save_screenshot(screenshot=e.screenshot)
+
+                        if e.xpath_data:
+                            watch.save_xpath_data(data=e.xpath_data)
 
                         # Only when enabled, send the notification
                         if watch.get('filter_failure_notification_send', False):