this logic also not needed

changedetectionio/__init__.py

@@ -175,7 +175,6 @@ def main():
     #         proxy_set_header Host "localhost";
     #         proxy_set_header X-Forwarded-Prefix /app;
 
-
     if os.getenv('USE_X_SETTINGS'):
         logger.info("USE_X_SETTINGS is ENABLED")
         from werkzeug.middleware.proxy_fix import ProxyFix

changedetectionio/blueprint/browser_steps/__init__.py

@@ -84,9 +84,7 @@ def construct_blueprint(datastore: ChangeDetectionStore):
         # Tell Playwright to connect to Chrome and setup a new session via our stepper interface
         browsersteps_start_session['browserstepper'] = browser_steps.browsersteps_live_ui(
             playwright_browser=browsersteps_start_session['browser'],
-            proxy=proxy,
-            start_url=datastore.data['watching'][watch_uuid].get('url')
-        )
+            proxy=proxy)
 
         # For test
         #browsersteps_start_session['browserstepper'].action_goto_url(value="http://example.com?time="+str(time.time()))
@@ -169,6 +167,11 @@ def construct_blueprint(datastore: ChangeDetectionStore):
             step_n = int(request.form.get('step_n'))
             is_last_step = strtobool(request.form.get('is_last_step'))
 
+            if step_operation == 'Goto site':
+                step_operation = 'goto_url'
+                step_optional_value = datastore.data['watching'][uuid].get('url')
+                step_selector = None
+
             # @todo try.. accept.. nice errors not popups..
             try:
 

changedetectionio/blueprint/browser_steps/browser_steps.py

@@ -49,10 +49,6 @@ browser_step_ui_config = {'Choose one': '0 0',
 # ONLY Works in Playwright because we need the fullscreen screenshot
 class steppable_browser_interface():
     page = None
-    start_url = None
-
-    def __init__(self, start_url):
-        self.start_url = start_url
 
     # Convert and perform "Click Button" for example
     def call_action(self, action_name, selector=None, optional_value=None):
@@ -91,10 +87,6 @@ class steppable_browser_interface():
         logger.debug(f"Time to goto URL {time.time()-now:.2f}s")
         return response
 
-    # Incase they request to go back to the start
-    def action_goto_site(self, selector=None, value=None):
-        return self.action_goto_url(value=self.start_url)
-
     def action_click_element_containing_text(self, selector=None, value=''):
         if not len(value.strip()):
             return
@@ -202,11 +194,10 @@ class browsersteps_live_ui(steppable_browser_interface):
 
     browser_type = os.getenv("PLAYWRIGHT_BROWSER_TYPE", 'chromium').strip('"')
 
-    def __init__(self, playwright_browser, proxy=None, headers=None, start_url=None):
+    def __init__(self, playwright_browser, proxy=None, headers=None):
         self.headers = headers or {}
         self.age_start = time.time()
         self.playwright_browser = playwright_browser
-        self.start_url = start_url
         if self.context is None:
             self.connect(proxy=proxy)
 

changedetectionio/content_fetchers/base.py

@@ -112,26 +112,23 @@ class Fetcher():
 
     def browser_steps_get_valid_steps(self):
         if self.browser_steps is not None and len(self.browser_steps):
-            valid_steps = list(filter(
-                lambda s: (s['operation'] and len(s['operation']) and s['operation'] != 'Choose one'),
-                self.browser_steps))
-
-            # Just incase they selected Goto site by accident with older JS
-            if valid_steps and valid_steps[0]['operation'] == 'Goto site':
-                del(valid_steps[0])
+            valid_steps = filter(
+                lambda s: (s['operation'] and len(s['operation']) and s['operation'] != 'Choose one' and s['operation'] != 'Goto site'),
+                self.browser_steps)
 
             return valid_steps
 
         return None
 
-    def iterate_browser_steps(self, start_url=None):
+    def iterate_browser_steps(self):
         from changedetectionio.blueprint.browser_steps.browser_steps import steppable_browser_interface
         from playwright._impl._errors import TimeoutError, Error
         from changedetectionio.safe_jinja import render as jinja_render
+
         step_n = 0
 
         if self.browser_steps is not None and len(self.browser_steps):
-            interface = steppable_browser_interface(start_url=start_url)
+            interface = steppable_browser_interface()
             interface.page = self.page
             valid_steps = self.browser_steps_get_valid_steps()
 

changedetectionio/content_fetchers/playwright.py

@@ -119,7 +119,7 @@ class fetcher(Fetcher):
 
             # Re-use as much code from browser steps as possible so its the same
             from changedetectionio.blueprint.browser_steps.browser_steps import steppable_browser_interface
-            browsersteps_interface = steppable_browser_interface(start_url=url)
+            browsersteps_interface = steppable_browser_interface()
             browsersteps_interface.page = self.page
 
             response = browsersteps_interface.action_goto_url(value=url)
@@ -172,7 +172,7 @@ class fetcher(Fetcher):
 
             # Run Browser Steps here
             if self.browser_steps_get_valid_steps():
-                self.iterate_browser_steps(start_url=url)
+                self.iterate_browser_steps()
 
             self.page.wait_for_timeout(extra_wait * 1000)
 

changedetectionio/flask_app.py

@@ -1063,8 +1063,6 @@ def changedetection_app(config=None, datastore_o=None):
         content = []
         ignored_line_numbers = []
         trigger_line_numbers = []
-        versions = []
-        timestamp = None
 
         # More for testing, possible to return the first/only
         if uuid == 'first':
@@ -1084,53 +1082,57 @@ def changedetection_app(config=None, datastore_o=None):
         if (watch.get('fetch_backend') == 'system' and system_uses_webdriver) or watch.get('fetch_backend') == 'html_webdriver' or watch.get('fetch_backend', '').startswith('extra_browser_'):
             is_html_webdriver = True
 
+        # Never requested successfully, but we detected a fetch error
         if datastore.data['watching'][uuid].history_n == 0 and (watch.get_error_text() or watch.get_error_snapshot()):
             flash("Preview unavailable - No fetch/check completed or triggers not reached", "error")
-        else:
-            # So prepare the latest preview or not
-            preferred_version = request.args.get('version')
-            versions = list(watch.history.keys())
-            timestamp = versions[-1]
-            if preferred_version and preferred_version in versions:
-                timestamp = preferred_version
+            output = render_template("preview.html",
+                                     content=content,
+                                     history_n=watch.history_n,
+                                     extra_stylesheets=extra_stylesheets,
+#                                     current_diff_url=watch['url'],
+                                     watch=watch,
+                                     uuid=uuid,
+                                     is_html_webdriver=is_html_webdriver,
+                                     last_error=watch['last_error'],
+                                     last_error_text=watch.get_error_text(),
+                                     last_error_screenshot=watch.get_error_snapshot())
+            return output
 
-            try:
-                versions = list(watch.history.keys())
-                tmp = watch.get_history_snapshot(timestamp).splitlines()
+        timestamp = list(watch.history.keys())[-1]
+        try:
+            tmp = watch.get_history_snapshot(timestamp).splitlines()
 
-                # Get what needs to be highlighted
-                ignore_rules = watch.get('ignore_text', []) + datastore.data['settings']['application']['global_ignore_text']
+            # Get what needs to be highlighted
+            ignore_rules = watch.get('ignore_text', []) + datastore.data['settings']['application']['global_ignore_text']
 
-                # .readlines will keep the \n, but we will parse it here again, in the future tidy this up
-                ignored_line_numbers = html_tools.strip_ignore_text(content="\n".join(tmp),
-                                                                    wordlist=ignore_rules,
-                                                                    mode='line numbers'
-                                                                    )
+            # .readlines will keep the \n, but we will parse it here again, in the future tidy this up
+            ignored_line_numbers = html_tools.strip_ignore_text(content="\n".join(tmp),
+                                                                wordlist=ignore_rules,
+                                                                mode='line numbers'
+                                                                )
 
-                trigger_line_numbers = html_tools.strip_ignore_text(content="\n".join(tmp),
-                                                                    wordlist=watch['trigger_text'],
-                                                                    mode='line numbers'
-                                                                    )
-                # Prepare the classes and lines used in the template
-                i=0
-                for l in tmp:
-                    classes=[]
-                    i+=1
-                    if i in ignored_line_numbers:
-                        classes.append('ignored')
-                    if i in trigger_line_numbers:
-                        classes.append('triggered')
-                    content.append({'line': l, 'classes': ' '.join(classes)})
+            trigger_line_numbers = html_tools.strip_ignore_text(content="\n".join(tmp),
+                                                                wordlist=watch['trigger_text'],
+                                                                mode='line numbers'
+                                                                )
+            # Prepare the classes and lines used in the template
+            i=0
+            for l in tmp:
+                classes=[]
+                i+=1
+                if i in ignored_line_numbers:
+                    classes.append('ignored')
+                if i in trigger_line_numbers:
+                    classes.append('triggered')
+                content.append({'line': l, 'classes': ' '.join(classes)})
 
-            except Exception as e:
-                content.append({'line': f"File doesnt exist or unable to read timestamp {timestamp}", 'classes': ''})
+        except Exception as e:
+            content.append({'line': f"File doesnt exist or unable to read timestamp {timestamp}", 'classes': ''})
 
         output = render_template("preview.html",
                                  content=content,
-                                 current_version=timestamp,
                                  history_n=watch.history_n,
                                  extra_stylesheets=extra_stylesheets,
-                                 extra_title=f" - Diff - {watch.label} @ {timestamp}",
                                  ignored_line_numbers=ignored_line_numbers,
                                  triggered_line_numbers=trigger_line_numbers,
                                  current_diff_url=watch['url'],
@@ -1140,10 +1142,7 @@ def changedetection_app(config=None, datastore_o=None):
                                  is_html_webdriver=is_html_webdriver,
                                  last_error=watch['last_error'],
                                  last_error_text=watch.get_error_text(),
-                                 last_error_screenshot=watch.get_error_snapshot(),
-                                 versions=versions
-                                )
-
+                                 last_error_screenshot=watch.get_error_snapshot())
 
         return output
 

changedetectionio/notification.py

@@ -48,7 +48,7 @@ from apprise.decorators import notify
 def apprise_custom_api_call_wrapper(body, title, notify_type, *args, **kwargs):
     import requests
     from apprise.utils import parse_url as apprise_parse_url
-    from apprise import URLBase
+    from apprise.URLBase import URLBase
 
     url = kwargs['meta'].get('url')
 
@@ -122,6 +122,10 @@ def process_notification(n_object, datastore):
     # Insert variables into the notification content
     notification_parameters = create_notification_parameters(n_object, datastore)
 
+    # Get the notification body from datastore
+    n_body = jinja_render(template_str=n_object.get('notification_body', ''), **notification_parameters)
+    n_title = jinja_render(template_str=n_object.get('notification_title', ''), **notification_parameters)
+
     n_format = valid_notification_formats.get(
         n_object.get('notification_format', default_notification_format),
         valid_notification_formats[default_notification_format],
@@ -147,11 +151,6 @@ def process_notification(n_object, datastore):
 
     with apprise.LogCapture(level=apprise.logging.DEBUG) as logs:
         for url in n_object['notification_urls']:
-
-            # Get the notification body from datastore
-            n_body = jinja_render(template_str=n_object.get('notification_body', ''), **notification_parameters)
-            n_title = jinja_render(template_str=n_object.get('notification_title', ''), **notification_parameters)
-
             url = url.strip()
             if not url:
                 logger.warning(f"Process Notification: skipping empty notification URL.")

changedetectionio/static/js/browser-steps.js

@@ -26,8 +26,7 @@ $(document).ready(function () {
         set_scale();
     });
     // Should always be disabled
-    $('#browser_steps-0-operation option[value="Goto site"]').prop("selected", "selected");
-    $('#browser_steps-0-operation').attr('disabled', 'disabled');
+    $('#browser_steps >li:first-child select').val('Goto site').attr('disabled', 'disabled');
 
     $('#browsersteps-click-start').click(function () {
         $("#browsersteps-click-start").fadeOut();

changedetectionio/static/js/diff-overview.js

@@ -8,13 +8,6 @@ $(document).ready(function () {
         }
     })
 
-    $('.needs-localtime').each(function () {
-        for (var option of this.options) {
-            var dateObject = new Date(option.value * 1000);
-            option.label = dateObject.toLocaleString(undefined, {dateStyle: "full", timeStyle: "medium"});
-        }
-    });
-
     // Load it when the #screenshot tab is in use, so we dont give a slow experience when waiting for the text diff to load
     window.addEventListener('hashchange', function (e) {
         toggle(location.hash);

changedetectionio/static/js/diff-render.js

@@ -79,7 +79,12 @@ $(document).ready(function () {
         $('#jump-next-diff').click();
     }
 
-
+    $('.needs-localtime').each(function () {
+        for (var option of this.options) {
+            var dateObject = new Date(option.value * 1000);
+            option.label = dateObject.toLocaleString(undefined, {dateStyle: "full", timeStyle: "medium"});
+        }
+    })
     onDiffTypeChange(
         document.querySelector('#settings [name="diff_type"]:checked'),
     );

changedetectionio/static/js/preview.js

@@ -1,49 +0,0 @@
-function redirect_to_version(version) {
-    var currentUrl = window.location.href;
-    var baseUrl = currentUrl.split('?')[0]; // Base URL without query parameters
-    var anchor = '';
-
-    // Check if there is an anchor
-    if (baseUrl.indexOf('#') !== -1) {
-        anchor = baseUrl.substring(baseUrl.indexOf('#'));
-        baseUrl = baseUrl.substring(0, baseUrl.indexOf('#'));
-    }
-    window.location.href = baseUrl + '?version=' + version + anchor;
-}
-
-document.addEventListener('keydown', function (event) {
-    var selectElement = document.getElementById('preview-version');
-    if (selectElement) {
-        var selectedOption = selectElement.querySelector('option:checked');
-        if (selectedOption) {
-            if (event.key === 'ArrowLeft') {
-                if (selectedOption.previousElementSibling) {
-                    redirect_to_version(selectedOption.previousElementSibling.value);
-                }
-            } else if (event.key === 'ArrowRight') {
-                if (selectedOption.nextElementSibling) {
-                    redirect_to_version(selectedOption.nextElementSibling.value);
-                }
-            }
-        }
-    }
-});
-
-
-document.getElementById('preview-version').addEventListener('change', function () {
-    redirect_to_version(this.value);
-});
-
-var selectElement = document.getElementById('preview-version');
-if (selectElement) {
-    var selectedOption = selectElement.querySelector('option:checked');
-    if (selectedOption) {
-        if (selectedOption.previousElementSibling) {
-            document.getElementById('btn-previous').href = "?version=" + selectedOption.previousElementSibling.value;
-        }
-        if (selectedOption.nextElementSibling) {
-            document.getElementById('btn-next').href = "?version=" + selectedOption.nextElementSibling.value;
-        }
-
-    }
-}

changedetectionio/templates/preview.html

@@ -1,103 +1,72 @@
 {% extends 'base.html' %}
 
 {% block content %}
-    <script>
-        const screenshot_url = "{{url_for('static_content', group='screenshot', filename=uuid)}}";
-        {% if last_error_screenshot %}
-            const error_screenshot_url = "{{url_for('static_content', group='screenshot', filename=uuid, error_screenshot=1) }}";
-        {% endif %}
-        const highlight_submit_ignore_url = "{{url_for('highlight_submit_ignore_url', uuid=uuid)}}";
-    </script>
-    <script src="{{ url_for('static_content', group='js', filename='diff-overview.js') }}" defer></script>
-    <script src="{{ url_for('static_content', group='js', filename='preview.js') }}" defer></script>
-    <script src="{{ url_for('static_content', group='js', filename='tabs.js') }}" defer></script>
-    {% if versions|length >= 2 %}
-        <div id="settings" style="text-align: center;">
-            <form class="pure-form " action="" method="POST">
-                <fieldset>
-                    <label for="preview-version">Select timestamp</label> <select id="preview-version"
-                                                                                 name="from_version"
-                                                                                 class="needs-localtime">
-                    {% for version in versions|reverse %}
-                        <option value="{{ version }}" {% if version == current_version %} selected="" {% endif %}>
-                            {{ version }}
-                        </option>
-                    {% endfor %}
-                </select>
-                    <button type="submit" class="pure-button pure-button-primary">Go</button>
-
-                </fieldset>
-            </form>
-                <br>
-                <strong>Keyboard: </strong><a href="" class="pure-button pure-button-primary" id="btn-previous">
-                &larr; Previous</a> &nbsp; <a class="pure-button pure-button-primary" id="btn-next" href="">
-                &rarr; Next</a>
-        </div>
+<script>
+    const screenshot_url="{{url_for('static_content', group='screenshot', filename=uuid)}}";
+    {% if last_error_screenshot %}
+    const error_screenshot_url="{{url_for('static_content', group='screenshot', filename=uuid, error_screenshot=1) }}";
     {% endif %}
+    const highlight_submit_ignore_url="{{url_for('highlight_submit_ignore_url', uuid=uuid)}}";
+</script>
+<script src="{{url_for('static_content', group='js', filename='diff-overview.js')}}" defer></script>
 
-    <div class="tabs">
-        <ul>
-            {% if last_error_text %}
-                <li class="tab" id="error-text-tab"><a href="#error-text">Error Text</a></li> {% endif %}
-            {% if last_error_screenshot %}
-                <li class="tab" id="error-screenshot-tab"><a href="#error-screenshot">Error Screenshot</a>
-                </li> {% endif %}
-            {% if history_n > 0 %}
-                <li class="tab" id="text-tab"><a href="#text">Text</a></li>
-                <li class="tab" id="screenshot-tab"><a href="#screenshot">Screenshot</a></li>
-            {% endif %}
-        </ul>
-    </div>
-
-
-    <div id="diff-ui">
-        <div class="tab-pane-inner" id="error-text">
-            <div class="snapshot-age error">{{ watch.error_text_ctime|format_seconds_ago }} seconds ago</div>
-            <pre>
+<script src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
+<div class="tabs">
+    <ul>
+        {% if last_error_text %}<li class="tab" id="error-text-tab"><a href="#error-text">Error Text</a></li> {% endif %}
+        {% if last_error_screenshot %}<li class="tab" id="error-screenshot-tab"><a href="#error-screenshot">Error Screenshot</a></li> {% endif %}
+        {% if history_n > 0 %}
+        <li class="tab" id="text-tab"><a href="#text">Text</a></li>
+        <li class="tab" id="screenshot-tab"><a href="#screenshot">Screenshot</a></li>
+        {% endif %}
+    </ul>
+</div>
+<form><input type="hidden" name="csrf_token" value="{{ csrf_token() }}"></form>
+<div id="diff-ui">
+    <div class="tab-pane-inner" id="error-text">
+        <div class="snapshot-age error">{{watch.error_text_ctime|format_seconds_ago}} seconds ago</div>
+        <pre>
             {{ last_error_text }}
         </pre>
-        </div>
-
-        <div class="tab-pane-inner" id="error-screenshot">
-            <div class="snapshot-age error">{{ watch.snapshot_error_screenshot_ctime|format_seconds_ago }} seconds ago
-            </div>
-            <img id="error-screenshot-img" style="max-width: 80%"
-                 alt="Current erroring screenshot from most recent request">
-        </div>
-
-        <div class="tab-pane-inner" id="text">
-            <div class="snapshot-age">{{ watch.snapshot_text_ctime|format_timestamp_timeago }}</div>
-            <span class="ignored">Grey lines are ignored</span> <span class="triggered">Blue lines are triggers</span>
-            <span class="tip"><strong>Pro-tip</strong>: Highlight text to add to ignore filters</span>
-
-            <table>
-                <tbody>
-                <tr>
-                    <td id="diff-col" class="highlightable-filter">
-                        {% for row in content %}
-                            <div class="{{ row.classes }}">{{ row.line }}</div>
-                        {% endfor %}
-                    </td>
-                </tr>
-                </tbody>
-            </table>
-        </div>
-
-        <div class="tab-pane-inner" id="screenshot">
-            <div class="tip">
-                For now, Differences are performed on text, not graphically, only the latest screenshot is available.
-            </div>
-            <br>
-            {% if is_html_webdriver %}
-                {% if screenshot %}
-                    <div class="snapshot-age">{{ watch.snapshot_screenshot_ctime|format_timestamp_timeago }}</div>
-                    <img style="max-width: 80%" id="screenshot-img" alt="Current screenshot from most recent request">
-                {% else %}
-                    No screenshot available just yet! Try rechecking the page.
-                {% endif %}
-            {% else %}
-                <strong>Screenshot requires Playwright/WebDriver enabled</strong>
-            {% endif %}
-        </div>
     </div>
+
+    <div class="tab-pane-inner" id="error-screenshot">
+        <div class="snapshot-age error">{{watch.snapshot_error_screenshot_ctime|format_seconds_ago}} seconds ago</div>
+        <img id="error-screenshot-img"  style="max-width: 80%" alt="Current erroring screenshot from most recent request" >
+    </div>
+
+    <div class="tab-pane-inner" id="text">
+        <div class="snapshot-age">{{watch.snapshot_text_ctime|format_timestamp_timeago}}</div>
+        <span class="ignored">Grey lines are ignored</span> <span class="triggered">Blue lines are triggers</span> <span class="tip"><strong>Pro-tip</strong>: Highlight text to add to ignore filters</span>
+
+        <table>
+            <tbody>
+            <tr>
+                <td id="diff-col" class="highlightable-filter">
+                    {% for row in content %}
+                    <div class="{{row.classes}}">{{row.line}}</div>
+                    {% endfor %}
+                </td>
+            </tr>
+            </tbody>
+        </table>
+    </div>
+
+     <div class="tab-pane-inner" id="screenshot">
+         <div class="tip">
+             For now, Differences are performed on text, not graphically, only the latest screenshot is available.
+         </div>
+         <br>
+         {% if is_html_webdriver %}
+           {% if screenshot %}
+             <div class="snapshot-age">{{watch.snapshot_screenshot_ctime|format_timestamp_timeago}}</div>
+             <img style="max-width: 80%" id="screenshot-img" alt="Current screenshot from most recent request" >
+           {% else %}
+              No screenshot available just yet! Try rechecking the page.
+           {% endif %}
+         {% else %}
+           <strong>Screenshot requires Playwright/WebDriver enabled</strong>
+         {% endif %}
+     </div>
+</div>
 {% endblock %}

changedetectionio/tests/visualselector/test_fetch_data.py

@@ -102,9 +102,10 @@ def test_basic_browserstep(client, live_server):
             "url": test_url,
             "tags": "",
             'fetch_backend': "html_webdriver",
-            'browser_steps-0-operation': 'Click element',
-            'browser_steps-0-selector': 'button[name=test-button]',
-            'browser_steps-0-optional_value': '',
+            'browser_steps-0-operation': 'Goto site',
+            'browser_steps-1-operation': 'Click element',
+            'browser_steps-1-selector': 'button[name=test-button]',
+            'browser_steps-1-optional_value': '',
             # For now, cookies doesnt work in headers because it must be a full cookiejar object
             'headers': "testheader: yes\buser-agent: MyCustomAgent",
         },
@@ -140,9 +141,10 @@ def test_basic_browserstep(client, live_server):
               "url": four_o_four_url,
               "tags": "",
               'fetch_backend': "html_webdriver",
-              'browser_steps-0-operation': 'Click element',
-              'browser_steps-0-selector': 'button[name=test-button]',
-              'browser_steps-0-optional_value': ''
+              'browser_steps-0-operation': 'Goto site',
+              'browser_steps-1-operation': 'Click element',
+              'browser_steps-1-selector': 'button[name=test-button]',
+              'browser_steps-1-optional_value': ''
         },
         follow_redirects=True
     )

requirements.txt

@@ -36,7 +36,7 @@ dnspython==2.3.0 # related to eventlet fixes
 # jq not available on Windows so must be installed manually
 
 # Notification library
-apprise~=1.8.0
+apprise~=1.7.4
 
 # apprise mqtt https://github.com/dgtlmoon/changedetection.io/issues/315
 # and 2.0.0 https://github.com/dgtlmoon/changedetection.io/issues/2241 not yet compatible
@@ -52,10 +52,7 @@ cryptography~=3.4
 beautifulsoup4
 
 # XPath filtering, lxml is required by bs4 anyway, but put it here to be safe.
-# #2328 - 5.2.0 and 5.2.1 had extra CPU flag CFLAGS set which was not compatible on older hardware
-#         It could be advantageous to run its own pypi package here with those performance flags set
-#         https://bugs.launchpad.net/lxml/+bug/2059910/comments/16
-lxml >=4.8.0,<6,!=5.2.0,!=5.2.1
+lxml >=4.8.0,<6
 
 # XPath 2.0-3.1 support - 4.2.0 broke something?
 elementpath==4.1.5