Ability to supply a special reformatting filter for 'source:'.. entries - '|pretty'

changedetectionio/blueprint/browser_steps/__init__.py

@@ -35,7 +35,7 @@ io_interface_context = None
 def construct_blueprint(datastore: ChangeDetectionStore):
     browser_steps_blueprint = Blueprint('browser_steps', __name__, template_folder="templates")
 
-    def start_browsersteps_session():
+    def start_browsersteps_session(watch_uuid):
         from . import nonContext
         from . import browser_steps
         import time
@@ -55,7 +55,7 @@ def construct_blueprint(datastore: ChangeDetectionStore):
             io_interface_context = nonContext.c_sync_playwright()
             # Start the Playwright context, which is actually a nodejs sub-process and communicates over STDIN/STDOUT pipes
             io_interface_context = io_interface_context.start()
-#Element is outside of the viewport
+
 
         # keep it alive for 10 seconds more than we advertise, sometimes it helps to keep it shutting down cleanly
         keepalive = "&timeout={}".format(((seconds_keepalive + 3) * 1000))
@@ -68,29 +68,53 @@ def construct_blueprint(datastore: ChangeDetectionStore):
             else:
                 return make_response(str(e), 401)
 
+        proxy_id = datastore.get_preferred_proxy_for_watch(uuid=watch_uuid)
+        proxy = None
+        if proxy_id:
+            proxy_url = datastore.proxy_list.get(proxy_id).get('url')
+            if proxy_url:
+
+                # Playwright needs separate username and password values
+                from urllib.parse import urlparse
+                parsed = urlparse(proxy_url)
+                proxy = {'server': proxy_url}
+
+                if parsed.username:
+                    proxy['username'] = parsed.username
+
+                if parsed.password:
+                    proxy['password'] = parsed.password
+
+                print("Browser Steps: UUID {} selected proxy {}".format(watch_uuid, proxy_url))
+
         # Tell Playwright to connect to Chrome and setup a new session via our stepper interface
         browsersteps_start_session['browserstepper'] = browser_steps.browsersteps_live_ui(
-            playwright_browser=browsersteps_start_session['browser'])
+            playwright_browser=browsersteps_start_session['browser'],
+            proxy=proxy)
 
         # For test
         #browsersteps_start_session['browserstepper'].action_goto_url(value="http://example.com?time="+str(time.time()))
 
         return browsersteps_start_session
 
+
     @login_optionally_required
     @browser_steps_blueprint.route("/browsersteps_start_session", methods=['GET'])
     def browsersteps_start_session():
         # A new session was requested, return sessionID
 
         import uuid
-        browsersteps_session_id = str(uuid.uuid4())
-
-        watch_uuid = request.args.get('uuid')
         global browsersteps_sessions
 
+        browsersteps_session_id = str(uuid.uuid4())
+        watch_uuid = request.args.get('uuid')
+
+        if not watch_uuid:
+            return make_response('No Watch UUID specified', 500)
+
         print("Starting connection with playwright")
         logging.debug("browser_steps.py connecting")
-        browsersteps_sessions[browsersteps_session_id] = start_browsersteps_session()
+        browsersteps_sessions[browsersteps_session_id] = start_browsersteps_session(watch_uuid)
         print("Starting connection with playwright - done")
         return {'browsersteps_session_id': browsersteps_session_id}
 

changedetectionio/blueprint/browser_steps/browser_steps.py

@@ -133,18 +133,18 @@ class steppable_browser_interface():
         self.page.wait_for_timeout(1000)
 
     def action_wait_for_seconds(self, selector, value):
-        self.page.wait_for_timeout(int(value) * 1000)
+        self.page.wait_for_timeout(float(value.strip()) * 1000)
 
     def action_wait_for_text(self, selector, value):
         import json
         v = json.dumps(value)
-        self.page.wait_for_function(f'document.querySelector("body").innerText.includes({v});', timeout=30000)
+        self.page.wait_for_function(f'document.querySelector("body").innerText.includes({v});', timeout=90000)
 
     def action_wait_for_text_in_element(self, selector, value):
         import json
         s = json.dumps(selector)
         v = json.dumps(value)
-        self.page.wait_for_function(f'document.querySelector({s}).innerText.includes({v});', timeout=30000)
+        self.page.wait_for_function(f'document.querySelector({s}).innerText.includes({v});', timeout=90000)
 
     # @todo - in the future make some popout interface to capture what needs to be set
     # https://playwright.dev/python/docs/api/class-keyboard

changedetectionio/processors/text_json_diff.py

@@ -54,6 +54,7 @@ class perform_site_check(difference_detection_processor):
         changed_detected = False
         screenshot = False  # as bytes
         stripped_text_from_html = ""
+        source_filter = None # A machine filter that can be applied to source: (url|filter)
 
         # DeepCopy so we can be sure we don't accidently change anything by reference
         watch = deepcopy(self.datastore.data['watching'].get(uuid))
@@ -249,6 +250,11 @@ class perform_site_check(difference_detection_processor):
 
                     for filter_rule in include_filters_rule:
                         # For HTML/XML we offer xpath as an option, just start a regular xPath "/.."
+                        if '|' in filter_rule:
+                            filter_rule, source_filter = filter_rule.split('|')
+                        else:
+                            source_filter = None
+
                         if filter_rule[0] == '/' or filter_rule.startswith('xpath:'):
                             html_content += html_tools.xpath_filter(xpath_filter=filter_rule.replace('xpath:', ''),
                                                                     html_content=fetcher.content,
@@ -258,6 +264,10 @@ class perform_site_check(difference_detection_processor):
                             html_content += html_tools.include_filters(include_filters=filter_rule,
                                                                        html_content=fetcher.content,
                                                                        append_pretty_line_formatting=not is_source)
+                        if source_filter == 'pretty':
+                            from bs4 import BeautifulSoup
+                            html_content = BeautifulSoup(html_content, 'html.parser').prettify()
+
 
                     if not html_content.strip():
                         raise FilterNotFoundInResponse(include_filters_rule)

changedetectionio/static/js/browser-steps.js

@@ -162,7 +162,9 @@ $(document).ready(function () {
                 ) {
                     // There could be many elements here, record them all and then we'll find out which is the most 'useful'
                     // (input, textarea, button, A etc)
-                    possible_elements.push(item);
+                    if (item.width < xpath_data['browser_width']) {
+                        possible_elements.push(item);
+                    }
                 }
             });
 
@@ -420,7 +422,6 @@ $(document).ready(function () {
             apply_buttons_disabled = false;
             $("#browsersteps-img").css('opacity', 1);
             $('ul#browser_steps li .control .apply').css('opacity', 1);
-            browserless_seconds_remaining = data.browser_time_remaining;
             $("#loading-status-text").hide();
             set_first_gotosite_disabled();
         }).fail(function (data) {

changedetectionio/static/js/visual-selector.js

@@ -61,7 +61,12 @@ $(document).ready(function () {
     function bootstrap_visualselector() {
         if (1) {
             // bootstrap it, this will trigger everything else
-            $("img#selector-background").bind('load', function () {
+            $("img#selector-background").on("error", function () {
+                $('.fetching-update-notice').html("<strong>Ooops!</strong> The VisualSelector tool needs atleast one fetched page, please unpause the watch and/or wait for the watch to complete fetching and then reload this page.");
+                $('.fetching-update-notice').css('color','#bb0000');
+                $('#selector-current-xpath').hide();
+                $('#clear-selector').hide();
+            }).bind('load', function () {
                 console.log("Loaded background...");
                 c = document.getElementById("selector-canvas");
                 // greyed out fill context
@@ -79,10 +84,11 @@ $(document).ready(function () {
             }).attr("src", screenshot_url);
         }
         // Tell visualSelector that the image should update
-        var s = $("img#selector-background").attr('src')+"?"+ new Date().getTime();
-        $("img#selector-background").attr('src',s)
+        var s = $("img#selector-background").attr('src') + "?" + new Date().getTime();
+        $("img#selector-background").attr('src', s)
     }
 
+    // This is fired once the img src is loaded in bootstrap_visualselector()
     function fetch_data() {
         // Image is ready
         $('.fetching-update-notice').html("Fetching element data..");
@@ -99,7 +105,8 @@ $(document).ready(function () {
             reflow_selector();
             $('.fetching-update-notice').fadeOut();
         });
-    };
+
+    }
 
 
     function set_scale() {

changedetectionio/templates/base.html

@@ -106,7 +106,7 @@
     </div>
     {% if hosted_sticky %}
       <div class="sticky-tab" id="hosted-sticky">
-        <a href="https://lemonade.changedetection.io/start?ref={{guid}}">Let us host your instance!</a>
+        <a href="https://changedetection.io/?ref={{guid}}">Let us host your instance!</a>
       </div>
     {% endif %}
     {% if left_sticky %}