remove script type

.github/workflows/test-only.yml

@@ -30,7 +30,7 @@ jobs:
 
           # Selenium+browserless
           docker run --network changedet-network -d --hostname selenium  -p 4444:4444 --rm --shm-size="2g"  selenium/standalone-chrome-debug:3.141.59
-          docker run --network changedet-network -d --hostname browserless -e "FUNCTION_BUILT_INS=[\"fs\",\"crypto\"]" -e "DEFAULT_LAUNCH_ARGS=[\"--window-size=1920,1080\"]" --rm  -p 3000:3000  --shm-size="2g"  browserless/chrome:1.53-chrome-stable
+          docker run --network changedet-network -d --hostname browserless -e "DEFAULT_LAUNCH_ARGS=[\"--window-size=1920,1080\"]" --rm  -p 3000:3000  --shm-size="2g"  browserless/chrome:1.53-chrome-stable
 
       - name: Build changedetection.io container for testing
         run: |         
@@ -58,11 +58,6 @@ jobs:
           # restock detection via playwright - added name=changedet here so that playwright/browserless can connect to it
           docker run --rm --name "changedet" -e "FLASK_SERVER_NAME=changedet" -e "PLAYWRIGHT_DRIVER_URL=ws://browserless:3000" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-port=5004 --live-server-host=0.0.0.0 tests/restock/test_restock.py'
 
-      - name: Test with puppeteer fetcher and disk cache
-        run: |
-          docker run --rm -e "PUPPETEER_DISK_CACHE=/tmp/data/" -e "USE_EXPERIMENTAL_PUPPETEER_FETCH=yes" -e "PLAYWRIGHT_DRIVER_URL=ws://browserless:3000" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio;pytest tests/fetchers/test_content.py && pytest tests/test_errorhandling.py && pytest tests/visualselector/test_fetch_data.py'
-          # Browserless would have had -e "FUNCTION_BUILT_INS=[\"fs\",\"crypto\"]" added above
-
       - name: Test proxy interaction
         run: |
           cd changedetectionio

changedetectionio/__init__.py

@@ -1313,13 +1313,6 @@ def changedetection_app(config=None, datastore_o=None):
                     update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid, 'skip_when_checksum_same': False}))
             flash("{} watches queued for rechecking".format(len(uuids)))
 
-        elif (op == 'clear-history'):
-            for uuid in uuids:
-                uuid = uuid.strip()
-                if datastore.data['watching'].get(uuid):
-                    datastore.clear_watch_history(uuid)
-            flash("{} watches cleared/reset.".format(len(uuids)))
-
         elif (op == 'notification-default'):
             from changedetectionio.notification import (
                 default_notification_format_for_watch

changedetectionio/blueprint/browser_steps/__init__.py

@@ -27,106 +27,58 @@ import os
 import logging
 from changedetectionio.store import ChangeDetectionStore
 from changedetectionio import login_optionally_required
+browsersteps_live_ui_o = {}
+browsersteps_playwright_browser_interface = None
+browsersteps_playwright_browser_interface_browser = None
+browsersteps_playwright_browser_interface_context = None
+browsersteps_playwright_browser_interface_end_time = None
+browsersteps_playwright_browser_interface_start_time = None
 
-browsersteps_sessions = {}
-io_interface_context = None
+def cleanup_playwright_session():
 
+    global browsersteps_live_ui_o
+    global browsersteps_playwright_browser_interface
+    global browsersteps_playwright_browser_interface_browser
+    global browsersteps_playwright_browser_interface_context
+    global browsersteps_playwright_browser_interface_end_time
+    global browsersteps_playwright_browser_interface_start_time
+
+    browsersteps_live_ui_o = {}
+    browsersteps_playwright_browser_interface = None
+    browsersteps_playwright_browser_interface_browser = None
+    browsersteps_playwright_browser_interface_end_time = None
+    browsersteps_playwright_browser_interface_start_time = None
+
+    print("Cleaning up old playwright session because time was up, calling .goodbye()")
+    try:
+        browsersteps_playwright_browser_interface_context.goodbye()
+    except Exception as e:
+        print ("Got exception in shutdown, probably OK")
+        print (str(e))
+
+    browsersteps_playwright_browser_interface_context = None
+
+    print ("Cleaning up old playwright session because time was up - done")
 
 def construct_blueprint(datastore: ChangeDetectionStore):
+
     browser_steps_blueprint = Blueprint('browser_steps', __name__, template_folder="templates")
 
-    def start_browsersteps_session(watch_uuid):
-        from . import nonContext
-        from . import browser_steps
-        import time
-        global browsersteps_sessions
-        global io_interface_context
-
-
-        # We keep the playwright session open for many minutes
-        seconds_keepalive = int(os.getenv('BROWSERSTEPS_MINUTES_KEEPALIVE', 10)) * 60
-
-        browsersteps_start_session = {'start_time': time.time()}
-
-        # You can only have one of these running
-        # This should be very fine to leave running for the life of the application
-        # @idea - Make it global so the pool of watch fetchers can use it also
-        if not io_interface_context:
-            io_interface_context = nonContext.c_sync_playwright()
-            # Start the Playwright context, which is actually a nodejs sub-process and communicates over STDIN/STDOUT pipes
-            io_interface_context = io_interface_context.start()
-
-
-        # keep it alive for 10 seconds more than we advertise, sometimes it helps to keep it shutting down cleanly
-        keepalive = "&timeout={}".format(((seconds_keepalive + 3) * 1000))
-        try:
-            browsersteps_start_session['browser'] = io_interface_context.chromium.connect_over_cdp(
-                os.getenv('PLAYWRIGHT_DRIVER_URL', '') + keepalive)
-        except Exception as e:
-            if 'ECONNREFUSED' in str(e):
-                return make_response('Unable to start the Playwright Browser session, is it running?', 401)
-            else:
-                return make_response(str(e), 401)
-
-        proxy_id = datastore.get_preferred_proxy_for_watch(uuid=watch_uuid)
-        proxy = None
-        if proxy_id:
-            proxy_url = datastore.proxy_list.get(proxy_id).get('url')
-            if proxy_url:
-
-                # Playwright needs separate username and password values
-                from urllib.parse import urlparse
-                parsed = urlparse(proxy_url)
-                proxy = {'server': proxy_url}
-
-                if parsed.username:
-                    proxy['username'] = parsed.username
-
-                if parsed.password:
-                    proxy['password'] = parsed.password
-
-                print("Browser Steps: UUID {} selected proxy {}".format(watch_uuid, proxy_url))
-
-        # Tell Playwright to connect to Chrome and setup a new session via our stepper interface
-        browsersteps_start_session['browserstepper'] = browser_steps.browsersteps_live_ui(
-            playwright_browser=browsersteps_start_session['browser'],
-            proxy=proxy)
-
-        # For test
-        #browsersteps_start_session['browserstepper'].action_goto_url(value="http://example.com?time="+str(time.time()))
-
-        return browsersteps_start_session
-
-
     @login_optionally_required
-    @browser_steps_blueprint.route("/browsersteps_start_session", methods=['GET'])
-    def browsersteps_start_session():
-        # A new session was requested, return sessionID
-
-        import uuid
-        global browsersteps_sessions
-
-        browsersteps_session_id = str(uuid.uuid4())
-        watch_uuid = request.args.get('uuid')
-
-        if not watch_uuid:
-            return make_response('No Watch UUID specified', 500)
-
-        print("Starting connection with playwright")
-        logging.debug("browser_steps.py connecting")
-        browsersteps_sessions[browsersteps_session_id] = start_browsersteps_session(watch_uuid)
-        print("Starting connection with playwright - done")
-        return {'browsersteps_session_id': browsersteps_session_id}
-
-    # A request for an action was received
-    @login_optionally_required
-    @browser_steps_blueprint.route("/browsersteps_update", methods=['POST'])
+    @browser_steps_blueprint.route("/browsersteps_update", methods=['GET', 'POST'])
     def browsersteps_ui_update():
         import base64
         import playwright._impl._api_types
-        global browsersteps_sessions
+        import time
+
         from changedetectionio.blueprint.browser_steps import browser_steps
 
+        global browsersteps_live_ui_o, browsersteps_playwright_browser_interface_end_time
+        global browsersteps_playwright_browser_interface_browser
+        global browsersteps_playwright_browser_interface
+        global browsersteps_playwright_browser_interface_start_time
+
+        step_n = None
         remaining =0
         uuid = request.args.get('uuid')
 
@@ -135,9 +87,13 @@ def construct_blueprint(datastore: ChangeDetectionStore):
         if not browsersteps_session_id:
             return make_response('No browsersteps_session_id specified', 500)
 
-        if not browsersteps_sessions.get(browsersteps_session_id):
-            return make_response('No session exists under that ID', 500)
-
+        # Because we don't "really" run in a context manager ( we make the playwright interface global/long-living )
+        # We need to manage the shutdown when the time is up
+        if browsersteps_playwright_browser_interface_end_time:
+            remaining = browsersteps_playwright_browser_interface_end_time-time.time()
+            if browsersteps_playwright_browser_interface_end_time and remaining <= 0:
+                cleanup_playwright_session()
+                return make_response('Browser session expired, please reload the Browser Steps interface', 401)
 
         # Actions - step/apply/etc, do the thing and return state
         if request.method == 'POST':
@@ -156,7 +112,12 @@ def construct_blueprint(datastore: ChangeDetectionStore):
             # @todo try.. accept.. nice errors not popups..
             try:
 
-                browsersteps_sessions[browsersteps_session_id]['browserstepper'].call_action(action_name=step_operation,
+                this_session = browsersteps_live_ui_o.get(browsersteps_session_id)
+                if not this_session:
+                    print("Browser exited")
+                    return make_response('Browser session ran out of time :( Please reload this page.', 401)
+
+                this_session.call_action(action_name=step_operation,
                                          selector=step_selector,
                                          optional_value=step_optional_value)
 
@@ -168,43 +129,99 @@ def construct_blueprint(datastore: ChangeDetectionStore):
             # Get visual selector ready/update its data (also use the current filter info from the page?)
             # When the last 'apply' button was pressed
             # @todo this adds overhead because the xpath selection is happening twice
-            u = browsersteps_sessions[browsersteps_session_id]['browserstepper'].page.url
+            u = this_session.page.url
             if is_last_step and u:
-                (screenshot, xpath_data) = browsersteps_sessions[browsersteps_session_id]['browserstepper'].request_visualselector_data()
+                (screenshot, xpath_data) = this_session.request_visualselector_data()
                 datastore.save_screenshot(watch_uuid=uuid, screenshot=screenshot)
                 datastore.save_xpath_data(watch_uuid=uuid, data=xpath_data)
 
-#        if not this_session.page:
-#            cleanup_playwright_session()
-#            return make_response('Browser session ran out of time :( Please reload this page.', 401)
+        # Setup interface
+        if request.method == 'GET':
 
-        # Screenshots and other info only needed on requesting a step (POST)
-        try:
-            state = browsersteps_sessions[browsersteps_session_id]['browserstepper'].get_current_state()
-        except playwright._impl._api_types.Error as e:
-            return make_response("Browser session ran out of time :( Please reload this page."+str(e), 401)
+            if not browsersteps_playwright_browser_interface:
+                print("Starting connection with playwright")
+                logging.debug("browser_steps.py connecting")
 
-        # Use send_file() which is way faster than read/write loop on bytes
-        import json
-        from tempfile import mkstemp
-        from flask import send_file
-        tmp_fd, tmp_file = mkstemp(text=True, suffix=".json", prefix="changedetectionio-")
+                global browsersteps_playwright_browser_interface_context
+                from . import nonContext
+                browsersteps_playwright_browser_interface_context = nonContext.c_sync_playwright()
+                browsersteps_playwright_browser_interface = browsersteps_playwright_browser_interface_context.start()
 
-        output = json.dumps({'screenshot': "data:image/jpeg;base64,{}".format(
-            base64.b64encode(state[0]).decode('ascii')),
-            'xpath_data': state[1],
-            'session_age_start': browsersteps_sessions[browsersteps_session_id]['browserstepper'].age_start,
-            'browser_time_remaining': round(remaining)
-        })
+                time.sleep(1)
+                # At 20 minutes, some other variable is closing it
+                # @todo find out what it is and set it
+                seconds_keepalive = int(os.getenv('BROWSERSTEPS_MINUTES_KEEPALIVE', 10)) * 60
 
-        with os.fdopen(tmp_fd, 'w') as f:
-            f.write(output)
+                # keep it alive for 10 seconds more than we advertise, sometimes it helps to keep it shutting down cleanly
+                keepalive = "&timeout={}".format(((seconds_keepalive+3) * 1000))
+                try:
+                    browsersteps_playwright_browser_interface_browser = browsersteps_playwright_browser_interface.chromium.connect_over_cdp(
+                        os.getenv('PLAYWRIGHT_DRIVER_URL', '') + keepalive)
+                except Exception as e:
+                    if 'ECONNREFUSED' in str(e):
+                        return make_response('Unable to start the Playwright session properly, is it running?', 401)
 
-        response = make_response(send_file(path_or_file=tmp_file,
-                                           mimetype='application/json; charset=UTF-8',
-                                           etag=True))
-        # No longer needed
-        os.unlink(tmp_file)
+                browsersteps_playwright_browser_interface_end_time = time.time() + (seconds_keepalive-3)
+                print("Starting connection with playwright - done")
+
+            if not browsersteps_live_ui_o.get(browsersteps_session_id):
+                # Boot up a new session
+                proxy_id = datastore.get_preferred_proxy_for_watch(uuid=uuid)
+                proxy = None
+                if proxy_id:
+                    proxy_url = datastore.proxy_list.get(proxy_id).get('url')
+                    if proxy_url:
+                        proxy = {'server': proxy_url}
+                        print("Browser Steps: UUID {} Using proxy {}".format(uuid, proxy_url))
+
+                # Begin the new "Playwright Context" that re-uses the playwright interface
+                # Each session is a "Playwright Context" as a list, that uses the playwright interface
+                browsersteps_live_ui_o[browsersteps_session_id] = browser_steps.browsersteps_live_ui(
+                    playwright_browser=browsersteps_playwright_browser_interface_browser,
+                    proxy=proxy)
+                this_session = browsersteps_live_ui_o[browsersteps_session_id]
+
+        if not this_session.page:
+            cleanup_playwright_session()
+            return make_response('Browser session ran out of time :( Please reload this page.', 401)
+
+        response = None
+
+        if request.method == 'POST':
+            # Screenshots and other info only needed on requesting a step (POST)
+            try:
+                state = this_session.get_current_state()
+            except playwright._impl._api_types.Error as e:
+                return make_response("Browser session ran out of time :( Please reload this page."+str(e), 401)
+
+            # Use send_file() which is way faster than read/write loop on bytes
+            import json
+            from tempfile import mkstemp
+            from flask import send_file
+            tmp_fd, tmp_file = mkstemp(text=True, suffix=".json", prefix="changedetectionio-")
+
+            output = json.dumps({'screenshot': "data:image/jpeg;base64,{}".format(
+                base64.b64encode(state[0]).decode('ascii')),
+                'xpath_data': state[1],
+                'session_age_start': this_session.age_start,
+                'browser_time_remaining': round(remaining)
+            })
+
+            with os.fdopen(tmp_fd, 'w') as f:
+                f.write(output)
+
+            response = make_response(send_file(path_or_file=tmp_file,
+                                               mimetype='application/json; charset=UTF-8',
+                                               etag=True))
+            # No longer needed
+            os.unlink(tmp_file)
+
+        elif request.method == 'GET':
+            # Just enough to get the session rolling, it will call for goto-site via POST next
+            response = make_response({
+                'session_age_start': this_session.age_start,
+                'browser_time_remaining': round(remaining)
+            })
 
         return response
 

changedetectionio/blueprint/browser_steps/browser_steps.py

@@ -71,10 +71,10 @@ class steppable_browser_interface():
             optional_value = str(jinja2_env.from_string(optional_value).render())
 
         action_handler(selector, optional_value)
-        self.page.wait_for_timeout(1.5 * 1000)
+        self.page.wait_for_timeout(3 * 1000)
         print("Call action done in", time.time() - now)
 
-    def action_goto_url(self, selector=None, value=None):
+    def action_goto_url(self, selector, value):
         # self.page.set_viewport_size({"width": 1280, "height": 5000})
         now = time.time()
         response = self.page.goto(value, timeout=0, wait_until='commit')
@@ -105,8 +105,7 @@ class steppable_browser_interface():
         print("Clicking element")
         if not len(selector.strip()):
             return
-
-        self.page.click(selector=selector, timeout=30 * 1000, delay=randint(200, 500))
+        self.page.click(selector, timeout=10 * 1000, delay=randint(200, 500))
 
     def action_click_element_if_exists(self, selector, value):
         import playwright._impl._api_types as _api_types
@@ -133,18 +132,18 @@ class steppable_browser_interface():
         self.page.wait_for_timeout(1000)
 
     def action_wait_for_seconds(self, selector, value):
-        self.page.wait_for_timeout(float(value.strip()) * 1000)
+        self.page.wait_for_timeout(int(value) * 1000)
 
     def action_wait_for_text(self, selector, value):
         import json
         v = json.dumps(value)
-        self.page.wait_for_function(f'document.querySelector("body").innerText.includes({v});', timeout=90000)
+        self.page.wait_for_function(f'document.querySelector("body").innerText.includes({v});', timeout=30000)
 
     def action_wait_for_text_in_element(self, selector, value):
         import json
         s = json.dumps(selector)
         v = json.dumps(value)
-        self.page.wait_for_function(f'document.querySelector({s}).innerText.includes({v});', timeout=90000)
+        self.page.wait_for_function(f'document.querySelector({s}).innerText.includes({v});', timeout=30000)
 
     # @todo - in the future make some popout interface to capture what needs to be set
     # https://playwright.dev/python/docs/api/class-keyboard

changedetectionio/content_fetcher.py

@@ -10,7 +10,6 @@ import time
 
 visualselector_xpath_selectors = 'div,span,form,table,tbody,tr,td,a,p,ul,li,h1,h2,h3,h4, header, footer, section, article, aside, details, main, nav, section, summary'
 
-
 class Non200ErrorCodeReceived(Exception):
     def __init__(self, status_code, url, screenshot=None, xpath_data=None, page_html=None):
         # Set this so we can use it in other parts of the app
@@ -25,12 +24,10 @@ class Non200ErrorCodeReceived(Exception):
             self.page_text = html_tools.html_to_text(page_html)
         return
 
-
 class checksumFromPreviousCheckWasTheSame(Exception):
     def __init__(self):
         return
 
-
 class JSActionExceptions(Exception):
     def __init__(self, status_code, url, screenshot, message=''):
         self.status_code = status_code
@@ -39,7 +36,6 @@ class JSActionExceptions(Exception):
         self.message = message
         return
 
-
 class BrowserStepsStepTimout(Exception):
     def __init__(self, step_n):
         self.step_n = step_n
@@ -55,7 +51,6 @@ class PageUnloadable(Exception):
         self.message = message
         return
 
-
 class EmptyReply(Exception):
     def __init__(self, status_code, url, screenshot=None):
         # Set this so we can use it in other parts of the app
@@ -64,7 +59,6 @@ class EmptyReply(Exception):
         self.screenshot = screenshot
         return
 
-
 class ScreenshotUnavailable(Exception):
     def __init__(self, status_code, url, page_html=None):
         # Set this so we can use it in other parts of the app
@@ -75,7 +69,6 @@ class ScreenshotUnavailable(Exception):
             self.page_text = html_to_text(page_html)
         return
 
-
 class ReplyWithContentButNoText(Exception):
     def __init__(self, status_code, url, screenshot=None):
         # Set this so we can use it in other parts of the app
@@ -84,14 +77,13 @@ class ReplyWithContentButNoText(Exception):
         self.screenshot = screenshot
         return
 
-
 class Fetcher():
     browser_steps = None
     browser_steps_screenshot_path = None
     content = None
     error = None
     fetcher_description = "No description"
-    headers = {}
+    headers = None
     status_code = None
     webdriver_js_execute_code = None
     xpath_data = None
@@ -113,6 +105,7 @@ class Fetcher():
         self.xpath_element_js = resource_string(__name__, "res/xpath_element_scraper.js").decode('utf-8')
         self.instock_data_js = resource_string(__name__, "res/stock-not-in-stock.js").decode('utf-8')
 
+
     @abstractmethod
     def get_error(self):
         return self.error
@@ -159,15 +152,13 @@ class Fetcher():
             interface = steppable_browser_interface()
             interface.page = self.page
 
-            valid_steps = filter(
-                lambda s: (s['operation'] and len(s['operation']) and s['operation'] != 'Choose one' and s['operation'] != 'Goto site'),
-                self.browser_steps)
+            valid_steps = filter(lambda s: (s['operation'] and len(s['operation']) and s['operation'] != 'Choose one' and s['operation'] != 'Goto site'), self.browser_steps)
 
             for step in valid_steps:
                 step_n += 1
                 print(">> Iterating check - browser Step n {} - {}...".format(step_n, step['operation']))
-                self.screenshot_step("before-" + str(step_n))
-                self.save_step_html("before-" + str(step_n))
+                self.screenshot_step("before-"+str(step_n))
+                self.save_step_html("before-"+str(step_n))
                 try:
                     optional_value = step['optional_value']
                     selector = step['selector']
@@ -182,11 +173,12 @@ class Fetcher():
                                                       optional_value=optional_value)
                     self.screenshot_step(step_n)
                     self.save_step_html(step_n)
-                except TimeoutError as e:
-                    print(str(e))
+                except TimeoutError:
                     # Stop processing here
                     raise BrowserStepsStepTimout(step_n=step_n)
 
+
+
     # It's always good to reset these
     def delete_browser_steps_screenshots(self):
         import glob
@@ -196,7 +188,6 @@ class Fetcher():
             for f in files:
                 os.unlink(f)
 
-
 #   Maybe for the future, each fetcher provides its own diff output, could be used for text, image
 #   the current one would return javascript output (as we use JS to generate the diff)
 #
@@ -214,7 +205,6 @@ def available_fetchers():
 
     return p
 
-
 class base_html_playwright(Fetcher):
     fetcher_description = "Playwright {}/Javascript".format(
         os.getenv("PLAYWRIGHT_BROWSER_TYPE", 'chromium').capitalize()
@@ -278,118 +268,6 @@ class base_html_playwright(Fetcher):
         with open(destination, 'w') as f:
             f.write(content)
 
-    def run_fetch_browserless_puppeteer(self,
-            url,
-            timeout,
-            request_headers,
-            request_body,
-            request_method,
-            ignore_status_codes=False,
-            current_include_filters=None,
-            is_binary=False):
-
-        from pkg_resources import resource_string
-
-        extra_wait_ms = (int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)) + self.render_extract_delay) * 1000
-
-        self.xpath_element_js = self.xpath_element_js.replace('%ELEMENTS%', visualselector_xpath_selectors)
-        code = resource_string(__name__, "res/puppeteer_fetch.js").decode('utf-8')
-        # In the future inject this is a proper JS package
-        code = code.replace('%xpath_scrape_code%', self.xpath_element_js)
-        code = code.replace('%instock_scrape_code%', self.instock_data_js)
-
-        from requests.exceptions import ConnectTimeout, ReadTimeout
-        wait_browserless_seconds = 240
-
-        browserless_function_url = os.getenv('BROWSERLESS_FUNCTION_URL')
-        from urllib.parse import urlparse
-        if not browserless_function_url:
-            # Convert/try to guess from PLAYWRIGHT_DRIVER_URL
-            o = urlparse(os.getenv('PLAYWRIGHT_DRIVER_URL'))
-            browserless_function_url = o._replace(scheme="http")._replace(path="function").geturl()
-
-
-        # Append proxy connect string
-        if self.proxy:
-            import urllib.parse
-            # Remove username/password if it exists in the URL or you will receive "ERR_NO_SUPPORTED_PROXIES" error
-            # Actual authentication handled by Puppeteer/node
-            o = urlparse(self.proxy.get('server'))
-            proxy_url = urllib.parse.quote(o._replace(netloc="{}:{}".format(o.hostname, o.port)).geturl())
-            browserless_function_url = f"{browserless_function_url}&--proxy-server={proxy_url}&dumpio=true"
-
-
-        try:
-            amp = '&' if '?' in browserless_function_url else '?'
-            response = requests.request(
-                method="POST",
-                json={
-                    "code": code,
-                    "context": {
-                        # Very primitive disk cache - USE WITH EXTREME CAUTION
-                        # Run browserless container  with -e "FUNCTION_BUILT_INS=[\"fs\",\"crypto\"]"
-                        'disk_cache_dir': os.getenv("PUPPETEER_DISK_CACHE", False), # or path to disk cache ending in /, ie /tmp/cache/
-                        'execute_js': self.webdriver_js_execute_code,
-                        'extra_wait_ms': extra_wait_ms,
-                        'include_filters': current_include_filters,
-                        'req_headers': request_headers,
-                        'screenshot_quality': int(os.getenv("PLAYWRIGHT_SCREENSHOT_QUALITY", 72)),
-                        'url': url,
-                        'user_agent': request_headers.get('User-Agent', 'Mozilla/5.0'),
-                        'proxy_username': self.proxy.get('username','') if self.proxy else False,
-                        'proxy_password': self.proxy.get('password', '') if self.proxy else False,
-                        'no_cache_list': [
-                            'twitter',
-                            '.pdf'
-                        ],
-                        # Could use https://github.com/easylist/easylist here, or install a plugin
-                        'block_url_list': [
-                            'adnxs.com',
-                            'analytics.twitter.com',
-                            'doubleclick.net',
-                            'google-analytics.com',
-                            'googletagmanager',
-                            'trustpilot.com'
-                        ]
-                    }
-                },
-                # @todo /function needs adding ws:// to http:// rebuild this
-                url=browserless_function_url+f"{amp}--disable-features=AudioServiceOutOfProcess&dumpio=true&--disable-remote-fonts",
-                timeout=wait_browserless_seconds)
-
-        except ReadTimeout:
-            raise PageUnloadable(url=url, status_code=None, message=f"No response from browserless in {wait_browserless_seconds}s")
-        except ConnectTimeout:
-            raise PageUnloadable(url=url, status_code=None, message=f"Timed out connecting to browserless, retrying..")
-        else:
-            # 200 Here means that the communication to browserless worked only, not the page state
-            if response.status_code == 200:
-                import base64
-
-                x = response.json()
-                if not x.get('screenshot'):
-                    # https://github.com/puppeteer/puppeteer/blob/v1.0.0/docs/troubleshooting.md#tips
-                    # https://github.com/puppeteer/puppeteer/issues/1834
-                    # https://github.com/puppeteer/puppeteer/issues/1834#issuecomment-381047051
-                    # Check your memory is shared and big enough
-                    raise ScreenshotUnavailable(url=url, status_code=None)
-
-                if not x.get('content', '').strip():
-                    raise EmptyReply(url=url, status_code=None)
-
-                if x.get('status_code', 200) != 200 and not ignore_status_codes:
-                    raise Non200ErrorCodeReceived(url=url, status_code=x.get('status_code', 200), page_html=x['content'])
-
-                self.content = x.get('content')
-                self.headers = x.get('headers')
-                self.instock_data = x.get('instock_data')
-                self.screenshot = base64.b64decode(x.get('screenshot'))
-                self.xpath_data = x.get('xpath_data')
-
-            else:
-                # Some other error from browserless
-                raise PageUnloadable(url=url, status_code=None, message=response.content.decode('utf-8'))
-
     def run(self,
             url,
             timeout,
@@ -400,24 +278,6 @@ class base_html_playwright(Fetcher):
             current_include_filters=None,
             is_binary=False):
 
-        # For now, USE_EXPERIMENTAL_PUPPETEER_FETCH is not supported by watches with BrowserSteps (for now!)
-        has_browser_steps = self.browser_steps and list(filter(
-                lambda s: (s['operation'] and len(s['operation']) and s['operation'] != 'Choose one' and s['operation'] != 'Goto site'),
-                self.browser_steps))
-
-        if not has_browser_steps:
-            if os.getenv('USE_EXPERIMENTAL_PUPPETEER_FETCH'):
-                # Temporary backup solution until we rewrite the playwright code
-                return self.run_fetch_browserless_puppeteer(
-                    url,
-                    timeout,
-                    request_headers,
-                    request_body,
-                    request_method,
-                    ignore_status_codes,
-                    current_include_filters,
-                    is_binary)
-
         from playwright.sync_api import sync_playwright
         import playwright._impl._api_types
 
@@ -434,7 +294,7 @@ class base_html_playwright(Fetcher):
             # Set user agent to prevent Cloudflare from blocking the browser
             # Use the default one configured in the App.py model that's passed from fetch_site_status.py
             context = browser.new_context(
-                user_agent=request_headers.get('User-Agent', 'Mozilla/5.0'),
+                user_agent=request_headers['User-Agent'] if request_headers.get('User-Agent') else 'Mozilla/5.0',
                 proxy=self.proxy,
                 # This is needed to enable JavaScript execution on GitHub and others
                 bypass_csp=True,
@@ -464,12 +324,12 @@ class base_html_playwright(Fetcher):
             except playwright._impl._api_types.Error as e:
                 # Retry once - https://github.com/browserless/chrome/issues/2485
                 # Sometimes errors related to invalid cert's and other can be random
-                print("Content Fetcher > retrying request got error - ", str(e))
+                print ("Content Fetcher > retrying request got error - ", str(e))
                 time.sleep(1)
                 response = self.page.goto(url, wait_until='commit')
 
             except Exception as e:
-                print("Content Fetcher > Other exception when page.goto", str(e))
+                print ("Content Fetcher > Other exception when page.goto", str(e))
                 context.close()
                 browser.close()
                 raise PageUnloadable(url=url, status_code=None, message=str(e))
@@ -488,7 +348,7 @@ class base_html_playwright(Fetcher):
                 # This can be ok, we will try to grab what we could retrieve
                 pass
             except Exception as e:
-                print("Content Fetcher > Other exception when executing custom JS code", str(e))
+                print ("Content Fetcher > Other exception when executing custom JS code", str(e))
                 context.close()
                 browser.close()
                 raise PageUnloadable(url=url, status_code=None, message=str(e))
@@ -496,7 +356,7 @@ class base_html_playwright(Fetcher):
             if response is None:
                 context.close()
                 browser.close()
-                print("Content Fetcher > Response object was none")
+                print ("Content Fetcher > Response object was none")
                 raise EmptyReply(url=url, status_code=None)
 
             # Run Browser Steps here
@@ -510,7 +370,7 @@ class base_html_playwright(Fetcher):
             if len(self.page.content().strip()) == 0:
                 context.close()
                 browser.close()
-                print("Content Fetcher > Content was empty")
+                print ("Content Fetcher > Content was empty")
                 raise EmptyReply(url=url, status_code=response.status)
 
             self.status_code = response.status
@@ -522,8 +382,7 @@ class base_html_playwright(Fetcher):
             else:
                 self.page.evaluate("var include_filters=''")
 
-            self.xpath_data = self.page.evaluate(
-                "async () => {" + self.xpath_element_js.replace('%ELEMENTS%', visualselector_xpath_selectors) + "}")
+            self.xpath_data = self.page.evaluate("async () => {" + self.xpath_element_js.replace('%ELEMENTS%', visualselector_xpath_selectors) + "}")
             self.instock_data = self.page.evaluate("async () => {" + self.instock_data_js + "}")
 
             # Bug 3 in Playwright screenshot handling
@@ -535,8 +394,7 @@ class base_html_playwright(Fetcher):
             # acceptable screenshot quality here
             try:
                 # The actual screenshot
-                self.screenshot = self.page.screenshot(type='jpeg', full_page=True,
-                                                       quality=int(os.getenv("PLAYWRIGHT_SCREENSHOT_QUALITY", 72)))
+                self.screenshot = self.page.screenshot(type='jpeg', full_page=True, quality=int(os.getenv("PLAYWRIGHT_SCREENSHOT_QUALITY", 72)))
             except Exception as e:
                 context.close()
                 browser.close()
@@ -545,7 +403,6 @@ class base_html_playwright(Fetcher):
             context.close()
             browser.close()
 
-
 class base_html_webdriver(Fetcher):
     if os.getenv("WEBDRIVER_URL"):
         fetcher_description = "WebDriver Chrome/Javascript via '{}'".format(os.getenv("WEBDRIVER_URL"))

changedetectionio/processors/text_json_diff.py

@@ -54,7 +54,6 @@ class perform_site_check(difference_detection_processor):
         changed_detected = False
         screenshot = False  # as bytes
         stripped_text_from_html = ""
-        source_filter = None # A machine filter that can be applied to source: (url|filter)
 
         # DeepCopy so we can be sure we don't accidently change anything by reference
         watch = deepcopy(self.datastore.data['watching'].get(uuid))
@@ -250,11 +249,6 @@ class perform_site_check(difference_detection_processor):
 
                     for filter_rule in include_filters_rule:
                         # For HTML/XML we offer xpath as an option, just start a regular xPath "/.."
-                        if '|' in filter_rule:
-                            filter_rule, source_filter = filter_rule.split('|')
-                        else:
-                            source_filter = None
-
                         if filter_rule[0] == '/' or filter_rule.startswith('xpath:'):
                             html_content += html_tools.xpath_filter(xpath_filter=filter_rule.replace('xpath:', ''),
                                                                     html_content=fetcher.content,
@@ -264,10 +258,6 @@ class perform_site_check(difference_detection_processor):
                             html_content += html_tools.include_filters(include_filters=filter_rule,
                                                                        html_content=fetcher.content,
                                                                        append_pretty_line_formatting=not is_source)
-                        if source_filter == 'pretty':
-                            from bs4 import BeautifulSoup
-                            html_content = BeautifulSoup(html_content, 'html.parser').prettify()
-
 
                     if not html_content.strip():
                         raise FilterNotFoundInResponse(include_filters_rule)

changedetectionio/res/puppeteer_fetch.js

@@ -1,183 +0,0 @@
-module.exports = async ({page, context}) => {
-
-    var {
-        url,
-        execute_js,
-        user_agent,
-        extra_wait_ms,
-        req_headers,
-        include_filters,
-        xpath_element_js,
-        screenshot_quality,
-        proxy_username,
-        proxy_password,
-        disk_cache_dir,
-        no_cache_list,
-        block_url_list,
-    } = context;
-
-    await page.setBypassCSP(true)
-    await page.setExtraHTTPHeaders(req_headers);
-    await page.setUserAgent(user_agent);
-    // https://ourcodeworld.com/articles/read/1106/how-to-solve-puppeteer-timeouterror-navigation-timeout-of-30000-ms-exceeded
-
-    await page.setDefaultNavigationTimeout(0);
-
-    if (proxy_username) {
-        await page.authenticate({
-            username: proxy_username,
-            password: proxy_password
-        });
-    }
-
-    await page.setViewport({
-        width: 1024,
-        height: 768,
-        deviceScaleFactor: 1,
-    });
-
-    await page.setRequestInterception(true);
-    if (disk_cache_dir) {
-        console.log(">>>>>>>>>>>>>>> LOCAL DISK CACHE ENABLED <<<<<<<<<<<<<<<<<<<<<");
-    }
-    const fs = require('fs');
-    const crypto = require('crypto');
-
-    function file_is_expired(file_path) {
-        if (!fs.existsSync(file_path)) {
-            return true;
-        }
-        var stats = fs.statSync(file_path);
-        const now_date = new Date();
-        const expire_seconds = 300;
-        if ((now_date / 1000) - (stats.mtime.getTime() / 1000) > expire_seconds) {
-            console.log("CACHE EXPIRED: " + file_path);
-            return true;
-        }
-        return false;
-
-    }
-
-    page.on('request', async (request) => {
-        // General blocking of requests that waste traffic
-        if (block_url_list.some(substring => request.url().toLowerCase().includes(substring))) return request.abort();
-
-        if (disk_cache_dir) {
-            const url = request.url();
-            const key = crypto.createHash('md5').update(url).digest("hex");
-            const dir_path = disk_cache_dir + key.slice(0, 1) + '/' + key.slice(1, 2) + '/' + key.slice(2, 3) + '/';
-
-            // https://stackoverflow.com/questions/4482686/check-synchronously-if-file-directory-exists-in-node-js
-
-            if (fs.existsSync(dir_path + key)) {
-                console.log("* CACHE HIT , using - " + dir_path + key + " - " + url);
-                const cached_data = fs.readFileSync(dir_path + key);
-                // @todo headers can come from dir_path+key+".meta" json file
-                request.respond({
-                    status: 200,
-                    //contentType: 'text/html', //@todo
-                    body: cached_data
-                });
-                return;
-            }
-        }
-        request.continue();
-    });
-
-
-    if (disk_cache_dir) {
-        page.on('response', async (response) => {
-            const url = response.url();
-            // Basic filtering for sane responses
-            if (response.request().method() != 'GET' || response.request().resourceType() == 'xhr' || response.request().resourceType() == 'document' || response.status() != 200) {
-                console.log("Skipping (not useful) - Status:" + response.status() + " Method:" + response.request().method() + " ResourceType:" + response.request().resourceType() + " " + url);
-                return;
-            }
-            if (no_cache_list.some(substring => url.toLowerCase().includes(substring))) {
-                console.log("Skipping (no_cache_list) - " + url);
-                return;
-            }
-            if (url.toLowerCase().includes('data:')) {
-                console.log("Skipping (embedded-data) - " + url);
-                return;
-            }
-            response.buffer().then(buffer => {
-                if (buffer.length > 100) {
-                    console.log("Cache - Saving " + response.request().method() + " - " + url + " - " + response.request().resourceType());
-
-                    const key = crypto.createHash('md5').update(url).digest("hex");
-                    const dir_path = disk_cache_dir + key.slice(0, 1) + '/' + key.slice(1, 2) + '/' + key.slice(2, 3) + '/';
-
-                    if (!fs.existsSync(dir_path)) {
-                        fs.mkdirSync(dir_path, {recursive: true})
-                    }
-
-                    if (fs.existsSync(dir_path + key)) {
-                        if (file_is_expired(dir_path + key)) {
-                            fs.writeFileSync(dir_path + key, buffer);
-                        }
-                    } else {
-                        fs.writeFileSync(dir_path + key, buffer);
-                    }
-                }
-            });
-        });
-    }
-
-    const r = await page.goto(url, {
-        waitUntil: 'load'
-    });
-
-    await page.waitForTimeout(1000);
-    await page.waitForTimeout(extra_wait_ms);
-
-    if (execute_js) {
-        await page.evaluate(execute_js);
-        await page.waitForTimeout(200);
-    }
-
-    var xpath_data;
-    var instock_data;
-    try {
-        // Not sure the best way here, in the future this should be a new package added to npm then run in browserless
-        // (Once the old playwright is removed)
-        xpath_data = await page.evaluate((include_filters) => {%xpath_scrape_code%}, include_filters);
-        instock_data = await page.evaluate(() => {%instock_scrape_code%});
-    } catch (e) {
-        console.log(e);
-    }
-
-    // Protocol error (Page.captureScreenshot): Cannot take screenshot with 0 width can come from a proxy auth failure
-    // Wrap it here (for now)
-
-    var b64s = false;
-    try {
-        b64s = await page.screenshot({encoding: "base64", fullPage: true, quality: screenshot_quality, type: 'jpeg'});
-    } catch (e) {
-        console.log(e);
-    }
-
-    // May fail on very large pages with 'WARNING: tile memory limits exceeded, some content may not draw'
-    if (!b64s) {
-        // @todo after text extract, we can place some overlay text with red background to say 'croppped'
-        console.error('ERROR: content-fetcher page was maybe too large for a screenshot, reverting to viewport only screenshot');
-        try {
-            b64s = await page.screenshot({encoding: "base64", quality: screenshot_quality, type: 'jpeg'});
-        } catch (e) {
-            console.log(e);
-        }
-    }
-
-    var html = await page.content();
-    return {
-        data: {
-            'content': html,
-            'headers': r.headers(),
-            'instock_data': instock_data,
-            'screenshot': b64s,
-            'status_code': r.status(),
-            'xpath_data': xpath_data
-        },
-        type: 'application/json',
-    };
-};

changedetectionio/res/xpath_element_scraper.js

@@ -8,14 +8,8 @@
 // Some pages like https://www.londonstockexchange.com/stock/NCCL/ncondezi-energy-limited/analysis
 // will automatically force a scroll somewhere, so include the position offset
 // Lets hope the position doesnt change while we iterate the bbox's, but this is better than nothing
-var scroll_y = 0;
-try {
-    scroll_y = +document.documentElement.scrollTop || document.body.scrollTop
-} catch (e) {
-    console.log(e);
-}
-
 
+var scroll_y=+document.documentElement.scrollTop || document.body.scrollTop
 
 // Include the getXpath script directly, easier than fetching
 function getxpath(e) {
@@ -44,15 +38,15 @@ const findUpTag = (el) => {
     if (el.name !== undefined && el.name.length) {
         var proposed = el.tagName + "[name=" + el.name + "]";
         var proposed_element = window.document.querySelectorAll(proposed);
-        if (proposed_element.length) {
+        if(proposed_element.length) {
             if (proposed_element.length === 1) {
                 return proposed;
             } else {
                 // Some sites change ID but name= stays the same, we can hit it if we know the index
                 // Find all the elements that match and work out the input[n]
-                var n = Array.from(proposed_element).indexOf(el);
+                var n=Array.from(proposed_element).indexOf(el);
                 // Return a Playwright selector for nthinput[name=zipcode]
-                return proposed + " >> nth=" + n;
+                return proposed+" >> nth="+n;
             }
         }
     }

changedetectionio/static/js/browser-steps.js

@@ -114,11 +114,11 @@ $(document).ready(function () {
             e.preventDefault()
         });
 
-        // When the mouse moves we know which element it should be above
-        // mousedown will link that to the UI (select the right action, highlight etc)
         $('#browsersteps-selector-canvas').bind('mousedown', function (e) {
             // https://developer.mozilla.org/en-US/docs/Web/API/MouseEvent
             e.preventDefault()
+            console.log(e);
+            console.log("current xpath in index is " + current_selected_i);
             last_click_xy = {'x': parseInt((1 / x_scale) * e.offsetX), 'y': parseInt((1 / y_scale) * e.offsetY)}
             process_selected(current_selected_i);
             current_selected_i = false;
@@ -132,7 +132,6 @@ $(document).ready(function () {
             }
         });
 
-        // Debounce and find the current most 'interesting' element we are hovering above
         $('#browsersteps-selector-canvas').bind('mousemove', function (e) {
             if (!xpath_data) {
                 return;
@@ -152,40 +151,41 @@ $(document).ready(function () {
             current_selected_i = false;
             // Reverse order - the most specific one should be deeper/"laster"
             // Basically, find the most 'deepest'
-            var possible_elements = [];
-            xpath_data['size_pos'].forEach(function (item, index) {
+            //$('#browsersteps-selector-canvas').css('cursor', 'pointer');
+            for (var i = xpath_data['size_pos'].length; i !== 0; i--) {
+                // draw all of them? let them choose somehow?
+                var sel = xpath_data['size_pos'][i - 1];
                 // If we are in a bounding-box
-                if (e.offsetY > item.top * y_scale && e.offsetY < item.top * y_scale + item.height * y_scale
+                if (e.offsetY > sel.top * y_scale && e.offsetY < sel.top * y_scale + sel.height * y_scale
                     &&
-                    e.offsetX > item.left * y_scale && e.offsetX < item.left * y_scale + item.width * y_scale
+                    e.offsetX > sel.left * y_scale && e.offsetX < sel.left * y_scale + sel.width * y_scale
 
                 ) {
-                    // There could be many elements here, record them all and then we'll find out which is the most 'useful'
-                    // (input, textarea, button, A etc)
-                    if (item.width < xpath_data['browser_width']) {
-                        possible_elements.push(item);
+                    // Only highlight these interesting types
+                    if (1) {
+                        ctx.strokeRect(sel.left * x_scale, sel.top * y_scale, sel.width * x_scale, sel.height * y_scale);
+                        ctx.fillRect(sel.left * x_scale, sel.top * y_scale, sel.width * x_scale, sel.height * y_scale);
+                        current_selected_i = i - 1;
+                        break;
+
+                        // find the smallest one at this x,y
+                        // does it mean sort the xpath list by size (w*h) i think so!
+                    } else {
+
+                        if (include_text_elements[0].checked === true) {
+                            // blue one with background instead?
+                            ctx.fillStyle = 'rgba(0,0,255, 0.1)';
+                            ctx.strokeStyle = 'rgba(0,0,200, 0.7)';
+                            $('#browsersteps-selector-canvas').css('cursor', 'grab');
+                            ctx.strokeRect(sel.left * x_scale, sel.top * y_scale, sel.width * x_scale, sel.height * y_scale);
+                            ctx.fillRect(sel.left * x_scale, sel.top * y_scale, sel.width * x_scale, sel.height * y_scale);
+                            current_selected_i = i - 1;
+                            break;
+                        }
                     }
                 }
-            });
-
-            // Find the best one
-            if (possible_elements.length) {
-                possible_elements.forEach(function (item, index) {
-                  if (["a", "input", "textarea", "button"].includes(item['tagName'])) {
-                      current_selected_i = item;
-                  }
-                });
-
-                if (!current_selected_i) {
-                    current_selected_i = possible_elements[0];
-                }
-
-                sel = xpath_data['size_pos'][current_selected_i];
-                ctx.strokeRect(current_selected_i.left * x_scale, current_selected_i.top * y_scale, current_selected_i.width * x_scale, current_selected_i.height * y_scale);
-                ctx.fillRect(current_selected_i.left * x_scale, current_selected_i.top * y_scale, current_selected_i.width * x_scale, current_selected_i.height * y_scale);
             }
 
-
         }.debounce(10));
     });
 
@@ -195,16 +195,16 @@ $(document).ready(function () {
 
 
     // callback for clicking on an xpath on the canvas
-    function process_selected(selected_in_xpath_list) {
+    function process_selected(xpath_data_index) {
         found_something = false;
         var first_available = $("ul#browser_steps li.empty").first();
 
 
-        if (selected_in_xpath_list !== false) {
+        if (xpath_data_index !== false) {
             // Nothing focused, so fill in a new one
             // if inpt type button or <button>
             // from the top, find the next not used one and use it
-            var x = selected_in_xpath_list;
+            var x = xpath_data['size_pos'][xpath_data_index];
             console.log(x);
             if (x && first_available.length) {
                 // @todo will it let you click shit that has a layer ontop? probably not.
@@ -214,18 +214,26 @@ $(document).ready(function () {
                     $('input[placeholder="Value"]', first_available).addClass('ok').click().focus();
                     found_something = true;
                 } else {
-                    // There's no good way (that I know) to find if this
-                    // see https://stackoverflow.com/questions/446892/how-to-find-event-listeners-on-a-dom-node-in-javascript-or-in-debugging
-                    // https://codepen.io/azaslavsky/pen/DEJVWv
-
-                    // So we dont know if its really a clickable element or not :-(
-                    // Assume it is - then we dont fill the pages with unreliable "Click X,Y" selections
-                    // If you switch to "Click X,y" after an element here is setup, it will give the last co-ords anyway
-                    //if (x['isClickable'] || x['tagName'].startsWith('h') || x['tagName'] === 'a' || x['tagName'] === 'button' || x['tagtype'] === 'submit' || x['tagtype'] === 'checkbox' || x['tagtype'] === 'radio' || x['tagtype'] === 'li') {
+                    if (x['isClickable'] || x['tagName'].startsWith('h') || x['tagName'] === 'a' || x['tagName'] === 'button' || x['tagtype'] === 'submit' || x['tagtype'] === 'checkbox' || x['tagtype'] === 'radio' || x['tagtype'] === 'li') {
                         $('select', first_available).val('Click element').change();
                         $('input[type=text]', first_available).first().val(x['xpath']);
                         found_something = true;
-                    //}
+                    }
+                }
+
+                first_available.xpath_data_index = xpath_data_index;
+
+                if (!found_something) {
+                    if (include_text_elements[0].checked === true) {
+                        // Suggest that we use as filter?
+                        // @todo filters should always be in the last steps, nothing non-filter after it
+                        found_something = true;
+                        ctx.strokeStyle = 'rgba(0,0,255, 0.9)';
+                        ctx.fillStyle = 'rgba(0,0,255, 0.1)';
+                        $('select', first_available).val('Extract text and use as filter').change();
+                        $('input[type=text]', first_available).first().val(x['xpath']);
+                        include_text_elements[0].checked = false;
+                    }
                 }
             }
         }
@@ -240,7 +248,7 @@ $(document).ready(function () {
 
     function start() {
         console.log("Starting browser-steps UI");
-        browsersteps_session_id = false;
+        browsersteps_session_id = Date.now();
         // @todo This setting of the first one should be done at the datalayer but wtforms doesnt wanna play nice
         $('#browser_steps >li:first-child').removeClass('empty');
         set_first_gotosite_disabled();
@@ -248,7 +256,7 @@ $(document).ready(function () {
         $('.clear,.remove', $('#browser_steps >li:first-child')).hide();
         $.ajax({
             type: "GET",
-            url: browser_steps_start_url,
+            url: browser_steps_sync_url + "&browsersteps_session_id=" + browsersteps_session_id,
             statusCode: {
                 400: function () {
                     // More than likely the CSRF token was lost when the server restarted
@@ -256,12 +264,12 @@ $(document).ready(function () {
                 }
             }
         }).done(function (data) {
+            xpath_data = data.xpath_data;
             $("#loading-status-text").fadeIn();
-            browsersteps_session_id = data.browsersteps_session_id;
             // This should trigger 'Goto site'
             console.log("Got startup response, requesting Goto-Site (first) step fake click");
             $('#browser_steps >li:first-child .apply').click();
-            browserless_seconds_remaining = 500;
+            browserless_seconds_remaining = data.browser_time_remaining;
             set_first_gotosite_disabled();
         }).fail(function (data) {
             console.log(data);
@@ -422,6 +430,7 @@ $(document).ready(function () {
             apply_buttons_disabled = false;
             $("#browsersteps-img").css('opacity', 1);
             $('ul#browser_steps li .control .apply').css('opacity', 1);
+            browserless_seconds_remaining = data.browser_time_remaining;
             $("#loading-status-text").hide();
             set_first_gotosite_disabled();
         }).fail(function (data) {

changedetectionio/static/js/tabs.js

@@ -12,7 +12,7 @@ window.addEventListener('hashchange', function () {
 var has_errors = document.querySelectorAll(".messages .error");
 if (!has_errors.length) {
     if (document.location.hash == "") {
-        location.replace(document.querySelector(".tabs ul li:first-child a").hash);
+        document.querySelector(".tabs ul li:first-child a").click();
     } else {
         set_active_tab();
     }

changedetectionio/static/js/visual-selector.js

@@ -61,12 +61,7 @@ $(document).ready(function () {
     function bootstrap_visualselector() {
         if (1) {
             // bootstrap it, this will trigger everything else
-            $("img#selector-background").on("error", function () {
-                $('.fetching-update-notice').html("<strong>Ooops!</strong> The VisualSelector tool needs atleast one fetched page, please unpause the watch and/or wait for the watch to complete fetching and then reload this page.");
-                $('.fetching-update-notice').css('color','#bb0000');
-                $('#selector-current-xpath').hide();
-                $('#clear-selector').hide();
-            }).bind('load', function () {
+            $("img#selector-background").bind('load', function () {
                 console.log("Loaded background...");
                 c = document.getElementById("selector-canvas");
                 // greyed out fill context
@@ -84,11 +79,10 @@ $(document).ready(function () {
             }).attr("src", screenshot_url);
         }
         // Tell visualSelector that the image should update
-        var s = $("img#selector-background").attr('src') + "?" + new Date().getTime();
-        $("img#selector-background").attr('src', s)
+        var s = $("img#selector-background").attr('src')+"?"+ new Date().getTime();
+        $("img#selector-background").attr('src',s)
     }
 
-    // This is fired once the img src is loaded in bootstrap_visualselector()
     function fetch_data() {
         // Image is ready
         $('.fetching-update-notice').html("Fetching element data..");
@@ -105,8 +99,7 @@ $(document).ready(function () {
             reflow_selector();
             $('.fetching-update-notice').fadeOut();
         });
-
-    }
+    };
 
 
     function set_scale() {

changedetectionio/templates/base.html

@@ -106,7 +106,7 @@
     </div>
     {% if hosted_sticky %}
       <div class="sticky-tab" id="hosted-sticky">
-        <a href="https://changedetection.io/?ref={{guid}}">Let us host your instance!</a>
+        <a href="https://lemonade.changedetection.io/start?ref={{guid}}">Let us host your instance!</a>
       </div>
     {% endif %}
     {% if left_sticky %}

changedetectionio/templates/edit.html

@@ -14,9 +14,7 @@
 {% endif %}
 
     const browser_steps_config=JSON.parse('{{ browser_steps_config|tojson }}');
-    const browser_steps_start_url="{{url_for('browser_steps.browsersteps_start_session', uuid=uuid)}}";
     const browser_steps_sync_url="{{url_for('browser_steps.browsersteps_ui_update', uuid=uuid)}}";
-
 </script>
 
 <script src="{{url_for('static_content', group='js', filename='watch-settings.js')}}" defer></script>
@@ -188,8 +186,7 @@ User-Agent: wonderbra 1.0") }}
                                     <span class="loader" >
                                         <span id="browsersteps-click-start">
                                             <h2 >Click here to Start</h2>
-                                            <svg style="height: 3.5rem;" version="1.1" viewBox="0 0 32 32"  xml:space="preserve" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink"><g id="Layer_1"/><g id="play_x5F_alt"><path d="M16,0C7.164,0,0,7.164,0,16s7.164,16,16,16s16-7.164,16-16S24.836,0,16,0z M10,24V8l16.008,8L10,24z" style="fill: var(--color-grey-400);"/></g></svg><br>
-                                            Please allow 10-15 seconds for the browser to connect.<br>
+                                            Please allow 10-15 seconds for the browser to connect.
                                         </span>
                                         <div class="spinner"  style="display: none;"></div>
                                     </span>

changedetectionio/templates/watch-overview.html

@@ -38,8 +38,7 @@
         <button class="pure-button button-secondary button-xsmall" name="op" value="recheck">Recheck</button>
         <button class="pure-button button-secondary button-xsmall" name="op" value="mark-viewed">Mark viewed</button>
         <button class="pure-button button-secondary button-xsmall" name="op" value="notification-default">Use default notification</button>
-        <button class="pure-button button-secondary button-xsmall" style="background: #dd4242;" name="op" value="clear-history">Clear/reset history</button>
-        <button class="pure-button button-secondary button-xsmall" style="background: #dd4242;" name="op" value="delete">Delete</button>
+        <button class="pure-button button-secondary button-xsmall" style="background: #dd4242; font-size: 70%" name="op" value="delete">Delete</button>
     </div>
     {% if watches|length >= pagination.per_page %}
         {{ pagination.info }}

changedetectionio/update_worker.py

@@ -315,8 +315,7 @@ class update_worker(threading.Thread):
                             self.datastore.save_screenshot(watch_uuid=uuid, screenshot=e.screenshot, as_error=True)
 
                         self.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text,
-                                                                           'last_check_status': e.status_code,
-                                                                           'has_ldjson_price_data': None})
+                                                                           'last_check_status': e.status_code})
                         process_changedetection_results = False
                     except Exception as e:
                         self.app.logger.error("Exception reached processing watch UUID: %s - %s", uuid, str(e))