remove script type

.github/workflows/test-only.yml

@@ -30,7 +30,7 @@ jobs:
 
           # Selenium+browserless
           docker run --network changedet-network -d --hostname selenium  -p 4444:4444 --rm --shm-size="2g"  selenium/standalone-chrome-debug:3.141.59
-          docker run --network changedet-network -d --hostname browserless -e "FUNCTION_BUILT_INS=[\"fs\",\"crypto\"]" -e "DEFAULT_LAUNCH_ARGS=[\"--window-size=1920,1080\"]" --rm  -p 3000:3000  --shm-size="2g"  browserless/chrome:1.53-chrome-stable
+          docker run --network changedet-network -d --hostname browserless -e "DEFAULT_LAUNCH_ARGS=[\"--window-size=1920,1080\"]" --rm  -p 3000:3000  --shm-size="2g"  browserless/chrome:1.53-chrome-stable
 
       - name: Build changedetection.io container for testing
         run: |         
@@ -55,19 +55,9 @@ jobs:
           # Playwright/Browserless fetch
           docker run --rm -e "PLAYWRIGHT_DRIVER_URL=ws://browserless:3000" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio;pytest tests/fetchers/test_content.py && pytest tests/test_errorhandling.py && pytest tests/visualselector/test_fetch_data.py'
           
-          # Settings headers playwright tests - Call back in from Browserless, check headers
-          docker run --name "changedet" --hostname changedet --rm -e "FLASK_SERVER_NAME=changedet" -e "PLAYWRIGHT_DRIVER_URL=ws://browserless:3000?dumpio=true" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0  --live-server-port=5004 tests/test_request.py'
-          docker run --name "changedet" --hostname changedet --rm -e "FLASK_SERVER_NAME=changedet" -e "WEBDRIVER_URL=http://selenium:4444/wd/hub" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0  --live-server-port=5004 tests/test_request.py'
-          docker run --name "changedet" --hostname changedet --rm -e "FLASK_SERVER_NAME=changedet" -e "USE_EXPERIMENTAL_PUPPETEER_FETCH=yes" -e "PLAYWRIGHT_DRIVER_URL=ws://browserless:3000?dumpio=true" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0  --live-server-port=5004 tests/test_request.py'
-          
           # restock detection via playwright - added name=changedet here so that playwright/browserless can connect to it
           docker run --rm --name "changedet" -e "FLASK_SERVER_NAME=changedet" -e "PLAYWRIGHT_DRIVER_URL=ws://browserless:3000" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-port=5004 --live-server-host=0.0.0.0 tests/restock/test_restock.py'
 
-      - name: Test with puppeteer fetcher and disk cache
-        run: |
-          docker run --rm -e "PUPPETEER_DISK_CACHE=/tmp/data/" -e "USE_EXPERIMENTAL_PUPPETEER_FETCH=yes" -e "PLAYWRIGHT_DRIVER_URL=ws://browserless:3000" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio;pytest tests/fetchers/test_content.py && pytest tests/test_errorhandling.py && pytest tests/visualselector/test_fetch_data.py'
-          # Browserless would have had -e "FUNCTION_BUILT_INS=[\"fs\",\"crypto\"]" added above
-
       - name: Test proxy interaction
         run: |
           cd changedetectionio

README.md

@@ -66,7 +66,6 @@ Requires Playwright to be enabled.
 - Proactively search for jobs, get notified when companies update their careers page, search job portals for keywords.
 - Get alerts when new job positions are open on Bamboo HR and other job platforms
 - Website defacement monitoring
-- Pokémon Card Restock Tracker / Pokémon TCG Tracker
 
 _Need an actual Chrome runner with Javascript support? We support fetching via WebDriver and Playwright!</a>_
 
@@ -145,7 +144,7 @@ See the wiki for more information https://github.com/dgtlmoon/changedetection.io
 ## Filters
 
 XPath, JSONPath, jq, and CSS support comes baked in! You can be as specific as you need, use XPath exported from various XPath element query creation tools. 
-(We support LXML `re:test`, `re:match` and `re:replace`.)
+(We support LXML `re:test`, `re:math` and `re:replace`.)
 
 ## Notifications
 

changedetectionio/__init__.py

@@ -38,7 +38,7 @@ from flask_paginate import Pagination, get_page_parameter
 from changedetectionio import html_tools
 from changedetectionio.api import api_v1
 
-__version__ = '0.42.2'
+__version__ = '0.41.1'
 
 datastore = None
 
@@ -124,15 +124,6 @@ def _jinja2_filter_datetimestamp(timestamp, format="%Y-%m-%d %H:%M:%S"):
 
     return timeago.format(timestamp, time.time())
 
-
-@app.template_filter('pagination_slice')
-def _jinja2_filter_pagination_slice(arr, skip):
-    per_page = datastore.data['settings']['application'].get('pager_size', 50)
-    if per_page:
-        return arr[skip:skip + per_page]
-
-    return arr
-
 @app.template_filter('format_seconds_ago')
 def _jinja2_filter_seconds_precise(timestamp):
     if timestamp == False:
@@ -412,40 +403,27 @@ def changedetection_app(config=None, datastore_o=None):
 
         # Sort by last_changed and add the uuid which is usually the key..
         sorted_watches = []
-        search_q = request.args.get('q').strip().lower() if request.args.get('q') else False
         for uuid, watch in datastore.data['watching'].items():
 
-            if limit_tag:
+            if limit_tag != None:
                 # Support for comma separated list of tags.
-                if not watch.get('tag'):
+                if watch['tag'] is None:
                     continue
-                for tag_in_watch in watch.get('tag', '').split(','):
+                for tag_in_watch in watch['tag'].split(','):
                     tag_in_watch = tag_in_watch.strip()
                     if tag_in_watch == limit_tag:
                         watch['uuid'] = uuid
-                        if search_q:
+                        sorted_watches.append(watch)
-                            if (watch.get('title') and search_q in watch.get('title').lower()) or search_q in watch.get('url', '').lower():
-                                sorted_watches.append(watch)
-                        else:
-                            sorted_watches.append(watch)
 
             else:
-                #watch['uuid'] = uuid
+                watch['uuid'] = uuid
-                if search_q:
+                sorted_watches.append(watch)
-                    if (watch.get('title') and search_q in watch.get('title').lower()) or search_q in watch.get('url', '').lower():
-                        sorted_watches.append(watch)
-                else:
-                    sorted_watches.append(watch)
 
         existing_tags = datastore.get_all_tags()
         form = forms.quickWatchForm(request.form)
         page = request.args.get(get_page_parameter(), type=int, default=1)
-        total_count = len(sorted_watches)
+        total_count = len(sorted_watches) if sorted_watches else len(datastore.data['watching'])
-
+        pagination = Pagination(page=page, total=total_count, per_page=int(os.getenv('pagination_per_page', 50)), css_framework = "semantic")
-        pagination = Pagination(page=page,
-                                total=total_count,
-                                per_page=datastore.data['settings']['application'].get('pager_size', 50), css_framework="semantic")
-
 
         output = render_template(
             "watch-overview.html",
@@ -459,7 +437,6 @@ def changedetection_app(config=None, datastore_o=None):
                                  hosted_sticky=os.getenv("SALTED_PASS", False) == False,
                                  pagination=pagination,
                                  queued_uuids=[q_uuid.item['uuid'] for q_uuid in update_q.queue],
-                                 search_q=request.args.get('q','').strip(),
                                  sort_attribute=request.args.get('sort') if request.args.get('sort') else request.cookies.get('sort'),
                                  sort_order=request.args.get('order') if request.args.get('order') else request.cookies.get('order'),
                                  system_default_fetcher=datastore.data['settings']['application'].get('fetch_backend'),
@@ -713,7 +690,6 @@ def changedetection_app(config=None, datastore_o=None):
                                      form=form,
                                      has_default_notification_urls=True if len(datastore.data['settings']['application']['notification_urls']) else False,
                                      has_empty_checktime=using_default_check_time,
-                                     has_extra_headers_file=watch.has_extra_headers_file or datastore.has_extra_headers_file,
                                      is_html_webdriver=is_html_webdriver,
                                      jq_support=jq_support,
                                      playwright_enabled=os.getenv('PLAYWRIGHT_DRIVER_URL', False),
@@ -1337,13 +1313,6 @@ def changedetection_app(config=None, datastore_o=None):
                     update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid, 'skip_when_checksum_same': False}))
             flash("{} watches queued for rechecking".format(len(uuids)))
 
-        elif (op == 'clear-history'):
-            for uuid in uuids:
-                uuid = uuid.strip()
-                if datastore.data['watching'].get(uuid):
-                    datastore.clear_watch_history(uuid)
-            flash("{} watches cleared/reset.".format(len(uuids)))
-
         elif (op == 'notification-default'):
             from changedetectionio.notification import (
                 default_notification_format_for_watch
@@ -1458,7 +1427,6 @@ def check_for_new_version():
         # Check daily
         app.config.exit.wait(86400)
 
-
 def notification_runner():
     global notification_debug_log
     from datetime import datetime

changedetectionio/blueprint/browser_steps/__init__.py

@@ -27,106 +27,58 @@ import os
 import logging
 from changedetectionio.store import ChangeDetectionStore
 from changedetectionio import login_optionally_required
+browsersteps_live_ui_o = {}
+browsersteps_playwright_browser_interface = None
+browsersteps_playwright_browser_interface_browser = None
+browsersteps_playwright_browser_interface_context = None
+browsersteps_playwright_browser_interface_end_time = None
+browsersteps_playwright_browser_interface_start_time = None
 
-browsersteps_sessions = {}
+def cleanup_playwright_session():
-io_interface_context = None
 
+    global browsersteps_live_ui_o
+    global browsersteps_playwright_browser_interface
+    global browsersteps_playwright_browser_interface_browser
+    global browsersteps_playwright_browser_interface_context
+    global browsersteps_playwright_browser_interface_end_time
+    global browsersteps_playwright_browser_interface_start_time
+
+    browsersteps_live_ui_o = {}
+    browsersteps_playwright_browser_interface = None
+    browsersteps_playwright_browser_interface_browser = None
+    browsersteps_playwright_browser_interface_end_time = None
+    browsersteps_playwright_browser_interface_start_time = None
+
+    print("Cleaning up old playwright session because time was up, calling .goodbye()")
+    try:
+        browsersteps_playwright_browser_interface_context.goodbye()
+    except Exception as e:
+        print ("Got exception in shutdown, probably OK")
+        print (str(e))
+
+    browsersteps_playwright_browser_interface_context = None
+
+    print ("Cleaning up old playwright session because time was up - done")
 
 def construct_blueprint(datastore: ChangeDetectionStore):
+
     browser_steps_blueprint = Blueprint('browser_steps', __name__, template_folder="templates")
 
-    def start_browsersteps_session(watch_uuid):
-        from . import nonContext
-        from . import browser_steps
-        import time
-        global browsersteps_sessions
-        global io_interface_context
-
-
-        # We keep the playwright session open for many minutes
-        seconds_keepalive = int(os.getenv('BROWSERSTEPS_MINUTES_KEEPALIVE', 10)) * 60
-
-        browsersteps_start_session = {'start_time': time.time()}
-
-        # You can only have one of these running
-        # This should be very fine to leave running for the life of the application
-        # @idea - Make it global so the pool of watch fetchers can use it also
-        if not io_interface_context:
-            io_interface_context = nonContext.c_sync_playwright()
-            # Start the Playwright context, which is actually a nodejs sub-process and communicates over STDIN/STDOUT pipes
-            io_interface_context = io_interface_context.start()
-
-
-        # keep it alive for 10 seconds more than we advertise, sometimes it helps to keep it shutting down cleanly
-        keepalive = "&timeout={}".format(((seconds_keepalive + 3) * 1000))
-        try:
-            browsersteps_start_session['browser'] = io_interface_context.chromium.connect_over_cdp(
-                os.getenv('PLAYWRIGHT_DRIVER_URL', '') + keepalive)
-        except Exception as e:
-            if 'ECONNREFUSED' in str(e):
-                return make_response('Unable to start the Playwright Browser session, is it running?', 401)
-            else:
-                return make_response(str(e), 401)
-
-        proxy_id = datastore.get_preferred_proxy_for_watch(uuid=watch_uuid)
-        proxy = None
-        if proxy_id:
-            proxy_url = datastore.proxy_list.get(proxy_id).get('url')
-            if proxy_url:
-
-                # Playwright needs separate username and password values
-                from urllib.parse import urlparse
-                parsed = urlparse(proxy_url)
-                proxy = {'server': proxy_url}
-
-                if parsed.username:
-                    proxy['username'] = parsed.username
-
-                if parsed.password:
-                    proxy['password'] = parsed.password
-
-                print("Browser Steps: UUID {} selected proxy {}".format(watch_uuid, proxy_url))
-
-        # Tell Playwright to connect to Chrome and setup a new session via our stepper interface
-        browsersteps_start_session['browserstepper'] = browser_steps.browsersteps_live_ui(
-            playwright_browser=browsersteps_start_session['browser'],
-            proxy=proxy)
-
-        # For test
-        #browsersteps_start_session['browserstepper'].action_goto_url(value="http://example.com?time="+str(time.time()))
-
-        return browsersteps_start_session
-
-
     @login_optionally_required
-    @browser_steps_blueprint.route("/browsersteps_start_session", methods=['GET'])
+    @browser_steps_blueprint.route("/browsersteps_update", methods=['GET', 'POST'])
-    def browsersteps_start_session():
-        # A new session was requested, return sessionID
-
-        import uuid
-        global browsersteps_sessions
-
-        browsersteps_session_id = str(uuid.uuid4())
-        watch_uuid = request.args.get('uuid')
-
-        if not watch_uuid:
-            return make_response('No Watch UUID specified', 500)
-
-        print("Starting connection with playwright")
-        logging.debug("browser_steps.py connecting")
-        browsersteps_sessions[browsersteps_session_id] = start_browsersteps_session(watch_uuid)
-        print("Starting connection with playwright - done")
-        return {'browsersteps_session_id': browsersteps_session_id}
-
-    # A request for an action was received
-    @login_optionally_required
-    @browser_steps_blueprint.route("/browsersteps_update", methods=['POST'])
     def browsersteps_ui_update():
         import base64
         import playwright._impl._api_types
-        global browsersteps_sessions
+        import time
+
         from changedetectionio.blueprint.browser_steps import browser_steps
 
+        global browsersteps_live_ui_o, browsersteps_playwright_browser_interface_end_time
+        global browsersteps_playwright_browser_interface_browser
+        global browsersteps_playwright_browser_interface
+        global browsersteps_playwright_browser_interface_start_time
+
+        step_n = None
         remaining =0
         uuid = request.args.get('uuid')
 
@@ -135,9 +87,13 @@ def construct_blueprint(datastore: ChangeDetectionStore):
         if not browsersteps_session_id:
             return make_response('No browsersteps_session_id specified', 500)
 
-        if not browsersteps_sessions.get(browsersteps_session_id):
+        # Because we don't "really" run in a context manager ( we make the playwright interface global/long-living )
-            return make_response('No session exists under that ID', 500)
+        # We need to manage the shutdown when the time is up
-
+        if browsersteps_playwright_browser_interface_end_time:
+            remaining = browsersteps_playwright_browser_interface_end_time-time.time()
+            if browsersteps_playwright_browser_interface_end_time and remaining <= 0:
+                cleanup_playwright_session()
+                return make_response('Browser session expired, please reload the Browser Steps interface', 401)
 
         # Actions - step/apply/etc, do the thing and return state
         if request.method == 'POST':
@@ -156,7 +112,12 @@ def construct_blueprint(datastore: ChangeDetectionStore):
             # @todo try.. accept.. nice errors not popups..
             try:
 
-                browsersteps_sessions[browsersteps_session_id]['browserstepper'].call_action(action_name=step_operation,
+                this_session = browsersteps_live_ui_o.get(browsersteps_session_id)
+                if not this_session:
+                    print("Browser exited")
+                    return make_response('Browser session ran out of time :( Please reload this page.', 401)
+
+                this_session.call_action(action_name=step_operation,
                                          selector=step_selector,
                                          optional_value=step_optional_value)
 
@@ -168,43 +129,99 @@ def construct_blueprint(datastore: ChangeDetectionStore):
             # Get visual selector ready/update its data (also use the current filter info from the page?)
             # When the last 'apply' button was pressed
             # @todo this adds overhead because the xpath selection is happening twice
-            u = browsersteps_sessions[browsersteps_session_id]['browserstepper'].page.url
+            u = this_session.page.url
             if is_last_step and u:
-                (screenshot, xpath_data) = browsersteps_sessions[browsersteps_session_id]['browserstepper'].request_visualselector_data()
+                (screenshot, xpath_data) = this_session.request_visualselector_data()
                 datastore.save_screenshot(watch_uuid=uuid, screenshot=screenshot)
                 datastore.save_xpath_data(watch_uuid=uuid, data=xpath_data)
 
-#        if not this_session.page:
+        # Setup interface
-#            cleanup_playwright_session()
+        if request.method == 'GET':
-#            return make_response('Browser session ran out of time :( Please reload this page.', 401)
 
-        # Screenshots and other info only needed on requesting a step (POST)
+            if not browsersteps_playwright_browser_interface:
-        try:
+                print("Starting connection with playwright")
-            state = browsersteps_sessions[browsersteps_session_id]['browserstepper'].get_current_state()
+                logging.debug("browser_steps.py connecting")
-        except playwright._impl._api_types.Error as e:
-            return make_response("Browser session ran out of time :( Please reload this page."+str(e), 401)
 
-        # Use send_file() which is way faster than read/write loop on bytes
+                global browsersteps_playwright_browser_interface_context
-        import json
+                from . import nonContext
-        from tempfile import mkstemp
+                browsersteps_playwright_browser_interface_context = nonContext.c_sync_playwright()
-        from flask import send_file
+                browsersteps_playwright_browser_interface = browsersteps_playwright_browser_interface_context.start()
-        tmp_fd, tmp_file = mkstemp(text=True, suffix=".json", prefix="changedetectionio-")
 
-        output = json.dumps({'screenshot': "data:image/jpeg;base64,{}".format(
+                time.sleep(1)
-            base64.b64encode(state[0]).decode('ascii')),
+                # At 20 minutes, some other variable is closing it
-            'xpath_data': state[1],
+                # @todo find out what it is and set it
-            'session_age_start': browsersteps_sessions[browsersteps_session_id]['browserstepper'].age_start,
+                seconds_keepalive = int(os.getenv('BROWSERSTEPS_MINUTES_KEEPALIVE', 10)) * 60
-            'browser_time_remaining': round(remaining)
-        })
 
-        with os.fdopen(tmp_fd, 'w') as f:
+                # keep it alive for 10 seconds more than we advertise, sometimes it helps to keep it shutting down cleanly
-            f.write(output)
+                keepalive = "&timeout={}".format(((seconds_keepalive+3) * 1000))
+                try:
+                    browsersteps_playwright_browser_interface_browser = browsersteps_playwright_browser_interface.chromium.connect_over_cdp(
+                        os.getenv('PLAYWRIGHT_DRIVER_URL', '') + keepalive)
+                except Exception as e:
+                    if 'ECONNREFUSED' in str(e):
+                        return make_response('Unable to start the Playwright session properly, is it running?', 401)
 
-        response = make_response(send_file(path_or_file=tmp_file,
+                browsersteps_playwright_browser_interface_end_time = time.time() + (seconds_keepalive-3)
-                                           mimetype='application/json; charset=UTF-8',
+                print("Starting connection with playwright - done")
-                                           etag=True))
+
-        # No longer needed
+            if not browsersteps_live_ui_o.get(browsersteps_session_id):
-        os.unlink(tmp_file)
+                # Boot up a new session
+                proxy_id = datastore.get_preferred_proxy_for_watch(uuid=uuid)
+                proxy = None
+                if proxy_id:
+                    proxy_url = datastore.proxy_list.get(proxy_id).get('url')
+                    if proxy_url:
+                        proxy = {'server': proxy_url}
+                        print("Browser Steps: UUID {} Using proxy {}".format(uuid, proxy_url))
+
+                # Begin the new "Playwright Context" that re-uses the playwright interface
+                # Each session is a "Playwright Context" as a list, that uses the playwright interface
+                browsersteps_live_ui_o[browsersteps_session_id] = browser_steps.browsersteps_live_ui(
+                    playwright_browser=browsersteps_playwright_browser_interface_browser,
+                    proxy=proxy)
+                this_session = browsersteps_live_ui_o[browsersteps_session_id]
+
+        if not this_session.page:
+            cleanup_playwright_session()
+            return make_response('Browser session ran out of time :( Please reload this page.', 401)
+
+        response = None
+
+        if request.method == 'POST':
+            # Screenshots and other info only needed on requesting a step (POST)
+            try:
+                state = this_session.get_current_state()
+            except playwright._impl._api_types.Error as e:
+                return make_response("Browser session ran out of time :( Please reload this page."+str(e), 401)
+
+            # Use send_file() which is way faster than read/write loop on bytes
+            import json
+            from tempfile import mkstemp
+            from flask import send_file
+            tmp_fd, tmp_file = mkstemp(text=True, suffix=".json", prefix="changedetectionio-")
+
+            output = json.dumps({'screenshot': "data:image/jpeg;base64,{}".format(
+                base64.b64encode(state[0]).decode('ascii')),
+                'xpath_data': state[1],
+                'session_age_start': this_session.age_start,
+                'browser_time_remaining': round(remaining)
+            })
+
+            with os.fdopen(tmp_fd, 'w') as f:
+                f.write(output)
+
+            response = make_response(send_file(path_or_file=tmp_file,
+                                               mimetype='application/json; charset=UTF-8',
+                                               etag=True))
+            # No longer needed
+            os.unlink(tmp_file)
+
+        elif request.method == 'GET':
+            # Just enough to get the session rolling, it will call for goto-site via POST next
+            response = make_response({
+                'session_age_start': this_session.age_start,
+                'browser_time_remaining': round(remaining)
+            })
 
         return response
 

changedetectionio/blueprint/browser_steps/browser_steps.py

@@ -71,10 +71,10 @@ class steppable_browser_interface():
             optional_value = str(jinja2_env.from_string(optional_value).render())
 
         action_handler(selector, optional_value)
-        self.page.wait_for_timeout(1.5 * 1000)
+        self.page.wait_for_timeout(3 * 1000)
         print("Call action done in", time.time() - now)
 
-    def action_goto_url(self, selector=None, value=None):
+    def action_goto_url(self, selector, value):
         # self.page.set_viewport_size({"width": 1280, "height": 5000})
         now = time.time()
         response = self.page.goto(value, timeout=0, wait_until='commit')
@@ -105,8 +105,7 @@ class steppable_browser_interface():
         print("Clicking element")
         if not len(selector.strip()):
             return
-
+        self.page.click(selector, timeout=10 * 1000, delay=randint(200, 500))
-        self.page.click(selector=selector, timeout=30 * 1000, delay=randint(200, 500))
 
     def action_click_element_if_exists(self, selector, value):
         import playwright._impl._api_types as _api_types
@@ -133,18 +132,18 @@ class steppable_browser_interface():
         self.page.wait_for_timeout(1000)
 
     def action_wait_for_seconds(self, selector, value):
-        self.page.wait_for_timeout(float(value.strip()) * 1000)
+        self.page.wait_for_timeout(int(value) * 1000)
 
     def action_wait_for_text(self, selector, value):
         import json
         v = json.dumps(value)
-        self.page.wait_for_function(f'document.querySelector("body").innerText.includes({v});', timeout=90000)
+        self.page.wait_for_function(f'document.querySelector("body").innerText.includes({v});', timeout=30000)
 
     def action_wait_for_text_in_element(self, selector, value):
         import json
         s = json.dumps(selector)
         v = json.dumps(value)
-        self.page.wait_for_function(f'document.querySelector({s}).innerText.includes({v});', timeout=90000)
+        self.page.wait_for_function(f'document.querySelector({s}).innerText.includes({v});', timeout=30000)
 
     # @todo - in the future make some popout interface to capture what needs to be set
     # https://playwright.dev/python/docs/api/class-keyboard

changedetectionio/content_fetcher.py

@@ -10,7 +10,6 @@ import time
 
 visualselector_xpath_selectors = 'div,span,form,table,tbody,tr,td,a,p,ul,li,h1,h2,h3,h4, header, footer, section, article, aside, details, main, nav, section, summary'
 
-
 class Non200ErrorCodeReceived(Exception):
     def __init__(self, status_code, url, screenshot=None, xpath_data=None, page_html=None):
         # Set this so we can use it in other parts of the app
@@ -25,12 +24,10 @@ class Non200ErrorCodeReceived(Exception):
             self.page_text = html_tools.html_to_text(page_html)
         return
 
-
 class checksumFromPreviousCheckWasTheSame(Exception):
     def __init__(self):
         return
 
-
 class JSActionExceptions(Exception):
     def __init__(self, status_code, url, screenshot, message=''):
         self.status_code = status_code
@@ -39,7 +36,6 @@ class JSActionExceptions(Exception):
         self.message = message
         return
 
-
 class BrowserStepsStepTimout(Exception):
     def __init__(self, step_n):
         self.step_n = step_n
@@ -55,7 +51,6 @@ class PageUnloadable(Exception):
         self.message = message
         return
 
-
 class EmptyReply(Exception):
     def __init__(self, status_code, url, screenshot=None):
         # Set this so we can use it in other parts of the app
@@ -64,7 +59,6 @@ class EmptyReply(Exception):
         self.screenshot = screenshot
         return
 
-
 class ScreenshotUnavailable(Exception):
     def __init__(self, status_code, url, page_html=None):
         # Set this so we can use it in other parts of the app
@@ -75,7 +69,6 @@ class ScreenshotUnavailable(Exception):
             self.page_text = html_to_text(page_html)
         return
 
-
 class ReplyWithContentButNoText(Exception):
     def __init__(self, status_code, url, screenshot=None):
         # Set this so we can use it in other parts of the app
@@ -84,14 +77,13 @@ class ReplyWithContentButNoText(Exception):
         self.screenshot = screenshot
         return
 
-
 class Fetcher():
     browser_steps = None
     browser_steps_screenshot_path = None
     content = None
     error = None
     fetcher_description = "No description"
-    headers = {}
+    headers = None
     status_code = None
     webdriver_js_execute_code = None
     xpath_data = None
@@ -113,6 +105,7 @@ class Fetcher():
         self.xpath_element_js = resource_string(__name__, "res/xpath_element_scraper.js").decode('utf-8')
         self.instock_data_js = resource_string(__name__, "res/stock-not-in-stock.js").decode('utf-8')
 
+
     @abstractmethod
     def get_error(self):
         return self.error
@@ -159,15 +152,13 @@ class Fetcher():
             interface = steppable_browser_interface()
             interface.page = self.page
 
-            valid_steps = filter(
+            valid_steps = filter(lambda s: (s['operation'] and len(s['operation']) and s['operation'] != 'Choose one' and s['operation'] != 'Goto site'), self.browser_steps)
-                lambda s: (s['operation'] and len(s['operation']) and s['operation'] != 'Choose one' and s['operation'] != 'Goto site'),
-                self.browser_steps)
 
             for step in valid_steps:
                 step_n += 1
                 print(">> Iterating check - browser Step n {} - {}...".format(step_n, step['operation']))
-                self.screenshot_step("before-" + str(step_n))
+                self.screenshot_step("before-"+str(step_n))
-                self.save_step_html("before-" + str(step_n))
+                self.save_step_html("before-"+str(step_n))
                 try:
                     optional_value = step['optional_value']
                     selector = step['selector']
@@ -182,11 +173,12 @@ class Fetcher():
                                                       optional_value=optional_value)
                     self.screenshot_step(step_n)
                     self.save_step_html(step_n)
-                except TimeoutError as e:
+                except TimeoutError:
-                    print(str(e))
                     # Stop processing here
                     raise BrowserStepsStepTimout(step_n=step_n)
 
+
+
     # It's always good to reset these
     def delete_browser_steps_screenshots(self):
         import glob
@@ -196,7 +188,6 @@ class Fetcher():
             for f in files:
                 os.unlink(f)
 
-
 #   Maybe for the future, each fetcher provides its own diff output, could be used for text, image
 #   the current one would return javascript output (as we use JS to generate the diff)
 #
@@ -214,7 +205,6 @@ def available_fetchers():
 
     return p
 
-
 class base_html_playwright(Fetcher):
     fetcher_description = "Playwright {}/Javascript".format(
         os.getenv("PLAYWRIGHT_BROWSER_TYPE", 'chromium').capitalize()
@@ -278,119 +268,6 @@ class base_html_playwright(Fetcher):
         with open(destination, 'w') as f:
             f.write(content)
 
-    def run_fetch_browserless_puppeteer(self,
-            url,
-            timeout,
-            request_headers,
-            request_body,
-            request_method,
-            ignore_status_codes=False,
-            current_include_filters=None,
-            is_binary=False):
-
-        from pkg_resources import resource_string
-
-        extra_wait_ms = (int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)) + self.render_extract_delay) * 1000
-
-        self.xpath_element_js = self.xpath_element_js.replace('%ELEMENTS%', visualselector_xpath_selectors)
-        code = resource_string(__name__, "res/puppeteer_fetch.js").decode('utf-8')
-        # In the future inject this is a proper JS package
-        code = code.replace('%xpath_scrape_code%', self.xpath_element_js)
-        code = code.replace('%instock_scrape_code%', self.instock_data_js)
-
-        from requests.exceptions import ConnectTimeout, ReadTimeout
-        wait_browserless_seconds = 240
-
-        browserless_function_url = os.getenv('BROWSERLESS_FUNCTION_URL')
-        from urllib.parse import urlparse
-        if not browserless_function_url:
-            # Convert/try to guess from PLAYWRIGHT_DRIVER_URL
-            o = urlparse(os.getenv('PLAYWRIGHT_DRIVER_URL'))
-            browserless_function_url = o._replace(scheme="http")._replace(path="function").geturl()
-
-
-        # Append proxy connect string
-        if self.proxy:
-            import urllib.parse
-            # Remove username/password if it exists in the URL or you will receive "ERR_NO_SUPPORTED_PROXIES" error
-            # Actual authentication handled by Puppeteer/node
-            o = urlparse(self.proxy.get('server'))
-            proxy_url = urllib.parse.quote(o._replace(netloc="{}:{}".format(o.hostname, o.port)).geturl())
-            browserless_function_url = f"{browserless_function_url}&--proxy-server={proxy_url}&dumpio=true"
-
-
-        try:
-            amp = '&' if '?' in browserless_function_url else '?'
-            response = requests.request(
-                method="POST",
-                json={
-                    "code": code,
-                    "context": {
-                        # Very primitive disk cache - USE WITH EXTREME CAUTION
-                        # Run browserless container  with -e "FUNCTION_BUILT_INS=[\"fs\",\"crypto\"]"
-                        'disk_cache_dir': os.getenv("PUPPETEER_DISK_CACHE", False), # or path to disk cache ending in /, ie /tmp/cache/
-                        'execute_js': self.webdriver_js_execute_code,
-                        'extra_wait_ms': extra_wait_ms,
-                        'include_filters': current_include_filters,
-                        'req_headers': request_headers,
-                        'screenshot_quality': int(os.getenv("PLAYWRIGHT_SCREENSHOT_QUALITY", 72)),
-                        'url': url,
-                        'user_agent': request_headers.get('User-Agent', 'Mozilla/5.0'),
-                        'proxy_username': self.proxy.get('username','') if self.proxy else False,
-                        'proxy_password': self.proxy.get('password', '') if self.proxy else False,
-                        'no_cache_list': [
-                            'twitter',
-                            '.pdf'
-                        ],
-                        # Could use https://github.com/easylist/easylist here, or install a plugin
-                        'block_url_list': [
-                            'adnxs.com',
-                            'analytics.twitter.com',
-                            'doubleclick.net',
-                            'google-analytics.com',
-                            'googletagmanager',
-                            'trustpilot.com'
-                        ]
-                    }
-                },
-                # @todo /function needs adding ws:// to http:// rebuild this
-                url=browserless_function_url+f"{amp}--disable-features=AudioServiceOutOfProcess&dumpio=true&--disable-remote-fonts",
-                timeout=wait_browserless_seconds)
-
-        except ReadTimeout:
-            raise PageUnloadable(url=url, status_code=None, message=f"No response from browserless in {wait_browserless_seconds}s")
-        except ConnectTimeout:
-            raise PageUnloadable(url=url, status_code=None, message=f"Timed out connecting to browserless, retrying..")
-        else:
-            # 200 Here means that the communication to browserless worked only, not the page state
-            if response.status_code == 200:
-                import base64
-
-                x = response.json()
-                if not x.get('screenshot'):
-                    # https://github.com/puppeteer/puppeteer/blob/v1.0.0/docs/troubleshooting.md#tips
-                    # https://github.com/puppeteer/puppeteer/issues/1834
-                    # https://github.com/puppeteer/puppeteer/issues/1834#issuecomment-381047051
-                    # Check your memory is shared and big enough
-                    raise ScreenshotUnavailable(url=url, status_code=None)
-
-                if not x.get('content', '').strip():
-                    raise EmptyReply(url=url, status_code=None)
-
-                if x.get('status_code', 200) != 200 and not ignore_status_codes:
-                    raise Non200ErrorCodeReceived(url=url, status_code=x.get('status_code', 200), page_html=x['content'])
-
-                self.content = x.get('content')
-                self.headers = x.get('headers')
-                self.instock_data = x.get('instock_data')
-                self.screenshot = base64.b64decode(x.get('screenshot'))
-                self.status_code = x.get('status_code')
-                self.xpath_data = x.get('xpath_data')
-
-            else:
-                # Some other error from browserless
-                raise PageUnloadable(url=url, status_code=None, message=response.content.decode('utf-8'))
-
     def run(self,
             url,
             timeout,
@@ -401,24 +278,6 @@ class base_html_playwright(Fetcher):
             current_include_filters=None,
             is_binary=False):
 
-        # For now, USE_EXPERIMENTAL_PUPPETEER_FETCH is not supported by watches with BrowserSteps (for now!)
-        has_browser_steps = self.browser_steps and list(filter(
-                lambda s: (s['operation'] and len(s['operation']) and s['operation'] != 'Choose one' and s['operation'] != 'Goto site'),
-                self.browser_steps))
-
-        if not has_browser_steps:
-            if os.getenv('USE_EXPERIMENTAL_PUPPETEER_FETCH'):
-                # Temporary backup solution until we rewrite the playwright code
-                return self.run_fetch_browserless_puppeteer(
-                    url,
-                    timeout,
-                    request_headers,
-                    request_body,
-                    request_method,
-                    ignore_status_codes,
-                    current_include_filters,
-                    is_binary)
-
         from playwright.sync_api import sync_playwright
         import playwright._impl._api_types
 
@@ -435,7 +294,7 @@ class base_html_playwright(Fetcher):
             # Set user agent to prevent Cloudflare from blocking the browser
             # Use the default one configured in the App.py model that's passed from fetch_site_status.py
             context = browser.new_context(
-                user_agent=request_headers.get('User-Agent', 'Mozilla/5.0'),
+                user_agent=request_headers['User-Agent'] if request_headers.get('User-Agent') else 'Mozilla/5.0',
                 proxy=self.proxy,
                 # This is needed to enable JavaScript execution on GitHub and others
                 bypass_csp=True,
@@ -465,12 +324,12 @@ class base_html_playwright(Fetcher):
             except playwright._impl._api_types.Error as e:
                 # Retry once - https://github.com/browserless/chrome/issues/2485
                 # Sometimes errors related to invalid cert's and other can be random
-                print("Content Fetcher > retrying request got error - ", str(e))
+                print ("Content Fetcher > retrying request got error - ", str(e))
                 time.sleep(1)
                 response = self.page.goto(url, wait_until='commit')
 
             except Exception as e:
-                print("Content Fetcher > Other exception when page.goto", str(e))
+                print ("Content Fetcher > Other exception when page.goto", str(e))
                 context.close()
                 browser.close()
                 raise PageUnloadable(url=url, status_code=None, message=str(e))
@@ -489,7 +348,7 @@ class base_html_playwright(Fetcher):
                 # This can be ok, we will try to grab what we could retrieve
                 pass
             except Exception as e:
-                print("Content Fetcher > Other exception when executing custom JS code", str(e))
+                print ("Content Fetcher > Other exception when executing custom JS code", str(e))
                 context.close()
                 browser.close()
                 raise PageUnloadable(url=url, status_code=None, message=str(e))
@@ -497,7 +356,7 @@ class base_html_playwright(Fetcher):
             if response is None:
                 context.close()
                 browser.close()
-                print("Content Fetcher > Response object was none")
+                print ("Content Fetcher > Response object was none")
                 raise EmptyReply(url=url, status_code=None)
 
             # Run Browser Steps here
@@ -511,7 +370,7 @@ class base_html_playwright(Fetcher):
             if len(self.page.content().strip()) == 0:
                 context.close()
                 browser.close()
-                print("Content Fetcher > Content was empty")
+                print ("Content Fetcher > Content was empty")
                 raise EmptyReply(url=url, status_code=response.status)
 
             self.status_code = response.status
@@ -523,8 +382,7 @@ class base_html_playwright(Fetcher):
             else:
                 self.page.evaluate("var include_filters=''")
 
-            self.xpath_data = self.page.evaluate(
+            self.xpath_data = self.page.evaluate("async () => {" + self.xpath_element_js.replace('%ELEMENTS%', visualselector_xpath_selectors) + "}")
-                "async () => {" + self.xpath_element_js.replace('%ELEMENTS%', visualselector_xpath_selectors) + "}")
             self.instock_data = self.page.evaluate("async () => {" + self.instock_data_js + "}")
 
             # Bug 3 in Playwright screenshot handling
@@ -536,8 +394,7 @@ class base_html_playwright(Fetcher):
             # acceptable screenshot quality here
             try:
                 # The actual screenshot
-                self.screenshot = self.page.screenshot(type='jpeg', full_page=True,
+                self.screenshot = self.page.screenshot(type='jpeg', full_page=True, quality=int(os.getenv("PLAYWRIGHT_SCREENSHOT_QUALITY", 72)))
-                                                       quality=int(os.getenv("PLAYWRIGHT_SCREENSHOT_QUALITY", 72)))
             except Exception as e:
                 context.close()
                 browser.close()
@@ -546,7 +403,6 @@ class base_html_playwright(Fetcher):
             context.close()
             browser.close()
 
-
 class base_html_webdriver(Fetcher):
     if os.getenv("WEBDRIVER_URL"):
         fetcher_description = "WebDriver Chrome/Javascript via '{}'".format(os.getenv("WEBDRIVER_URL"))

changedetectionio/forms.py

@@ -481,10 +481,6 @@ class globalSettingsApplicationForm(commonSettingsForm):
     global_subtractive_selectors = StringListField('Remove elements', [ValidateCSSJSONXPATHInput(allow_xpath=False, allow_json=False)])
     ignore_whitespace = BooleanField('Ignore whitespace')
     password = SaltyPasswordField()
-    pager_size = IntegerField('Pager size',
-                              render_kw={"style": "width: 5em;"},
-                              validators=[validators.NumberRange(min=0,
-                                                                 message="Should be atleast zero (disabled)")])
     removepassword_button = SubmitField('Remove password', render_kw={"class": "pure-button pure-button-primary"})
     render_anchor_tag_content = BooleanField('Render anchor tag content', default=False)
     shared_diff_access = BooleanField('Allow access to view diff page when password is enabled', default=False, validators=[validators.Optional()])

changedetectionio/importer.py

@@ -52,8 +52,7 @@ class import_url_list(Importer):
 
             # Flask wtform validators wont work with basic auth, use validators package
             # Up to 5000 per batch so we dont flood the server
-            # @todo validators.url failed on local hostnames (such as referring to ourself when using browserless)
+            if len(url) and validators.url(url.replace('source:', '')) and good < 5000:
-            if len(url) and 'http' in url.lower() and good < 5000:
                 extras = None
                 if processor:
                     extras = {'processor': processor}

changedetectionio/model/App.py

@@ -23,26 +23,25 @@ class model(dict):
                     'workers': int(getenv("DEFAULT_SETTINGS_REQUESTS_WORKERS", "10")),  # Number of threads, lower is better for slow connections
                 },
                 'application': {
-                    # Custom notification content
                     'api_access_token_enabled': True,
+                    'password': False,
                     'base_url' : None,
-                    'empty_pages_are_a_change': False,
                     'extract_title_as_title': False,
+                    'empty_pages_are_a_change': False,
                     'fetch_backend': getenv("DEFAULT_FETCH_BACKEND", "html_requests"),
                     'filter_failure_notification_threshold_attempts': _FILTER_FAILURE_THRESHOLD_ATTEMPTS_DEFAULT,
                     'global_ignore_text': [], # List of text to ignore when calculating the comparison checksum
                     'global_subtractive_selectors': [],
                     'ignore_whitespace': True,
+                    'render_anchor_tag_content': False,
+                    'notification_urls': [], # Apprise URL list
+                    # Custom notification content
+                    'notification_title': default_notification_title,
                     'notification_body': default_notification_body,
                     'notification_format': default_notification_format,
-                    'notification_title': default_notification_title,
-                    'notification_urls': [], # Apprise URL list
-                    'pager_size': 50,
-                    'password': False,
-                    'render_anchor_tag_content': False,
                     'schema_version' : 0,
                     'shared_diff_access': False,
-                    'webdriver_delay': None , # Extra delay in seconds before extracting text
+                    'webdriver_delay': None  # Extra delay in seconds before extracting text
                 }
             }
         }
@@ -50,15 +49,3 @@ class model(dict):
     def __init__(self, *arg, **kw):
         super(model, self).__init__(*arg, **kw)
         self.update(self.base_config)
-
-
-def parse_headers_from_text_file(filepath):
-    headers = {}
-    with open(filepath, 'r') as f:
-        for l in f.readlines():
-            l = l.strip()
-            if not l.startswith('#') and ':' in l:
-                (k, v) = l.split(':')
-                headers[k.strip()] = v.strip()
-
-    return headers

changedetectionio/model/Watch.py

@@ -473,40 +473,6 @@ class model(dict):
         # None is set
         return False
 
-    @property
-    def has_extra_headers_file(self):
-        if os.path.isfile(os.path.join(self.watch_data_dir, 'headers.txt')):
-            return True
-
-        for f in self.all_tags:
-            fname = "headers-"+re.sub(r'[\W_]', '', f).lower().strip() + ".txt"
-            filepath = os.path.join(self.__datastore_path, fname)
-            if os.path.isfile(filepath):
-                return True
-
-        return False
-
-    def get_all_headers(self):
-        from .App import parse_headers_from_text_file
-        headers = self.get('headers', {}).copy()
-        # Available headers on the disk could 'headers.txt' in the watch data dir
-        filepath = os.path.join(self.watch_data_dir, 'headers.txt')
-        try:
-            if os.path.isfile(filepath):
-                headers.update(parse_headers_from_text_file(filepath))
-        except Exception as e:
-            print(f"ERROR reading headers.txt at {filepath}", str(e))
-
-        # Or each by tag, as tagname.txt in the main datadir
-        for f in self.all_tags:
-            fname = "headers-"+re.sub(r'[\W_]', '', f).lower().strip() + ".txt"
-            filepath = os.path.join(self.__datastore_path, fname)
-            try:
-                if os.path.isfile(filepath):
-                    headers.update(parse_headers_from_text_file(filepath))
-            except Exception as e:
-                print(f"ERROR reading headers.txt at {filepath}", str(e))
-        return headers
 
     def get_last_fetched_before_filters(self):
         import brotli

changedetectionio/processors/restock_diff.py

@@ -12,12 +12,6 @@ urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 name = 'Re-stock detection for single product pages'
 description = 'Detects if the product goes back to in-stock'
 
-class UnableToExtractRestockData(Exception):
-    def __init__(self, status_code):
-        # Set this so we can use it in other parts of the app
-        self.status_code = status_code
-        return
-
 class perform_site_check(difference_detection_processor):
     screenshot = None
     xpath_data = None
@@ -111,8 +105,7 @@ class perform_site_check(difference_detection_processor):
             fetched_md5 = hashlib.md5(fetcher.instock_data.encode('utf-8')).hexdigest()
             # 'Possibly in stock' comes from stock-not-in-stock.js when no string found above the fold.
             update_obj["in_stock"] = True if fetcher.instock_data == 'Possibly in stock' else False
-        else:
+
-            raise UnableToExtractRestockData(status_code=fetcher.status_code)
 
         # The main thing that all this at the moment comes down to :)
         changed_detected = False

changedetectionio/processors/text_json_diff.py

@@ -70,9 +70,10 @@ class perform_site_check(difference_detection_processor):
         # Unset any existing notification error
         update_obj = {'last_notification_error': False, 'last_error': False}
 
+        extra_headers = watch.get('headers', [])
+
         # Tweak the base config with the per-watch ones
-        extra_headers = watch.get_all_headers()
+        request_headers = deepcopy(self.datastore.data['settings']['headers'])
-        request_headers = self.datastore.get_all_headers()
         request_headers.update(extra_headers)
 
         # https://github.com/psf/requests/issues/4525

changedetectionio/res/puppeteer_fetch.js

@@ -1,183 +0,0 @@
-module.exports = async ({page, context}) => {
-
-    var {
-        url,
-        execute_js,
-        user_agent,
-        extra_wait_ms,
-        req_headers,
-        include_filters,
-        xpath_element_js,
-        screenshot_quality,
-        proxy_username,
-        proxy_password,
-        disk_cache_dir,
-        no_cache_list,
-        block_url_list,
-    } = context;
-
-    await page.setBypassCSP(true)
-    await page.setExtraHTTPHeaders(req_headers);
-    await page.setUserAgent(user_agent);
-    // https://ourcodeworld.com/articles/read/1106/how-to-solve-puppeteer-timeouterror-navigation-timeout-of-30000-ms-exceeded
-
-    await page.setDefaultNavigationTimeout(0);
-
-    if (proxy_username) {
-        await page.authenticate({
-            username: proxy_username,
-            password: proxy_password
-        });
-    }
-
-    await page.setViewport({
-        width: 1024,
-        height: 768,
-        deviceScaleFactor: 1,
-    });
-
-    await page.setRequestInterception(true);
-    if (disk_cache_dir) {
-        console.log(">>>>>>>>>>>>>>> LOCAL DISK CACHE ENABLED <<<<<<<<<<<<<<<<<<<<<");
-    }
-    const fs = require('fs');
-    const crypto = require('crypto');
-
-    function file_is_expired(file_path) {
-        if (!fs.existsSync(file_path)) {
-            return true;
-        }
-        var stats = fs.statSync(file_path);
-        const now_date = new Date();
-        const expire_seconds = 300;
-        if ((now_date / 1000) - (stats.mtime.getTime() / 1000) > expire_seconds) {
-            console.log("CACHE EXPIRED: " + file_path);
-            return true;
-        }
-        return false;
-
-    }
-
-    page.on('request', async (request) => {
-        // General blocking of requests that waste traffic
-        if (block_url_list.some(substring => request.url().toLowerCase().includes(substring))) return request.abort();
-
-        if (disk_cache_dir) {
-            const url = request.url();
-            const key = crypto.createHash('md5').update(url).digest("hex");
-            const dir_path = disk_cache_dir + key.slice(0, 1) + '/' + key.slice(1, 2) + '/' + key.slice(2, 3) + '/';
-
-            // https://stackoverflow.com/questions/4482686/check-synchronously-if-file-directory-exists-in-node-js
-
-            if (fs.existsSync(dir_path + key)) {
-                console.log("* CACHE HIT , using - " + dir_path + key + " - " + url);
-                const cached_data = fs.readFileSync(dir_path + key);
-                // @todo headers can come from dir_path+key+".meta" json file
-                request.respond({
-                    status: 200,
-                    //contentType: 'text/html', //@todo
-                    body: cached_data
-                });
-                return;
-            }
-        }
-        request.continue();
-    });
-
-
-    if (disk_cache_dir) {
-        page.on('response', async (response) => {
-            const url = response.url();
-            // Basic filtering for sane responses
-            if (response.request().method() != 'GET' || response.request().resourceType() == 'xhr' || response.request().resourceType() == 'document' || response.status() != 200) {
-                console.log("Skipping (not useful) - Status:" + response.status() + " Method:" + response.request().method() + " ResourceType:" + response.request().resourceType() + " " + url);
-                return;
-            }
-            if (no_cache_list.some(substring => url.toLowerCase().includes(substring))) {
-                console.log("Skipping (no_cache_list) - " + url);
-                return;
-            }
-            if (url.toLowerCase().includes('data:')) {
-                console.log("Skipping (embedded-data) - " + url);
-                return;
-            }
-            response.buffer().then(buffer => {
-                if (buffer.length > 100) {
-                    console.log("Cache - Saving " + response.request().method() + " - " + url + " - " + response.request().resourceType());
-
-                    const key = crypto.createHash('md5').update(url).digest("hex");
-                    const dir_path = disk_cache_dir + key.slice(0, 1) + '/' + key.slice(1, 2) + '/' + key.slice(2, 3) + '/';
-
-                    if (!fs.existsSync(dir_path)) {
-                        fs.mkdirSync(dir_path, {recursive: true})
-                    }
-
-                    if (fs.existsSync(dir_path + key)) {
-                        if (file_is_expired(dir_path + key)) {
-                            fs.writeFileSync(dir_path + key, buffer);
-                        }
-                    } else {
-                        fs.writeFileSync(dir_path + key, buffer);
-                    }
-                }
-            });
-        });
-    }
-
-    const r = await page.goto(url, {
-        waitUntil: 'load'
-    });
-
-    await page.waitForTimeout(1000);
-    await page.waitForTimeout(extra_wait_ms);
-
-    if (execute_js) {
-        await page.evaluate(execute_js);
-        await page.waitForTimeout(200);
-    }
-
-    var xpath_data;
-    var instock_data;
-    try {
-        // Not sure the best way here, in the future this should be a new package added to npm then run in browserless
-        // (Once the old playwright is removed)
-        xpath_data = await page.evaluate((include_filters) => {%xpath_scrape_code%}, include_filters);
-        instock_data = await page.evaluate(() => {%instock_scrape_code%});
-    } catch (e) {
-        console.log(e);
-    }
-
-    // Protocol error (Page.captureScreenshot): Cannot take screenshot with 0 width can come from a proxy auth failure
-    // Wrap it here (for now)
-
-    var b64s = false;
-    try {
-        b64s = await page.screenshot({encoding: "base64", fullPage: true, quality: screenshot_quality, type: 'jpeg'});
-    } catch (e) {
-        console.log(e);
-    }
-
-    // May fail on very large pages with 'WARNING: tile memory limits exceeded, some content may not draw'
-    if (!b64s) {
-        // @todo after text extract, we can place some overlay text with red background to say 'croppped'
-        console.error('ERROR: content-fetcher page was maybe too large for a screenshot, reverting to viewport only screenshot');
-        try {
-            b64s = await page.screenshot({encoding: "base64", quality: screenshot_quality, type: 'jpeg'});
-        } catch (e) {
-            console.log(e);
-        }
-    }
-
-    var html = await page.content();
-    return {
-        data: {
-            'content': html,
-            'headers': r.headers(),
-            'instock_data': instock_data,
-            'screenshot': b64s,
-            'status_code': r.status(),
-            'xpath_data': xpath_data
-        },
-        type: 'application/json',
-    };
-};

changedetectionio/res/stock-not-in-stock.js

@@ -10,7 +10,6 @@ function isItemInStock() {
     'brak na stanie',
     'brak w magazynie',
     'coming soon',
-    'currently have any tickets for this',
     'currently unavailable',
     'en rupture de stock',
     'item is no longer available',
@@ -21,9 +20,7 @@ function isItemInStock() {
     'nicht zur verfügung',
     'no disponible temporalmente',
     'no longer in stock',
-    'no tickets available',
     'not available',
-    'not currently available',
     'not in stock',
     'notify me when available',
     'não estamos a aceitar encomendas',
@@ -33,8 +30,6 @@ function isItemInStock() {
     'sold out',
     'temporarily out of stock',
     'temporarily unavailable',
-    'tickets unavailable',
-    'unavailable tickets',
     'we do not currently have an estimate of when this product will be back in stock.',
     'zur zeit nicht an lager',
   ];

changedetectionio/res/xpath_element_scraper.js

@@ -8,14 +8,8 @@
 // Some pages like https://www.londonstockexchange.com/stock/NCCL/ncondezi-energy-limited/analysis
 // will automatically force a scroll somewhere, so include the position offset
 // Lets hope the position doesnt change while we iterate the bbox's, but this is better than nothing
-var scroll_y = 0;
-try {
-    scroll_y = +document.documentElement.scrollTop || document.body.scrollTop
-} catch (e) {
-    console.log(e);
-}
-
 
+var scroll_y=+document.documentElement.scrollTop || document.body.scrollTop
 
 // Include the getXpath script directly, easier than fetching
 function getxpath(e) {
@@ -44,15 +38,15 @@ const findUpTag = (el) => {
     if (el.name !== undefined && el.name.length) {
         var proposed = el.tagName + "[name=" + el.name + "]";
         var proposed_element = window.document.querySelectorAll(proposed);
-        if (proposed_element.length) {
+        if(proposed_element.length) {
             if (proposed_element.length === 1) {
                 return proposed;
             } else {
                 // Some sites change ID but name= stays the same, we can hit it if we know the index
                 // Find all the elements that match and work out the input[n]
-                var n = Array.from(proposed_element).indexOf(el);
+                var n=Array.from(proposed_element).indexOf(el);
                 // Return a Playwright selector for nthinput[name=zipcode]
-                return proposed + " >> nth=" + n;
+                return proposed+" >> nth="+n;
             }
         }
     }

changedetectionio/static/js/browser-steps.js

@@ -114,11 +114,11 @@ $(document).ready(function () {
             e.preventDefault()
         });
 
-        // When the mouse moves we know which element it should be above
-        // mousedown will link that to the UI (select the right action, highlight etc)
         $('#browsersteps-selector-canvas').bind('mousedown', function (e) {
             // https://developer.mozilla.org/en-US/docs/Web/API/MouseEvent
             e.preventDefault()
+            console.log(e);
+            console.log("current xpath in index is " + current_selected_i);
             last_click_xy = {'x': parseInt((1 / x_scale) * e.offsetX), 'y': parseInt((1 / y_scale) * e.offsetY)}
             process_selected(current_selected_i);
             current_selected_i = false;
@@ -132,7 +132,6 @@ $(document).ready(function () {
             }
         });
 
-        // Debounce and find the current most 'interesting' element we are hovering above
         $('#browsersteps-selector-canvas').bind('mousemove', function (e) {
             if (!xpath_data) {
                 return;
@@ -152,40 +151,41 @@ $(document).ready(function () {
             current_selected_i = false;
             // Reverse order - the most specific one should be deeper/"laster"
             // Basically, find the most 'deepest'
-            var possible_elements = [];
+            //$('#browsersteps-selector-canvas').css('cursor', 'pointer');
-            xpath_data['size_pos'].forEach(function (item, index) {
+            for (var i = xpath_data['size_pos'].length; i !== 0; i--) {
+                // draw all of them? let them choose somehow?
+                var sel = xpath_data['size_pos'][i - 1];
                 // If we are in a bounding-box
-                if (e.offsetY > item.top * y_scale && e.offsetY < item.top * y_scale + item.height * y_scale
+                if (e.offsetY > sel.top * y_scale && e.offsetY < sel.top * y_scale + sel.height * y_scale
                     &&
-                    e.offsetX > item.left * y_scale && e.offsetX < item.left * y_scale + item.width * y_scale
+                    e.offsetX > sel.left * y_scale && e.offsetX < sel.left * y_scale + sel.width * y_scale
 
                 ) {
-                    // There could be many elements here, record them all and then we'll find out which is the most 'useful'
+                    // Only highlight these interesting types
-                    // (input, textarea, button, A etc)
+                    if (1) {
-                    if (item.width < xpath_data['browser_width']) {
+                        ctx.strokeRect(sel.left * x_scale, sel.top * y_scale, sel.width * x_scale, sel.height * y_scale);
-                        possible_elements.push(item);
+                        ctx.fillRect(sel.left * x_scale, sel.top * y_scale, sel.width * x_scale, sel.height * y_scale);
+                        current_selected_i = i - 1;
+                        break;
+
+                        // find the smallest one at this x,y
+                        // does it mean sort the xpath list by size (w*h) i think so!
+                    } else {
+
+                        if (include_text_elements[0].checked === true) {
+                            // blue one with background instead?
+                            ctx.fillStyle = 'rgba(0,0,255, 0.1)';
+                            ctx.strokeStyle = 'rgba(0,0,200, 0.7)';
+                            $('#browsersteps-selector-canvas').css('cursor', 'grab');
+                            ctx.strokeRect(sel.left * x_scale, sel.top * y_scale, sel.width * x_scale, sel.height * y_scale);
+                            ctx.fillRect(sel.left * x_scale, sel.top * y_scale, sel.width * x_scale, sel.height * y_scale);
+                            current_selected_i = i - 1;
+                            break;
+                        }
                     }
                 }
-            });
-
-            // Find the best one
-            if (possible_elements.length) {
-                possible_elements.forEach(function (item, index) {
-                  if (["a", "input", "textarea", "button"].includes(item['tagName'])) {
-                      current_selected_i = item;
-                  }
-                });
-
-                if (!current_selected_i) {
-                    current_selected_i = possible_elements[0];
-                }
-
-                sel = xpath_data['size_pos'][current_selected_i];
-                ctx.strokeRect(current_selected_i.left * x_scale, current_selected_i.top * y_scale, current_selected_i.width * x_scale, current_selected_i.height * y_scale);
-                ctx.fillRect(current_selected_i.left * x_scale, current_selected_i.top * y_scale, current_selected_i.width * x_scale, current_selected_i.height * y_scale);
             }
 
-
         }.debounce(10));
     });
 
@@ -195,16 +195,16 @@ $(document).ready(function () {
 
 
     // callback for clicking on an xpath on the canvas
-    function process_selected(selected_in_xpath_list) {
+    function process_selected(xpath_data_index) {
         found_something = false;
         var first_available = $("ul#browser_steps li.empty").first();
 
 
-        if (selected_in_xpath_list !== false) {
+        if (xpath_data_index !== false) {
             // Nothing focused, so fill in a new one
             // if inpt type button or <button>
             // from the top, find the next not used one and use it
-            var x = selected_in_xpath_list;
+            var x = xpath_data['size_pos'][xpath_data_index];
             console.log(x);
             if (x && first_available.length) {
                 // @todo will it let you click shit that has a layer ontop? probably not.
@@ -214,18 +214,26 @@ $(document).ready(function () {
                     $('input[placeholder="Value"]', first_available).addClass('ok').click().focus();
                     found_something = true;
                 } else {
-                    // There's no good way (that I know) to find if this
+                    if (x['isClickable'] || x['tagName'].startsWith('h') || x['tagName'] === 'a' || x['tagName'] === 'button' || x['tagtype'] === 'submit' || x['tagtype'] === 'checkbox' || x['tagtype'] === 'radio' || x['tagtype'] === 'li') {
-                    // see https://stackoverflow.com/questions/446892/how-to-find-event-listeners-on-a-dom-node-in-javascript-or-in-debugging
-                    // https://codepen.io/azaslavsky/pen/DEJVWv
-
-                    // So we dont know if its really a clickable element or not :-(
-                    // Assume it is - then we dont fill the pages with unreliable "Click X,Y" selections
-                    // If you switch to "Click X,y" after an element here is setup, it will give the last co-ords anyway
-                    //if (x['isClickable'] || x['tagName'].startsWith('h') || x['tagName'] === 'a' || x['tagName'] === 'button' || x['tagtype'] === 'submit' || x['tagtype'] === 'checkbox' || x['tagtype'] === 'radio' || x['tagtype'] === 'li') {
                         $('select', first_available).val('Click element').change();
                         $('input[type=text]', first_available).first().val(x['xpath']);
                         found_something = true;
-                    //}
+                    }
+                }
+
+                first_available.xpath_data_index = xpath_data_index;
+
+                if (!found_something) {
+                    if (include_text_elements[0].checked === true) {
+                        // Suggest that we use as filter?
+                        // @todo filters should always be in the last steps, nothing non-filter after it
+                        found_something = true;
+                        ctx.strokeStyle = 'rgba(0,0,255, 0.9)';
+                        ctx.fillStyle = 'rgba(0,0,255, 0.1)';
+                        $('select', first_available).val('Extract text and use as filter').change();
+                        $('input[type=text]', first_available).first().val(x['xpath']);
+                        include_text_elements[0].checked = false;
+                    }
                 }
             }
         }
@@ -240,7 +248,7 @@ $(document).ready(function () {
 
     function start() {
         console.log("Starting browser-steps UI");
-        browsersteps_session_id = false;
+        browsersteps_session_id = Date.now();
         // @todo This setting of the first one should be done at the datalayer but wtforms doesnt wanna play nice
         $('#browser_steps >li:first-child').removeClass('empty');
         set_first_gotosite_disabled();
@@ -248,7 +256,7 @@ $(document).ready(function () {
         $('.clear,.remove', $('#browser_steps >li:first-child')).hide();
         $.ajax({
             type: "GET",
-            url: browser_steps_start_url,
+            url: browser_steps_sync_url + "&browsersteps_session_id=" + browsersteps_session_id,
             statusCode: {
                 400: function () {
                     // More than likely the CSRF token was lost when the server restarted
@@ -256,12 +264,12 @@ $(document).ready(function () {
                 }
             }
         }).done(function (data) {
+            xpath_data = data.xpath_data;
             $("#loading-status-text").fadeIn();
-            browsersteps_session_id = data.browsersteps_session_id;
             // This should trigger 'Goto site'
             console.log("Got startup response, requesting Goto-Site (first) step fake click");
             $('#browser_steps >li:first-child .apply').click();
-            browserless_seconds_remaining = 500;
+            browserless_seconds_remaining = data.browser_time_remaining;
             set_first_gotosite_disabled();
         }).fail(function (data) {
             console.log(data);
@@ -422,6 +430,7 @@ $(document).ready(function () {
             apply_buttons_disabled = false;
             $("#browsersteps-img").css('opacity', 1);
             $('ul#browser_steps li .control .apply').css('opacity', 1);
+            browserless_seconds_remaining = data.browser_time_remaining;
             $("#loading-status-text").hide();
             set_first_gotosite_disabled();
         }).fail(function (data) {

changedetectionio/static/js/tabs.js

@@ -12,7 +12,7 @@ window.addEventListener('hashchange', function () {
 var has_errors = document.querySelectorAll(".messages .error");
 if (!has_errors.length) {
     if (document.location.hash == "") {
-        location.replace(document.querySelector(".tabs ul li:first-child a").hash);
+        document.querySelector(".tabs ul li:first-child a").click();
     } else {
         set_active_tab();
     }

changedetectionio/static/js/toggle-theme.js

@@ -3,7 +3,7 @@
  * Toggles theme between light and dark mode.
  */
 $(document).ready(function () {
-  const button = document.getElementById("toggle-light-mode");
+  const button = document.getElementsByClassName("toggle-theme")[0];
 
   button.onclick = () => {
     const htmlElement = document.getElementsByTagName("html");
@@ -21,33 +21,4 @@ $(document).ready(function () {
   const setCookieValue = (value) => {
     document.cookie = `css_dark_mode=${value};max-age=31536000;path=/`
   }
-
-  // Search input box behaviour
-    const toggle_search = document.getElementById("toggle-search");
-  const search_q = document.getElementById("search-q");
-  window.addEventListener('keydown', function (e) {
-
-    if (e.altKey == true && e.keyCode == 83)
-      search_q.classList.toggle('expanded');
-      search_q.focus();
-  });
-
-
-  search_q.onkeydown = (e) => {
-    var key = e.keyCode || e.which;
-    if (key === 13) {
-      document.searchForm.submit();
-    }
-  };
-  toggle_search.onclick = () => {
-    // Could be that they want to search something once text is in there
-    if (search_q.value.length) {
-      document.searchForm.submit();
-    } else {
-      // If not..
-      search_q.classList.toggle('expanded');
-      search_q.focus();
-    }
-  };
-
 });

changedetectionio/static/js/visual-selector.js

@@ -61,12 +61,7 @@ $(document).ready(function () {
     function bootstrap_visualselector() {
         if (1) {
             // bootstrap it, this will trigger everything else
-            $("img#selector-background").on("error", function () {
+            $("img#selector-background").bind('load', function () {
-                $('.fetching-update-notice').html("<strong>Ooops!</strong> The VisualSelector tool needs atleast one fetched page, please unpause the watch and/or wait for the watch to complete fetching and then reload this page.");
-                $('.fetching-update-notice').css('color','#bb0000');
-                $('#selector-current-xpath').hide();
-                $('#clear-selector').hide();
-            }).bind('load', function () {
                 console.log("Loaded background...");
                 c = document.getElementById("selector-canvas");
                 // greyed out fill context
@@ -84,11 +79,10 @@ $(document).ready(function () {
             }).attr("src", screenshot_url);
         }
         // Tell visualSelector that the image should update
-        var s = $("img#selector-background").attr('src') + "?" + new Date().getTime();
+        var s = $("img#selector-background").attr('src')+"?"+ new Date().getTime();
-        $("img#selector-background").attr('src', s)
+        $("img#selector-background").attr('src',s)
     }
 
-    // This is fired once the img src is loaded in bootstrap_visualselector()
     function fetch_data() {
         // Image is ready
         $('.fetching-update-notice').html("Fetching element data..");
@@ -105,8 +99,7 @@ $(document).ready(function () {
             reflow_selector();
             $('.fetching-update-notice').fadeOut();
         });
-
+    };
-    }
 
 
     function set_scale() {

changedetectionio/static/styles/scss/styles.scss

@@ -54,47 +54,8 @@ a.github-link {
   }
 }
 
-#toggle-light-mode {
+button.toggle-theme {
-  width: 3rem;
+  width: 4rem;
-  .icon-dark {
-    display: none;
-  }
-
-  &.dark {
-    .icon-light {
-      display: none;
-    }
-
-    .icon-dark {
-      display: block;
-    }
-  }
-}
-
-#toggle-search {
-  width: 2rem;
-}
-
-#search-q {
-  opacity: 0;
-  -webkit-transition: all .9s ease;
-  -moz-transition: all .9s ease;
-  transition: all .9s ease;
-  width: 0;
-  display: none;
-  &.expanded {
-    width: auto;
-    display: inline-block;
-
-    opacity: 1;
-  }
-}
-#search-result-info {
-  color: #fff;
-}
-
-button.toggle-button {
-  vertical-align: middle;
   background: transparent;
   border: none;
   cursor: pointer;
@@ -113,7 +74,19 @@ button.toggle-button {
     display: block;
   }
 
+  .icon-dark {
+    display: none;
+  }
 
+  &.dark {
+    .icon-light {
+      display: none;
+    }
+
+    .icon-dark {
+      display: block;
+    }
+  }
 }
 
 .pure-menu-horizontal {

changedetectionio/static/styles/styles.css

@@ -331,44 +331,23 @@ a.github-link {
   a.github-link:hover {
     color: var(--color-icon-github-hover); }
 
-#toggle-light-mode {
+button.toggle-theme {
-  width: 3rem; }
+  width: 4rem;
-  #toggle-light-mode .icon-dark {
-    display: none; }
-  #toggle-light-mode.dark .icon-light {
-    display: none; }
-  #toggle-light-mode.dark .icon-dark {
-    display: block; }
-
-#toggle-search {
-  width: 2rem; }
-
-#search-q {
-  opacity: 0;
-  -webkit-transition: all .9s ease;
-  -moz-transition: all .9s ease;
-  transition: all .9s ease;
-  width: 0;
-  display: none; }
-  #search-q.expanded {
-    width: auto;
-    display: inline-block;
-    opacity: 1; }
-
-#search-result-info {
-  color: #fff; }
-
-button.toggle-button {
-  vertical-align: middle;
   background: transparent;
   border: none;
   cursor: pointer;
   color: var(--color-icon-github); }
-  button.toggle-button:hover {
+  button.toggle-theme:hover {
     color: var(--color-icon-github-hover); }
-  button.toggle-button svg {
+  button.toggle-theme svg {
     fill: currentColor; }
-  button.toggle-button .icon-light {
+  button.toggle-theme .icon-light {
+    display: block; }
+  button.toggle-theme .icon-dark {
+    display: none; }
+  button.toggle-theme.dark .icon-light {
+    display: none; }
+  button.toggle-theme.dark .icon-dark {
     display: block; }
 
 .pure-menu-horizontal {

changedetectionio/store.py

@@ -3,7 +3,7 @@ from flask import (
 )
 
 from . model import App, Watch
-from copy import deepcopy, copy
+from copy import deepcopy
 from os import path, unlink
 from threading import Lock
 import json
@@ -366,21 +366,19 @@ class ChangeDetectionStore:
     def save_error_text(self, watch_uuid, contents):
         if not self.data['watching'].get(watch_uuid):
             return
-
-        self.data['watching'][watch_uuid].ensure_data_dir_exists()
         target_path = os.path.join(self.datastore_path, watch_uuid, "last-error.txt")
+
         with open(target_path, 'w') as f:
             f.write(contents)
 
     def save_xpath_data(self, watch_uuid, data, as_error=False):
-
         if not self.data['watching'].get(watch_uuid):
             return
         if as_error:
             target_path = os.path.join(self.datastore_path, watch_uuid, "elements-error.json")
         else:
             target_path = os.path.join(self.datastore_path, watch_uuid, "elements.json")
-        self.data['watching'][watch_uuid].ensure_data_dir_exists()
+
         with open(target_path, 'w') as f:
             f.write(json.dumps(data))
             f.close()
@@ -474,6 +472,8 @@ class ChangeDetectionStore:
         return proxy_list if len(proxy_list) else None
 
 
+
+
     def get_preferred_proxy_for_watch(self, uuid):
         """
         Returns the preferred proxy by ID key
@@ -505,25 +505,6 @@ class ChangeDetectionStore:
 
         return None
 
-    @property
-    def has_extra_headers_file(self):
-        filepath = os.path.join(self.datastore_path, 'headers.txt')
-        return os.path.isfile(filepath)
-
-    def get_all_headers(self):
-        from .model.App import parse_headers_from_text_file
-        headers = copy(self.data['settings'].get('headers', {}))
-
-        filepath = os.path.join(self.datastore_path, 'headers.txt')
-        try:
-            if os.path.isfile(filepath):
-                headers.update(parse_headers_from_text_file(filepath))
-        except Exception as e:
-            print(f"ERROR reading headers.txt at {filepath}", str(e))
-
-        return headers
-
-
     # Run all updates
     # IMPORTANT - Each update could be run even when they have a new install and the schema is correct
     #             So therefor - each `update_n` should be very careful about checking if it needs to actually run

changedetectionio/templates/_common_fields.jinja

@@ -115,7 +115,7 @@
                                     URLs generated by changedetection.io (such as <code>{{ '{{diff_url}}' }}</code>) require the <code>BASE_URL</code> environment variable set.<br>
                                     Your <code>BASE_URL</code> var is currently "{{settings_application['current_base_url']}}"
 									<br>
-									Warning: Contents of <code>{{ '{{diff}}' }}</code>, <code>{{ '{{diff_removed}}' }}</code>, and <code>{{ '{{diff_added}}' }}</code> depend on how the difference algorithm perceives the change. For example, an addition or removal could be perceived as a change in some cases. <a target="_new" href="https://github.com/dgtlmoon/changedetection.io/wiki/Using-the-%7B%7Bdiff%7D%7D,-%7B%7Bdiff_added%7D%7D,-and-%7B%7Bdiff_removed%7D%7D-notification-tokens">More Here</a> <br>
+									Warning: Contents of <code>{{ '{{diff}}' }}</code>, <code>{{ '{{diff_removed}}' }}</code>, and <code>{{ '{{diff_added}}' }}</code> depend on how the difference algorithm perceives the change. For example, an addition or removal could be perceived as a change in some cases. <a target="_new" href="https://github.com/dgtlmoon/changedetection.io/wiki/Using-the-%7B%7Bdiff%7D%7D,-%7B%7Bdiff_added%7D%7D,-and-%7B%7Bdiff_removal%7D%7D-notification-tokens">More Here</a> <br>
                                 </div>
                             </div>
                         </div>

changedetectionio/templates/base.html

@@ -82,21 +82,11 @@
               <a href="{{url_for('logout')}}" class="pure-menu-link">LOG OUT</a>
             </li>
           {% endif %}
-          <li class="pure-menu-item pure-form" id="search-menu-item">
-            <!-- We use GET here so it offers people a chance to set bookmarks etc -->
-            <form name="searchForm" action="" method="GET">
-              <input id="search-q" class="" name="q" placeholder="URL or Title {% if active_tag %}in '{{ active_tag }}'{% endif %}" required="" type="text" value="">
-              <input name="tag" type="hidden" value="{% if active_tag %}{{active_tag}}{% endif %}">
-              <button class="toggle-button " id="toggle-search" type="button" title="Search, or Use Alt+S Key" >
-                {% include "svgs/search-icon.svg" %}
-              </button>
-            </form>
-          </li>
           <li class="pure-menu-item">
             {% if dark_mode %}
             {% set darkClass = 'dark' %}
             {% endif %}
-            <button class="toggle-button {{darkClass}}"  id ="toggle-light-mode" type="button" title="Toggle Light/Dark Mode">
+            <button class="toggle-theme {{darkClass}}" type="button" title="Toggle Light/Dark Mode">
               <span class="visually-hidden">Toggle light/dark mode</span>
               <span class="icon-light">
                 {% include "svgs/light-mode-toggle-icon.svg" %}
@@ -116,7 +106,7 @@
     </div>
     {% if hosted_sticky %}
       <div class="sticky-tab" id="hosted-sticky">
-        <a href="https://changedetection.io/?ref={{guid}}">Let us host your instance!</a>
+        <a href="https://lemonade.changedetection.io/start?ref={{guid}}">Let us host your instance!</a>
       </div>
     {% endif %}
     {% if left_sticky %}

changedetectionio/templates/edit.html

@@ -14,9 +14,7 @@
 {% endif %}
 
     const browser_steps_config=JSON.parse('{{ browser_steps_config|tojson }}');
-    const browser_steps_start_url="{{url_for('browser_steps.browsersteps_start_session', uuid=uuid)}}";
     const browser_steps_sync_url="{{url_for('browser_steps.browsersteps_ui_update', uuid=uuid)}}";
-
 </script>
 
 <script src="{{url_for('static_content', group='js', filename='watch-settings.js')}}" defer></script>
@@ -152,17 +150,6 @@
 {{ render_field(form.headers, rows=5, placeholder="Example
 Cookie: foobar
 User-Agent: wonderbra 1.0") }}
-
-                        <div class="pure-form-message-inline">
-                            {% if has_extra_headers_file %}
-                                <strong>Alert! Extra headers file found and will be added to this watch!</strong>
-                            {% else %}
-                                Headers can be also read from a file in your data-directory <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Adding-headers-from-an-external-file">Read more here</a>
-                            {% endif %}
-                            <br>
-                            (Not supported by Selenium browser)
-                        </div>
-
                     </div>
                     <div class="pure-control-group" id="request-body">
                                         {{ render_field(form.body, rows=5, placeholder="Example
@@ -199,8 +186,7 @@ User-Agent: wonderbra 1.0") }}
                                     <span class="loader" >
                                         <span id="browsersteps-click-start">
                                             <h2 >Click here to Start</h2>
-                                            <svg style="height: 3.5rem;" version="1.1" viewBox="0 0 32 32"  xml:space="preserve" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink"><g id="Layer_1"/><g id="play_x5F_alt"><path d="M16,0C7.164,0,0,7.164,0,16s7.164,16,16,16s16-7.164,16-16S24.836,0,16,0z M10,24V8l16.008,8L10,24z" style="fill: var(--color-grey-400);"/></g></svg><br>
+                                            Please allow 10-15 seconds for the browser to connect.
-                                            Please allow 10-15 seconds for the browser to connect.<br>
                                         </span>
                                         <div class="spinner"  style="display: none;"></div>
                                     </span>

changedetectionio/templates/settings.html

@@ -70,10 +70,6 @@
                             <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Configurable-BASE_URL-setting">read more here</a>.
                         </span>
                     </div>
-                    <div class="pure-control-group">
-                        {{ render_field(form.application.form.pager_size) }}
-                        <span class="pure-form-message-inline">Number of items per page in the watch overview list, 0 to disable.</span>
-                    </div>
 
                     <div class="pure-control-group">
                         {{ render_checkbox_field(form.application.form.extract_title_as_title) }}

changedetectionio/templates/svgs/search-icon.svg

@@ -1 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?><svg version="1.1" id="Layer_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 0 122.879 119.799" enable-background="new 0 0 122.879 119.799" xml:space="preserve"><g><path d="M49.988,0h0.016v0.007C63.803,0.011,76.298,5.608,85.34,14.652c9.027,9.031,14.619,21.515,14.628,35.303h0.007v0.033v0.04 h-0.007c-0.005,5.557-0.917,10.905-2.594,15.892c-0.281,0.837-0.575,1.641-0.877,2.409v0.007c-1.446,3.66-3.315,7.12-5.547,10.307 l29.082,26.139l0.018,0.016l0.157,0.146l0.011,0.011c1.642,1.563,2.536,3.656,2.649,5.78c0.11,2.1-0.543,4.248-1.979,5.971 l-0.011,0.016l-0.175,0.203l-0.035,0.035l-0.146,0.16l-0.016,0.021c-1.565,1.642-3.654,2.534-5.78,2.646 c-2.097,0.111-4.247-0.54-5.971-1.978l-0.015-0.011l-0.204-0.175l-0.029-0.024L78.761,90.865c-0.88,0.62-1.778,1.209-2.687,1.765 c-1.233,0.755-2.51,1.466-3.813,2.115c-6.699,3.342-14.269,5.222-22.272,5.222v0.007h-0.016v-0.007 c-13.799-0.004-26.296-5.601-35.338-14.645C5.605,76.291,0.016,63.805,0.007,50.021H0v-0.033v-0.016h0.007 c0.004-13.799,5.601-26.296,14.645-35.338C23.683,5.608,36.167,0.016,49.955,0.007V0H49.988L49.988,0z M50.004,11.21v0.007h-0.016 h-0.033V11.21c-10.686,0.007-20.372,4.35-27.384,11.359C15.56,29.578,11.213,39.274,11.21,49.973h0.007v0.016v0.033H11.21 c0.007,10.686,4.347,20.367,11.359,27.381c7.009,7.012,16.705,11.359,27.403,11.361v-0.007h0.016h0.033v0.007 c10.686-0.007,20.368-4.348,27.382-11.359c7.011-7.009,11.358-16.702,11.36-27.4h-0.006v-0.016v-0.033h0.006 c-0.006-10.686-4.35-20.372-11.358-27.384C70.396,15.56,60.703,11.213,50.004,11.21L50.004,11.21z"/></g></svg>

changedetectionio/templates/watch-overview.html

@@ -38,13 +38,11 @@
         <button class="pure-button button-secondary button-xsmall" name="op" value="recheck">Recheck</button>
         <button class="pure-button button-secondary button-xsmall" name="op" value="mark-viewed">Mark viewed</button>
         <button class="pure-button button-secondary button-xsmall" name="op" value="notification-default">Use default notification</button>
-        <button class="pure-button button-secondary button-xsmall" style="background: #dd4242;" name="op" value="clear-history">Clear/reset history</button>
+        <button class="pure-button button-secondary button-xsmall" style="background: #dd4242; font-size: 70%" name="op" value="delete">Delete</button>
-        <button class="pure-button button-secondary button-xsmall" style="background: #dd4242;" name="op" value="delete">Delete</button>
     </div>
     {% if watches|length >= pagination.per_page %}
         {{ pagination.info }}
     {% endif %}
-    {% if search_q %}<div id="search-result-info">Searching "<strong><i>{{search_q}}</i></strong>"</div>{% endif %}
     <div>
         <a href="{{url_for('index')}}" class="pure-button button-tag {{'active' if not active_tag }}">All</a>
         {% for tag in tags %}
@@ -74,12 +72,8 @@
             </tr>
             </thead>
             <tbody>
-            {% if not watches|length %}
+
-            <tr>
+            {% for watch in (watches|sort(attribute=sort_attribute, reverse=sort_order == 'asc'))[pagination.skip:pagination.skip+pagination.per_page] %}
-                <td colspan="6">No website watches configured, please add a URL in the box above, or <a href="{{ url_for('import_page')}}" >import a list</a>.</td>
-            </tr>
-            {% endif %}
-            {% for watch in (watches|sort(attribute=sort_attribute, reverse=sort_order == 'asc'))|pagination_slice(skip=pagination.skip) %}
             <tr id="{{ watch.uuid }}"
                 class="{{ loop.cycle('pure-table-odd', 'pure-table-even') }} processor-{{ watch['processor'] }}
                 {% if watch.last_error is defined and watch.last_error != False %}error{% endif %}

changedetectionio/tests/conftest.py

@@ -14,16 +14,13 @@ global app
 
 def cleanup(datastore_path):
     # Unlink test output files
-    files = [
+    files = ['output.txt',
-        'count.txt',
+             'url-watches.json',
-        'endpoint-content.txt'
+             'secret.txt',
-        'headers.txt',
+             'notification.txt',
-        'headers-testtag.txt',
+             'count.txt',
-        'notification.txt',
+             'endpoint-content.txt'
-        'secret.txt',
+                 ]
-        'url-watches.json',
-        'output.txt',
-    ]
     for file in files:
         try:
             os.unlink("{}/{}".format(datastore_path, file))

changedetectionio/tests/test_request.py

@@ -1,8 +1,7 @@
 import json
-import os
 import time
 from flask import url_for
-from . util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks, extract_UUID_from_client
+from . util import set_original_response, set_modified_response, live_server_setup
 
 def test_setup(live_server):
     live_server_setup(live_server)
@@ -10,12 +9,8 @@ def test_setup(live_server):
 # Hard to just add more live server URLs when one test is already running (I think)
 # So we add our test here (was in a different file)
 def test_headers_in_request(client, live_server):
-    #live_server_setup(live_server)
     # Add our URL to the import page
     test_url = url_for('test_headers', _external=True)
-    if os.getenv('PLAYWRIGHT_DRIVER_URL'):
-        # Because its no longer calling back to localhost but from browserless, set in test-only.yml
-        test_url = test_url.replace('localhost', 'changedet')
 
     # Add the test URL twice, we will check
     res = client.post(
@@ -34,7 +29,7 @@ def test_headers_in_request(client, live_server):
     )
     assert b"1 Imported" in res.data
 
-    wait_for_all_checks(client)
+    time.sleep(3)
     cookie_header = '_ga=GA1.2.1022228332; cookie-preferences=analytics:accepted;'
 
 
@@ -44,7 +39,7 @@ def test_headers_in_request(client, live_server):
         data={
               "url": test_url,
               "tag": "",
-              "fetch_backend": 'html_webdriver' if os.getenv('PLAYWRIGHT_DRIVER_URL') else 'html_requests',
+              "fetch_backend": "html_requests",
               "headers": "xxx:ooo\ncool:yeah\r\ncookie:"+cookie_header},
         follow_redirects=True
     )
@@ -52,7 +47,7 @@ def test_headers_in_request(client, live_server):
 
 
     # Give the thread time to pick up the first version
-    wait_for_all_checks(client)
+    time.sleep(5)
 
     # The service should echo back the request headers
     res = client.get(
@@ -68,7 +63,7 @@ def test_headers_in_request(client, live_server):
     from html import escape
     assert escape(cookie_header).encode('utf-8') in res.data
 
-    wait_for_all_checks(client)
+    time.sleep(5)
 
     # Re #137 -  Examine the JSON index file, it should have only one set of headers entered
     watches_with_headers = 0
@@ -84,9 +79,6 @@ def test_headers_in_request(client, live_server):
 def test_body_in_request(client, live_server):
     # Add our URL to the import page
     test_url = url_for('test_body', _external=True)
-    if os.getenv('PLAYWRIGHT_DRIVER_URL'):
-        # Because its no longer calling back to localhost but from browserless, set in test-only.yml
-        test_url = test_url.replace('localhost', 'cdio')
 
     res = client.post(
         url_for("import_page"),
@@ -175,9 +167,6 @@ def test_body_in_request(client, live_server):
 def test_method_in_request(client, live_server):
     # Add our URL to the import page
     test_url = url_for('test_method', _external=True)
-    if os.getenv('PLAYWRIGHT_DRIVER_URL'):
-        # Because its no longer calling back to localhost but from browserless, set in test-only.yml
-        test_url = test_url.replace('localhost', 'cdio')
 
     # Add the test URL twice, we will check
     res = client.post(
@@ -245,76 +234,3 @@ def test_method_in_request(client, live_server):
     # Should be only one with method set to PATCH
     assert watches_with_method == 1
 
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
-    assert b'Deleted' in res.data
-
-def test_headers_textfile_in_request(client, live_server):
-    #live_server_setup(live_server)
-    # Add our URL to the import page
-    test_url = url_for('test_headers', _external=True)
-    if os.getenv('PLAYWRIGHT_DRIVER_URL'):
-        # Because its no longer calling back to localhost but from browserless, set in test-only.yml
-        test_url = test_url.replace('localhost', 'cdio')
-
-    print ("TEST URL IS ",test_url)
-    # Add the test URL twice, we will check
-    res = client.post(
-        url_for("import_page"),
-        data={"urls": test_url},
-        follow_redirects=True
-    )
-    assert b"1 Imported" in res.data
-
-    time.sleep(1)
-
-
-    # Add some headers to a request
-    res = client.post(
-        url_for("edit_page", uuid="first"),
-        data={
-              "url": test_url,
-              "tag": "testtag",
-              "fetch_backend": 'html_webdriver' if os.getenv('PLAYWRIGHT_DRIVER_URL') else 'html_requests',
-              "headers": "xxx:ooo\ncool:yeah\r\n"},
-        follow_redirects=True
-    )
-    assert b"Updated watch." in res.data
-    wait_for_all_checks(client)
-
-    with open('test-datastore/headers-testtag.txt', 'w') as f:
-        f.write("tag-header: test")
-
-    with open('test-datastore/headers.txt', 'w') as f:
-        f.write("global-header: nice\r\nnext-global-header: nice")
-
-    with open('test-datastore/'+extract_UUID_from_client(client)+'/headers.txt', 'w') as f:
-        f.write("watch-header: nice")
-
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
-
-    # Give the thread time to pick it up
-    wait_for_all_checks(client)
-
-    res = client.get(url_for("edit_page", uuid="first"))
-    assert b"Extra headers file found and will be added to this watch" in res.data
-
-    # Not needed anymore
-    os.unlink('test-datastore/headers.txt')
-    os.unlink('test-datastore/headers-testtag.txt')
-    os.unlink('test-datastore/'+extract_UUID_from_client(client)+'/headers.txt')
-    # The service should echo back the request verb
-    res = client.get(
-        url_for("preview_page", uuid="first"),
-        follow_redirects=True
-    )
-
-    assert b"Global-Header:nice" in res.data
-    assert b"Next-Global-Header:nice" in res.data
-    assert b"Xxx:ooo" in res.data
-    assert b"Watch-Header:nice" in res.data
-    assert b"Tag-Header:test" in res.data
-
-
-    #unlink headers.txt on start/stop
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
-    assert b'Deleted' in res.data

changedetectionio/update_worker.py

@@ -5,7 +5,7 @@ import time
 
 from changedetectionio import content_fetcher
 from .processors.text_json_diff import FilterNotFoundInResponse
-from .processors.restock_diff import UnableToExtractRestockData
+
 
 # A single update worker
 #
@@ -238,7 +238,7 @@ class update_worker(threading.Thread):
                         if not self.datastore.data['watching'].get(uuid):
                             continue
 
-                        err_text = "Warning, no filters were found, no change detection ran - Did the page change layout? update your Visual Filter if necessary."
+                        err_text = "Warning, no filters were found, no change detection ran."
                         self.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text})
 
                         # Only when enabled, send the notification
@@ -262,7 +262,6 @@ class update_worker(threading.Thread):
                         # Yes fine, so nothing todo, don't continue to process.
                         process_changedetection_results = False
                         changed_detected = False
-                        self.datastore.update_watch(uuid=uuid, update_obj={'last_error': False})
 
                     except content_fetcher.BrowserStepsStepTimout as e:
 
@@ -316,13 +315,7 @@ class update_worker(threading.Thread):
                             self.datastore.save_screenshot(watch_uuid=uuid, screenshot=e.screenshot, as_error=True)
 
                         self.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text,
-                                                                           'last_check_status': e.status_code,
+                                                                           'last_check_status': e.status_code})
-                                                                           'has_ldjson_price_data': None})
-                        process_changedetection_results = False
-                    except UnableToExtractRestockData as e:
-                        # Usually when fetcher.instock_data returns empty
-                        self.app.logger.error("Exception reached processing watch UUID: %s - %s", uuid, str(e))
-                        self.datastore.update_watch(uuid=uuid, update_obj={'last_error': f"Unable to extract restock data for this page unfortunately. (Got code {e.status_code} from server)"})
                         process_changedetection_results = False
                     except Exception as e:
                         self.app.logger.error("Exception reached processing watch UUID: %s - %s", uuid, str(e))