Update wiki link for 'More info' about sharing a watch and its configuration

changedetectionio/__init__.py

@@ -36,14 +36,11 @@ from flask import (
     url_for,
 )
 from flask_login import login_required
-from flask_restful import abort, Api
-
 from flask_wtf import CSRFProtect
 
 from changedetectionio import html_tools
-from changedetectionio.api import api_v1
 
-__version__ = '0.39.14'
+__version__ = '0.39.13.1'
 
 datastore = None
 
@@ -81,8 +78,6 @@ csrf.init_app(app)
 
 notification_debug_log=[]
 
-watch_api = Api(app, decorators=[csrf.exempt])
-
 def init_app_secret(datastore_path):
     secret = ""
 
@@ -184,25 +179,6 @@ def changedetection_app(config=None, datastore_o=None):
     login_manager.login_view = 'login'
     app.secret_key = init_app_secret(config['datastore_path'])
 
-
-    watch_api.add_resource(api_v1.WatchSingleHistory,
-                           '/api/v1/watch/<string:uuid>/history/<string:timestamp>',
-                           resource_class_kwargs={'datastore': datastore, 'update_q': update_q})
-
-    watch_api.add_resource(api_v1.WatchHistory,
-                           '/api/v1/watch/<string:uuid>/history',
-                           resource_class_kwargs={'datastore': datastore})
-
-    watch_api.add_resource(api_v1.CreateWatch, '/api/v1/watch',
-                           resource_class_kwargs={'datastore': datastore, 'update_q': update_q})
-
-    watch_api.add_resource(api_v1.Watch, '/api/v1/watch/<string:uuid>',
-                           resource_class_kwargs={'datastore': datastore, 'update_q': update_q})
-
-
-
-
-
     # Setup cors headers to allow all domains
     # https://flask-cors.readthedocs.io/en/latest/
     #    CORS(app)
@@ -391,8 +367,6 @@ def changedetection_app(config=None, datastore_o=None):
 
             if limit_tag != None:
                 # Support for comma separated list of tags.
-                if watch['tag'] is None:
-                    continue
                 for tag_in_watch in watch['tag'].split(','):
                     tag_in_watch = tag_in_watch.strip()
                     if tag_in_watch == limit_tag:
@@ -546,9 +520,16 @@ def changedetection_app(config=None, datastore_o=None):
 
         # Defaults for proxy choice
         if datastore.proxy_list is not None:  # When enabled
-            # Radio needs '' not None, or incase that the chosen one no longer exists
-            if default['proxy'] is None or not any(default['proxy'] in tup for tup in datastore.proxy_list):
-                default['proxy'] = ''
+            system_proxy = datastore.data['settings']['requests']['proxy']
+            if default['proxy'] is None:
+                default['proxy'] = system_proxy
+            else:
+                # Does the chosen one exist?
+                if not any(default['proxy'] in tup for tup in datastore.proxy_list):
+                    default['proxy'] = datastore.proxy_list[0][0]
+
+            # Used by the form handler to keep or remove the proxy settings
+            default['proxy_list'] = datastore.proxy_list
 
         # proxy_override set to the json/text list of the items
         form = forms.watchForm(formdata=request.form if request.method == 'POST' else None,
@@ -559,7 +540,9 @@ def changedetection_app(config=None, datastore_o=None):
             # @todo - Couldn't get setattr() etc dynamic addition working, so remove it instead
             del form.proxy
         else:
-            form.proxy.choices = [('', 'Default')] + datastore.proxy_list
+            form.proxy.choices = datastore.proxy_list
+            if default['proxy'] is None:
+                form.proxy.default='http://hello'
 
         if request.method == 'POST' and form.validate():
             extra_update_obj = {}
@@ -596,10 +579,6 @@ def changedetection_app(config=None, datastore_o=None):
                 if len(datastore.data['watching'][uuid]['history']):
                     extra_update_obj['previous_md5'] = get_current_checksum_include_ignore_text(uuid=uuid)
 
-            # Be sure proxy value is None
-            if datastore.proxy_list is not None and form.data['proxy'] == '':
-                extra_update_obj['proxy'] = None
-
             datastore.data['watching'][uuid].update(form.data)
             datastore.data['watching'][uuid].update(extra_update_obj)
 
@@ -626,12 +605,6 @@ def changedetection_app(config=None, datastore_o=None):
             if request.method == 'POST' and not form.validate():
                 flash("An error occurred, please see below.", "error")
 
-            visualselector_data_is_ready = datastore.visualselector_data_is_ready(uuid)
-
-            # Only works reliably with Playwright
-            visualselector_enabled = os.getenv('PLAYWRIGHT_DRIVER_URL', False) and default['fetch_backend'] == 'html_webdriver'
-
-
             output = render_template("edit.html",
                                      uuid=uuid,
                                      watch=datastore.data['watching'][uuid],
@@ -639,9 +612,7 @@ def changedetection_app(config=None, datastore_o=None):
                                      has_empty_checktime=using_default_check_time,
                                      using_global_webdriver_wait=default['webdriver_delay'] is None,
                                      current_base_url=datastore.data['settings']['application']['base_url'],
-                                     emailprefix=os.getenv('NOTIFICATION_MAIL_BUTTON_PREFIX', False),
-                                     visualselector_data_is_ready=visualselector_data_is_ready,
-                                     visualselector_enabled=visualselector_enabled
+                                     emailprefix=os.getenv('NOTIFICATION_MAIL_BUTTON_PREFIX', False)
                                      )
 
         return output
@@ -705,7 +676,6 @@ def changedetection_app(config=None, datastore_o=None):
                                  form=form,
                                  current_base_url = datastore.data['settings']['application']['base_url'],
                                  hide_remove_pass=os.getenv("SALTED_PASS", False),
-                                 api_key=datastore.data['settings']['application'].get('api_access_token'),
                                  emailprefix=os.getenv('NOTIFICATION_MAIL_BUTTON_PREFIX', False))
 
         return output
@@ -904,6 +874,27 @@ def changedetection_app(config=None, datastore_o=None):
 
         return output
 
+    @app.route("/api/<string:uuid>/snapshot/current", methods=['GET'])
+    @login_required
+    def api_snapshot(uuid):
+
+        # More for testing, possible to return the first/only
+        if uuid == 'first':
+            uuid = list(datastore.data['watching'].keys()).pop()
+
+        try:
+            watch = datastore.data['watching'][uuid]
+        except KeyError:
+            return abort(400, "No history found for the specified link, bad link?")
+
+        newest = list(watch['history'].keys())[-1]
+        with open(watch['history'][newest], 'r') as f:
+            content = f.read()
+
+        resp = make_response(content)
+        resp.headers['Content-Type'] = 'text/plain'
+        return resp
+
     @app.route("/favicon.ico", methods=['GET'])
     def favicon():
         return send_from_directory("static/images", path="favicon.ico")
@@ -984,9 +975,10 @@ def changedetection_app(config=None, datastore_o=None):
 
     @app.route("/static/<string:group>/<string:filename>", methods=['GET'])
     def static_content(group, filename):
-        from flask import make_response
-
         if group == 'screenshot':
+
+            from flask import make_response
+
             # Could be sensitive, follow password requirements
             if datastore.data['settings']['application']['password'] and not flask_login.current_user.is_authenticated:
                 abort(403)
@@ -1005,26 +997,6 @@ def changedetection_app(config=None, datastore_o=None):
             except FileNotFoundError:
                 abort(404)
 
-
-        if group == 'visual_selector_data':
-            # Could be sensitive, follow password requirements
-            if datastore.data['settings']['application']['password'] and not flask_login.current_user.is_authenticated:
-                abort(403)
-
-            # These files should be in our subdirectory
-            try:
-                # set nocache, set content-type
-                watch_dir = datastore_o.datastore_path + "/" + filename
-                response = make_response(send_from_directory(filename="elements.json", directory=watch_dir, path=watch_dir + "/elements.json"))
-                response.headers['Content-type'] = 'application/json'
-                response.headers['Cache-Control'] = 'no-cache, no-store, must-revalidate'
-                response.headers['Pragma'] = 'no-cache'
-                response.headers['Expires'] = 0
-                return response
-
-            except FileNotFoundError:
-                abort(404)
-
         # These files should be in our subdirectory
         try:
             return send_from_directory("static/{}".format(group), path=filename)
@@ -1033,7 +1005,7 @@ def changedetection_app(config=None, datastore_o=None):
 
     @app.route("/api/add", methods=['POST'])
     @login_required
-    def form_watch_add():
+    def api_watch_add():
         from changedetectionio import forms
         form = forms.quickWatchForm(request.form)
 
@@ -1059,7 +1031,7 @@ def changedetection_app(config=None, datastore_o=None):
 
     @app.route("/api/delete", methods=['GET'])
     @login_required
-    def form_delete():
+    def api_delete():
         uuid = request.args.get('uuid')
 
         if uuid != 'all' and not uuid in datastore.data['watching'].keys():
@@ -1076,7 +1048,7 @@ def changedetection_app(config=None, datastore_o=None):
 
     @app.route("/api/clone", methods=['GET'])
     @login_required
-    def form_clone():
+    def api_clone():
         uuid = request.args.get('uuid')
         # More for testing, possible to return the first/only
         if uuid == 'first':
@@ -1090,7 +1062,7 @@ def changedetection_app(config=None, datastore_o=None):
 
     @app.route("/api/checknow", methods=['GET'])
     @login_required
-    def form_watch_checknow():
+    def api_watch_checknow():
 
         tag = request.args.get('tag')
         uuid = request.args.get('uuid')
@@ -1127,7 +1099,7 @@ def changedetection_app(config=None, datastore_o=None):
 
     @app.route("/api/share-url", methods=['GET'])
     @login_required
-    def form_share_put_watch():
+    def api_share_put_watch():
         """Given a watch UUID, upload the info and return a share-link
            the share-link can be imported/added"""
         import requests
@@ -1177,6 +1149,7 @@ def changedetection_app(config=None, datastore_o=None):
         # paste in etc
         return redirect(url_for('index'))
 
+
     # @todo handle ctrl break
     ticker_thread = threading.Thread(target=ticker_thread_check_time_launch_checks).start()
 

changedetectionio/api/api_v1.py

@@ -1,125 +0,0 @@
-from flask_restful import abort, Resource
-from flask import request, make_response
-import validators
-from . import auth
-
-
-
-# https://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html
-
-class Watch(Resource):
-    def __init__(self, **kwargs):
-        # datastore is a black box dependency
-        self.datastore = kwargs['datastore']
-        self.update_q = kwargs['update_q']
-
-    # Get information about a single watch, excluding the history list (can be large)
-    # curl http://localhost:4000/api/v1/watch/<string:uuid>
-    # ?recheck=true
-    @auth.check_token
-    def get(self, uuid):
-        from copy import deepcopy
-        watch = deepcopy(self.datastore.data['watching'].get(uuid))
-        if not watch:
-            abort(404, message='No watch exists with the UUID of {}'.format(uuid))
-
-        if request.args.get('recheck'):
-            self.update_q.put(uuid)
-            return "OK", 200
-
-        # Return without history, get that via another API call
-        watch['history_n'] = len(watch['history'])
-        del (watch['history'])
-        return watch
-
-    @auth.check_token
-    def delete(self, uuid):
-        if not self.datastore.data['watching'].get(uuid):
-            abort(400, message='No watch exists with the UUID of {}'.format(uuid))
-
-        self.datastore.delete(uuid)
-        return 'OK', 204
-
-
-class WatchHistory(Resource):
-    def __init__(self, **kwargs):
-        # datastore is a black box dependency
-        self.datastore = kwargs['datastore']
-
-    # Get a list of available history for a watch by UUID
-    # curl http://localhost:4000/api/v1/watch/<string:uuid>/history
-    def get(self, uuid):
-        watch = self.datastore.data['watching'].get(uuid)
-        if not watch:
-            abort(404, message='No watch exists with the UUID of {}'.format(uuid))
-        return watch['history'], 200
-
-
-class WatchSingleHistory(Resource):
-    def __init__(self, **kwargs):
-        # datastore is a black box dependency
-        self.datastore = kwargs['datastore']
-
-    # Read a given history snapshot and return its content
-    # <string:timestamp> or "latest"
-    # curl http://localhost:4000/api/v1/watch/<string:uuid>/history/<int:timestamp>
-    @auth.check_token
-    def get(self, uuid, timestamp):
-        watch = self.datastore.data['watching'].get(uuid)
-        if not watch:
-            abort(404, message='No watch exists with the UUID of {}'.format(uuid))
-
-        if not len(watch['history']):
-            abort(404, message='Watch found but no history exists for the UUID {}'.format(uuid))
-
-        if timestamp == 'latest':
-            timestamp = list(watch['history'].keys())[-1]
-
-        with open(watch['history'][timestamp], 'r') as f:
-            content = f.read()
-
-        response = make_response(content, 200)
-        response.mimetype = "text/plain"
-        return response
-
-
-class CreateWatch(Resource):
-    def __init__(self, **kwargs):
-        # datastore is a black box dependency
-        self.datastore = kwargs['datastore']
-        self.update_q = kwargs['update_q']
-
-    @auth.check_token
-    def post(self):
-        # curl http://localhost:4000/api/v1/watch -H "Content-Type: application/json" -d '{"url": "https://my-nice.com", "tag": "one, two" }'
-        json_data = request.get_json()
-        tag = json_data['tag'].strip() if json_data.get('tag') else ''
-
-        if not validators.url(json_data['url'].strip()):
-            return "Invalid or unsupported URL", 400
-
-        extras = {'title': json_data['title'].strip()} if json_data.get('title') else {}
-
-        new_uuid = self.datastore.add_watch(url=json_data['url'].strip(), tag=tag, extras=extras)
-        self.update_q.put(new_uuid)
-        return {'uuid': new_uuid}, 201
-
-    # Return concise list of available watches and some very basic info
-    # curl http://localhost:4000/api/v1/watch|python -mjson.tool
-    # ?recheck_all=1 to recheck all
-    @auth.check_token
-    def get(self):
-        list = {}
-        for k, v in self.datastore.data['watching'].items():
-            list[k] = {'url': v['url'],
-                       'title': v['title'],
-                       'last_checked': v['last_checked'],
-                       'last_changed': v['last_changed'],
-                       'last_error': v['last_error']}
-
-        if request.args.get('recheck_all'):
-            for uuid in self.datastore.data['watching'].keys():
-                self.update_q.put(uuid)
-            return {'status': "OK"}, 200
-
-        return list, 200

changedetectionio/api/auth.py

@@ -1,33 +0,0 @@
-from flask import request, make_response, jsonify
-from functools import wraps
-
-
-# Simple API auth key comparison
-# @todo - Maybe short lived token in the future?
-
-def check_token(f):
-    @wraps(f)
-    def decorated(*args, **kwargs):
-        datastore = args[0].datastore
-
-        config_api_token_enabled = datastore.data['settings']['application'].get('api_access_token_enabled')
-        if not config_api_token_enabled:
-            return
-
-        try:
-            api_key_header = request.headers['x-api-key']
-        except KeyError:
-            return make_response(
-                jsonify("No authorization x-api-key header."), 403
-            )
-
-        config_api_token = datastore.data['settings']['application'].get('api_access_token')
-
-        if api_key_header != config_api_token:
-            return make_response(
-                jsonify("Invalid access - API key invalid."), 403
-            )
-
-        return f(*args, **kwargs)
-
-    return decorated

changedetectionio/content_fetcher.py

@@ -27,117 +27,6 @@ class Fetcher():
     status_code = None
     content = None
     headers = None
-
-    fetcher_description = "No description"
-    xpath_element_js = """               
-                // Include the getXpath script directly, easier than fetching
-                !function(e,n){"object"==typeof exports&&"undefined"!=typeof module?module.exports=n():"function"==typeof define&&define.amd?define(n):(e=e||self).getXPath=n()}(this,function(){return function(e){var n=e;if(n&&n.id)return'//*[@id="'+n.id+'"]';for(var o=[];n&&Node.ELEMENT_NODE===n.nodeType;){for(var i=0,r=!1,d=n.previousSibling;d;)d.nodeType!==Node.DOCUMENT_TYPE_NODE&&d.nodeName===n.nodeName&&i++,d=d.previousSibling;for(d=n.nextSibling;d;){if(d.nodeName===n.nodeName){r=!0;break}d=d.nextSibling}o.push((n.prefix?n.prefix+":":"")+n.localName+(i||r?"["+(i+1)+"]":"")),n=n.parentNode}return o.length?"/"+o.reverse().join("/"):""}});
-
-
-                const findUpTag = (el) => {
-                  let r = el
-                  chained_css = [];
-                  depth=0;
-            
-                // Strategy 1: Keep going up until we hit an ID tag, imagine it's like  #list-widget div h4
-                  while (r.parentNode) {
-                    if(depth==5) {
-                      break;
-                    }
-                    if('' !==r.id) {
-                      chained_css.unshift("#"+r.id);
-                      final_selector= chained_css.join('>');
-                      // Be sure theres only one, some sites have multiples of the same ID tag :-(
-                      if (window.document.querySelectorAll(final_selector).length ==1 ) {
-                        return final_selector;
-                      }
-                      return null;
-                    } else {
-                      chained_css.unshift(r.tagName.toLowerCase());
-                    }
-                    r=r.parentNode;
-                    depth+=1;
-                  }
-                  return null;
-                }
-
-
-                // @todo - if it's SVG or IMG, go into image diff mode
-                var elements = window.document.querySelectorAll("div,span,form,table,tbody,tr,td,a,p,ul,li,h1,h2,h3,h4, header, footer, section, article, aside, details, main, nav, section, summary");
-                var size_pos=[];
-                // after page fetch, inject this JS
-                // build a map of all elements and their positions (maybe that only include text?)
-                var bbox;
-                for (var i = 0; i < elements.length; i++) {   
-                 bbox = elements[i].getBoundingClientRect();
-
-                 // forget really small ones
-                 if (bbox['width'] <20 && bbox['height'] < 20 ) {
-                   continue;
-                 }
-
-                 // @todo the getXpath kind of sucks, it doesnt know when there is for example just one ID sometimes
-                 // it should not traverse when we know we can anchor off just an ID one level up etc..
-                 // maybe, get current class or id, keep traversing up looking for only class or id until there is just one match 
-
-                 // 1st primitive - if it has class, try joining it all and select, if theres only one.. well thats us.
-                 xpath_result=false;
-                 
-                 try {
-                   var d= findUpTag(elements[i]);
-                   if (d) {
-                     xpath_result =d;
-                   }                
-                 } catch (e) {
-                   var x=1;
-                 }
-                 
-// You could swap it and default to getXpath and then try the smarter one
-                 // default back to the less intelligent one
-                 if (!xpath_result) {
-                   xpath_result = getXPath(elements[i]);                   
-                 }
-                 if(window.getComputedStyle(elements[i]).visibility === "hidden") {
-                   continue;
-                 }
-
-                 size_pos.push({
-                   xpath: xpath_result,
-                   width: Math.round(bbox['width']), 
-                   height: Math.round(bbox['height']), 
-                   left: Math.floor(bbox['left']), 
-                   top: Math.floor(bbox['top']), 
-                   childCount: elements[i].childElementCount
-                 });                 
-                }
-
-
-                // inject the current one set in the css_filter, which may be a CSS rule
-                // used for displaying the current one in VisualSelector, where its not one we generated.
-                if (css_filter.length) {
-                   // is it xpath?
-                   if (css_filter.startsWith('/') ) {
-                     q=document.evaluate(css_filter, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null).singleNodeValue;
-                   } else {
-                     q=document.querySelector(css_filter);
-                   }
-                   bbox = q.getBoundingClientRect();                
-                   if (bbox && bbox['width'] >0 && bbox['height']>0) {                       
-                       size_pos.push({
-                           xpath: css_filter,
-                           width: bbox['width'], 
-                           height: bbox['height'],
-                           left: bbox['left'],
-                           top: bbox['top'],
-                           childCount: q.childElementCount
-                         });
-                     }
-                }
-// https://stackoverflow.com/questions/1145850/how-to-get-height-of-entire-document-with-javascript
-                return {'size_pos':size_pos, 'browser_width': window.innerWidth, 'browser_height':document.body.scrollHeight};
-    """
-    xpath_data = None
-
     # Will be needed in the future by the VisualSelector, always get this where possible.
     screenshot = False
     fetcher_description = "No description"
@@ -158,8 +47,7 @@ class Fetcher():
             request_headers,
             request_body,
             request_method,
-            ignore_status_codes=False,
-            current_css_filter=None):
+            ignore_status_codes=False):
         # Should set self.error, self.status_code and self.content
         pass
 
@@ -240,8 +128,7 @@ class base_html_playwright(Fetcher):
             request_headers,
             request_body,
             request_method,
-            ignore_status_codes=False,
-            current_css_filter=None):
+            ignore_status_codes=False):
 
         from playwright.sync_api import sync_playwright
         import playwright._impl._api_types
@@ -261,8 +148,8 @@ class base_html_playwright(Fetcher):
                 proxy=self.proxy
             )
             page = context.new_page()
+            page.set_viewport_size({"width": 1280, "height": 1024})
             try:
-               # Bug - never set viewport size BEFORE page.goto
                 response = page.goto(url, timeout=timeout * 1000, wait_until='commit')
                 # Wait_until = commit
                 # - `'commit'` - consider operation to be finished when network response is received and the document started loading.
@@ -279,27 +166,14 @@ class base_html_playwright(Fetcher):
             if len(page.content().strip()) == 0:
                 raise EmptyReply(url=url, status_code=None)
 
-            # Bug 2(?) Set the viewport size AFTER loading the page
-            page.set_viewport_size({"width": 1280, "height": 1024})
-            # Bugish - Let the page redraw/reflow
-            page.set_viewport_size({"width": 1280, "height": 1024})
-
             self.status_code = response.status
             self.content = page.content()
             self.headers = response.all_headers()
 
-            if current_css_filter is not None:
-                page.evaluate("var css_filter='{}'".format(current_css_filter))
-            else:
-                page.evaluate("var css_filter=''")
-
-            self.xpath_data = page.evaluate("async () => {" + self.xpath_element_js + "}")
-            # Bug 3 in Playwright screenshot handling
             # Some bug where it gives the wrong screenshot size, but making a request with the clip set first seems to solve it
             # JPEG is better here because the screenshots can be very very large
             page.screenshot(type='jpeg', clip={'x': 1.0, 'y': 1.0, 'width': 1280, 'height': 1024})
-            self.screenshot = page.screenshot(type='jpeg', full_page=True, quality=92)
-
+            self.screenshot = page.screenshot(type='jpeg', full_page=True, quality=90)
             context.close()
             browser.close()
 
@@ -351,8 +225,7 @@ class base_html_webdriver(Fetcher):
             request_headers,
             request_body,
             request_method,
-            ignore_status_codes=False,
-            current_css_filter=None):
+            ignore_status_codes=False):
 
         from selenium import webdriver
         from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
@@ -372,10 +245,6 @@ class base_html_webdriver(Fetcher):
             self.quit()
             raise
 
-        self.driver.set_window_size(1280, 1024)
-        self.driver.implicitly_wait(int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)))
-        self.screenshot = self.driver.get_screenshot_as_png()
-
         # @todo - how to check this? is it possible?
         self.status_code = 200
         # @todo somehow we should try to get this working for WebDriver
@@ -385,6 +254,8 @@ class base_html_webdriver(Fetcher):
         time.sleep(int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)) + self.render_extract_delay)
         self.content = self.driver.page_source
         self.headers = {}
+        self.screenshot = self.driver.get_screenshot_as_png()
+        self.quit()
 
     # Does the connection to the webdriver work? run a test connection.
     def is_ready(self):
@@ -421,8 +292,7 @@ class html_requests(Fetcher):
             request_headers,
             request_body,
             request_method,
-            ignore_status_codes=False,
-            current_css_filter=None):
+            ignore_status_codes=False):
 
         proxies={}
 

changedetectionio/fetch_site_status.py

@@ -94,7 +94,6 @@ class perform_site_check():
             # If the klass doesnt exist, just use a default
             klass = getattr(content_fetcher, "html_requests")
 
-
         proxy_args = self.set_proxy_from_list(watch)
         fetcher = klass(proxy_override=proxy_args)
 
@@ -105,8 +104,7 @@ class perform_site_check():
         elif system_webdriver_delay is not None:
             fetcher.render_extract_delay = system_webdriver_delay
 
-        fetcher.run(url, timeout, request_headers, request_body, request_method, ignore_status_code, watch['css_filter'])
-        fetcher.quit()
+        fetcher.run(url, timeout, request_headers, request_body, request_method, ignore_status_code)
 
         # Fetching complete, now filters
         # @todo move to class / maybe inside of fetcher abstract base?
@@ -238,4 +236,4 @@ class perform_site_check():
                 if not watch['title'] or not len(watch['title']):
                     update_obj['title'] = html_tools.extract_element(find='title', html_content=fetcher.content)
 
-        return changed_detected, update_obj, text_content_before_ignored_filter, fetcher.screenshot, fetcher.xpath_data
+        return changed_detected, update_obj, text_content_before_ignored_filter, fetcher.screenshot

changedetectionio/forms.py

@@ -374,7 +374,6 @@ class globalSettingsApplicationForm(commonSettingsForm):
     empty_pages_are_a_change =  BooleanField('Treat empty pages as a change?', default=False)
     render_anchor_tag_content = BooleanField('Render anchor tag content', default=False)
     fetch_backend = RadioField('Fetch Method', default="html_requests", choices=content_fetcher.available_fetchers(), validators=[ValidateContentFetcherIsReady()])
-    api_access_token_enabled = BooleanField('API access token security check enabled', default=True, validators=[validators.Optional()])
     password = SaltyPasswordField()
 
 

changedetectionio/html_tools.py

@@ -39,7 +39,7 @@ def element_removal(selectors: List[str], html_content):
 def xpath_filter(xpath_filter, html_content):
     from lxml import etree, html
 
-    tree = html.fromstring(bytes(html_content, encoding='utf-8'))
+    tree = html.fromstring(html_content)
     html_block = ""
 
     for item in tree.xpath(xpath_filter.strip(), namespaces={'re':'http://exslt.org/regular-expressions'}):

changedetectionio/model/App.py

@@ -27,7 +27,6 @@ class model(dict):
                     'proxy': None # Preferred proxy connection
                 },
                 'application': {
-                    'api_access_token_enabled': True,
                     'password': False,
                     'base_url' : None,
                     'extract_title_as_title': False,

changedetectionio/run_all_tests.sh

@@ -22,26 +22,3 @@ echo "RUNNING WITH BASE_URL SET"
 export BASE_URL="https://really-unique-domain.io"
 pytest tests/test_notification.py
 
-
-# Now for the selenium and playwright/browserless fetchers
-# Note - this is not UI functional tests - just checking that each one can fetch the content
-
-echo "TESTING WEBDRIVER FETCH > SELENIUM/WEBDRIVER..."
-docker run -d --name $$-test_selenium  -p 4444:4444 --rm --shm-size="2g"  selenium/standalone-chrome-debug:3.141.59
-# takes a while to spin up
-sleep 5
-export WEBDRIVER_URL=http://localhost:4444/wd/hub
-pytest tests/fetchers/test_content.py
-unset WEBDRIVER_URL
-docker kill $$-test_selenium
-
-echo "TESTING WEBDRIVER FETCH > PLAYWRIGHT/BROWSERLESS..."
-# Not all platforms support playwright (not ARM/rPI), so it's not packaged in requirements.txt
-pip3 install playwright~=1.22
-docker run -d --name $$-test_browserless -e "DEFAULT_LAUNCH_ARGS=[\"--window-size=1920,1080\"]" --rm  -p 3000:3000  --shm-size="2g"  browserless/chrome:1.53-chrome-stable
-# takes a while to spin up
-sleep 5
-export PLAYWRIGHT_DRIVER_URL=ws://127.0.0.1:3000
-pytest tests/fetchers/test_content.py
-unset PLAYWRIGHT_DRIVER_URL
-docker kill $$-test_browserless

changedetectionio/static/js/global-settings.js

@@ -1,4 +1,4 @@
-$(document).ready(function () {
+$(document).ready(function() {
     function toggle() {
         if ($('input[name="application-fetch_backend"]:checked').val() != 'html_requests') {
             $('#requests-override-options').hide();
@@ -8,29 +8,9 @@ $(document).ready(function () {
             $('#webdriver-override-options').hide();
         }
     }
-
     $('input[name="application-fetch_backend"]').click(function (e) {
         toggle();
     });
     toggle();
 
-    $("#api-key").hover(
-        function () {
-            $("#api-key-copy").html('copy').fadeIn();
-        },
-        function () {
-            $("#api-key-copy").hide();
-        }
-    ).click(function (e) {
-        $("#api-key-copy").html('copied');
-        var range = document.createRange();
-        var n = $("#api-key")[0];
-        range.selectNode(n);
-        window.getSelection().removeAllRanges();
-        window.getSelection().addRange(range);
-        document.execCommand("copy");
-        window.getSelection().removeAllRanges();
-
-    });
 });
-

changedetectionio/static/js/limit.js

@@ -1,56 +0,0 @@
-/**
- * debounce
- * @param {integer} milliseconds This param indicates the number of milliseconds
- *     to wait after the last call before calling the original function.
- * @param {object} What "this" refers to in the returned function.
- * @return {function} This returns a function that when called will wait the
- *     indicated number of milliseconds after the last call before
- *     calling the original function.
- */
-Function.prototype.debounce = function (milliseconds, context) {
-    var baseFunction = this,
-        timer = null,
-        wait = milliseconds;
-
-    return function () {
-        var self = context || this,
-            args = arguments;
-
-        function complete() {
-            baseFunction.apply(self, args);
-            timer = null;
-        }
-
-        if (timer) {
-            clearTimeout(timer);
-        }
-
-        timer = setTimeout(complete, wait);
-    };
-};
-
-/**
-* throttle
-* @param {integer} milliseconds This param indicates the number of milliseconds
-*     to wait between calls before calling the original function.
-* @param {object} What "this" refers to in the returned function.
-* @return {function} This returns a function that when called will wait the
-*     indicated number of milliseconds between calls before
-*     calling the original function.
-*/
-Function.prototype.throttle = function (milliseconds, context) {
-    var baseFunction = this,
-        lastEventTimestamp = null,
-        limit = milliseconds;
-
-    return function () {
-        var self = context || this,
-            args = arguments,
-            now = Date.now();
-
-        if (!lastEventTimestamp || now - lastEventTimestamp >= limit) {
-            lastEventTimestamp = now;
-            baseFunction.apply(self, args);
-        }
-    };
-};

changedetectionio/static/js/visual-selector.js

@@ -1,219 +0,0 @@
-// Horrible proof of concept code :)
-// yes - this is really a hack, if you are a front-ender and want to help, please get in touch!
-
-$(document).ready(function() {
-
-    $('#visualselector-tab').click(function () {
-        $("img#selector-background").off('load');
-        bootstrap_visualselector();
-    });
-
-    $(document).on('keydown', function(event) {
-        if ($("img#selector-background").is(":visible")) {
-            if (event.key == "Escape") {
-                state_clicked=false;
-                ctx.clearRect(0, 0, c.width, c.height);
-            }
-        }
-    });
-
-    // For when the page loads
-    if(!window.location.hash || window.location.hash != '#visualselector') {
-        $("img#selector-background").attr('src','');
-        return;
-    }
-
-    // Handle clearing button/link
-    $('#clear-selector').on('click', function(event) {
-        if(!state_clicked) {
-            alert('Oops, Nothing selected!');
-        }
-        state_clicked=false;
-        ctx.clearRect(0, 0, c.width, c.height);
-    });
-
-
-    bootstrap_visualselector();
-
-    var current_selected_i;
-    var state_clicked=false;
-
-    var c;
-
-    // greyed out fill context
-    var xctx;
-    // redline highlight context
-    var ctx;
-
-    var current_default_xpath;
-    var x_scale=1;
-    var y_scale=1;
-    var selector_image;
-    var selector_image_rect;
-    var vh;
-    var selector_data;
-
-
-    function bootstrap_visualselector() {
-        if ( 1 ) {
-            // bootstrap it, this will trigger everything else
-            $("img#selector-background").bind('load', function () {
-                console.log("Loaded background...");
-               c = document.getElementById("selector-canvas");
-                // greyed out fill context
-               xctx = c.getContext("2d");
-                // redline highlight context
-               ctx = c.getContext("2d");
-               current_default_xpath =$("#css_filter").val();
-               fetch_data();
-               $('#selector-canvas').off("mousemove");
-               // screenshot_url defined in the edit.html template
-            }).attr("src", screenshot_url);
-        }
-    }
-
-    function fetch_data() {
-      // Image is ready
-      $('.fetching-update-notice').html("Fetching element data..");
-
-      $.ajax({
-        url: watch_visual_selector_data_url,
-        context: document.body
-      }).done(function (data) {
-        $('.fetching-update-notice').html("Rendering..");
-        selector_data = data;
-        console.log("Reported browser width from backend: "+data['browser_width']);
-        state_clicked=false;
-        set_scale();
-        reflow_selector();
-        $('.fetching-update-notice').fadeOut();
-      });
-    };
-
-
-
-    function set_scale() {
-
-      // some things to check if the scaling doesnt work
-      // - that the widths/sizes really are about the actual screen size cat elements.json |grep -o width......|sort|uniq
-      selector_image = $("img#selector-background")[0];
-      selector_image_rect = selector_image.getBoundingClientRect();
-
-      // make the canvas the same size as the image
-      $('#selector-canvas').attr('height', selector_image_rect.height);
-      $('#selector-canvas').attr('width', selector_image_rect.width);
-      $('#selector-wrapper').attr('width', selector_image_rect.width);
-      x_scale = selector_image_rect.width / selector_data['browser_width'];
-      y_scale = selector_image_rect.height / selector_image.naturalHeight;
-      ctx.strokeStyle = 'rgba(255,0,0, 0.9)';
-      ctx.fillStyle = 'rgba(255,0,0, 0.1)';
-      ctx.lineWidth = 3;
-      console.log("scaling set  x: "+x_scale+" by y:"+y_scale);
-      $("#selector-current-xpath").css('max-width', selector_image_rect.width);
-    }
-
-    function reflow_selector() {
-        $(window).resize(function() {
-            set_scale();
-            highlight_current_selected_i();
-        });
-      var selector_currnt_xpath_text=$("#selector-current-xpath span");
-
-      set_scale();
-
-      console.log(selector_data['size_pos'].length + " selectors found");
-
-      // highlight the default one if we can find it in the xPath list
-      // or the xpath matches the default one
-      found = false;
-      if(current_default_xpath.length) {
-          for (var i = selector_data['size_pos'].length; i!==0; i--) {
-            var sel = selector_data['size_pos'][i-1];
-            if(selector_data['size_pos'][i - 1].xpath == current_default_xpath) {
-            console.log("highlighting "+current_default_xpath);
-              current_selected_i = i-1;
-              highlight_current_selected_i();
-              found = true;
-              break;
-            }
-          }
-        if(!found) {
-          alert("unfortunately your existing CSS/xPath Filter was no longer found!");
-        }
-      }
-
-
-      $('#selector-canvas').bind('mousemove', function (e) {
-        if(state_clicked) {
-          return;
-        }
-        ctx.clearRect(0, 0, c.width, c.height);
-        current_selected_i=null;
-
-        // Reverse order - the most specific one should be deeper/"laster"
-        // Basically, find the most 'deepest'
-        var found=0;
-        ctx.fillStyle = 'rgba(205,0,0,0.35)';
-        for (var i = selector_data['size_pos'].length; i!==0; i--) {
-          // draw all of them? let them choose somehow?
-          var sel = selector_data['size_pos'][i-1];
-          // If we are in a bounding-box
-          if (e.offsetY > sel.top * y_scale && e.offsetY < sel.top * y_scale + sel.height * y_scale
-              &&
-              e.offsetX > sel.left * y_scale && e.offsetX < sel.left * y_scale + sel.width * y_scale
-
-          ) {
-
-            // FOUND ONE
-            set_current_selected_text(sel.xpath);
-            ctx.strokeRect(sel.left * x_scale, sel.top * y_scale, sel.width * x_scale, sel.height * y_scale);
-            ctx.fillRect(sel.left * x_scale, sel.top * y_scale, sel.width * x_scale, sel.height * y_scale);
-
-            // no need to keep digging
-            // @todo or, O to go out/up, I to go in
-            // or double click to go up/out the selector?
-            current_selected_i=i-1;
-            found+=1;
-            break;
-          }
-        }
-
-      }.debounce(5));
-
-      function set_current_selected_text(s) {
-        selector_currnt_xpath_text[0].innerHTML=s;
-      }
-
-      function highlight_current_selected_i() {
-        if(state_clicked) {
-          state_clicked=false;
-          xctx.clearRect(0,0,c.width, c.height);
-          return;
-        }
-
-        var sel = selector_data['size_pos'][current_selected_i];
-        if (sel[0] == '/') {
-        // @todo - not sure just checking / is right
-            $("#css_filter").val('xpath:'+sel.xpath);
-        } else {
-            $("#css_filter").val(sel.xpath);
-        }
-        xctx.fillStyle = 'rgba(205,205,205,0.95)';
-        xctx.strokeStyle = 'rgba(225,0,0,0.9)';
-        xctx.lineWidth = 3;
-        xctx.fillRect(0,0,c.width, c.height);
-        // Clear out what only should be seen (make a clear/clean spot)
-        xctx.clearRect(sel.left * x_scale, sel.top * y_scale, sel.width * x_scale, sel.height * y_scale);
-        xctx.strokeRect(sel.left * x_scale, sel.top * y_scale, sel.width * x_scale, sel.height * y_scale);
-        state_clicked=true;
-        set_current_selected_text(sel.xpath);
-
-      }
-
-
-      $('#selector-canvas').bind('mousedown', function (e) {
-        highlight_current_selected_i();
-      });
-    }
-
-});

changedetectionio/static/js/watch-overview.js

@@ -4,7 +4,6 @@ $(function () {
     $(this).closest('.unviewed').removeClass('unviewed');
   });
 
-
   $('.with-share-link > *').click(function () {
       $("#copied-clipboard").remove();
 
@@ -21,6 +20,5 @@ $(function () {
        $(this).remove();
       });
   });
-
 });
 

changedetectionio/static/styles/styles.css

@@ -284,11 +284,6 @@ footer {
 .pure-form {
   /* The input fields with errors */
   /* The list of errors */ }
-  .pure-form fieldset {
-    padding-top: 0px; }
-    .pure-form fieldset ul {
-      padding-bottom: 0px;
-      margin-bottom: 0px; }
   .pure-form .pure-control-group, .pure-form .pure-group, .pure-form .pure-controls {
     padding-bottom: 1em; }
     .pure-form .pure-control-group div, .pure-form .pure-group div, .pure-form .pure-controls div {
@@ -338,8 +333,7 @@ footer {
     padding-top: 110px; }
   div.tabs.collapsable ul li {
     display: block;
-    border-radius: 0px;
-    margin-right: 0px; }
+    border-radius: 0px; }
   input[type='text'] {
     width: 100%; }
   /*
@@ -430,15 +424,6 @@ and also iPads specifically.
   .tab-pane-inner:target {
     display: block; }
 
-#beta-logo {
-  height: 50px;
-  right: -3px;
-  top: -3px;
-  position: absolute; }
-
-#selector-header {
-  padding-bottom: 1em; }
-
 .edit-form {
   min-width: 70%;
   /* so it cant overflow */
@@ -462,31 +447,4 @@ ul {
 .time-check-widget tr {
   display: inline; }
   .time-check-widget tr input[type="number"] {
-    width: 5em; }
-
-#selector-wrapper {
-  height: 600px;
-  overflow-y: scroll;
-  position: relative; }
-  #selector-wrapper > img {
-    position: absolute;
-    z-index: 4;
-    max-width: 100%; }
-  #selector-wrapper > canvas {
-    position: relative;
-    z-index: 5;
-    max-width: 100%; }
-    #selector-wrapper > canvas:hover {
-      cursor: pointer; }
-
-#selector-current-xpath {
-  font-size: 80%; }
-
-#webdriver-override-options input[type="number"] {
-  width: 5em; }
-
-#api-key:hover {
-  cursor: pointer; }
-
-#api-key-copy {
-  color: #0078e7; }
+    width: 4em; }

changedetectionio/static/styles/styles.scss

@@ -375,13 +375,6 @@ footer {
 
 
 .pure-form {
-    fieldset {
-        padding-top: 0px;
-        ul {
-            padding-bottom: 0px;
-            margin-bottom: 0px;
-        }
-    }
     .pure-control-group, .pure-group, .pure-controls {
         padding-bottom: 1em;
         div {
@@ -469,7 +462,6 @@ footer {
   div.tabs.collapsable ul li {
     display: block;
     border-radius: 0px;
-    margin-right: 0px;
   }
 
   input[type='text'] {
@@ -614,18 +606,6 @@ $form-edge-padding: 20px;
     padding: 0px;
 }
 
-#beta-logo {
-    height: 50px;
-    // looks better when it's hanging off a little
-    right: -3px;
-    top: -3px;
-    position: absolute;
-}
-
-#selector-header {
-    padding-bottom: 1em;
-}
-
 .edit-form {
   min-width: 70%;
   /* so it cant overflow */
@@ -657,47 +637,7 @@ ul {
     tr {
         display: inline;
         input[type="number"] {
-            width: 5em;
+            width: 4em;
         }
     }
-}
-
-#selector-wrapper {
- height: 600px;
- overflow-y: scroll;
- position: relative;
-    //width: 100%;
- > img {
-    position: absolute;
-    z-index: 4;
-    max-width: 100%;
- }
- >canvas {
-    position: relative;
-    z-index: 5;
-     max-width: 100%;
-     &:hover {
-     cursor: pointer;
-     }
- }
-}
-
-#selector-current-xpath {
-  font-size: 80%;
-}
-
-#webdriver-override-options {
-        input[type="number"] {
-            width: 5em;
-        }
-}
-
-#api-key {
-  &:hover {
-    cursor: pointer;
-  }
-}
-
-#api-key-copy {
-  color: #0078e7;
-}
+}

changedetectionio/store.py

@@ -12,7 +12,6 @@ from os import mkdir, path, unlink
 from threading import Lock
 import re
 import requests
-import secrets
 
 from . model import App, Watch
 
@@ -108,13 +107,10 @@ class ChangeDetectionStore:
 
         # Generate the URL access token for RSS feeds
         if not 'rss_access_token' in self.__data['settings']['application']:
+            import secrets
             secret = secrets.token_hex(16)
             self.__data['settings']['application']['rss_access_token'] = secret
 
-        # Generate the API access token
-        if not 'api_access_token' in self.__data['settings']['application']:
-            secret = secrets.token_hex(16)
-            self.__data['settings']['application']['api_access_token'] = secret
 
         # Proxy list support - available as a selection in settings when text file is imported
         # CSV list
@@ -215,8 +211,7 @@ class ChangeDetectionStore:
     def get_all_tags(self):
         tags = []
         for uuid, watch in self.data['watching'].items():
-            if watch['tag'] is None:
-                continue
+
             # Support for comma separated list of tags.
             for tag in watch['tag'].split(','):
                 tag = tag.strip()
@@ -285,10 +280,6 @@ class ChangeDetectionStore:
     def add_watch(self, url, tag="", extras=None, write_to_disk_now=True):
         if extras is None:
             extras = {}
-        # should always be str
-        if tag is None or not tag:
-            tag=''
-
         # Incase these are copied across, assume it's a reference and deepcopy()
         apply_extras = deepcopy(extras)
 
@@ -372,15 +363,6 @@ class ChangeDetectionStore:
 
         return False
 
-    def visualselector_data_is_ready(self, watch_uuid):
-        output_path = "{}/{}".format(self.datastore_path, watch_uuid)
-        screenshot_filename = "{}/last-screenshot.png".format(output_path)
-        elements_index_filename = "{}/elements.json".format(output_path)
-        if path.isfile(screenshot_filename) and  path.isfile(elements_index_filename) :
-            return True
-
-        return False
-
     # Save as PNG, PNG is larger but better for doing visual diff in the future
     def save_screenshot(self, watch_uuid, screenshot: bytes):
         output_path = "{}/{}".format(self.datastore_path, watch_uuid)
@@ -389,14 +371,6 @@ class ChangeDetectionStore:
             f.write(screenshot)
             f.close()
 
-    def save_xpath_data(self, watch_uuid, data):
-        output_path = "{}/{}".format(self.datastore_path, watch_uuid)
-        fname = "{}/elements.json".format(output_path)
-        with open(fname, 'w') as f:
-            f.write(json.dumps(data))
-            f.close()
-
-
     def sync_to_json(self):
         logging.info("Saving JSON..")
         print("Saving JSON..")

changedetectionio/templates/diff.html

@@ -39,6 +39,9 @@
 <div class="tabs">
     <ul>
         <li class="tab" id="default-tab"><a href="#text">Text</a></li>
+{% if screenshot %}
+        <li class="tab"><a href="#screenshot">Current screenshot</a></li>
+{% endif %}
     </ul>
 </div>
 
@@ -60,6 +63,17 @@
          </table>
          Diff algorithm from the amazing <a href="https://github.com/kpdecker/jsdiff">github.com/kpdecker/jsdiff</a>
      </div>
+
+{% if screenshot %}
+     <div class="tab-pane-inner" id="screenshot">
+         <p>
+         <i>For now, only the most recent screenshot is saved and displayed.</i>
+             </p>
+
+        <img src="{{url_for('static_content', group='screenshot', filename=uuid)}}">
+     </div>
+{% endif %}
+
 </div>
 
 

changedetectionio/templates/edit.html

@@ -5,18 +5,12 @@
 <script type="text/javascript" src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
 <script>
     const notification_base_url="{{url_for('ajax_callback_send_notification_test')}}";
-    const watch_visual_selector_data_url="{{url_for('static_content', group='visual_selector_data', filename=uuid)}}";
-    const screenshot_url="{{url_for('static_content', group='screenshot', filename=uuid)}}";
-
 {% if emailprefix %}
     const email_notification_prefix=JSON.parse('{{ emailprefix|tojson }}');
 {% endif %}
-
 </script>
 <script type="text/javascript" src="{{url_for('static_content', group='js', filename='watch-settings.js')}}" defer></script>
 <script type="text/javascript" src="{{url_for('static_content', group='js', filename='notifications.js')}}" defer></script>
-<script type="text/javascript" src="{{url_for('static_content', group='js', filename='visual-selector.js')}}" defer></script>
-<script type="text/javascript" src="{{url_for('static_content', group='js', filename='limit.js')}}" defer></script>
 
 <div class="edit-form monospaced-textarea">
 
@@ -24,7 +18,6 @@
         <ul>
             <li class="tab" id="default-tab"><a href="#general">General</a></li>
             <li class="tab"><a href="#request">Request</a></li>
-            <li class="tab"><a id="visualselector-tab" href="#visualselector">Visual Selector</a></li>
             <li class="tab"><a href="#filters-and-triggers">Filters &amp; Triggers</a></li>
             <li class="tab"><a href="#notifications">Notifications</a></li>
         </ul>
@@ -80,7 +73,7 @@
                         </span>
                     </div>
                 {% endif %}
-                <fieldset id="webdriver-override-options">
+                <fieldset class="pure-group" id="webdriver-override-options">
                     <div class="pure-form-message-inline">
                         <strong>If you're having trouble waiting for the page to be fully rendered (text missing etc), try increasing the 'wait' time here.</strong>
                         <br/>
@@ -132,7 +125,8 @@ User-Agent: wonderbra 1.0") }}
             </div>
 
             <div class="tab-pane-inner" id="filters-and-triggers">
-                    <div class="pure-control-group">
+                <fieldset>
+                        <div class="pure-control-group">
                             <strong>Pro-tips:</strong><br/>
                             <ul>
                                 <li>
@@ -143,6 +137,7 @@ User-Agent: wonderbra 1.0") }}
                                 </li>
                             </ul>
                     </div>
+
                     <div class="pure-control-group">
                         {{ render_field(form.css_filter, placeholder=".class-name or #some-id, or other CSS selector rule.",
                         class="m-d") }}
@@ -158,7 +153,7 @@ User-Agent: wonderbra 1.0") }}
                                 href="https://github.com/dgtlmoon/changedetection.io/wiki/CSS-Selector-help">here for more CSS selector help</a>.<br/>
                 </span>
                     </div>
-                    <div class="pure-control-group">
+                    <fieldset class="pure-group">
                       {{ render_field(form.subtractive_selectors, rows=5, placeholder="header
 footer
 nav
@@ -169,7 +164,8 @@ nav
                           <li> Add multiple elements or CSS selectors per line to ignore multiple parts of the HTML. </li>
                         </ul>
                       </span>
-                    </div>
+                    </fieldset>
+                </fieldset>
                 <fieldset class="pure-group">
                     {{ render_field(form.ignore_text, rows=5, placeholder="Some text to ignore in a line
 /some.regex\d{2}/ for case-INsensitive regex
@@ -201,54 +197,14 @@ nav
                 </fieldset>
             </div>
 
-            <div class="tab-pane-inner visual-selector-ui" id="visualselector">
-                <img id="beta-logo" src="{{url_for('static_content', group='images', filename='beta-logo.png')}}">
-
-                <fieldset>
-                    <div class="pure-control-group">
-                        {% if visualselector_enabled %}
-                            {% if visualselector_data_is_ready %}
-                                <div id="selector-header">
-                                    <a id="clear-selector" class="pure-button button-secondary button-xsmall" style="font-size: 70%">Clear selection</a>
-                                    <i class="fetching-update-notice" style="font-size: 80%;">One moment, fetching screenshot and element information..</i>
-                                </div>
-                                <div id="selector-wrapper">
-                                    <!-- request the screenshot and get the element offset info ready -->
-                                    <!-- use img src ready load to know everything is ready to map out -->
-                                    <!-- @todo: maybe something interesting like a field to select 'elements that contain text... and their parents n' -->
-                                    <img id="selector-background" />
-                                    <canvas id="selector-canvas"></canvas>
-
-                                </div>
-                                <div id="selector-current-xpath" style="overflow-x: hidden"><strong>Currently:</strong>&nbsp;<span class="text">Loading...</span></div>
-
-                            <span class="pure-form-message-inline">
-                                <p><span style="font-weight: bold">Beta!</span> The Visual Selector is new and there may be minor bugs, please report pages that dont work, help us to improve this software!</p>
-                            </span>
-
-                            {% else %}
-                                <span class="pure-form-message-inline">Screenshot and element data is not available or not yet ready.</span>
-                            {% endif %}
-                        {% else %}
-                            <span class="pure-form-message-inline">
-                                <p>Sorry, this functionality only works with Playwright/Chrome enabled watches.</p>
-                                <p>Enable the Playwright Chrome fetcher, or alternatively try our <a href="https://lemonade.changedetection.io/start">very affordable subscription based service</a>.</p>
-                                <p>This is because Selenium/WebDriver can not extract full page screenshots reliably.</p>
-
-                            </span>
-                        {% endif %}
-                    </div>
-                </fieldset>
-            </div>
-
             <div id="actions">
                 <div class="pure-control-group">
 
                       {{ render_button(form.save_button) }} {{ render_button(form.save_and_preview_button) }}
 
-                    <a href="{{url_for('form_delete', uuid=uuid)}}"
+                    <a href="{{url_for('api_delete', uuid=uuid)}}"
                        class="pure-button button-small button-error ">Delete</a>
-                    <a href="{{url_for('form_clone', uuid=uuid)}}"
+                    <a href="{{url_for('api_clone', uuid=uuid)}}"
                        class="pure-button button-small ">Create Copy</a>
                 </div>
             </div>

changedetectionio/templates/preview.html

@@ -10,6 +10,9 @@
 <div class="tabs">
     <ul>
         <li class="tab" id="default-tab"><a href="#text">Text</a></li>
+{% if screenshot %}
+        <li class="tab"><a href="#screenshot">Current screenshot</a></li>
+{% endif %}
     </ul>
 </div>
 
@@ -28,5 +31,15 @@
             </tbody>
         </table>
     </div>
+
+{% if screenshot %}
+     <div class="tab-pane-inner" id="screenshot">
+         <p>
+         <i>For now, only the most recent screenshot is saved and displayed.</i>
+             </p>
+
+        <img src="{{url_for('static_content', group='screenshot', filename=uuid)}}">
+     </div>
+{% endif %}
 </div>
 {% endblock %}

changedetectionio/templates/settings.html

@@ -20,7 +20,6 @@
             <li class="tab"><a href="#notifications">Notifications</a></li>
             <li class="tab"><a href="#fetching">Fetching</a></li>
             <li class="tab"><a href="#filters">Global Filters</a></li>
-            <li class="tab"><a href="#api">API</a></li>
         </ul>
     </div>
     <div class="box-wrap inner">
@@ -44,7 +43,6 @@
                             <span class="pure-form-message-inline">Password is locked.</span>
                         {% endif %}
                     </div>
-
                     <div class="pure-control-group">
                         {{ render_field(form.application.form.base_url, placeholder="http://yoursite.com:5000/",
                         class="m-d") }}
@@ -107,6 +105,7 @@
                 </fieldset>
             </div>
 
+
             <div class="tab-pane-inner" id="filters">
 
                     <fieldset class="pure-group">
@@ -151,26 +150,12 @@ nav
                     </fieldset>
            </div>
 
-            <div class="tab-pane-inner" id="api">
-
-                <p>Drive your changedetection.io via API, More about <a href="https://github.com/dgtlmoon/changedetection.io/wiki/API-Reference">API access here</a></p>
-
-                <div class="pure-control-group">
-                    {{ render_checkbox_field(form.application.form.api_access_token_enabled) }}
-                    <div class="pure-form-message-inline">Restrict API access limit by using <code>x-api-key</code> header</div><br/>
-                    <div class="pure-form-message-inline"><br/>API Key <span id="api-key">{{api_key}}</span>
-                        <span style="display:none;" id="api-key-copy" >copy</span>
-                    </div>
-                </div>
-            </div>
-
             <div id="actions">
                 <div class="pure-control-group">
                     {{ render_button(form.save_button) }}
                     <a href="{{url_for('index')}}" class="pure-button button-small button-cancel">Back</a>
                     <a href="{{url_for('scrub_page')}}" class="pure-button button-small button-cancel">Delete History Snapshot Data</a>
                 </div>
-
             </div>
         </form>
     </div>

changedetectionio/templates/watch-overview.html

@@ -3,10 +3,9 @@
 {% from '_helpers.jinja' import render_simple_field %}
 <script type="text/javascript" src="{{url_for('static_content', group='js', filename='jquery-3.6.0.min.js')}}"></script>
 <script type="text/javascript" src="{{url_for('static_content', group='js', filename='watch-overview.js')}}" defer></script>
-
 <div class="box">
 
-    <form class="pure-form" action="{{ url_for('form_watch_add') }}" method="POST" id="new-watch-form">
+    <form class="pure-form" action="{{ url_for('api_watch_add') }}" method="POST" id="new-watch-form">
         <input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
         <fieldset>
             <legend>Add a new change detection watch</legend>
@@ -53,7 +52,7 @@
 
                 <td class="title-col inline">{{watch.title if watch.title is not none and watch.title|length > 0 else watch.url}}
                     <a class="external" target="_blank" rel="noopener" href="{{ watch.url.replace('source:','') }}"></a>
-                    <a href="{{url_for('form_share_put_watch', uuid=watch.uuid)}}"><img style="height: 1em;display:inline-block;" src="{{url_for('static_content', group='images', filename='spread.svg')}}" /></a>
+                    <a href="{{url_for('api_share_put_watch', uuid=watch.uuid)}}"><img style="height: 1em;display:inline-block;" src="{{url_for('static_content', group='images', filename='spread.svg')}}" /></a>
 
                     {%if watch.fetch_backend == "html_webdriver" %}<img style="height: 1em; display:inline-block;" src="{{url_for('static_content', group='images', filename='Google-Chrome-icon.png')}}" />{% endif %}
 
@@ -75,7 +74,7 @@
                     {% endif %}
                 </td>
                 <td>
-                    <a {% if watch.uuid in queued_uuids %}disabled="true"{% endif %} href="{{ url_for('form_watch_checknow', uuid=watch.uuid, tag=request.args.get('tag')) }}"
+                    <a {% if watch.uuid in queued_uuids %}disabled="true"{% endif %} href="{{ url_for('api_watch_checknow', uuid=watch.uuid, tag=request.args.get('tag')) }}"
                        class="recheck pure-button button-small pure-button-primary">{% if watch.uuid in queued_uuids %}Queued{% else %}Recheck{% endif %}</a>
                     <a href="{{ url_for('edit_page', uuid=watch.uuid)}}" class="pure-button button-small pure-button-primary">Edit</a>
                     {% if watch.history|length >= 2 %}
@@ -97,7 +96,7 @@
             </li>
             {% endif %}
             <li>
-               <a href="{{ url_for('form_watch_checknow', tag=active_tag) }}" class="pure-button button-tag ">Recheck
+               <a href="{{ url_for('api_watch_checknow', tag=active_tag) }}" class="pure-button button-tag ">Recheck
                 all {% if active_tag%}in "{{active_tag}}"{%endif%}</a>
             </li>
             <li>

changedetectionio/tests/fetchers/__init__.py

@@ -1,2 +0,0 @@
-"""Tests for the app."""
-

changedetectionio/tests/fetchers/conftest.py

@@ -1,3 +0,0 @@
-#!/usr/bin/python3
-
-from .. import conftest

changedetectionio/tests/fetchers/test_content.py

@@ -1,48 +0,0 @@
-#!/usr/bin/python3
-
-import time
-from flask import url_for
-from ..util import live_server_setup
-import logging
-
-
-def test_fetch_webdriver_content(client, live_server):
-    live_server_setup(live_server)
-
-    #####################
-    res = client.post(
-        url_for("settings_page"),
-        data={"application-empty_pages_are_a_change": "",
-              "requests-time_between_check-minutes": 180,
-              'application-fetch_backend': "html_webdriver"},
-        follow_redirects=True
-    )
-
-    assert b"Settings updated." in res.data
-
-    # Add our URL to the import page
-    res = client.post(
-        url_for("import_page"),
-        data={"urls": "https://changedetection.io/ci-test.html"},
-        follow_redirects=True
-    )
-
-    assert b"1 Imported" in res.data
-    time.sleep(3)
-    attempt = 0
-    while attempt < 20:
-        res = client.get(url_for("index"))
-        if not b'Checking now' in res.data:
-            break
-        logging.getLogger().info("Waiting for check to not say 'Checking now'..")
-        time.sleep(3)
-        attempt += 1
-
-
-    res = client.get(
-        url_for("preview_page", uuid="first"),
-        follow_redirects=True
-    )
-    logging.getLogger().info("Looking for correct fetched HTML (text) from server")
-
-    assert b'cool it works' in res.data

changedetectionio/tests/test_api.py

@@ -2,205 +2,73 @@
 
 import time
 from flask import url_for
-from .util import live_server_setup
+from . util import live_server_setup
 
-import json
-import uuid
-
-
-def set_original_response():
-    test_return_data = """<html>
-       <body>
-     Some initial text</br>
-     <p>Which is across multiple lines</p>
-     </br>
-     So let's see what happens.  </br>
-     <div id="sametext">Some text thats the same</div>
-     <div id="changetext">Some text that will change</div>
-     </body>
-     </html>
-    """
-
-    with open("test-datastore/endpoint-content.txt", "w") as f:
-        f.write(test_return_data)
-    return None
-
-
-def set_modified_response():
-    test_return_data = """<html>
-       <body>
-     Some initial text</br>
-     <p>which has this one new line</p>
-     </br>
-     So let's see what happens.  </br>
-     <div id="sametext">Some text thats the same</div>
-     <div id="changetext">Some text that changes</div>
-     </body>
-     </html>
-    """
-
-    with open("test-datastore/endpoint-content.txt", "w") as f:
-        f.write(test_return_data)
-
-    return None
-
-
-def is_valid_uuid(val):
-    try:
-        uuid.UUID(str(val))
-        return True
-    except ValueError:
-        return False
-
-
-# kinda funky, but works for now
-def _extract_api_key_from_UI(client):
-    import re
-    res = client.get(
-        url_for("settings_page"),
-    )
-    # <span id="api-key">{{api_key}}</span>
-
-    m = re.search('<span id="api-key">(.+?)</span>', str(res.data))
-    api_key = m.group(1)
-    return api_key.strip()
-
-
-def test_api_simple(client, live_server):
+def test_setup(live_server):
     live_server_setup(live_server)
 
-    api_key = _extract_api_key_from_UI(client)
 
-    # Create a watch
-    set_original_response()
-    watch_uuid = None
+def set_response_data(test_return_data):
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write(test_return_data)
 
-    # Validate bad URL
-    test_url = url_for('test_endpoint', _external=True,
-                       headers={'x-api-key': api_key}, )
+
+def test_snapshot_api_detects_change(client, live_server):
+    test_return_data = "Some initial text"
+
+    test_return_data_modified = "Some NEW nice initial text"
+
+    sleep_time_for_fetch_thread = 3
+
+    set_response_data(test_return_data)
+
+    # Give the endpoint time to spin up
+    time.sleep(1)
+
+    # Add our URL to the import page
+    test_url = url_for('test_endpoint', content_type="text/plain",
+                       _external=True)
     res = client.post(
-        url_for("createwatch"),
-        data=json.dumps({"url": "h://xxxxxxxxxom"}),
-        headers={'content-type': 'application/json', 'x-api-key': api_key},
+        url_for("import_page"),
+        data={"urls": test_url},
         follow_redirects=True
     )
+    assert b"1 Imported" in res.data
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
+
+    res = client.get(
+        url_for("api_snapshot", uuid="first"),
+        follow_redirects=True
+    )
+
+    assert test_return_data.encode() == res.data
+
+    #  Make a change
+    set_response_data(test_return_data_modified)
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
+
+    res = client.get(
+        url_for("api_snapshot", uuid="first"),
+        follow_redirects=True
+    )
+
+    assert test_return_data_modified.encode() == res.data
+
+def test_snapshot_api_invalid_uuid(client, live_server):
+
+    res = client.get(
+        url_for("api_snapshot", uuid="invalid"),
+        follow_redirects=True
+    )
+
     assert res.status_code == 400
 
-    # Create new
-    res = client.post(
-        url_for("createwatch"),
-        data=json.dumps({"url": test_url, 'tag': "One, Two", "title": "My test URL"}),
-        headers={'content-type': 'application/json', 'x-api-key': api_key},
-        follow_redirects=True
-    )
-    s = json.loads(res.data)
-    assert is_valid_uuid(s['uuid'])
-    watch_uuid = s['uuid']
-    assert res.status_code == 201
-
-    time.sleep(3)
-
-    # Verify its in the list and that recheck worked
-    res = client.get(
-        url_for("createwatch"),
-        headers={'x-api-key': api_key}
-    )
-    assert watch_uuid in json.loads(res.data).keys()
-    before_recheck_info = json.loads(res.data)[watch_uuid]
-    assert before_recheck_info['last_checked'] != 0
-    assert before_recheck_info['title'] == 'My test URL'
-
-    set_modified_response()
-    # Trigger recheck of all ?recheck_all=1
-    client.get(
-        url_for("createwatch", recheck_all='1'),
-        headers={'x-api-key': api_key},
-    )
-    time.sleep(3)
-
-    # Did the recheck fire?
-    res = client.get(
-        url_for("createwatch"),
-        headers={'x-api-key': api_key},
-    )
-    after_recheck_info = json.loads(res.data)[watch_uuid]
-    assert after_recheck_info['last_checked'] != before_recheck_info['last_checked']
-    assert after_recheck_info['last_changed'] != 0
-
-    # Check history index list
-    res = client.get(
-        url_for("watchhistory", uuid=watch_uuid),
-        headers={'x-api-key': api_key},
-    )
-    history = json.loads(res.data)
-    assert len(history) == 2, "Should have two history entries (the original and the changed)"
-
-    # Fetch a snapshot by timestamp, check the right one was found
-    res = client.get(
-        url_for("watchsinglehistory", uuid=watch_uuid, timestamp=list(history.keys())[-1]),
-        headers={'x-api-key': api_key},
-    )
-    assert b'which has this one new line' in res.data
-
-    # Fetch a snapshot by 'latest'', check the right one was found
-    res = client.get(
-        url_for("watchsinglehistory", uuid=watch_uuid, timestamp='latest'),
-        headers={'x-api-key': api_key},
-    )
-    assert b'which has this one new line' in res.data
-
-    # Fetch the whole watch
-    res = client.get(
-        url_for("watch", uuid=watch_uuid),
-        headers={'x-api-key': api_key}
-    )
-    watch = json.loads(res.data)
-    # @todo how to handle None/default global values?
-    assert watch['history_n'] == 2, "Found replacement history section, which is in its own API"
-
-    # Finally delete the watch
-    res = client.delete(
-        url_for("watch", uuid=watch_uuid),
-        headers={'x-api-key': api_key},
-    )
-    assert res.status_code == 204
-
-    # Check via a relist
-    res = client.get(
-        url_for("createwatch"),
-        headers={'x-api-key': api_key}
-    )
-    watch_list = json.loads(res.data)
-    assert len(watch_list) == 0, "Watch list should be empty"
-
-
-def test_access_denied(client, live_server):
-    # `config_api_token_enabled` Should be On by default
-    res = client.get(
-        url_for("createwatch")
-    )
-    assert res.status_code == 403
-
-    res = client.get(
-        url_for("createwatch"),
-        headers={'x-api-key': "something horrible"}
-    )
-    assert res.status_code == 403
-
-    # Disable config_api_token_enabled and it should work
-    res = client.post(
-        url_for("settings_page"),
-        data={
-            "requests-time_between_check-minutes": 180,
-            "application-fetch_backend": "html_requests",
-            "application-api_access_token_enabled": ""
-        },
-        follow_redirects=True
-    )
-
-    assert b"Settings updated." in res.data
-
-    res = client.get(
-        url_for("createwatch")
-    )
-    assert res.status_code == 200

changedetectionio/tests/test_auth.py

@@ -29,7 +29,7 @@ def test_basic_auth(client, live_server):
     assert b"Updated watch." in res.data
 
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
     time.sleep(1)
     res = client.get(
         url_for("preview_page", uuid="first"),

changedetectionio/tests/test_backend.py

@@ -32,7 +32,7 @@ def test_check_basic_change_detection_functionality(client, live_server):
 
     # Do this a few times.. ensures we dont accidently set the status
     for n in range(3):
-        client.get(url_for("form_watch_checknow"), follow_redirects=True)
+        client.get(url_for("api_watch_checknow"), follow_redirects=True)
 
         # Give the thread time to pick it up
         time.sleep(sleep_time_for_fetch_thread)
@@ -65,7 +65,7 @@ def test_check_basic_change_detection_functionality(client, live_server):
     assert b'which has this one new line' in res.read()
 
     # Force recheck
-    res = client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    res = client.get(url_for("api_watch_checknow"), follow_redirects=True)
     assert b'1 watches are queued for rechecking.' in res.data
 
     time.sleep(sleep_time_for_fetch_thread)
@@ -93,7 +93,7 @@ def test_check_basic_change_detection_functionality(client, live_server):
 
     # Do this a few times.. ensures we dont accidently set the status
     for n in range(2):
-        client.get(url_for("form_watch_checknow"), follow_redirects=True)
+        client.get(url_for("api_watch_checknow"), follow_redirects=True)
 
         # Give the thread time to pick it up
         time.sleep(sleep_time_for_fetch_thread)
@@ -113,7 +113,7 @@ def test_check_basic_change_detection_functionality(client, live_server):
         follow_redirects=True
     )
 
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
     time.sleep(sleep_time_for_fetch_thread)
 
     res = client.get(url_for("index"))
@@ -123,6 +123,6 @@ def test_check_basic_change_detection_functionality(client, live_server):
 
     #
     # Cleanup everything
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    res = client.get(url_for("api_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
 

changedetectionio/tests/test_clone.py

@@ -23,7 +23,7 @@ def test_trigger_functionality(client, live_server):
 
 
     res = client.get(
-        url_for("form_clone", uuid="first"),
+        url_for("api_clone", uuid="first"),
         follow_redirects=True
     )
 

changedetectionio/tests/test_css_selector.py

@@ -89,7 +89,7 @@ def test_check_markup_css_filter_restriction(client, live_server):
     assert b"1 Imported" in res.data
 
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
@@ -110,7 +110,7 @@ def test_check_markup_css_filter_restriction(client, live_server):
     assert bytes(css_filter.encode('utf-8')) in res.data
 
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
@@ -118,7 +118,7 @@ def test_check_markup_css_filter_restriction(client, live_server):
     set_modified_response()
 
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
 

changedetectionio/tests/test_element_removal.py

@@ -145,7 +145,7 @@ def test_element_removal_full(client, live_server):
     assert bytes(subtractive_selectors_data.encode("utf-8")) in res.data
 
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
@@ -158,7 +158,7 @@ def test_element_removal_full(client, live_server):
     set_modified_response()
 
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)

changedetectionio/tests/test_encoding.py

@@ -39,7 +39,7 @@ def test_check_encoding_detection(client, live_server):
     )
 
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(2)
@@ -71,7 +71,7 @@ def test_check_encoding_detection_missing_content_type_header(client, live_serve
     )
 
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(2)

changedetectionio/tests/test_errorhandling.py

@@ -29,7 +29,7 @@ def test_error_handler(client, live_server):
     assert b"1 Imported" in res.data
 
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(3)
@@ -54,7 +54,7 @@ def test_error_text_handler(client, live_server):
     assert b"1 Imported" in res.data
 
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(3)

changedetectionio/tests/test_ignore_text.py

@@ -102,7 +102,7 @@ def test_check_ignore_text_functionality(client, live_server):
     assert b"1 Imported" in res.data
 
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
@@ -123,7 +123,7 @@ def test_check_ignore_text_functionality(client, live_server):
     assert bytes(ignore_text.encode('utf-8')) in res.data
 
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
@@ -137,7 +137,7 @@ def test_check_ignore_text_functionality(client, live_server):
     set_modified_ignore_response()
 
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
 
@@ -152,7 +152,7 @@ def test_check_ignore_text_functionality(client, live_server):
 
     # Just to be sure.. set a regular modified change..
     set_modified_original_ignore_response()
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
     time.sleep(sleep_time_for_fetch_thread)
 
     res = client.get(url_for("index"))
@@ -165,7 +165,7 @@ def test_check_ignore_text_functionality(client, live_server):
     # We should be able to see what we ignored
     assert b'<div class="ignored">new ignore stuff' in res.data
 
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    res = client.get(url_for("api_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
 
 def test_check_global_ignore_text_functionality(client, live_server):
@@ -200,7 +200,7 @@ def test_check_global_ignore_text_functionality(client, live_server):
     assert b"1 Imported" in res.data
 
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
@@ -222,7 +222,7 @@ def test_check_global_ignore_text_functionality(client, live_server):
     assert bytes(ignore_text.encode('utf-8')) in res.data
 
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
@@ -240,7 +240,7 @@ def test_check_global_ignore_text_functionality(client, live_server):
     set_modified_ignore_response()
 
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
 
@@ -251,10 +251,10 @@ def test_check_global_ignore_text_functionality(client, live_server):
 
     # Just to be sure.. set a regular modified change that will trigger it
     set_modified_original_ignore_response()
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
     time.sleep(sleep_time_for_fetch_thread)
     res = client.get(url_for("index"))
     assert b'unviewed' in res.data
 
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    res = client.get(url_for("api_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data

changedetectionio/tests/test_ignorehyperlinks.py

@@ -72,14 +72,14 @@ def test_render_anchor_tag_content_true(client, live_server):
 
     time.sleep(sleep_time_for_fetch_thread)
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
 
     # set a new html text with a modified link
     set_modified_ignore_response()
     time.sleep(sleep_time_for_fetch_thread)
 
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
@@ -101,7 +101,7 @@ def test_render_anchor_tag_content_true(client, live_server):
     assert b"Settings updated." in res.data
 
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
@@ -119,7 +119,7 @@ def test_render_anchor_tag_content_true(client, live_server):
     assert b"/test-endpoint" in res.data
 
     # Cleanup everything
-    res = client.get(url_for("form_delete", uuid="all"),
+    res = client.get(url_for("api_delete", uuid="all"),
                      follow_redirects=True)
     assert b'Deleted' in res.data
 

changedetectionio/tests/test_ignorestatuscode.py

@@ -70,12 +70,12 @@ def test_normal_page_check_works_with_ignore_status_code(client, live_server):
 
     time.sleep(sleep_time_for_fetch_thread)
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
 
     set_some_changed_response()
     time.sleep(sleep_time_for_fetch_thread)
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
@@ -105,7 +105,7 @@ def test_403_page_check_works_with_ignore_status_code(client, live_server):
     assert b"1 Imported" in res.data
 
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
@@ -120,7 +120,7 @@ def test_403_page_check_works_with_ignore_status_code(client, live_server):
     assert b"Updated watch." in res.data
 
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
@@ -128,7 +128,7 @@ def test_403_page_check_works_with_ignore_status_code(client, live_server):
     set_some_changed_response()
 
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
 
@@ -157,7 +157,7 @@ def test_403_page_check_fails_without_ignore_status_code(client, live_server):
     assert b"1 Imported" in res.data
 
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
@@ -172,7 +172,7 @@ def test_403_page_check_fails_without_ignore_status_code(client, live_server):
     assert b"Updated watch." in res.data
 
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
@@ -180,7 +180,7 @@ def test_403_page_check_fails_without_ignore_status_code(client, live_server):
     set_some_changed_response()
 
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
 

changedetectionio/tests/test_ignorewhitespace.py

@@ -80,12 +80,12 @@ def test_check_ignore_whitespace(client, live_server):
 
     time.sleep(sleep_time_for_fetch_thread)
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
 
     set_original_ignore_response_but_with_whitespace()
     time.sleep(sleep_time_for_fetch_thread)
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)

changedetectionio/tests/test_import.py

@@ -25,7 +25,7 @@ https://example.com tag1, other tag"""
     assert b"3 Imported" in res.data
     assert b"tag1" in res.data
     assert b"other tag" in res.data
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    res = client.get(url_for("api_delete", uuid="all"), follow_redirects=True)
 
     # Clear flask alerts
     res = client.get( url_for("index"))
@@ -50,7 +50,7 @@ def xtest_import_skip_url(client, live_server):
     assert b"1 Imported" in res.data
     assert b"ht000000broken" in res.data
     assert b"1 Skipped" in res.data
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    res = client.get(url_for("api_delete", uuid="all"), follow_redirects=True)
     # Clear flask alerts
     res = client.get( url_for("index"))
 
@@ -79,7 +79,7 @@ def test_import_distillio(client, live_server):
 
     # Give the endpoint time to spin up
     time.sleep(1)
-    client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    client.get(url_for("api_delete", uuid="all"), follow_redirects=True)
     res = client.post(
         url_for("import_page"),
         data={
@@ -115,6 +115,6 @@ def test_import_distillio(client, live_server):
     assert b"nice stuff" in res.data
     assert b"nerd-news" in res.data
 
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    res = client.get(url_for("api_delete", uuid="all"), follow_redirects=True)
     # Clear flask alerts
     res = client.get(url_for("index"))

changedetectionio/tests/test_jsonpath_selector.py

@@ -171,7 +171,7 @@ def test_check_json_without_filter(client, live_server):
     )
 
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(3)
@@ -203,7 +203,7 @@ def test_check_json_filter(client, live_server):
     assert b"1 Imported" in res.data
 
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(3)
@@ -229,7 +229,7 @@ def test_check_json_filter(client, live_server):
     assert bytes(json_filter.encode('utf-8')) in res.data
 
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(3)
@@ -237,7 +237,7 @@ def test_check_json_filter(client, live_server):
     set_modified_response()
 
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
     # Give the thread time to pick it up
     time.sleep(4)
 
@@ -288,7 +288,7 @@ def test_check_json_filter_bool_val(client, live_server):
     time.sleep(3)
 
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(3)
@@ -296,7 +296,7 @@ def test_check_json_filter_bool_val(client, live_server):
     set_modified_response()
 
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
     # Give the thread time to pick it up
     time.sleep(3)
 
@@ -327,7 +327,7 @@ def test_check_json_ext_filter(client, live_server):
     assert b"1 Imported" in res.data
 
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(3)
@@ -353,7 +353,7 @@ def test_check_json_ext_filter(client, live_server):
     assert bytes(json_filter.encode('utf-8')) in res.data
 
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(3)
@@ -361,7 +361,7 @@ def test_check_json_ext_filter(client, live_server):
     set_modified_ext_response()
 
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
     # Give the thread time to pick it up
     time.sleep(4)
 

changedetectionio/tests/test_nonrenderable_pages.py

@@ -39,7 +39,7 @@ def test_check_basic_change_detection_functionality(client, live_server):
 
     # Do this a few times.. ensures we dont accidently set the status
     for n in range(3):
-        client.get(url_for("form_watch_checknow"), follow_redirects=True)
+        client.get(url_for("api_watch_checknow"), follow_redirects=True)
 
         # Give the thread time to pick it up
         time.sleep(sleep_time_for_fetch_thread)
@@ -61,7 +61,7 @@ def test_check_basic_change_detection_functionality(client, live_server):
     # this should not trigger a change, because no good text could be converted from the HTML
     set_nonrenderable_response()
 
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
@@ -83,7 +83,7 @@ def test_check_basic_change_detection_functionality(client, live_server):
     set_modified_response()
 
 
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
@@ -97,6 +97,6 @@ def test_check_basic_change_detection_functionality(client, live_server):
 
     #
     # Cleanup everything
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    res = client.get(url_for("api_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
 

changedetectionio/tests/test_notification.py

@@ -36,7 +36,7 @@ def test_check_notification(client, live_server):
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
-        url_for("form_watch_add"),
+        url_for("api_watch_add"),
         data={"url": test_url, "tag": ''},
         follow_redirects=True
     )
@@ -98,7 +98,7 @@ def test_check_notification(client, live_server):
     notification_submission = None
 
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
     time.sleep(3)
     # Verify what was sent as a notification, this file should exist
     with open("test-datastore/notification.txt", "r") as f:
@@ -133,7 +133,7 @@ def test_check_notification(client, live_server):
 
     # This should insert the {current_snapshot}
     set_more_modified_response()
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
     time.sleep(3)
     # Verify what was sent as a notification, this file should exist
     with open("test-datastore/notification.txt", "r") as f:
@@ -146,17 +146,17 @@ def test_check_notification(client, live_server):
     os.unlink("test-datastore/notification.txt")
 
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
     time.sleep(1)
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
     time.sleep(1)
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
     time.sleep(1)
     assert os.path.exists("test-datastore/notification.txt") == False
 
     # cleanup for the next
     client.get(
-        url_for("form_delete", uuid="all"),
+        url_for("api_delete", uuid="all"),
         follow_redirects=True
     )
 
@@ -168,7 +168,7 @@ def test_notification_validation(client, live_server):
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
-        url_for("form_watch_add"),
+        url_for("api_watch_add"),
         data={"url": test_url, "tag": 'nice one'},
         follow_redirects=True
     )
@@ -208,6 +208,6 @@ def test_notification_validation(client, live_server):
 
     # cleanup for the next
     client.get(
-        url_for("form_delete", uuid="all"),
+        url_for("api_delete", uuid="all"),
         follow_redirects=True
     )

changedetectionio/tests/test_notification_errors.py

@@ -16,7 +16,7 @@ def test_check_notification_error_handling(client, live_server):
     # use a different URL so that it doesnt interfere with the actual check until we are ready
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
-        url_for("form_watch_add"),
+        url_for("api_watch_add"),
         data={"url": "https://changedetection.io/CHANGELOG.txt", "tag": ''},
         follow_redirects=True
     )

changedetectionio/tests/test_share_watch.py

@@ -41,7 +41,7 @@ def test_share_watch(client, live_server):
 
     # click share the link
     res = client.get(
-        url_for("form_share_put_watch", uuid="first"),
+        url_for("api_share_put_watch", uuid="first"),
         follow_redirects=True
     )
 
@@ -54,7 +54,7 @@ def test_share_watch(client, live_server):
 
     # Now delete what we have, we will try to re-import it
     # Cleanup everything
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    res = client.get(url_for("api_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
 
     # Add our URL to the import page

changedetectionio/tests/test_source.py

@@ -39,7 +39,7 @@ def test_check_basic_change_detection_functionality_source(client, live_server):
     set_modified_response()
 
     # Force recheck
-    res = client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    res = client.get(url_for("api_watch_checknow"), follow_redirects=True)
     assert b'1 watches are queued for rechecking.' in res.data
 
     time.sleep(5)

changedetectionio/tests/test_trigger.py

@@ -76,7 +76,7 @@ def test_trigger_functionality(client, live_server):
     assert b"1 Imported" in res.data
 
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
@@ -99,7 +99,7 @@ def test_trigger_functionality(client, live_server):
     assert bytes(trigger_text.encode('utf-8')) in res.data
 
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
@@ -113,7 +113,7 @@ def test_trigger_functionality(client, live_server):
     set_modified_original_ignore_response()
 
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
 
@@ -121,21 +121,15 @@ def test_trigger_functionality(client, live_server):
     res = client.get(url_for("index"))
     assert b'unviewed' not in res.data
 
-    # Now set the content which contains the trigger text
+    # Just to be sure.. set a regular modified change..
     time.sleep(sleep_time_for_fetch_thread)
     set_modified_with_trigger_text_response()
 
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
     time.sleep(sleep_time_for_fetch_thread)
     res = client.get(url_for("index"))
     assert b'unviewed' in res.data
 
-    # https://github.com/dgtlmoon/changedetection.io/issues/616
-    # Apparently the actual snapshot that contains the trigger never shows
-    res = client.get(url_for("diff_history_page", uuid="first"))
-    assert b'foobar123' in res.data
-
-
     # Check the preview/highlighter, we should be able to see what we triggered on, but it should be highlighted
     res = client.get(url_for("preview_page", uuid="first"))
     # We should be able to see what we ignored

changedetectionio/tests/test_trigger_regex.py

@@ -43,7 +43,7 @@ def test_trigger_regex_functionality(client, live_server):
     assert b"1 Imported" in res.data
 
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
@@ -65,7 +65,7 @@ def test_trigger_regex_functionality(client, live_server):
     with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write("some new noise")
 
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
     time.sleep(sleep_time_for_fetch_thread)
 
     # It should report nothing found (nothing should match the regex)
@@ -75,7 +75,7 @@ def test_trigger_regex_functionality(client, live_server):
     with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write("regex test123<br/>\nsomething 123")
 
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
     time.sleep(sleep_time_for_fetch_thread)
     res = client.get(url_for("index"))
     assert b'unviewed' in res.data

changedetectionio/tests/test_trigger_regex_with_filter.py

@@ -43,7 +43,7 @@ def test_trigger_regex_functionality(client, live_server):
     assert b"1 Imported" in res.data
 
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
@@ -66,7 +66,7 @@ def test_trigger_regex_functionality(client, live_server):
     with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write("<html>some new noise with cool stuff2 ok</html>")
 
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
     time.sleep(sleep_time_for_fetch_thread)
 
     # It should report nothing found (nothing should match the regex and filter)
@@ -76,7 +76,7 @@ def test_trigger_regex_functionality(client, live_server):
     with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write("<html>some new noise with <span id=in-here>cool stuff6</span> ok</html>")
 
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
     time.sleep(sleep_time_for_fetch_thread)
     res = client.get(url_for("index"))
     assert b'unviewed' in res.data

changedetectionio/tests/test_xpath_selector.py

@@ -44,61 +44,6 @@ def set_modified_response():
 
     return None
 
-# Handle utf-8 charset replies https://github.com/dgtlmoon/changedetection.io/pull/613
-def test_check_xpath_filter_utf8(client, live_server):
-    filter='//item/*[self::description]'
-
-    d='''<?xml version="1.0" encoding="UTF-8"?>
-<rss xmlns:taxo="http://purl.org/rss/1.0/modules/taxonomy/" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" xmlns:itunes="http://www.itunes.com/dtds/podcast-1.0.dtd" xmlns:dc="http://purl.org/dc/elements/1.1/" version="2.0">
-	<channel>
-		<title>rpilocator.com</title>
-		<link>https://rpilocator.com</link>
-		<description>Find Raspberry Pi Computers in Stock</description>
-		<lastBuildDate>Thu, 19 May 2022 23:27:30 GMT</lastBuildDate>
-		<image>
-			<url>https://rpilocator.com/favicon.png</url>
-			<title>rpilocator.com</title>
-			<link>https://rpilocator.com/</link>
-			<width>32</width>
-			<height>32</height>
-		</image>
-		<item>
-			<title>Stock Alert (UK): RPi CM4 - 1GB RAM, No MMC, No Wifi is In Stock at Pimoroni</title>
-			<description>Stock Alert (UK): RPi CM4 - 1GB RAM, No MMC, No Wifi is In Stock at Pimoroni</description>
-			<link>https://rpilocator.com?vendor=pimoroni&amp;utm_source=feed&amp;utm_medium=rss</link>
-			<category>pimoroni</category>
-			<category>UK</category>
-			<category>CM4</category>
-			<guid isPermaLink="false">F9FAB0D9-DF6F-40C8-8DEE5FC0646BB722</guid>
-			<pubDate>Thu, 19 May 2022 14:32:32 GMT</pubDate>
-		</item>
-	</channel>
-</rss>'''
-
-    with open("test-datastore/endpoint-content.txt", "w") as f:
-        f.write(d)
-
-    # Add our URL to the import page
-    test_url = url_for('test_endpoint', _external=True, content_type="application/rss+xml;charset=UTF-8")
-    res = client.post(
-        url_for("import_page"),
-        data={"urls": test_url},
-        follow_redirects=True
-    )
-    assert b"1 Imported" in res.data
-    res = client.post(
-        url_for("edit_page", uuid="first"),
-        data={"css_filter": filter, "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
-        follow_redirects=True
-    )
-    assert b"Updated watch." in res.data
-    time.sleep(3)
-    res = client.get(url_for("index"))
-    assert b'Unicode strings with encoding declaration are not supported.' not in res.data
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
-    assert b'Deleted' in res.data
-
-
 
 def test_check_markup_xpath_filter_restriction(client, live_server):
     sleep_time_for_fetch_thread = 3
@@ -120,7 +65,7 @@ def test_check_markup_xpath_filter_restriction(client, live_server):
     assert b"1 Imported" in res.data
 
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
@@ -144,14 +89,12 @@ def test_check_markup_xpath_filter_restriction(client, live_server):
     set_modified_response()
 
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
 
     res = client.get(url_for("index"))
     assert b'unviewed' not in res.data
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
-    assert b'Deleted' in res.data
 
 
 def test_xpath_validation(client, live_server):
@@ -174,13 +117,11 @@ def test_xpath_validation(client, live_server):
         follow_redirects=True
     )
     assert b"is not a valid XPath expression" in res.data
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
-    assert b'Deleted' in res.data
 
 
 # actually only really used by the distll.io importer, but could be handy too
 def test_check_with_prefix_css_filter(client, live_server):
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    res = client.get(url_for("api_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
 
     # Give the endpoint time to spin up
@@ -212,7 +153,9 @@ def test_check_with_prefix_css_filter(client, live_server):
         follow_redirects=True
     )
 
+    with open('/tmp/fuck.html', 'wb') as f:
+        f.write(res.data)
     assert b"Some text thats the same" in res.data #in selector
     assert b"Some text that will change" not in res.data #not in selector
 
-    client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    client.get(url_for("api_delete", uuid="all"), follow_redirects=True)

changedetectionio/update_worker.py

@@ -40,11 +40,10 @@ class update_worker(threading.Thread):
                     contents = ""
                     screenshot = False
                     update_obj= {}
-                    xpath_data = False
                     now = time.time()
 
                     try:
-                        changed_detected, update_obj, contents, screenshot, xpath_data = update_handler.run(uuid)
+                        changed_detected, update_obj, contents, screenshot = update_handler.run(uuid)
 
                         # Re #342
                         # In Python 3, all strings are sequences of Unicode characters. There is a bytes type that holds raw bytes.
@@ -56,7 +55,6 @@ class update_worker(threading.Thread):
                     except content_fetcher.ReplyWithContentButNoText as e:
                         # Totally fine, it's by choice - just continue on, nothing more to care about
                         # Page had elements/content but no renderable text
-                        self.datastore.update_watch(uuid=uuid, update_obj={'last_error': "Got HTML content but no text found."})
                         pass
                     except content_fetcher.EmptyReply as e:
                         # Some kind of custom to-str handler in the exception handler that does this?
@@ -150,9 +148,6 @@ class update_worker(threading.Thread):
                         # Always save the screenshot if it's available
                         if screenshot:
                             self.datastore.save_screenshot(watch_uuid=uuid, screenshot=screenshot)
-                        if xpath_data:
-                            self.datastore.save_xpath_data(watch_uuid=uuid, data=xpath_data)
-
 
                 self.current_uuid = None  # Done
                 self.q.task_done()

requirements.txt

@@ -6,7 +6,6 @@ timeago ~=1.0
 inscriptis ~= 2.2
 feedgen ~= 0.9
 flask-login ~= 0.5
-flask_restful
 pytz
 
 # Set these versions together to avoid a RequestsDependencyWarning