Handle disabled situation

WIP on access
Work on handling auth token
2025-11-08 10:36:32 +00:00 · 2022-05-20 16:03:10 +02:00 · 2022-05-20 16:00:34 +02:00 · 2022-05-20 15:37:43 +02:00 · 2022-05-20 12:47:50 +02:00 · 2022-05-20 12:44:07 +02:00
51 changed files with 1452 additions and 334 deletions
--- a/5
+++ b/5
@@ -20,6 +20,11 @@ COPY requirements.txt /requirements.txt

 RUN pip install --target=/dependencies -r /requirements.txt

+# Playwright is an alternative to Selenium
+# Excluded this package from requirements.txt to prevent arm/v6 and arm/v7 builds from failing
+RUN pip install --target=/dependencies playwright~=1.20 \
+    || echo "WARN: Failed to install Playwright. The application can still run, but the Playwright option will be disabled."
+
 # Final image stage
 FROM python:3.8-slim

--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -1,5 +1,6 @@
 recursive-include changedetectionio/templates *
 recursive-include changedetectionio/static *
+recursive-include changedetectionio/model *
 include changedetection.py
 global-exclude *.pyc
 global-exclude node_modules
--- a/README-pip.md
+++ b/README-pip.md
@@ -16,6 +16,13 @@ Live your data-life *pro-actively* instead of *re-actively*, do not rely on mani

 <img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/screenshot.png" style="max-width:100%;" alt="Self-hosted web page change monitoring"  title="Self-hosted web page change monitoring"  />

+
+**Get your own private instance now! Let us host it for you!**
+
+[**Try our $6.99/month subscription - unlimited checks, watches and notifications!**](https://lemonade.changedetection.io/start), choose from different geographical locations, let us handle everything for you. 
+
+
+
 #### Example use cases

 Know when ...
@@ -58,14 +65,3 @@ Then visit http://127.0.0.1:5000 , You should now be able to access the UI.

 See https://github.com/dgtlmoon/changedetection.io for more information.

-
-
-### Support us
-
-Do you use changedetection.io to make money? does it save you time or money? Does it make your life easier? less stressful? Remember, we write this software when we should be doing actual paid work, we have to buy food and pay rent just like you.
-
-Please support us, even small amounts help a LOT.
-
-BTC `1PLFN327GyUarpJd7nVe7Reqg9qHx5frNn`
-
-<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/btc-support.png" style="max-width:50%;" alt="Support us!"  />
--- a/changedetectionio/init.py
+++ b/changedetectionio/init.py
@@ -36,11 +36,14 @@ from flask import (
    url_for,
 )
 from flask_login import login_required
+from flask_restful import abort, Api
+
 from flask_wtf import CSRFProtect

 from changedetectionio import html_tools
+from changedetectionio.api import api_v1

-__version__ = '0.39.12'
+__version__ = '0.39.13.1'

 datastore = None

@@ -78,6 +81,8 @@ csrf.init_app(app)

 notification_debug_log=[]

+watch_api = Api(app, decorators=[csrf.exempt])
+
 def init_app_secret(datastore_path):
    secret = ""

@@ -179,6 +184,25 @@ def changedetection_app(config=None, datastore_o=None):
    login_manager.login_view = 'login'
    app.secret_key = init_app_secret(config['datastore_path'])

+
+    watch_api.add_resource(api_v1.WatchSingleHistory,
+                           '/api/v1/watch/<string:uuid>/history/<string:timestamp>',
+                           resource_class_kwargs={'datastore': datastore, 'update_q': update_q})
+
+    watch_api.add_resource(api_v1.WatchHistory,
+                           '/api/v1/watch/<string:uuid>/history',
+                           resource_class_kwargs={'datastore': datastore})
+
+    watch_api.add_resource(api_v1.CreateWatch, '/api/v1/watch',
+                           resource_class_kwargs={'datastore': datastore, 'update_q': update_q})
+
+    watch_api.add_resource(api_v1.Watch, '/api/v1/watch/<string:uuid>',
+                           resource_class_kwargs={'datastore': datastore, 'update_q': update_q})
+
+
+
+
+
    # Setup cors headers to allow all domains
    # https://flask-cors.readthedocs.io/en/latest/
    #    CORS(app)
@@ -367,6 +391,8 @@ def changedetection_app(config=None, datastore_o=None):

            if limit_tag != None:
                # Support for comma separated list of tags.
+                if watch['tag'] is None:
+                    continue
                for tag_in_watch in watch['tag'].split(','):
                    tag_in_watch = tag_in_watch.strip()
                    if tag_in_watch == limit_tag:
@@ -518,10 +544,22 @@ def changedetection_app(config=None, datastore_o=None):
        if all(value == 0 or value == None for value in datastore.data['watching'][uuid]['time_between_check'].values()):
            default['time_between_check'] = deepcopy(datastore.data['settings']['requests']['time_between_check'])

-        form = forms.watchForm(formdata=request.form if request.method == 'POST' else None,
-                                        data=default
-                                        )
+        # Defaults for proxy choice
+        if datastore.proxy_list is not None:  # When enabled
+            # Radio needs '' not None, or incase that the chosen one no longer exists
+            if default['proxy'] is None or not any(default['proxy'] in tup for tup in datastore.proxy_list):
+                default['proxy'] = ''

+        # proxy_override set to the json/text list of the items
+        form = forms.watchForm(formdata=request.form if request.method == 'POST' else None,
+                               data=default,
+                               )
+
+        if datastore.proxy_list is None:
+            # @todo - Couldn't get setattr() etc dynamic addition working, so remove it instead
+            del form.proxy
+        else:
+            form.proxy.choices = [('', 'Default')] + datastore.proxy_list

        if request.method == 'POST' and form.validate():
            extra_update_obj = {}
@@ -558,6 +596,10 @@ def changedetection_app(config=None, datastore_o=None):
                if len(datastore.data['watching'][uuid]['history']):
                    extra_update_obj['previous_md5'] = get_current_checksum_include_ignore_text(uuid=uuid)

+            # Be sure proxy value is None
+            if datastore.proxy_list is not None and form.data['proxy'] == '':
+                extra_update_obj['proxy'] = None
+
            datastore.data['watching'][uuid].update(form.data)
            datastore.data['watching'][uuid].update(extra_update_obj)

@@ -584,12 +626,12 @@ def changedetection_app(config=None, datastore_o=None):
            if request.method == 'POST' and not form.validate():
                flash("An error occurred, please see below.", "error")

-
            output = render_template("edit.html",
                                     uuid=uuid,
                                     watch=datastore.data['watching'][uuid],
                                     form=form,
                                     has_empty_checktime=using_default_check_time,
+                                     using_global_webdriver_wait=default['webdriver_delay'] is None,
                                     current_base_url=datastore.data['settings']['application']['base_url'],
                                     emailprefix=os.getenv('NOTIFICATION_MAIL_BUTTON_PREFIX', False)
                                     )
@@ -601,10 +643,28 @@ def changedetection_app(config=None, datastore_o=None):
    def settings_page():
        from changedetectionio import content_fetcher, forms

+        default = deepcopy(datastore.data['settings'])
+        if datastore.proxy_list is not None:
+            # When enabled
+            system_proxy = datastore.data['settings']['requests']['proxy']
+            # In the case it doesnt exist anymore
+            if not any([system_proxy in tup for tup in datastore.proxy_list]):
+                system_proxy = None
+
+            default['requests']['proxy'] = system_proxy if system_proxy is not None else datastore.proxy_list[0][0]
+            # Used by the form handler to keep or remove the proxy settings
+            default['proxy_list'] = datastore.proxy_list
+
+
        # Don't use form.data on POST so that it doesnt overrid the checkbox status from the POST status
        form = forms.globalSettingsForm(formdata=request.form if request.method == 'POST' else None,
-                                        data=datastore.data['settings']
+                                        data=default
                                        )
+        if datastore.proxy_list is None:
+            # @todo - Couldn't get setattr() etc dynamic addition working, so remove it instead
+            del form.requests.form.proxy
+        else:
+            form.requests.form.proxy.choices = datastore.proxy_list

        if request.method == 'POST':
            # Password unset is a GET, but we can lock the session to a salted env password to always need the password
@@ -637,6 +697,7 @@ def changedetection_app(config=None, datastore_o=None):
                                 form=form,
                                 current_base_url = datastore.data['settings']['application']['base_url'],
                                 hide_remove_pass=os.getenv("SALTED_PASS", False),
+                                 api_key=datastore.data['settings']['application'].get('api_access_token'),
                                 emailprefix=os.getenv('NOTIFICATION_MAIL_BUTTON_PREFIX', False))

        return output
@@ -644,44 +705,37 @@ def changedetection_app(config=None, datastore_o=None):
    @app.route("/import", methods=['GET', "POST"])
    @login_required
    def import_page():
-        import validators
        remaining_urls = []
-
-        good = 0
-
        if request.method == 'POST':
-            now=time.time()
-            urls = request.values.get('urls').split("\n")
+            from .importer import import_url_list, import_distill_io_json

-            if (len(urls) > 5000):
-                flash("Importing 5,000 of the first URLs from your list, the rest can be imported again.")
+            # URL List import
+            if request.values.get('urls') and len(request.values.get('urls').strip()):
+                # Import and push into the queue for immediate update check
+                importer = import_url_list()
+                importer.run(data=request.values.get('urls'), flash=flash, datastore=datastore)
+                for uuid in importer.new_uuids:
+                    update_q.put(uuid)

-            for url in urls:
-                url = url.strip()
-                url, *tags = url.split(" ")
-                # Flask wtform validators wont work with basic auth, use validators package
-                # Up to 5000 per batch so we dont flood the server
-                if len(url) and validators.url(url.replace('source:', '')) and good < 5000:
-                    new_uuid = datastore.add_watch(url=url.strip(), tag=" ".join(tags), write_to_disk_now=False)
-                    if new_uuid:
-                        # Straight into the queue.
-                        update_q.put(new_uuid)
-                        good += 1
-                        continue
+                if len(importer.remaining_data) == 0:
+                    return redirect(url_for('index'))
+                else:
+                    remaining_urls = importer.remaining_data

-                if len(url.strip()):
-                    remaining_urls.append(url)
+            # Distill.io import
+            if request.values.get('distill-io') and len(request.values.get('distill-io').strip()):
+                # Import and push into the queue for immediate update check
+                d_importer = import_distill_io_json()
+                d_importer.run(data=request.values.get('distill-io'), flash=flash, datastore=datastore)
+                for uuid in d_importer.new_uuids:
+                    update_q.put(uuid)

-            flash("{} Imported in {:.2f}s, {} Skipped.".format(good, time.time()-now,len(remaining_urls)))
-            datastore.needs_write = True

-            if len(remaining_urls) == 0:
-                # Looking good, redirect to index.
-                return redirect(url_for('index'))

        # Could be some remaining, or we could be on GET
        output = render_template("import.html",
-                                 remaining="\n".join(remaining_urls)
+                                 import_url_list_remaining="\n".join(remaining_urls),
+                                 original_distill_json=''
                                 )
        return output

@@ -842,27 +896,6 @@ def changedetection_app(config=None, datastore_o=None):

        return output

-    @app.route("/api/<string:uuid>/snapshot/current", methods=['GET'])
-    @login_required
-    def api_snapshot(uuid):
-
-        # More for testing, possible to return the first/only
-        if uuid == 'first':
-            uuid = list(datastore.data['watching'].keys()).pop()
-
-        try:
-            watch = datastore.data['watching'][uuid]
-        except KeyError:
-            return abort(400, "No history found for the specified link, bad link?")
-
-        newest = list(watch['history'].keys())[-1]
-        with open(watch['history'][newest], 'r') as f:
-            content = f.read()
-
-        resp = make_response(content)
-        resp.headers['Content-Type'] = 'text/plain'
-        return resp
-
    @app.route("/favicon.ico", methods=['GET'])
    def favicon():
        return send_from_directory("static/images", path="favicon.ico")
@@ -973,7 +1006,7 @@ def changedetection_app(config=None, datastore_o=None):

    @app.route("/api/add", methods=['POST'])
    @login_required
-    def api_watch_add():
+    def form_watch_add():
        from changedetectionio import forms
        form = forms.quickWatchForm(request.form)

@@ -999,7 +1032,7 @@ def changedetection_app(config=None, datastore_o=None):

    @app.route("/api/delete", methods=['GET'])
    @login_required
-    def api_delete():
+    def form_delete():
        uuid = request.args.get('uuid')

        if uuid != 'all' and not uuid in datastore.data['watching'].keys():
@@ -1016,7 +1049,7 @@ def changedetection_app(config=None, datastore_o=None):

    @app.route("/api/clone", methods=['GET'])
    @login_required
-    def api_clone():
+    def form_clone():
        uuid = request.args.get('uuid')
        # More for testing, possible to return the first/only
        if uuid == 'first':
@@ -1030,7 +1063,7 @@ def changedetection_app(config=None, datastore_o=None):

    @app.route("/api/checknow", methods=['GET'])
    @login_required
-    def api_watch_checknow():
+    def form_watch_checknow():

        tag = request.args.get('tag')
        uuid = request.args.get('uuid')
@@ -1067,7 +1100,7 @@ def changedetection_app(config=None, datastore_o=None):

    @app.route("/api/share-url", methods=['GET'])
    @login_required
-    def api_share_put_watch():
+    def form_share_put_watch():
        """Given a watch UUID, upload the info and return a share-link
           the share-link can be imported/added"""
        import requests
--- a/changedetectionio/api/init.py
+++ b/changedetectionio/api/init.py
--- a/changedetectionio/api/api_v1.py
+++ b/changedetectionio/api/api_v1.py
@@ -0,0 +1,125 @@
+from flask_restful import abort, Resource
+from flask import request, make_response
+import validators
+from . import auth
+
+
+
+# https://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html
+
+class Watch(Resource):
+    def __init__(self, **kwargs):
+        # datastore is a black box dependency
+        self.datastore = kwargs['datastore']
+        self.update_q = kwargs['update_q']
+
+    # Get information about a single watch, excluding the history list (can be large)
+    # curl http://localhost:4000/api/v1/watch/<string:uuid>
+    # ?recheck=true
+    @auth.check_token
+    def get(self, uuid):
+        from copy import deepcopy
+        watch = deepcopy(self.datastore.data['watching'].get(uuid))
+        if not watch:
+            abort(404, message='No watch exists with the UUID of {}'.format(uuid))
+
+        if request.args.get('recheck'):
+            self.update_q.put(uuid)
+            return "OK", 200
+
+        # Return without history, get that via another API call
+        watch['history_n'] = len(watch['history'])
+        del (watch['history'])
+        return watch
+
+    @auth.check_token
+    def delete(self, uuid):
+        if not self.datastore.data['watching'].get(uuid):
+            abort(400, message='No watch exists with the UUID of {}'.format(uuid))
+
+        self.datastore.delete(uuid)
+        return 'OK', 204
+
+
+class WatchHistory(Resource):
+    def __init__(self, **kwargs):
+        # datastore is a black box dependency
+        self.datastore = kwargs['datastore']
+
+    # Get a list of available history for a watch by UUID
+    # curl http://localhost:4000/api/v1/watch/<string:uuid>/history
+    def get(self, uuid):
+        watch = self.datastore.data['watching'].get(uuid)
+        if not watch:
+            abort(404, message='No watch exists with the UUID of {}'.format(uuid))
+        return watch['history'], 200
+
+
+class WatchSingleHistory(Resource):
+    def __init__(self, **kwargs):
+        # datastore is a black box dependency
+        self.datastore = kwargs['datastore']
+
+    # Read a given history snapshot and return its content
+    # <string:timestamp> or "latest"
+    # curl http://localhost:4000/api/v1/watch/<string:uuid>/history/<int:timestamp>
+    @auth.check_token
+    def get(self, uuid, timestamp):
+        watch = self.datastore.data['watching'].get(uuid)
+        if not watch:
+            abort(404, message='No watch exists with the UUID of {}'.format(uuid))
+
+        if not len(watch['history']):
+            abort(404, message='Watch found but no history exists for the UUID {}'.format(uuid))
+
+        if timestamp == 'latest':
+            timestamp = list(watch['history'].keys())[-1]
+
+        with open(watch['history'][timestamp], 'r') as f:
+            content = f.read()
+
+        response = make_response(content, 200)
+        response.mimetype = "text/plain"
+        return response
+
+
+class CreateWatch(Resource):
+    def __init__(self, **kwargs):
+        # datastore is a black box dependency
+        self.datastore = kwargs['datastore']
+        self.update_q = kwargs['update_q']
+
+    @auth.check_token
+    def post(self):
+        # curl http://localhost:4000/api/v1/watch -H "Content-Type: application/json" -d '{"url": "https://my-nice.com", "tag": "one, two" }'
+        json_data = request.get_json()
+        tag = json_data['tag'].strip() if json_data.get('tag') else ''
+
+        if not validators.url(json_data['url'].strip()):
+            return "Invalid or unsupported URL", 400
+
+        extras = {'title': json_data['title'].strip()} if json_data.get('title') else {}
+
+        new_uuid = self.datastore.add_watch(url=json_data['url'].strip(), tag=tag, extras=extras)
+        self.update_q.put(new_uuid)
+        return {'uuid': new_uuid}, 201
+
+    # Return concise list of available watches and some very basic info
+    # curl http://localhost:4000/api/v1/watch|python -mjson.tool
+    # ?recheck_all=1 to recheck all
+    @auth.check_token
+    def get(self):
+        list = {}
+        for k, v in self.datastore.data['watching'].items():
+            list[k] = {'url': v['url'],
+                       'title': v['title'],
+                       'last_checked': v['last_checked'],
+                       'last_changed': v['last_changed'],
+                       'last_error': v['last_error']}
+
+        if request.args.get('recheck_all'):
+            for uuid in self.datastore.data['watching'].keys():
+                self.update_q.put(uuid)
+            return {'status': "OK"}, 200
+
+        return list, 200
--- a/changedetectionio/api/auth.py
+++ b/changedetectionio/api/auth.py
@@ -0,0 +1,33 @@
+from flask import request, make_response, jsonify
+from functools import wraps
+
+
+# Simple API auth key comparison
+# @todo - Maybe short lived token in the future?
+
+def check_token(f):
+    @wraps(f)
+    def decorated(*args, **kwargs):
+        datastore = args[0].datastore
+
+        config_api_token_enabled = datastore.data['settings']['application'].get('api_access_token_enabled')
+        if not config_api_token_enabled:
+            return
+
+        try:
+            api_key_header = request.headers['x-api-key']
+        except KeyError:
+            return make_response(
+                jsonify("No authorization x-api-key header."), 403
+            )
+
+        config_api_token = datastore.data['settings']['application'].get('api_access_token')
+
+        if api_key_header != config_api_token:
+            return make_response(
+                jsonify("Invalid access - API key invalid."), 403
+            )
+
+        return f(*args, **kwargs)
+
+    return decorated
--- a/changedetectionio/changedetection.py
+++ b/changedetectionio/changedetection.py
@@ -8,7 +8,7 @@ import sys

 import eventlet
 import eventlet.wsgi
-from . import store, changedetection_app
+from . import store, changedetection_app, content_fetcher
 from . import __version__

 def main():
--- a/changedetectionio/content_fetcher.py
+++ b/changedetectionio/content_fetcher.py
@@ -1,14 +1,9 @@
 from abc import ABC, abstractmethod
 import chardet
 import os
-from selenium import webdriver
-from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
-from selenium.webdriver.common.proxy import Proxy as SeleniumProxy
-from selenium.common.exceptions import WebDriverException
 import requests
 import time
-import urllib3.exceptions
-
+import sys

 class EmptyReply(Exception):
    def __init__(self, status_code, url):
@@ -16,16 +11,30 @@ class EmptyReply(Exception):
        self.status_code = status_code
        self.url = url
        return
-
    pass

+class ReplyWithContentButNoText(Exception):
+    def __init__(self, status_code, url):
+        # Set this so we can use it in other parts of the app
+        self.status_code = status_code
+        self.url = url
+        return
+    pass
+
+
 class Fetcher():
    error = None
    status_code = None
    content = None
    headers = None
+    # Will be needed in the future by the VisualSelector, always get this where possible.
+    screenshot = False
+    fetcher_description = "No description"
+    system_http_proxy = os.getenv('HTTP_PROXY')
+    system_https_proxy = os.getenv('HTTPS_PROXY')

-    fetcher_description ="No description"
+    # Time ONTOP of the system defined env minimum time
+    render_extract_delay=0

    @abstractmethod
    def get_error(self):
@@ -46,10 +55,6 @@ class Fetcher():
    def quit(self):
        return

-    @abstractmethod
-    def screenshot(self):
-        return
-
    @abstractmethod
    def get_last_status_code(self):
        return self.status_code
@@ -59,29 +64,121 @@ class Fetcher():
    def is_ready(self):
        return True

+
 #   Maybe for the future, each fetcher provides its own diff output, could be used for text, image
 #   the current one would return javascript output (as we use JS to generate the diff)
 #
-#   Returns tuple(mime_type, stream)
-#    @abstractmethod
-#    def return_diff(self, stream_a, stream_b):
-#        return
-
 def available_fetchers():
-        import inspect
-        from changedetectionio import content_fetcher
-        p=[]
-        for name, obj in inspect.getmembers(content_fetcher):
-            if inspect.isclass(obj):
-                # @todo html_ is maybe better as fetcher_ or something
-                # In this case, make sure to edit the default one in store.py and fetch_site_status.py
-                if "html_" in name:
-                    t=tuple([name,obj.fetcher_description])
-                    p.append(t)
+    # See the if statement at the bottom of this file for how we switch between playwright and webdriver
+    import inspect
+    p = []
+    for name, obj in inspect.getmembers(sys.modules[__name__], inspect.isclass):
+        if inspect.isclass(obj):
+            # @todo html_ is maybe better as fetcher_ or something
+            # In this case, make sure to edit the default one in store.py and fetch_site_status.py
+            if name.startswith('html_'):
+                t = tuple([name, obj.fetcher_description])
+                p.append(t)

-        return p
+    return p

-class html_webdriver(Fetcher):
+
+class base_html_playwright(Fetcher):
+    fetcher_description = "Playwright {}/Javascript".format(
+        os.getenv("PLAYWRIGHT_BROWSER_TYPE", 'chromium').capitalize()
+    )
+    if os.getenv("PLAYWRIGHT_DRIVER_URL"):
+        fetcher_description += " via '{}'".format(os.getenv("PLAYWRIGHT_DRIVER_URL"))
+
+    browser_type = ''
+    command_executor = ''
+
+    # Configs for Proxy setup
+    # In the ENV vars, is prefixed with "playwright_proxy_", so it is for example "playwright_proxy_server"
+    playwright_proxy_settings_mappings = ['bypass', 'server', 'username', 'password']
+
+    proxy = None
+
+    def __init__(self, proxy_override=None):
+
+        # .strip('"') is going to save someone a lot of time when they accidently wrap the env value
+        self.browser_type = os.getenv("PLAYWRIGHT_BROWSER_TYPE", 'chromium').strip('"')
+        self.command_executor = os.getenv(
+            "PLAYWRIGHT_DRIVER_URL",
+            'ws://playwright-chrome:3000'
+        ).strip('"')
+
+        # If any proxy settings are enabled, then we should setup the proxy object
+        proxy_args = {}
+        for k in self.playwright_proxy_settings_mappings:
+            v = os.getenv('playwright_proxy_' + k, False)
+            if v:
+                proxy_args[k] = v.strip('"')
+
+        if proxy_args:
+            self.proxy = proxy_args
+
+        # allow per-watch proxy selection override
+        if proxy_override:
+            self.proxy = {'server': proxy_override}
+
+    def run(self,
+            url,
+            timeout,
+            request_headers,
+            request_body,
+            request_method,
+            ignore_status_codes=False):
+
+        from playwright.sync_api import sync_playwright
+        import playwright._impl._api_types
+        from playwright._impl._api_types import Error, TimeoutError
+
+        with sync_playwright() as p:
+            browser_type = getattr(p, self.browser_type)
+
+            # Seemed to cause a connection Exception even tho I can see it connect
+            # self.browser = browser_type.connect(self.command_executor, timeout=timeout*1000)
+            browser = browser_type.connect_over_cdp(self.command_executor, timeout=timeout * 1000)
+
+            # Set user agent to prevent Cloudflare from blocking the browser
+            # Use the default one configured in the App.py model that's passed from fetch_site_status.py
+            context = browser.new_context(
+                user_agent=request_headers['User-Agent'] if request_headers.get('User-Agent') else 'Mozilla/5.0',
+                proxy=self.proxy
+            )
+            page = context.new_page()
+            page.set_viewport_size({"width": 1280, "height": 1024})
+            try:
+                response = page.goto(url, timeout=timeout * 1000, wait_until='commit')
+                # Wait_until = commit
+                # - `'commit'` - consider operation to be finished when network response is received and the document started loading.
+                # Better to not use any smarts from Playwright and just wait an arbitrary number of seconds
+                # This seemed to solve nearly all 'TimeoutErrors'
+                extra_wait = int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)) + self.render_extract_delay
+                page.wait_for_timeout(extra_wait * 1000)
+            except playwright._impl._api_types.TimeoutError as e:
+                raise EmptyReply(url=url, status_code=None)
+
+            if response is None:
+                raise EmptyReply(url=url, status_code=None)
+
+            if len(page.content().strip()) == 0:
+                raise EmptyReply(url=url, status_code=None)
+
+            self.status_code = response.status
+            self.content = page.content()
+            self.headers = response.all_headers()
+
+            # Some bug where it gives the wrong screenshot size, but making a request with the clip set first seems to solve it
+            # JPEG is better here because the screenshots can be very very large
+            page.screenshot(type='jpeg', clip={'x': 1.0, 'y': 1.0, 'width': 1280, 'height': 1024})
+            self.screenshot = page.screenshot(type='jpeg', full_page=True, quality=90)
+            context.close()
+            browser.close()
+
+
+class base_html_webdriver(Fetcher):
    if os.getenv("WEBDRIVER_URL"):
        fetcher_description = "WebDriver Chrome/Javascript via '{}'".format(os.getenv("WEBDRIVER_URL"))
    else:
@@ -94,12 +191,11 @@ class html_webdriver(Fetcher):
    selenium_proxy_settings_mappings = ['proxyType', 'ftpProxy', 'httpProxy', 'noProxy',
                                        'proxyAutoconfigUrl', 'sslProxy', 'autodetect',
                                        'socksProxy', 'socksVersion', 'socksUsername', 'socksPassword']
+    proxy = None

+    def __init__(self, proxy_override=None):
+        from selenium.webdriver.common.proxy import Proxy as SeleniumProxy

-
-    proxy=None
-
-    def __init__(self):
        # .strip('"') is going to save someone a lot of time when they accidently wrap the env value
        self.command_executor = os.getenv("WEBDRIVER_URL", 'http://browser-chrome:4444/wd/hub').strip('"')

@@ -110,6 +206,16 @@ class html_webdriver(Fetcher):
            if v:
                proxy_args[k] = v.strip('"')

+        # Map back standard HTTP_ and HTTPS_PROXY to webDriver httpProxy/sslProxy
+        if not proxy_args.get('webdriver_httpProxy') and self.system_http_proxy:
+            proxy_args['httpProxy'] = self.system_http_proxy
+        if not proxy_args.get('webdriver_sslProxy') and self.system_https_proxy:
+            proxy_args['httpsProxy'] = self.system_https_proxy
+
+        # Allows override the proxy on a per-request basis
+        if proxy_override is not None:
+            proxy_args['httpProxy'] = proxy_override
+
        if proxy_args:
            self.proxy = SeleniumProxy(raw=proxy_args)

@@ -121,6 +227,9 @@ class html_webdriver(Fetcher):
            request_method,
            ignore_status_codes=False):

+        from selenium import webdriver
+        from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
+        from selenium.common.exceptions import WebDriverException
        # request_body, request_method unused for now, until some magic in the future happens.

        # check env for WEBDRIVER_URL
@@ -142,12 +251,11 @@ class html_webdriver(Fetcher):
        # raise EmptyReply(url=url, status_code=r.status_code)

        # @todo - dom wait loaded?
-        time.sleep(int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)))
+        time.sleep(int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)) + self.render_extract_delay)
        self.content = self.driver.page_source
        self.headers = {}
-
-    def screenshot(self):
-        return self.driver.get_screenshot_as_png()
+        self.screenshot = self.driver.get_screenshot_as_png()
+        self.quit()

    # Does the connection to the webdriver work? run a test connection.
    def is_ready(self):
@@ -170,10 +278,14 @@ class html_webdriver(Fetcher):
            except Exception as e:
                print("Exception in chrome shutdown/quit" + str(e))

+
 # "html_requests" is listed as the default fetcher in store.py!
 class html_requests(Fetcher):
    fetcher_description = "Basic fast Plaintext/HTTP Client"

+    def __init__(self, proxy_override=None):
+        self.proxy_override = proxy_override
+
    def run(self,
            url,
            timeout,
@@ -182,12 +294,24 @@ class html_requests(Fetcher):
            request_method,
            ignore_status_codes=False):

+        proxies={}
+
+        # Allows override the proxy on a per-request basis
+        if self.proxy_override:
+            proxies = {'http': self.proxy_override, 'https': self.proxy_override, 'ftp': self.proxy_override}
+        else:
+            if self.system_http_proxy:
+                proxies['http'] = self.system_http_proxy
+            if self.system_https_proxy:
+                proxies['https'] = self.system_https_proxy
+
        r = requests.request(method=request_method,
-                         data=request_body,
-                         url=url,
-                         headers=request_headers,
-                         timeout=timeout,
-                         verify=False)
+                             data=request_body,
+                             url=url,
+                             headers=request_headers,
+                             timeout=timeout,
+                             proxies=proxies,
+                             verify=False)

        # If the response did not tell us what encoding format to expect, Then use chardet to override what `requests` thinks.
        # For example - some sites don't tell us it's utf-8, but return utf-8 content
@@ -207,3 +331,11 @@ class html_requests(Fetcher):
        self.content = r.text
        self.headers = r.headers

+
+# Decide which is the 'real' HTML webdriver, this is more a system wide config
+# rather than site-specific.
+use_playwright_as_chrome_fetcher = os.getenv('PLAYWRIGHT_DRIVER_URL', False)
+if use_playwright_as_chrome_fetcher:
+    html_webdriver = base_html_playwright
+else:
+    html_webdriver = base_html_webdriver
--- a/changedetectionio/fetch_site_status.py
+++ b/changedetectionio/fetch_site_status.py
@@ -16,6 +16,34 @@ class perform_site_check():
        super().__init__(*args, **kwargs)
        self.datastore = datastore

+    # If there was a proxy list enabled, figure out what proxy_args/which proxy to use
+    # if watch.proxy use that
+    # fetcher.proxy_override = watch.proxy or main config proxy
+    # Allows override the proxy on a per-request basis
+    # ALWAYS use the first one is nothing selected
+
+    def set_proxy_from_list(self, watch):
+        proxy_args = None
+        if self.datastore.proxy_list is None:
+            return None
+
+        # If its a valid one
+        if any([watch['proxy'] in p for p in self.datastore.proxy_list]):
+            proxy_args = watch['proxy']
+
+        # not valid (including None), try the system one
+        else:
+            system_proxy = self.datastore.data['settings']['requests']['proxy']
+            # Is not None and exists
+            if any([system_proxy in p for p in self.datastore.proxy_list]):
+                proxy_args = system_proxy
+
+        # Fallback - Did not resolve anything, use the first available
+        if proxy_args is None:
+            proxy_args = self.datastore.proxy_list[0][0]
+
+        return proxy_args
+
    def run(self, uuid):
        timestamp = int(time.time())  # used for storage etc too

@@ -66,8 +94,18 @@ class perform_site_check():
            # If the klass doesnt exist, just use a default
            klass = getattr(content_fetcher, "html_requests")

-        fetcher = klass()
+        proxy_args = self.set_proxy_from_list(watch)
+        fetcher = klass(proxy_override=proxy_args)
+
+        # Configurable per-watch or global extra delay before extracting text (for webDriver types)
+        system_webdriver_delay = self.datastore.data['settings']['application'].get('webdriver_delay', None)
+        if watch['webdriver_delay'] is not None:
+            fetcher.render_extract_delay = watch['webdriver_delay']
+        elif system_webdriver_delay is not None:
+            fetcher.render_extract_delay = system_webdriver_delay
+
        fetcher.run(url, timeout, request_headers, request_body, request_method, ignore_status_code)
+
        # Fetching complete, now filters
        # @todo move to class / maybe inside of fetcher abstract base?

@@ -117,11 +155,13 @@ class perform_site_check():
                # Then we assume HTML
                if has_filter_rule:
                    # For HTML/XML we offer xpath as an option, just start a regular xPath "/.."
-                    if css_filter_rule[0] == '/':
-                        html_content = html_tools.xpath_filter(xpath_filter=css_filter_rule, html_content=fetcher.content)
+                    if css_filter_rule[0] == '/' or css_filter_rule.startswith('xpath:'):
+                        html_content = html_tools.xpath_filter(xpath_filter=css_filter_rule.replace('xpath:', ''),
+                                                               html_content=fetcher.content)
                    else:
                        # CSS Filter, extract the HTML that matches and feed that into the existing inscriptis::get_text
                        html_content = html_tools.css_filter(css_filter=css_filter_rule, html_content=fetcher.content)
+
                if has_subtractive_selectors:
                    html_content = html_tools.element_removal(subtractive_selectors, html_content)

@@ -141,10 +181,14 @@ class perform_site_check():
            # Re #340 - return the content before the 'ignore text' was applied
            text_content_before_ignored_filter = stripped_text_from_html.encode('utf-8')

-
        # Re #340 - return the content before the 'ignore text' was applied
        text_content_before_ignored_filter = stripped_text_from_html.encode('utf-8')

+        # Treat pages with no renderable text content as a change? No by default
+        empty_pages_are_a_change = self.datastore.data['settings']['application'].get('empty_pages_are_a_change', False)
+        if not is_json and not empty_pages_are_a_change and len(stripped_text_from_html.strip()) == 0:
+            raise content_fetcher.ReplyWithContentButNoText(url=url, status_code=200)
+
        # We rely on the actual text in the html output.. many sites have random script vars etc,
        # in the future we'll implement other mechanisms.

@@ -192,9 +236,4 @@ class perform_site_check():
                if not watch['title'] or not len(watch['title']):
                    update_obj['title'] = html_tools.extract_element(find='title', html_content=fetcher.content)

-        if self.datastore.data['settings']['application'].get('real_browser_save_screenshot', True):
-            screenshot = fetcher.screenshot()
-
-        fetcher.quit()
-
-        return changed_detected, update_obj, text_content_before_ignored_filter, screenshot
+        return changed_detected, update_obj, text_content_before_ignored_filter, fetcher.screenshot
--- a/changedetectionio/forms.py
+++ b/changedetectionio/forms.py
@@ -318,6 +318,7 @@ class commonSettingsForm(Form):
    notification_format = SelectField('Notification format', choices=valid_notification_formats.keys(), default=default_notification_format)
    fetch_backend = RadioField(u'Fetch method', choices=content_fetcher.available_fetchers(), validators=[ValidateContentFetcherIsReady()])
    extract_title_as_title = BooleanField('Extract <title> from document and use as watch title', default=False)
+    webdriver_delay = IntegerField('Wait seconds before extracting text', validators=[validators.Optional(), validators.NumberRange(min=1, message="Should contain one or more seconds")] )

 class watchForm(commonSettingsForm):

@@ -337,9 +338,9 @@ class watchForm(commonSettingsForm):
    method = SelectField('Request method', choices=valid_method, default=default_method)
    ignore_status_codes = BooleanField('Ignore status codes (process non-2xx status codes as normal)', default=False)
    trigger_text = StringListField('Trigger/wait for text', [validators.Optional(), ValidateListRegex()])
-
    save_button = SubmitField('Save', render_kw={"class": "pure-button pure-button-primary"})
    save_and_preview_button = SubmitField('Save & Preview', render_kw={"class": "pure-button pure-button-primary"})
+    proxy = RadioField('Proxy')

    def validate(self, **kwargs):
        if not super().validate():
@@ -358,6 +359,7 @@ class watchForm(commonSettingsForm):
 # datastore.data['settings']['requests']..
 class globalSettingsRequestForm(Form):
    time_between_check = FormField(TimeBetweenCheckForm)
+    proxy = RadioField('Proxy')


 # datastore.data['settings']['application']..
@@ -369,8 +371,10 @@ class globalSettingsApplicationForm(commonSettingsForm):
    ignore_whitespace = BooleanField('Ignore whitespace')
    real_browser_save_screenshot = BooleanField('Save last screenshot when using Chrome?')
    removepassword_button = SubmitField('Remove password', render_kw={"class": "pure-button pure-button-primary"})
+    empty_pages_are_a_change =  BooleanField('Treat empty pages as a change?', default=False)
    render_anchor_tag_content = BooleanField('Render anchor tag content', default=False)
    fetch_backend = RadioField('Fetch Method', default="html_requests", choices=content_fetcher.available_fetchers(), validators=[ValidateContentFetcherIsReady()])
+    api_access_token_enabled = BooleanField('API access token security check enabled', default=True, validators=[validators.Optional()])
    password = SaltyPasswordField()


@@ -382,4 +386,3 @@ class globalSettingsForm(Form):
    requests = FormField(globalSettingsRequestForm)
    application = FormField(globalSettingsApplicationForm)
    save_button = SubmitField('Save', render_kw={"class": "pure-button pure-button-primary"})
-
--- a/changedetectionio/importer.py
+++ b/changedetectionio/importer.py
@@ -0,0 +1,133 @@
+from abc import ABC, abstractmethod
+import time
+import validators
+
+
+class Importer():
+    remaining_data = []
+    new_uuids = []
+    good = 0
+
+    def __init__(self):
+        self.new_uuids = []
+        self.good = 0
+        self.remaining_data = []
+
+    @abstractmethod
+    def run(self,
+            data,
+            flash,
+            datastore):
+        pass
+
+
+class import_url_list(Importer):
+    """
+    Imports a list, can be in <code>https://example.com tag1, tag2, last tag</code> format
+    """
+    def run(self,
+            data,
+            flash,
+            datastore,
+            ):
+
+        urls = data.split("\n")
+        good = 0
+        now = time.time()
+
+        if (len(urls) > 5000):
+            flash("Importing 5,000 of the first URLs from your list, the rest can be imported again.")
+
+        for url in urls:
+            url = url.strip()
+            if not len(url):
+                continue
+
+            tags = ""
+
+            # 'tags' should be a csv list after the URL
+            if ' ' in url:
+                url, tags = url.split(" ", 1)
+
+            # Flask wtform validators wont work with basic auth, use validators package
+            # Up to 5000 per batch so we dont flood the server
+            if len(url) and validators.url(url.replace('source:', '')) and good < 5000:
+                new_uuid = datastore.add_watch(url=url.strip(), tag=tags, write_to_disk_now=False)
+                if new_uuid:
+                    # Straight into the queue.
+                    self.new_uuids.append(new_uuid)
+                    good += 1
+                    continue
+
+            # Worked past the 'continue' above, append it to the bad list
+            if self.remaining_data is None:
+                self.remaining_data = []
+            self.remaining_data.append(url)
+
+        flash("{} Imported from list in {:.2f}s, {} Skipped.".format(good, time.time() - now, len(self.remaining_data)))
+
+
+class import_distill_io_json(Importer):
+    def run(self,
+            data,
+            flash,
+            datastore,
+            ):
+
+        import json
+        good = 0
+        now = time.time()
+        self.new_uuids=[]
+
+
+        try:
+            data = json.loads(data.strip())
+        except json.decoder.JSONDecodeError:
+            flash("Unable to read JSON file, was it broken?", 'error')
+            return
+
+        if not data.get('data'):
+            flash("JSON structure looks invalid, was it broken?", 'error')
+            return
+
+        for d in data.get('data'):
+            d_config = json.loads(d['config'])
+            extras = {'title': d['name']}
+
+            if len(d['uri']) and good < 5000:
+                try:
+                    # @todo we only support CSS ones at the moment
+                    if d_config['selections'][0]['frames'][0]['excludes'][0]['type'] == 'css':
+                        extras['subtractive_selectors'] = d_config['selections'][0]['frames'][0]['excludes'][0]['expr']
+                except KeyError:
+                    pass
+                except IndexError:
+                    pass
+
+                try:
+                    extras['css_filter'] = d_config['selections'][0]['frames'][0]['includes'][0]['expr']
+                    if d_config['selections'][0]['frames'][0]['includes'][0]['type'] == 'xpath':
+                        extras['css_filter'] = 'xpath:' + extras['css_filter']
+
+                except KeyError:
+                    pass
+                except IndexError:
+                    pass
+
+                try:
+                    extras['tag'] = ", ".join(d['tags'])
+                except KeyError:
+                    pass
+                except IndexError:
+                    pass
+
+                new_uuid = datastore.add_watch(url=d['uri'].strip(),
+                                               extras=extras,
+                                               write_to_disk_now=False)
+
+                if new_uuid:
+                    # Straight into the queue.
+                    self.new_uuids.append(new_uuid)
+                    good += 1
+
+        flash("{} Imported from Distill.io in {:.2f}s, {} Skipped.".format(len(self.new_uuids), time.time() - now, len(self.remaining_data)))
--- a/changedetectionio/model/App.py
+++ b/changedetectionio/model/App.py
@@ -23,12 +23,15 @@ class model(dict):
                'requests': {
                    'timeout': 15,  # Default 15 seconds
                    'time_between_check': {'weeks': None, 'days': None, 'hours': 3, 'minutes': None, 'seconds': None},
-                    'workers': 10  # Number of threads, lower is better for slow connections
+                    'workers': 10,  # Number of threads, lower is better for slow connections
+                    'proxy': None # Preferred proxy connection
                },
                'application': {
+                    'api_access_token_enabled': True,
                    'password': False,
                    'base_url' : None,
                    'extract_title_as_title': False,
+                    'empty_pages_are_a_change': False,
                    'fetch_backend': os.getenv("DEFAULT_FETCH_BACKEND", "html_requests"),
                    'global_ignore_text': [], # List of text to ignore when calculating the comparison checksum
                    'global_subtractive_selectors': [],
@@ -40,7 +43,8 @@ class model(dict):
                    'notification_body': default_notification_body,
                    'notification_format': default_notification_format,
                    'real_browser_save_screenshot': True,
-                    'schema_version' : 0
+                    'schema_version' : 0,
+                    'webdriver_delay': None  # Extra delay in seconds before extracting text
                }
            }
        }
--- a/changedetectionio/model/Watch.py
+++ b/changedetectionio/model/Watch.py
@@ -39,10 +39,12 @@ class model(dict):
            'trigger_text': [],  # List of text or regex to wait for until a change is detected
            'fetch_backend': None,
            'extract_title_as_title': False,
+            'proxy': None, # Preferred proxy connection
            # Re #110, so then if this is set to None, we know to use the default value instead
            # Requires setting to None on submit if it's the same as the default
            # Should be all None by default, so we use the system default in this case.
-            'time_between_check': {'weeks': None, 'days': None, 'hours': None, 'minutes': None, 'seconds': None}
+            'time_between_check': {'weeks': None, 'days': None, 'hours': None, 'minutes': None, 'seconds': None},
+            'webdriver_delay': None
        }

    def __init__(self, *arg, **kw):
--- a/changedetectionio/static/js/global-settings.js
+++ b/changedetectionio/static/js/global-settings.js
@@ -0,0 +1,36 @@
+$(document).ready(function () {
+    function toggle() {
+        if ($('input[name="application-fetch_backend"]:checked').val() != 'html_requests') {
+            $('#requests-override-options').hide();
+            $('#webdriver-override-options').show();
+        } else {
+            $('#requests-override-options').show();
+            $('#webdriver-override-options').hide();
+        }
+    }
+
+    $('input[name="application-fetch_backend"]').click(function (e) {
+        toggle();
+    });
+    toggle();
+
+    $("#api-key").hover(
+        function () {
+            $("#api-key-copy").html('copy').fadeIn();
+        },
+        function () {
+            $("#api-key-copy").hide();
+        }
+    ).click(function (e) {
+        $("#api-key-copy").html('copied');
+        var range = document.createRange();
+        var n = $("#api-key")[0];
+        range.selectNode(n);
+        window.getSelection().removeAllRanges();
+        window.getSelection().addRange(range);
+        document.execCommand("copy");
+        window.getSelection().removeAllRanges();
+
+    });
+});
+
--- a/changedetectionio/static/js/settings.js
+++ b/changedetectionio/static/js/settings.js
@@ -1,13 +0,0 @@
-window.addEventListener("load", (event) => {
-  // just an example for now
-  function toggleVisible(elem) {
-    // theres better ways todo this
-    var x = document.getElementById(elem);
-    if (x.style.display === "block") {
-      x.style.display = "none";
-    } else {
-      x.style.display = "block";
-    }
-  }
-});
-
--- a/changedetectionio/static/js/watch-settings.js
+++ b/changedetectionio/static/js/watch-settings.js
@@ -0,0 +1,16 @@
+$(document).ready(function() {
+    function toggle() {
+        if ($('input[name="fetch_backend"]:checked').val() != 'html_requests') {
+            $('#requests-override-options').hide();
+            $('#webdriver-override-options').show();
+        } else {
+            $('#requests-override-options').show();
+            $('#webdriver-override-options').hide();
+        }
+    }
+    $('input[name="fetch_backend"]').click(function (e) {
+        toggle();
+    });
+    toggle();
+
+});
--- a/changedetectionio/static/styles/styles.css
+++ b/changedetectionio/static/styles/styles.css
@@ -284,6 +284,11 @@ footer {
 .pure-form {
  /* The input fields with errors */
  /* The list of errors */ }
+  .pure-form fieldset {
+    padding-top: 0px; }
+    .pure-form fieldset ul {
+      padding-bottom: 0px;
+      margin-bottom: 0px; }
  .pure-form .pure-control-group, .pure-form .pure-group, .pure-form .pure-controls {
    padding-bottom: 1em; }
    .pure-form .pure-control-group div, .pure-form .pure-group div, .pure-form .pure-controls div {
@@ -309,10 +314,10 @@ footer {
    font-weight: bold; }
  .pure-form textarea {
    width: 100%; }
-  .pure-form ul.fetch-backend {
+  .pure-form .inline-radio ul {
    margin: 0px;
    list-style: none; }
-    .pure-form ul.fetch-backend li > * {
+    .pure-form .inline-radio ul li > * {
      display: inline-block; }

@media only screen and (max-width: 760px), (min-device-width: 768px) and (max-device-width: 1024px) {
@@ -447,4 +452,13 @@ ul {
 .time-check-widget tr {
  display: inline; }
  .time-check-widget tr input[type="number"] {
-    width: 4em; }
+    width: 5em; }
+
+#webdriver-override-options input[type="number"] {
+  width: 5em; }
+
+#api-key:hover {
+  cursor: pointer; }
+
+#api-key-copy {
+  color: #0078e7; }
--- a/changedetectionio/static/styles/styles.scss
+++ b/changedetectionio/static/styles/styles.scss
@@ -375,6 +375,13 @@ footer {


 .pure-form {
+    fieldset {
+        padding-top: 0px;
+        ul {
+            padding-bottom: 0px;
+            margin-bottom: 0px;
+        }
+    }
    .pure-control-group, .pure-group, .pure-controls {
        padding-bottom: 1em;
        div {
@@ -418,14 +425,16 @@ footer {
  textarea {
    width: 100%;
  }
-  ul.fetch-backend {
-    margin: 0px;
-    list-style: none;
-    li {
-        > * {
-            display: inline-block;
+  .inline-radio {
+      ul {
+        margin: 0px;
+        list-style: none;
+        li {
+            > * {
+                display: inline-block;
+            }
        }
-    }
+      }
  }
 }

@@ -635,7 +644,23 @@ ul {
    tr {
        display: inline;
        input[type="number"] {
-            width: 4em;
+            width: 5em;
        }
    }
-}
+}
+
+#webdriver-override-options {
+        input[type="number"] {
+            width: 5em;
+        }
+}
+
+#api-key {
+  &:hover {
+    cursor: pointer;
+  }
+}
+
+#api-key-copy {
+  color: #0078e7;
+}
--- a/changedetectionio/store.py
+++ b/changedetectionio/store.py
@@ -12,9 +12,9 @@ from os import mkdir, path, unlink
 from threading import Lock
 import re
 import requests
+import secrets

-from changedetectionio.model import Watch, App
-
+from . model import App, Watch

 # Is there an existing library to ensure some data store (JSON etc) is in sync with CRUD methods?
 # Open a github issue if you know something :)
@@ -33,6 +33,7 @@ class ChangeDetectionStore:
        self.needs_write = False
        self.datastore_path = datastore_path
        self.json_store_path = "{}/url-watches.json".format(self.datastore_path)
+        self.proxy_list = None
        self.stop_thread = False

        self.__data = App.model()
@@ -107,10 +108,21 @@ class ChangeDetectionStore:

        # Generate the URL access token for RSS feeds
        if not 'rss_access_token' in self.__data['settings']['application']:
-            import secrets
            secret = secrets.token_hex(16)
            self.__data['settings']['application']['rss_access_token'] = secret

+        # Generate the API access token
+        if not 'api_access_token' in self.__data['settings']['application']:
+            secret = secrets.token_hex(16)
+            self.__data['settings']['application']['api_access_token'] = secret
+
+        # Proxy list support - available as a selection in settings when text file is imported
+        # CSV list
+        # "name, address", or just "name"
+        proxy_list_file = "{}/proxies.txt".format(self.datastore_path)
+        if path.isfile(proxy_list_file):
+            self.import_proxy_list(proxy_list_file)
+
        # Bump the update version by running updates
        self.run_updates()

@@ -145,6 +157,10 @@ class ChangeDetectionStore:

    def update_watch(self, uuid, update_obj):

+        # It's possible that the watch could be deleted before update
+        if not self.__data['watching'].get(uuid):
+            return
+
        with self.lock:

            # In python 3.9 we have the |= dict operator, but that still will lose data on nested structures...
@@ -199,7 +215,8 @@ class ChangeDetectionStore:
    def get_all_tags(self):
        tags = []
        for uuid, watch in self.data['watching'].items():
-
+            if watch['tag'] is None:
+                continue
            # Support for comma separated list of tags.
            for tag in watch['tag'].split(','):
                tag = tag.strip()
@@ -268,6 +285,10 @@ class ChangeDetectionStore:
    def add_watch(self, url, tag="", extras=None, write_to_disk_now=True):
        if extras is None:
            extras = {}
+        # should always be str
+        if tag is None or not tag:
+            tag=''
+
        # Incase these are copied across, assume it's a reference and deepcopy()
        apply_extras = deepcopy(extras)

@@ -423,6 +444,21 @@ class ChangeDetectionStore:
                    print ("Removing",item)
                    unlink(item)

+    def import_proxy_list(self, filename):
+        import csv
+        with open(filename, newline='') as f:
+            reader = csv.reader(f, skipinitialspace=True)
+            # @todo This loop can could be improved
+            l = []
+            for row in reader:
+                if len(row):
+                    if len(row)>=2:
+                        l.append(tuple(row[:2]))
+                    else:
+                        l.append(tuple([row[0], row[0]]))
+            self.proxy_list = l if len(l) else None
+
+
    # Run all updates
    # IMPORTANT - Each update could be run even when they have a new install and the schema is correct
    #             So therefor - each `update_n` should be very careful about checking if it needs to actually run
--- a/changedetectionio/templates/_common_fields.jinja
+++ b/changedetectionio/templates/_common_fields.jinja
@@ -2,7 +2,6 @@
 {% from '_helpers.jinja' import render_field %}

 {% macro render_common_settings_form(form, current_base_url, emailprefix) %}
-
                        <div class="pure-control-group">
                            {{ render_field(form.notification_urls, rows=5, placeholder="Examples:
    Gitter - gitter://token/room
--- a/changedetectionio/templates/edit.html
+++ b/changedetectionio/templates/edit.html
@@ -9,6 +9,7 @@
    const email_notification_prefix=JSON.parse('{{ emailprefix|tojson }}');
 {% endif %}
 </script>
+<script type="text/javascript" src="{{url_for('static_content', group='js', filename='watch-settings.js')}}" defer></script>
 <script type="text/javascript" src="{{url_for('static_content', group='js', filename='notifications.js')}}" defer></script>

 <div class="edit-form monospaced-textarea">
@@ -57,20 +58,40 @@
            </div>

            <div class="tab-pane-inner" id="request">
-                    <div class="pure-control-group">
+                    <div class="pure-control-group inline-radio">
                        {{ render_field(form.fetch_backend, class="fetch-backend") }}
                        <span class="pure-form-message-inline">
                            <p>Use the <strong>Basic</strong> method (default) where your watched site doesn't need Javascript to render.</p>
                            <p>The <strong>Chrome/Javascript</strong> method requires a network connection to a running WebDriver+Chrome server, set by the ENV var 'WEBDRIVER_URL'. </p>
                        </span>
                    </div>
-
-                <hr/>
-                <fieldset class="pure-group">
-
-                    <span class="pure-form-message-inline">
+                {% if form.proxy %}
+                    <div class="pure-control-group inline-radio">
+                        {{ render_field(form.proxy, class="fetch-backend-proxy") }}
+                        <span class="pure-form-message-inline">
+                        Choose a proxy for this watch
+                        </span>
+                    </div>
+                {% endif %}
+                <fieldset id="webdriver-override-options">
+                    <div class="pure-form-message-inline">
+                        <strong>If you're having trouble waiting for the page to be fully rendered (text missing etc), try increasing the 'wait' time here.</strong>
+                        <br/>
+                        This will wait <i>n</i> seconds before extracting the text.
+                    </div>
+                    <div class="pure-control-group">
+                        {{ render_field(form.webdriver_delay) }}
+                    </div>
+                    {% if using_global_webdriver_wait %}
+                    <div class="pure-form-message-inline">
+                        <strong>Using the current global default settings</strong>
+                    </div>
+                    {% endif %}
+                </fieldset>
+                <fieldset class="pure-group" id="requests-override-options">
+                    <div class="pure-form-message-inline">
                        <strong>Request override is currently only used by the <i>Basic fast Plaintext/HTTP Client</i> method.</strong>
-                    </span>
+                    </div>
                    <div class="pure-control-group">
                        {{ render_field(form.method) }}
                    </div>
@@ -104,8 +125,7 @@ User-Agent: wonderbra 1.0") }}
            </div>

            <div class="tab-pane-inner" id="filters-and-triggers">
-                <fieldset>
-                        <div class="pure-control-group">
+                    <div class="pure-control-group">
                            <strong>Pro-tips:</strong><br/>
                            <ul>
                                <li>
@@ -116,7 +136,6 @@ User-Agent: wonderbra 1.0") }}
                                </li>
                            </ul>
                    </div>
-
                    <div class="pure-control-group">
                        {{ render_field(form.css_filter, placeholder=".class-name or #some-id, or other CSS selector rule.",
                        class="m-d") }}
@@ -125,14 +144,14 @@ User-Agent: wonderbra 1.0") }}
                        <li>CSS - Limit text to this CSS rule, only text matching this CSS rule is included.</li>
                        <li>JSON - Limit text to this JSON rule, using <a href="https://pypi.org/project/jsonpath-ng/">JSONPath</a>, prefix with <code>"json:"</code>, use <code>json:$</code> to force re-formatting if required,  <a
                                href="https://jsonpath.com/" target="new">test your JSONPath here</a></li>
-                        <li>XPath - Limit text to this XPath rule, simply start with a forward-slash, example  <code>//*[contains(@class, 'sametext')]</code>, <a
+                        <li>XPath - Limit text to this XPath rule, simply start with a forward-slash, example  <code>//*[contains(@class, 'sametext')]</code> or <code>xpath://*[contains(@class, 'sametext')]</code>, <a
                                href="http://xpather.com/" target="new">test your XPath here</a></li>
                    </ul>
                    Please be sure that you thoroughly understand how to write CSS or JSONPath, XPath selector rules before filing an issue on GitHub! <a
                                href="https://github.com/dgtlmoon/changedetection.io/wiki/CSS-Selector-help">here for more CSS selector help</a>.<br/>
                </span>
                    </div>
-                    <fieldset class="pure-group">
+                    <div class="pure-control-group">
                      {{ render_field(form.subtractive_selectors, rows=5, placeholder="header
 footer
 nav
@@ -143,8 +162,7 @@ nav
                          <li> Add multiple elements or CSS selectors per line to ignore multiple parts of the HTML. </li>
                        </ul>
                      </span>
-                    </fieldset>
-                </fieldset>
+                    </div>
                <fieldset class="pure-group">
                    {{ render_field(form.ignore_text, rows=5, placeholder="Some text to ignore in a line
 /some.regex\d{2}/ for case-INsensitive regex
@@ -181,9 +199,9 @@ nav

                      {{ render_button(form.save_button) }} {{ render_button(form.save_and_preview_button) }}

-                    <a href="{{url_for('api_delete', uuid=uuid)}}"
+                    <a href="{{url_for('form_delete', uuid=uuid)}}"
                       class="pure-button button-small button-error ">Delete</a>
-                    <a href="{{url_for('api_clone', uuid=uuid)}}"
+                    <a href="{{url_for('form_clone', uuid=uuid)}}"
                       class="pure-button button-small ">Create Copy</a>
                </div>
            </div>
--- a/changedetectionio/templates/import.html
+++ b/changedetectionio/templates/import.html
@@ -1,30 +1,86 @@
 {% extends 'base.html' %}
-
 {% block content %}
-<div class="edit-form">
-     <div class="inner">
+<script type="text/javascript" src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
+<div class="edit-form monospaced-textarea">
+
+    <div class="tabs collapsable">
+        <ul>
+            <li class="tab" id="default-tab"><a href="#url-list">URL List</a></li>
+            <li class="tab"><a href="#distill-io">Distill.io</a></li>
+        </ul>
+    </div>
+
+    <div class="box-wrap inner">
        <form class="pure-form pure-form-aligned" action="{{url_for('import_page')}}" method="POST">
            <input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
-            <fieldset class="pure-group">
-              <legend>
-                Enter one URL per line, and optionally add tags for each URL after a space, delineated by comma (,):
-                <br>
-                <code>https://example.com tag1, tag2, last tag</code>
-                <br>
-                URLs which do not pass validation will stay in the textarea.
-              </legend>
-              
+            <div class="tab-pane-inner" id="url-list">
+                <fieldset class="pure-group">
+                    <legend>
+                        Enter one URL per line, and optionally add tags for each URL after a space, delineated by comma
+                        (,):
+                        <br>
+                        <code>https://example.com tag1, tag2, last tag</code>
+                        <br>
+                        URLs which do not pass validation will stay in the textarea.
+                    </legend>

-                <textarea name="urls" class="pure-input-1-2" placeholder="https://"
-                          style="width: 100%;
+
+                    <textarea name="urls" class="pure-input-1-2" placeholder="https://"
+                              style="width: 100%;
                                font-family:monospace;
                                white-space: pre;
                                overflow-wrap: normal;
-                                overflow-x: scroll;" rows="25">{{ remaining }}</textarea>
-            </fieldset>
+                                overflow-x: scroll;" rows="25">{{ import_url_list_remaining }}</textarea>
+                </fieldset>
+
+
+            </div>
+
+            <div class="tab-pane-inner" id="distill-io">
+
+
+                <fieldset class="pure-group">
+                    <legend>
+                        Copy and Paste your Distill.io watch 'export' file, this should be a JSON file.</br>
+                        This is <i>experimental</i>, supported fields are <code>name</code>, <code>uri</code>, <code>tags</code>, <code>config:selections</code>, the rest (including <code>schedule</code>) are ignored.
+                        <br/>
+                        <p>
+                        How to export? <a href="https://distill.io/docs/web-monitor/how-export-and-import-monitors/">https://distill.io/docs/web-monitor/how-export-and-import-monitors/</a><br/>
+                        Be sure to set your default fetcher to Chrome if required.</br>
+                        </p>
+                    </legend>
+
+
+                    <textarea name="distill-io" class="pure-input-1-2" style="width: 100%;
+                                font-family:monospace;
+                                white-space: pre;
+                                overflow-wrap: normal;
+                                overflow-x: scroll;" placeholder="Example Distill.io JSON export file
+
+{
+    &quot;client&quot;: {
+        &quot;local&quot;: 1
+    },
+    &quot;data&quot;: [
+        {
+            &quot;name&quot;: &quot;Unraid | News&quot;,
+            &quot;uri&quot;: &quot;https://unraid.net/blog&quot;,
+            &quot;config&quot;: &quot;{\&quot;selections\&quot;:[{\&quot;frames\&quot;:[{\&quot;index\&quot;:0,\&quot;excludes\&quot;:[],\&quot;includes\&quot;:[{\&quot;type\&quot;:\&quot;xpath\&quot;,\&quot;expr\&quot;:\&quot;(//div[@id='App']/div[contains(@class,'flex')]/main[contains(@class,'relative')]/section[contains(@class,'relative')]/div[@class='container']/div[contains(@class,'flex')]/div[contains(@class,'w-full')])[1]\&quot;}]}],\&quot;dynamic\&quot;:true,\&quot;delay\&quot;:2}],\&quot;ignoreEmptyText\&quot;:true,\&quot;includeStyle\&quot;:false,\&quot;dataAttr\&quot;:\&quot;text\&quot;}&quot;,
+            &quot;tags&quot;: [],
+            &quot;content_type&quot;: 2,
+            &quot;state&quot;: 40,
+            &quot;schedule&quot;: &quot;{\&quot;type\&quot;:\&quot;INTERVAL\&quot;,\&quot;params\&quot;:{\&quot;interval\&quot;:4447}}&quot;,
+            &quot;ts&quot;: &quot;2022-03-27T15:51:15.667Z&quot;
+        }
+    ]
+}
+" rows="25">{{ original_distill_json }}</textarea>
+                </fieldset>
+            </div>
            <button type="submit" class="pure-button pure-input-1-2 pure-button-primary">Import</button>
        </form>
-     </div>
+
+    </div>
 </div>

 {% endblock %}
--- a/changedetectionio/templates/settings.html
+++ b/changedetectionio/templates/settings.html
@@ -9,10 +9,10 @@
    const email_notification_prefix=JSON.parse('{{emailprefix|tojson}}');
 {% endif %}
 </script>
-<script type="text/javascript" src="{{url_for('static_content', group='js', filename='settings.js')}}" defer></script>
 <script type="text/javascript" src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
 <script type="text/javascript" src="{{url_for('static_content', group='js', filename='notifications.js')}}" defer></script>

+<script type="text/javascript" src="{{url_for('static_content', group='js', filename='global-settings.js')}}" defer></script>
 <div class="edit-form">
    <div class="tabs collapsable">
        <ul>
@@ -20,6 +20,7 @@
            <li class="tab"><a href="#notifications">Notifications</a></li>
            <li class="tab"><a href="#fetching">Fetching</a></li>
            <li class="tab"><a href="#filters">Global Filters</a></li>
+            <li class="tab"><a href="#api">API</a></li>
        </ul>
    </div>
    <div class="box-wrap inner">
@@ -43,6 +44,7 @@
                            <span class="pure-form-message-inline">Password is locked.</span>
                        {% endif %}
                    </div>
+
                    <div class="pure-control-group">
                        {{ render_field(form.application.form.base_url, placeholder="http://yoursite.com:5000/",
                        class="m-d") }}
@@ -62,6 +64,18 @@
                        <span class="pure-form-message-inline">When using a Chrome browser, a screenshot from the last check will be available on the Diff page</span>
                    </div>

+                    <div class="pure-control-group">
+                        {{ render_checkbox_field(form.application.form.empty_pages_are_a_change) }}
+                        <span class="pure-form-message-inline">When a page contains HTML, but no renderable text appears (empty page), is this considered a change?</span>
+                    </div>
+                {% if form.requests.proxy %}
+                    <div class="pure-control-group inline-radio">
+                        {{ render_field(form.requests.form.proxy, class="fetch-backend-proxy") }}
+                        <span class="pure-form-message-inline">
+                        Choose a default proxy for all watches
+                        </span>
+                    </div>
+                {% endif %}
                </fieldset>
            </div>

@@ -74,16 +88,25 @@
            </div>

            <div class="tab-pane-inner" id="fetching">
-                <div class="pure-control-group">
+                <div class="pure-control-group inline-radio">
                    {{ render_field(form.application.form.fetch_backend, class="fetch-backend") }}
                    <span class="pure-form-message-inline">
                        <p>Use the <strong>Basic</strong> method (default) where your watched sites don't need Javascript to render.</p>
                        <p>The <strong>Chrome/Javascript</strong> method requires a network connection to a running WebDriver+Chrome server, set by the ENV var 'WEBDRIVER_URL'. </p>
                    </span>
                </div>
+                <fieldset class="pure-group" id="webdriver-override-options">
+                    <div class="pure-form-message-inline">
+                        <strong>If you're having trouble waiting for the page to be fully rendered (text missing etc), try increasing the 'wait' time here.</strong>
+                        <br/>
+                        This will wait <i>n</i> seconds before extracting the text.
+                    </div>
+                    <div class="pure-control-group">
+                        {{ render_field(form.application.form.webdriver_delay) }}
+                    </div>
+                </fieldset>
            </div>

-
            <div class="tab-pane-inner" id="filters">

                    <fieldset class="pure-group">
@@ -128,12 +151,26 @@ nav
                    </fieldset>
           </div>

+            <div class="tab-pane-inner" id="api">
+
+                <p>Drive your changedetection.io via API, More about <a href="https://github.com/dgtlmoon/changedetection.io/wiki/API-Reference">API access here</a></p>
+
+                <div class="pure-control-group">
+                    {{ render_checkbox_field(form.application.form.api_access_token_enabled) }}
+                    <div class="pure-form-message-inline">Restrict API access limit by using <code>x-api-key</code> header</div><br/>
+                    <div class="pure-form-message-inline"><br/>API Key <span id="api-key">{{api_key}}</span>
+                        <span style="display:none;" id="api-key-copy" >copy</span>
+                    </div>
+                </div>
+            </div>
+
            <div id="actions">
                <div class="pure-control-group">
                    {{ render_button(form.save_button) }}
                    <a href="{{url_for('index')}}" class="pure-button button-small button-cancel">Back</a>
                    <a href="{{url_for('scrub_page')}}" class="pure-button button-small button-cancel">Delete History Snapshot Data</a>
                </div>
+
            </div>
        </form>
    </div>
--- a/changedetectionio/templates/watch-overview.html
+++ b/changedetectionio/templates/watch-overview.html
@@ -5,7 +5,7 @@
 <script type="text/javascript" src="{{url_for('static_content', group='js', filename='watch-overview.js')}}" defer></script>
 <div class="box">

-    <form class="pure-form" action="{{ url_for('api_watch_add') }}" method="POST" id="new-watch-form">
+    <form class="pure-form" action="{{ url_for('form_watch_add') }}" method="POST" id="new-watch-form">
        <input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
        <fieldset>
            <legend>Add a new change detection watch</legend>
@@ -13,7 +13,7 @@
                {{ render_simple_field(form.tag, value=active_tag if active_tag else '', placeholder="watch group") }}
            <button type="submit" class="pure-button pure-button-primary">Watch</button>
        </fieldset>
-        <span style="color:#eee; font-size: 80%;"><img style="height: 1em;display:inline-block;" src="{{url_for('static_content', group='images', filename='spread.svg')}}" /> Tip: You can also add 'shared' watches. <a href="#">More info</a></a></span>
+        <span style="color:#eee; font-size: 80%;"><img style="height: 1em;display:inline-block;" src="{{url_for('static_content', group='images', filename='spread.svg')}}" /> Tip: You can also add 'shared' watches. <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Sharing-a-Watch">More info</a></a></span>
    </form>
    <div>
        <a href="{{url_for('index')}}" class="pure-button button-tag {{'active' if not active_tag }}">All</a>
@@ -52,7 +52,7 @@

                <td class="title-col inline">{{watch.title if watch.title is not none and watch.title|length > 0 else watch.url}}
                    <a class="external" target="_blank" rel="noopener" href="{{ watch.url.replace('source:','') }}"></a>
-                    <a href="{{url_for('api_share_put_watch', uuid=watch.uuid)}}"><img style="height: 1em;display:inline-block;" src="{{url_for('static_content', group='images', filename='spread.svg')}}" /></a>
+                    <a href="{{url_for('form_share_put_watch', uuid=watch.uuid)}}"><img style="height: 1em;display:inline-block;" src="{{url_for('static_content', group='images', filename='spread.svg')}}" /></a>

                    {%if watch.fetch_backend == "html_webdriver" %}<img style="height: 1em; display:inline-block;" src="{{url_for('static_content', group='images', filename='Google-Chrome-icon.png')}}" />{% endif %}

@@ -74,7 +74,7 @@
                    {% endif %}
                </td>
                <td>
-                    <a {% if watch.uuid in queued_uuids %}disabled="true"{% endif %} href="{{ url_for('api_watch_checknow', uuid=watch.uuid, tag=request.args.get('tag')) }}"
+                    <a {% if watch.uuid in queued_uuids %}disabled="true"{% endif %} href="{{ url_for('form_watch_checknow', uuid=watch.uuid, tag=request.args.get('tag')) }}"
                       class="recheck pure-button button-small pure-button-primary">{% if watch.uuid in queued_uuids %}Queued{% else %}Recheck{% endif %}</a>
                    <a href="{{ url_for('edit_page', uuid=watch.uuid)}}" class="pure-button button-small pure-button-primary">Edit</a>
                    {% if watch.history|length >= 2 %}
@@ -96,7 +96,7 @@
            </li>
            {% endif %}
            <li>
-               <a href="{{ url_for('api_watch_checknow', tag=active_tag) }}" class="pure-button button-tag ">Recheck
+               <a href="{{ url_for('form_watch_checknow', tag=active_tag) }}" class="pure-button button-tag ">Recheck
                all {% if active_tag%}in "{{active_tag}}"{%endif%}</a>
            </li>
            <li>
--- a/changedetectionio/tests/test_api.py
+++ b/changedetectionio/tests/test_api.py
@@ -2,73 +2,205 @@

 import time
 from flask import url_for
-from . util import live_server_setup
+from .util import live_server_setup

-def test_setup(live_server):
-    live_server_setup(live_server)
+import json
+import uuid


-def set_response_data(test_return_data):
+def set_original_response():
+    test_return_data = """<html>
+       <body>
+     Some initial text</br>
+     <p>Which is across multiple lines</p>
+     </br>
+     So let's see what happens.  </br>
+     <div id="sametext">Some text thats the same</div>
+     <div id="changetext">Some text that will change</div>
+     </body>
+     </html>
+    """
+
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write(test_return_data)
+    return None
+
+
+def set_modified_response():
+    test_return_data = """<html>
+       <body>
+     Some initial text</br>
+     <p>which has this one new line</p>
+     </br>
+     So let's see what happens.  </br>
+     <div id="sametext">Some text thats the same</div>
+     <div id="changetext">Some text that changes</div>
+     </body>
+     </html>
+    """
+
    with open("test-datastore/endpoint-content.txt", "w") as f:
        f.write(test_return_data)

+    return None

-def test_snapshot_api_detects_change(client, live_server):
-    test_return_data = "Some initial text"

-    test_return_data_modified = "Some NEW nice initial text"
+def is_valid_uuid(val):
+    try:
+        uuid.UUID(str(val))
+        return True
+    except ValueError:
+        return False

-    sleep_time_for_fetch_thread = 3

-    set_response_data(test_return_data)
+# kinda funky, but works for now
+def _extract_api_key_from_UI(client):
+    import re
+    res = client.get(
+        url_for("settings_page"),
+    )
+    # <span id="api-key">{{api_key}}</span>

-    # Give the endpoint time to spin up
-    time.sleep(1)
+    m = re.search('<span id="api-key">(.+?)</span>', str(res.data))
+    api_key = m.group(1)
+    return api_key.strip()

-    # Add our URL to the import page
-    test_url = url_for('test_endpoint', content_type="text/plain",
-                       _external=True)
+
+def test_api_simple(client, live_server):
+    live_server_setup(live_server)
+
+    api_key = _extract_api_key_from_UI(client)
+
+    # Create a watch
+    set_original_response()
+    watch_uuid = None
+
+    # Validate bad URL
+    test_url = url_for('test_endpoint', _external=True,
+                       headers={'x-api-key': api_key}, )
    res = client.post(
-        url_for("import_page"),
-        data={"urls": test_url},
+        url_for("createwatch"),
+        data=json.dumps({"url": "h://xxxxxxxxxom"}),
+        headers={'content-type': 'application/json', 'x-api-key': api_key},
        follow_redirects=True
    )
-    assert b"1 Imported" in res.data
-
-    # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
-
-    # Give the thread time to pick it up
-    time.sleep(sleep_time_for_fetch_thread)
-
-    res = client.get(
-        url_for("api_snapshot", uuid="first"),
-        follow_redirects=True
-    )
-
-    assert test_return_data.encode() == res.data
-
-    #  Make a change
-    set_response_data(test_return_data_modified)
-
-    # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
-    # Give the thread time to pick it up
-    time.sleep(sleep_time_for_fetch_thread)
-
-    res = client.get(
-        url_for("api_snapshot", uuid="first"),
-        follow_redirects=True
-    )
-
-    assert test_return_data_modified.encode() == res.data
-
-def test_snapshot_api_invalid_uuid(client, live_server):
-
-    res = client.get(
-        url_for("api_snapshot", uuid="invalid"),
-        follow_redirects=True
-    )
-
    assert res.status_code == 400

+    # Create new
+    res = client.post(
+        url_for("createwatch"),
+        data=json.dumps({"url": test_url, 'tag': "One, Two", "title": "My test URL"}),
+        headers={'content-type': 'application/json', 'x-api-key': api_key},
+        follow_redirects=True
+    )
+    s = json.loads(res.data)
+    assert is_valid_uuid(s['uuid'])
+    watch_uuid = s['uuid']
+    assert res.status_code == 201
+
+    time.sleep(3)
+
+    # Verify its in the list and that recheck worked
+    res = client.get(
+        url_for("createwatch"),
+        headers={'x-api-key': api_key}
+    )
+    assert watch_uuid in json.loads(res.data).keys()
+    before_recheck_info = json.loads(res.data)[watch_uuid]
+    assert before_recheck_info['last_checked'] != 0
+    assert before_recheck_info['title'] == 'My test URL'
+
+    set_modified_response()
+    # Trigger recheck of all ?recheck_all=1
+    client.get(
+        url_for("createwatch", recheck_all='1'),
+        headers={'x-api-key': api_key},
+    )
+    time.sleep(3)
+
+    # Did the recheck fire?
+    res = client.get(
+        url_for("createwatch"),
+        headers={'x-api-key': api_key},
+    )
+    after_recheck_info = json.loads(res.data)[watch_uuid]
+    assert after_recheck_info['last_checked'] != before_recheck_info['last_checked']
+    assert after_recheck_info['last_changed'] != 0
+
+    # Check history index list
+    res = client.get(
+        url_for("watchhistory", uuid=watch_uuid),
+        headers={'x-api-key': api_key},
+    )
+    history = json.loads(res.data)
+    assert len(history) == 2, "Should have two history entries (the original and the changed)"
+
+    # Fetch a snapshot by timestamp, check the right one was found
+    res = client.get(
+        url_for("watchsinglehistory", uuid=watch_uuid, timestamp=list(history.keys())[-1]),
+        headers={'x-api-key': api_key},
+    )
+    assert b'which has this one new line' in res.data
+
+    # Fetch a snapshot by 'latest'', check the right one was found
+    res = client.get(
+        url_for("watchsinglehistory", uuid=watch_uuid, timestamp='latest'),
+        headers={'x-api-key': api_key},
+    )
+    assert b'which has this one new line' in res.data
+
+    # Fetch the whole watch
+    res = client.get(
+        url_for("watch", uuid=watch_uuid),
+        headers={'x-api-key': api_key}
+    )
+    watch = json.loads(res.data)
+    # @todo how to handle None/default global values?
+    assert watch['history_n'] == 2, "Found replacement history section, which is in its own API"
+
+    # Finally delete the watch
+    res = client.delete(
+        url_for("watch", uuid=watch_uuid),
+        headers={'x-api-key': api_key},
+    )
+    assert res.status_code == 204
+
+    # Check via a relist
+    res = client.get(
+        url_for("createwatch"),
+        headers={'x-api-key': api_key}
+    )
+    watch_list = json.loads(res.data)
+    assert len(watch_list) == 0, "Watch list should be empty"
+
+
+def test_access_denied(client, live_server):
+    # `config_api_token_enabled` Should be On by default
+    res = client.get(
+        url_for("createwatch")
+    )
+    assert res.status_code == 403
+
+    res = client.get(
+        url_for("createwatch"),
+        headers={'x-api-key': "something horrible"}
+    )
+    assert res.status_code == 403
+
+    # Disable config_api_token_enabled and it should work
+    res = client.post(
+        url_for("settings_page"),
+        data={
+            "requests-time_between_check-minutes": 180,
+            "application-fetch_backend": "html_requests",
+            "application-api_access_token_enabled": ""
+        },
+        follow_redirects=True
+    )
+
+    assert b"Settings updated." in res.data
+
+    res = client.get(
+        url_for("createwatch")
+    )
+    assert res.status_code == 200
--- a/changedetectionio/tests/test_auth.py
+++ b/changedetectionio/tests/test_auth.py
@@ -29,7 +29,7 @@ def test_basic_auth(client, live_server):
    assert b"Updated watch." in res.data

    # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
    time.sleep(1)
    res = client.get(
        url_for("preview_page", uuid="first"),
--- a/changedetectionio/tests/test_backend.py
+++ b/changedetectionio/tests/test_backend.py
@@ -32,7 +32,7 @@ def test_check_basic_change_detection_functionality(client, live_server):

    # Do this a few times.. ensures we dont accidently set the status
    for n in range(3):
-        client.get(url_for("api_watch_checknow"), follow_redirects=True)
+        client.get(url_for("form_watch_checknow"), follow_redirects=True)

        # Give the thread time to pick it up
        time.sleep(sleep_time_for_fetch_thread)
@@ -65,7 +65,7 @@ def test_check_basic_change_detection_functionality(client, live_server):
    assert b'which has this one new line' in res.read()

    # Force recheck
-    res = client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    res = client.get(url_for("form_watch_checknow"), follow_redirects=True)
    assert b'1 watches are queued for rechecking.' in res.data

    time.sleep(sleep_time_for_fetch_thread)
@@ -93,7 +93,7 @@ def test_check_basic_change_detection_functionality(client, live_server):

    # Do this a few times.. ensures we dont accidently set the status
    for n in range(2):
-        client.get(url_for("api_watch_checknow"), follow_redirects=True)
+        client.get(url_for("form_watch_checknow"), follow_redirects=True)

        # Give the thread time to pick it up
        time.sleep(sleep_time_for_fetch_thread)
@@ -113,7 +113,7 @@ def test_check_basic_change_detection_functionality(client, live_server):
        follow_redirects=True
    )

-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
    time.sleep(sleep_time_for_fetch_thread)

    res = client.get(url_for("index"))
@@ -123,6 +123,6 @@ def test_check_basic_change_detection_functionality(client, live_server):

    #
    # Cleanup everything
-    res = client.get(url_for("api_delete", uuid="all"), follow_redirects=True)
+    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
    assert b'Deleted' in res.data

--- a/changedetectionio/tests/test_clone.py
+++ b/changedetectionio/tests/test_clone.py
@@ -23,7 +23,7 @@ def test_trigger_functionality(client, live_server):


    res = client.get(
-        url_for("api_clone", uuid="first"),
+        url_for("form_clone", uuid="first"),
        follow_redirects=True
    )

--- a/changedetectionio/tests/test_css_selector.py
+++ b/changedetectionio/tests/test_css_selector.py
@@ -89,7 +89,7 @@ def test_check_markup_css_filter_restriction(client, live_server):
    assert b"1 Imported" in res.data

    # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)

    # Give the thread time to pick it up
    time.sleep(sleep_time_for_fetch_thread)
@@ -110,7 +110,7 @@ def test_check_markup_css_filter_restriction(client, live_server):
    assert bytes(css_filter.encode('utf-8')) in res.data

    # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)

    # Give the thread time to pick it up
    time.sleep(sleep_time_for_fetch_thread)
@@ -118,7 +118,7 @@ def test_check_markup_css_filter_restriction(client, live_server):
    set_modified_response()

    # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
    # Give the thread time to pick it up
    time.sleep(sleep_time_for_fetch_thread)

--- a/changedetectionio/tests/test_element_removal.py
+++ b/changedetectionio/tests/test_element_removal.py
@@ -145,7 +145,7 @@ def test_element_removal_full(client, live_server):
    assert bytes(subtractive_selectors_data.encode("utf-8")) in res.data

    # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)

    # Give the thread time to pick it up
    time.sleep(sleep_time_for_fetch_thread)
@@ -158,7 +158,7 @@ def test_element_removal_full(client, live_server):
    set_modified_response()

    # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)

    # Give the thread time to pick it up
    time.sleep(sleep_time_for_fetch_thread)
--- a/changedetectionio/tests/test_encoding.py
+++ b/changedetectionio/tests/test_encoding.py
@@ -39,7 +39,7 @@ def test_check_encoding_detection(client, live_server):
    )

    # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)

    # Give the thread time to pick it up
    time.sleep(2)
@@ -71,7 +71,7 @@ def test_check_encoding_detection_missing_content_type_header(client, live_serve
    )

    # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)

    # Give the thread time to pick it up
    time.sleep(2)
--- a/changedetectionio/tests/test_errorhandling.py
+++ b/changedetectionio/tests/test_errorhandling.py
@@ -29,7 +29,7 @@ def test_error_handler(client, live_server):
    assert b"1 Imported" in res.data

    # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)

    # Give the thread time to pick it up
    time.sleep(3)
@@ -54,7 +54,7 @@ def test_error_text_handler(client, live_server):
    assert b"1 Imported" in res.data

    # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)

    # Give the thread time to pick it up
    time.sleep(3)
--- a/changedetectionio/tests/test_ignore_text.py
+++ b/changedetectionio/tests/test_ignore_text.py
@@ -102,7 +102,7 @@ def test_check_ignore_text_functionality(client, live_server):
    assert b"1 Imported" in res.data

    # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)

    # Give the thread time to pick it up
    time.sleep(sleep_time_for_fetch_thread)
@@ -123,7 +123,7 @@ def test_check_ignore_text_functionality(client, live_server):
    assert bytes(ignore_text.encode('utf-8')) in res.data

    # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)

    # Give the thread time to pick it up
    time.sleep(sleep_time_for_fetch_thread)
@@ -137,7 +137,7 @@ def test_check_ignore_text_functionality(client, live_server):
    set_modified_ignore_response()

    # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
    # Give the thread time to pick it up
    time.sleep(sleep_time_for_fetch_thread)

@@ -152,7 +152,7 @@ def test_check_ignore_text_functionality(client, live_server):

    # Just to be sure.. set a regular modified change..
    set_modified_original_ignore_response()
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
    time.sleep(sleep_time_for_fetch_thread)

    res = client.get(url_for("index"))
@@ -165,7 +165,7 @@ def test_check_ignore_text_functionality(client, live_server):
    # We should be able to see what we ignored
    assert b'<div class="ignored">new ignore stuff' in res.data

-    res = client.get(url_for("api_delete", uuid="all"), follow_redirects=True)
+    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
    assert b'Deleted' in res.data

 def test_check_global_ignore_text_functionality(client, live_server):
@@ -200,7 +200,7 @@ def test_check_global_ignore_text_functionality(client, live_server):
    assert b"1 Imported" in res.data

    # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)

    # Give the thread time to pick it up
    time.sleep(sleep_time_for_fetch_thread)
@@ -222,7 +222,7 @@ def test_check_global_ignore_text_functionality(client, live_server):
    assert bytes(ignore_text.encode('utf-8')) in res.data

    # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)

    # Give the thread time to pick it up
    time.sleep(sleep_time_for_fetch_thread)
@@ -240,7 +240,7 @@ def test_check_global_ignore_text_functionality(client, live_server):
    set_modified_ignore_response()

    # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
    # Give the thread time to pick it up
    time.sleep(sleep_time_for_fetch_thread)

@@ -251,10 +251,10 @@ def test_check_global_ignore_text_functionality(client, live_server):

    # Just to be sure.. set a regular modified change that will trigger it
    set_modified_original_ignore_response()
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
    time.sleep(sleep_time_for_fetch_thread)
    res = client.get(url_for("index"))
    assert b'unviewed' in res.data

-    res = client.get(url_for("api_delete", uuid="all"), follow_redirects=True)
+    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
    assert b'Deleted' in res.data
--- a/changedetectionio/tests/test_ignorehyperlinks.py
+++ b/changedetectionio/tests/test_ignorehyperlinks.py
@@ -72,14 +72,14 @@ def test_render_anchor_tag_content_true(client, live_server):

    time.sleep(sleep_time_for_fetch_thread)
    # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)

    # set a new html text with a modified link
    set_modified_ignore_response()
    time.sleep(sleep_time_for_fetch_thread)

    # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)

    # Give the thread time to pick it up
    time.sleep(sleep_time_for_fetch_thread)
@@ -101,7 +101,7 @@ def test_render_anchor_tag_content_true(client, live_server):
    assert b"Settings updated." in res.data

    # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)

    # Give the thread time to pick it up
    time.sleep(sleep_time_for_fetch_thread)
@@ -119,7 +119,7 @@ def test_render_anchor_tag_content_true(client, live_server):
    assert b"/test-endpoint" in res.data

    # Cleanup everything
-    res = client.get(url_for("api_delete", uuid="all"),
+    res = client.get(url_for("form_delete", uuid="all"),
                     follow_redirects=True)
    assert b'Deleted' in res.data

--- a/changedetectionio/tests/test_ignorestatuscode.py
+++ b/changedetectionio/tests/test_ignorestatuscode.py
@@ -70,12 +70,12 @@ def test_normal_page_check_works_with_ignore_status_code(client, live_server):

    time.sleep(sleep_time_for_fetch_thread)
    # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)

    set_some_changed_response()
    time.sleep(sleep_time_for_fetch_thread)
    # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)

    # Give the thread time to pick it up
    time.sleep(sleep_time_for_fetch_thread)
@@ -105,7 +105,7 @@ def test_403_page_check_works_with_ignore_status_code(client, live_server):
    assert b"1 Imported" in res.data

    # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)

    # Give the thread time to pick it up
    time.sleep(sleep_time_for_fetch_thread)
@@ -120,7 +120,7 @@ def test_403_page_check_works_with_ignore_status_code(client, live_server):
    assert b"Updated watch." in res.data

    # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)

    # Give the thread time to pick it up
    time.sleep(sleep_time_for_fetch_thread)
@@ -128,7 +128,7 @@ def test_403_page_check_works_with_ignore_status_code(client, live_server):
    set_some_changed_response()

    # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
    # Give the thread time to pick it up
    time.sleep(sleep_time_for_fetch_thread)

@@ -157,7 +157,7 @@ def test_403_page_check_fails_without_ignore_status_code(client, live_server):
    assert b"1 Imported" in res.data

    # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)

    # Give the thread time to pick it up
    time.sleep(sleep_time_for_fetch_thread)
@@ -172,7 +172,7 @@ def test_403_page_check_fails_without_ignore_status_code(client, live_server):
    assert b"Updated watch." in res.data

    # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)

    # Give the thread time to pick it up
    time.sleep(sleep_time_for_fetch_thread)
@@ -180,7 +180,7 @@ def test_403_page_check_fails_without_ignore_status_code(client, live_server):
    set_some_changed_response()

    # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
    # Give the thread time to pick it up
    time.sleep(sleep_time_for_fetch_thread)

--- a/changedetectionio/tests/test_ignorewhitespace.py
+++ b/changedetectionio/tests/test_ignorewhitespace.py
@@ -80,12 +80,12 @@ def test_check_ignore_whitespace(client, live_server):

    time.sleep(sleep_time_for_fetch_thread)
    # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)

    set_original_ignore_response_but_with_whitespace()
    time.sleep(sleep_time_for_fetch_thread)
    # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)

    # Give the thread time to pick it up
    time.sleep(sleep_time_for_fetch_thread)
--- a/changedetectionio/tests/test_import.py
+++ b/changedetectionio/tests/test_import.py
@@ -5,18 +5,17 @@ import time
 from flask import url_for

 from .util import live_server_setup
-
-
-def test_import(client, live_server):
-
+def test_setup(client, live_server):
    live_server_setup(live_server)

+def test_import(client, live_server):
    # Give the endpoint time to spin up
    time.sleep(1)

    res = client.post(
        url_for("import_page"),
        data={
+            "distill-io": "",
            "urls": """https://example.com
 https://example.com tag1
 https://example.com tag1, other tag"""
@@ -26,3 +25,96 @@ https://example.com tag1, other tag"""
    assert b"3 Imported" in res.data
    assert b"tag1" in res.data
    assert b"other tag" in res.data
+    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+
+    # Clear flask alerts
+    res = client.get( url_for("index"))
+    res = client.get( url_for("index"))
+
+def xtest_import_skip_url(client, live_server):
+
+
+    # Give the endpoint time to spin up
+    time.sleep(1)
+
+    res = client.post(
+        url_for("import_page"),
+        data={
+            "distill-io": "",
+            "urls": """https://example.com
+:ht000000broken
+"""
+        },
+        follow_redirects=True,
+    )
+    assert b"1 Imported" in res.data
+    assert b"ht000000broken" in res.data
+    assert b"1 Skipped" in res.data
+    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    # Clear flask alerts
+    res = client.get( url_for("index"))
+
+def test_import_distillio(client, live_server):
+
+    distill_data='''
+{
+    "client": {
+        "local": 1
+    },
+    "data": [
+        {
+            "name": "Unraid | News",
+            "uri": "https://unraid.net/blog",
+            "config": "{\\"selections\\":[{\\"frames\\":[{\\"index\\":0,\\"excludes\\":[],\\"includes\\":[{\\"type\\":\\"xpath\\",\\"expr\\":\\"(//div[@id='App']/div[contains(@class,'flex')]/main[contains(@class,'relative')]/section[contains(@class,'relative')]/div[@class='container']/div[contains(@class,'flex')]/div[contains(@class,'w-full')])[1]\\"}]}],\\"dynamic\\":true,\\"delay\\":2}],\\"ignoreEmptyText\\":true,\\"includeStyle\\":false,\\"dataAttr\\":\\"text\\"}",
+            "tags": ["nice stuff", "nerd-news"],
+            "content_type": 2,
+            "state": 40,
+            "schedule": "{\\"type\\":\\"INTERVAL\\",\\"params\\":{\\"interval\\":4447}}",
+            "ts": "2022-03-27T15:51:15.667Z"
+        }
+    ]
+}		   
+
+'''
+
+    # Give the endpoint time to spin up
+    time.sleep(1)
+    client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    res = client.post(
+        url_for("import_page"),
+        data={
+            "distill-io": distill_data,
+            "urls" : ''
+        },
+        follow_redirects=True,
+    )
+
+
+    assert b"Unable to read JSON file, was it broken?" not in res.data
+    assert b"1 Imported from Distill.io" in res.data
+
+    res = client.get( url_for("edit_page", uuid="first"))
+
+    assert b"https://unraid.net/blog" in res.data
+    assert b"Unraid | News" in res.data
+
+
+    # flask/wtforms should recode this, check we see it
+    # wtforms encodes it like id=&#39 ,but html.escape makes it like id=&#x27
+    # - so just check it manually :(
+    #import json
+    #import html
+    #d = json.loads(distill_data)
+    # embedded_d=json.loads(d['data'][0]['config'])
+    # x=html.escape(embedded_d['selections'][0]['frames'][0]['includes'][0]['expr']).encode('utf-8')
+    assert b"xpath:(//div[@id=&#39;App&#39;]/div[contains(@class,&#39;flex&#39;)]/main[contains(@class,&#39;relative&#39;)]/section[contains(@class,&#39;relative&#39;)]/div[@class=&#39;container&#39;]/div[contains(@class,&#39;flex&#39;)]/div[contains(@class,&#39;w-full&#39;)])[1]" in res.data
+
+    # did the tags work?
+    res = client.get( url_for("index"))
+
+    assert b"nice stuff" in res.data
+    assert b"nerd-news" in res.data
+
+    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    # Clear flask alerts
+    res = client.get(url_for("index"))
--- a/changedetectionio/tests/test_jsonpath_selector.py
+++ b/changedetectionio/tests/test_jsonpath_selector.py
@@ -171,7 +171,7 @@ def test_check_json_without_filter(client, live_server):
    )

    # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)

    # Give the thread time to pick it up
    time.sleep(3)
@@ -203,7 +203,7 @@ def test_check_json_filter(client, live_server):
    assert b"1 Imported" in res.data

    # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)

    # Give the thread time to pick it up
    time.sleep(3)
@@ -229,7 +229,7 @@ def test_check_json_filter(client, live_server):
    assert bytes(json_filter.encode('utf-8')) in res.data

    # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)

    # Give the thread time to pick it up
    time.sleep(3)
@@ -237,7 +237,7 @@ def test_check_json_filter(client, live_server):
    set_modified_response()

    # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
    # Give the thread time to pick it up
    time.sleep(4)

@@ -288,7 +288,7 @@ def test_check_json_filter_bool_val(client, live_server):
    time.sleep(3)

    # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)

    # Give the thread time to pick it up
    time.sleep(3)
@@ -296,7 +296,7 @@ def test_check_json_filter_bool_val(client, live_server):
    set_modified_response()

    # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
    # Give the thread time to pick it up
    time.sleep(3)

@@ -327,7 +327,7 @@ def test_check_json_ext_filter(client, live_server):
    assert b"1 Imported" in res.data

    # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)

    # Give the thread time to pick it up
    time.sleep(3)
@@ -353,7 +353,7 @@ def test_check_json_ext_filter(client, live_server):
    assert bytes(json_filter.encode('utf-8')) in res.data

    # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)

    # Give the thread time to pick it up
    time.sleep(3)
@@ -361,7 +361,7 @@ def test_check_json_ext_filter(client, live_server):
    set_modified_ext_response()

    # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
    # Give the thread time to pick it up
    time.sleep(4)

--- a/changedetectionio/tests/test_nonrenderable_pages.py
+++ b/changedetectionio/tests/test_nonrenderable_pages.py
@@ -0,0 +1,102 @@
+#!/usr/bin/python3
+
+import time
+from flask import url_for
+from urllib.request import urlopen
+from .util import set_original_response, set_modified_response, live_server_setup
+
+sleep_time_for_fetch_thread = 3
+
+
+def set_nonrenderable_response():
+    test_return_data = """<html>
+    <head><title>modified head title</title></head>
+    <!-- like when some angular app was broken and doesnt render or whatever -->
+    <body>
+     </body>
+     </html>
+    """
+
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write(test_return_data)
+
+    return None
+
+def test_check_basic_change_detection_functionality(client, live_server):
+    set_original_response()
+    live_server_setup(live_server)
+
+    # Add our URL to the import page
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": url_for('test_endpoint', _external=True)},
+        follow_redirects=True
+    )
+
+    assert b"1 Imported" in res.data
+
+    time.sleep(sleep_time_for_fetch_thread)
+
+    # Do this a few times.. ensures we dont accidently set the status
+    for n in range(3):
+        client.get(url_for("form_watch_checknow"), follow_redirects=True)
+
+        # Give the thread time to pick it up
+        time.sleep(sleep_time_for_fetch_thread)
+
+        # It should report nothing found (no new 'unviewed' class)
+        res = client.get(url_for("index"))
+        assert b'unviewed' not in res.data
+
+
+    #####################
+    client.post(
+        url_for("settings_page"),
+        data={"application-empty_pages_are_a_change": "",
+              "requests-time_between_check-minutes": 180,
+              'application-fetch_backend': "html_requests"},
+        follow_redirects=True
+    )
+
+    # this should not trigger a change, because no good text could be converted from the HTML
+    set_nonrenderable_response()
+
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
+
+    # It should report nothing found (no new 'unviewed' class)
+    res = client.get(url_for("index"))
+    assert b'unviewed' not in res.data
+
+
+    # ok now do the opposite
+
+    client.post(
+        url_for("settings_page"),
+        data={"application-empty_pages_are_a_change": "y",
+              "requests-time_between_check-minutes": 180,
+              'application-fetch_backend': "html_requests"},
+        follow_redirects=True
+    )
+    set_modified_response()
+
+
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
+
+    # It should report nothing found (no new 'unviewed' class)
+    res = client.get(url_for("index"))
+    assert b'unviewed' in res.data
+
+
+
+
+    #
+    # Cleanup everything
+    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    assert b'Deleted' in res.data
+
--- a/changedetectionio/tests/test_notification.py
+++ b/changedetectionio/tests/test_notification.py
@@ -36,7 +36,7 @@ def test_check_notification(client, live_server):
    # Add our URL to the import page
    test_url = url_for('test_endpoint', _external=True)
    res = client.post(
-        url_for("api_watch_add"),
+        url_for("form_watch_add"),
        data={"url": test_url, "tag": ''},
        follow_redirects=True
    )
@@ -98,7 +98,7 @@ def test_check_notification(client, live_server):
    notification_submission = None

    # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
    time.sleep(3)
    # Verify what was sent as a notification, this file should exist
    with open("test-datastore/notification.txt", "r") as f:
@@ -133,7 +133,7 @@ def test_check_notification(client, live_server):

    # This should insert the {current_snapshot}
    set_more_modified_response()
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
    time.sleep(3)
    # Verify what was sent as a notification, this file should exist
    with open("test-datastore/notification.txt", "r") as f:
@@ -146,17 +146,17 @@ def test_check_notification(client, live_server):
    os.unlink("test-datastore/notification.txt")

    # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
    time.sleep(1)
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
    time.sleep(1)
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
    time.sleep(1)
    assert os.path.exists("test-datastore/notification.txt") == False

    # cleanup for the next
    client.get(
-        url_for("api_delete", uuid="all"),
+        url_for("form_delete", uuid="all"),
        follow_redirects=True
    )

@@ -168,7 +168,7 @@ def test_notification_validation(client, live_server):
    # Add our URL to the import page
    test_url = url_for('test_endpoint', _external=True)
    res = client.post(
-        url_for("api_watch_add"),
+        url_for("form_watch_add"),
        data={"url": test_url, "tag": 'nice one'},
        follow_redirects=True
    )
@@ -208,6 +208,6 @@ def test_notification_validation(client, live_server):

    # cleanup for the next
    client.get(
-        url_for("api_delete", uuid="all"),
+        url_for("form_delete", uuid="all"),
        follow_redirects=True
    )
--- a/changedetectionio/tests/test_notification_errors.py
+++ b/changedetectionio/tests/test_notification_errors.py
@@ -16,7 +16,7 @@ def test_check_notification_error_handling(client, live_server):
    # use a different URL so that it doesnt interfere with the actual check until we are ready
    test_url = url_for('test_endpoint', _external=True)
    res = client.post(
-        url_for("api_watch_add"),
+        url_for("form_watch_add"),
        data={"url": "https://changedetection.io/CHANGELOG.txt", "tag": ''},
        follow_redirects=True
    )
--- a/changedetectionio/tests/test_share_watch.py
+++ b/changedetectionio/tests/test_share_watch.py
@@ -41,7 +41,7 @@ def test_share_watch(client, live_server):

    # click share the link
    res = client.get(
-        url_for("api_share_put_watch", uuid="first"),
+        url_for("form_share_put_watch", uuid="first"),
        follow_redirects=True
    )

@@ -54,7 +54,7 @@ def test_share_watch(client, live_server):

    # Now delete what we have, we will try to re-import it
    # Cleanup everything
-    res = client.get(url_for("api_delete", uuid="all"), follow_redirects=True)
+    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
    assert b'Deleted' in res.data

    # Add our URL to the import page
--- a/changedetectionio/tests/test_source.py
+++ b/changedetectionio/tests/test_source.py
@@ -39,7 +39,7 @@ def test_check_basic_change_detection_functionality_source(client, live_server):
    set_modified_response()

    # Force recheck
-    res = client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    res = client.get(url_for("form_watch_checknow"), follow_redirects=True)
    assert b'1 watches are queued for rechecking.' in res.data

    time.sleep(5)
--- a/changedetectionio/tests/test_trigger.py
+++ b/changedetectionio/tests/test_trigger.py
@@ -76,7 +76,7 @@ def test_trigger_functionality(client, live_server):
    assert b"1 Imported" in res.data

    # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)

    # Give the thread time to pick it up
    time.sleep(sleep_time_for_fetch_thread)
@@ -99,7 +99,7 @@ def test_trigger_functionality(client, live_server):
    assert bytes(trigger_text.encode('utf-8')) in res.data

    # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)

    # Give the thread time to pick it up
    time.sleep(sleep_time_for_fetch_thread)
@@ -113,7 +113,7 @@ def test_trigger_functionality(client, live_server):
    set_modified_original_ignore_response()

    # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
    # Give the thread time to pick it up
    time.sleep(sleep_time_for_fetch_thread)

@@ -125,7 +125,7 @@ def test_trigger_functionality(client, live_server):
    time.sleep(sleep_time_for_fetch_thread)
    set_modified_with_trigger_text_response()

-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
    time.sleep(sleep_time_for_fetch_thread)
    res = client.get(url_for("index"))
    assert b'unviewed' in res.data
--- a/changedetectionio/tests/test_trigger_regex.py
+++ b/changedetectionio/tests/test_trigger_regex.py
@@ -43,7 +43,7 @@ def test_trigger_regex_functionality(client, live_server):
    assert b"1 Imported" in res.data

    # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)

    # Give the thread time to pick it up
    time.sleep(sleep_time_for_fetch_thread)
@@ -65,7 +65,7 @@ def test_trigger_regex_functionality(client, live_server):
    with open("test-datastore/endpoint-content.txt", "w") as f:
        f.write("some new noise")

-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
    time.sleep(sleep_time_for_fetch_thread)

    # It should report nothing found (nothing should match the regex)
@@ -75,7 +75,7 @@ def test_trigger_regex_functionality(client, live_server):
    with open("test-datastore/endpoint-content.txt", "w") as f:
        f.write("regex test123<br/>\nsomething 123")

-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
    time.sleep(sleep_time_for_fetch_thread)
    res = client.get(url_for("index"))
    assert b'unviewed' in res.data
--- a/changedetectionio/tests/test_trigger_regex_with_filter.py
+++ b/changedetectionio/tests/test_trigger_regex_with_filter.py
@@ -43,7 +43,7 @@ def test_trigger_regex_functionality(client, live_server):
    assert b"1 Imported" in res.data

    # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)

    # Give the thread time to pick it up
    time.sleep(sleep_time_for_fetch_thread)
@@ -66,7 +66,7 @@ def test_trigger_regex_functionality(client, live_server):
    with open("test-datastore/endpoint-content.txt", "w") as f:
        f.write("<html>some new noise with cool stuff2 ok</html>")

-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
    time.sleep(sleep_time_for_fetch_thread)

    # It should report nothing found (nothing should match the regex and filter)
@@ -76,7 +76,7 @@ def test_trigger_regex_functionality(client, live_server):
    with open("test-datastore/endpoint-content.txt", "w") as f:
        f.write("<html>some new noise with <span id=in-here>cool stuff6</span> ok</html>")

-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
    time.sleep(sleep_time_for_fetch_thread)
    res = client.get(url_for("index"))
    assert b'unviewed' in res.data
--- a/changedetectionio/tests/test_xpath_selector.py
+++ b/changedetectionio/tests/test_xpath_selector.py
@@ -65,7 +65,7 @@ def test_check_markup_xpath_filter_restriction(client, live_server):
    assert b"1 Imported" in res.data

    # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)

    # Give the thread time to pick it up
    time.sleep(sleep_time_for_fetch_thread)
@@ -89,7 +89,7 @@ def test_check_markup_xpath_filter_restriction(client, live_server):
    set_modified_response()

    # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
    # Give the thread time to pick it up
    time.sleep(sleep_time_for_fetch_thread)

@@ -116,4 +116,46 @@ def test_xpath_validation(client, live_server):
        data={"css_filter": "/something horrible", "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
        follow_redirects=True
    )
-    assert b"is not a valid XPath expression" in res.data
+    assert b"is not a valid XPath expression" in res.data
+
+
+# actually only really used by the distll.io importer, but could be handy too
+def test_check_with_prefix_css_filter(client, live_server):
+    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    assert b'Deleted' in res.data
+
+    # Give the endpoint time to spin up
+    time.sleep(1)
+
+    set_original_response()
+
+    # Add our URL to the import page
+    test_url = url_for('test_endpoint', _external=True)
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+    time.sleep(3)
+
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={"css_filter":  "xpath://*[contains(@class, 'sametext')]", "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
+        follow_redirects=True
+    )
+
+    assert b"Updated watch." in res.data
+    time.sleep(3)
+
+    res = client.get(
+        url_for("preview_page", uuid="first"),
+        follow_redirects=True
+    )
+
+    with open('/tmp/fuck.html', 'wb') as f:
+        f.write(res.data)
+    assert b"Some text thats the same" in res.data #in selector
+    assert b"Some text that will change" not in res.data #not in selector
+
+    client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
--- a/changedetectionio/update_worker.py
+++ b/changedetectionio/update_worker.py
@@ -52,6 +52,10 @@ class update_worker(threading.Thread):
                            raise Exception("Error - returned data from the fetch handler SHOULD be bytes")
                    except PermissionError as e:
                        self.app.logger.error("File permission error updating", uuid, str(e))
+                    except content_fetcher.ReplyWithContentButNoText as e:
+                        # Totally fine, it's by choice - just continue on, nothing more to care about
+                        # Page had elements/content but no renderable text
+                        pass
                    except content_fetcher.EmptyReply as e:
                        # Some kind of custom to-str handler in the exception handler that does this?
                        err_text = "EmptyReply: Status Code {}".format(e.status_code)
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -17,12 +17,19 @@ services:
  #       Alternative WebDriver/selenium URL, do not use "'s or 's!
  #      - WEBDRIVER_URL=http://browser-chrome:4444/wd/hub
  #
-  #       WebDriver proxy settings webdriver_proxyType, webdriver_ftpProxy, webdriver_httpProxy, webdriver_noProxy,
-  #                                webdriver_proxyAutoconfigUrl, webdriver_sslProxy, webdriver_autodetect,
+  #       WebDriver proxy settings webdriver_proxyType, webdriver_ftpProxy, webdriver_noProxy,
+  #                                webdriver_proxyAutoconfigUrl, webdriver_autodetect,
  #                                webdriver_socksProxy, webdriver_socksUsername, webdriver_socksVersion, webdriver_socksPassword
  #
  #             https://selenium-python.readthedocs.io/api.html#module-selenium.webdriver.common.proxy
  #
+  #       Alternative Playwright URL, do not use "'s or 's!
+  #      - PLAYWRIGHT_DRIVER_URL=ws://playwright-chrome:3000/
+  #
+  #       Playwright proxy settings playwright_proxy_server, playwright_proxy_bypass, playwright_proxy_username, playwright_proxy_password
+  #
+  #             https://playwright.dev/python/docs/api/class-browsertype#browser-type-launch-option-proxy
+  #
  #        Plain requsts - proxy support example.
  #      - HTTP_PROXY=socks5h://10.10.1.10:1080
  #      - HTTPS_PROXY=socks5h://10.10.1.10:1080
@@ -58,6 +65,13 @@ services:
 #            # Workaround to avoid the browser crashing inside a docker container
 #            # See https://github.com/SeleniumHQ/docker-selenium#quick-start
 #            - /dev/shm:/dev/shm
+#        restart: unless-stopped
+
+     # Used for fetching pages via Playwright+Chrome where you need Javascript support.
+
+#    playwright-chrome:
+#        hostname: playwright-chrome
+#        image: browserless/chrome
 #        restart: unless-stopped

 volumes:
--- a/requirements.txt
+++ b/requirements.txt
@@ -6,6 +6,7 @@ timeago ~=1.0
 inscriptis ~= 2.2
 feedgen ~= 0.9
 flask-login ~= 0.5
+flask_restful
 pytz

 # Set these versions together to avoid a RequestsDependencyWarning
@@ -40,3 +41,4 @@ selenium ~= 4.1.0
 # need to revisit flask login versions
 werkzeug ~= 2.0.0

+# playwright is installed at Dockerfile build time because it's not available on all platforms
Author	SHA1	Message	Date
dgtlmoon	fba4b6747e	Handle disabled situation	2022-05-20 16:03:10 +02:00
dgtlmoon	4cf2d9d7aa	WIP on access	2022-05-20 16:00:34 +02:00
dgtlmoon	6bc1c681ec	Work on handling auth token	2022-05-20 15:37:43 +02:00
dgtlmoon	1267512858	Adjust tests api_ -> form_	2022-05-20 12:47:50 +02:00
dgtlmoon	886ef0c7c1	rename def api_ to def form_ because these arent API related	2022-05-20 12:44:07 +02:00
dgtlmoon	97c6db5e56	Code cleanup	2022-05-20 12:41:01 +02:00
dgtlmoon	23dde28399	More test coverage	2022-05-20 12:40:32 +02:00
dgtlmoon	91fe2dd420	bumping test	2022-05-20 09:33:35 +02:00
dgtlmoon	408c8878f3	bump old old API	2022-05-19 18:08:15 +02:00
dgtlmoon	37614224e5	WIP	2022-05-19 18:00:27 +02:00
dgtlmoon	1caff23d2c	WIP	2022-05-19 15:47:36 +02:00
dgtlmoon	b01ee24d55	POC	2022-05-19 15:14:36 +02:00
dgtlmoon	dbb5c666f0	Fixing edit template HTML	2022-05-18 14:09:39 +02:00
dgtlmoon	70b3493866	Proxy settings on watch should have a "[ ] default" option (#610 )	2022-05-18 13:59:54 +02:00
dgtlmoon	3b11c474d1	Input field tidyup (#611 )	2022-05-18 13:59:17 +02:00
dgtlmoon	890e1e6dcd	Update wiki link for 'More info' about sharing a watch and its configuration	2022-05-17 22:44:36 +02:00
dgtlmoon	6734fb91a2	Option to control if pages with no renderable content are a change (example: JS webapps that dont render any text sometimes) (#608 )	2022-05-17 22:22:00 +02:00
dgtlmoon	16809b48f8	Playwright - raise EmptyReply on empty reply, no need to process further	2022-05-17 18:40:15 +02:00
dgtlmoon	67c833d2bc	Re #214 - configurable wait extra seconds for webdriver requests before extracting text (#606 )	2022-05-17 18:35:33 +02:00
weeix	31fea55ee4	Fix PLAYWRIGHT_DRIVER_URL default value (cf. #587 ) (#599 )	2022-05-14 22:34:44 +02:00
dgtlmoon	b6c50d3b1a	Update PIP readme.md	2022-05-10 22:46:59 +02:00
dgtlmoon	034507f14f	Fixing Pip install problem - Update MANIFEST to include model/ subdir, improving imports (#593 )	2022-05-10 22:45:08 +02:00
dgtlmoon	0e385b1c22	0.39.13	2022-05-10 17:24:38 +02:00
dgtlmoon	f28c260576	Distill.io JSON export file importer (#592 )	2022-05-10 17:15:41 +02:00
dgtlmoon	18f0b63b7d	Ability to specify a list of proxies to choose from, always using the first one by default, See wiki (#591 )	2022-05-08 20:35:36 +02:00
Thilo-Alexander Ginkel	97045e7a7b	Improving Playwright docs (#588 )	2022-05-07 22:23:17 +02:00
dgtlmoon	9807cf0cda	Playwright - code fix	2022-05-07 17:29:59 +02:00
dgtlmoon	d4b5237103	Playwright fetcher - more reliable by just waiting arbitrary seconds after the last network IO	2022-05-07 17:14:40 +02:00
dgtlmoon	dc6f76ba64	Make proxy configuration more consistent - see https://github.com/dgtlmoon/changedetection.io/wiki/Proxy-configuration (#585 )	2022-05-07 16:37:56 +02:00
dgtlmoon	1f2f93184e	Playwright fetcher - use the correct default User-Agent	2022-05-06 23:59:38 +02:00
dgtlmoon	0f08c8dda3	Toggle visibility of extra requests options/settings when not in use (#584 )	2022-05-06 23:40:32 +02:00
dgtlmoon	68db20168e	Add new fetch method: Playwright Chromium (Selenium/WebDriver alternative) (#489 )	2022-05-02 21:40:40 +02:00
dgtlmoon	1d4474f5a3	Simplify scrub operation (simply cleans all) (#575 )	2022-05-02 21:10:23 +02:00
dgtlmoon	613308881c	Bugfix - dont update record when deleted during check	2022-05-01 21:41:29 +02:00