Handle disabled situation

changedetectionio/__init__.py

@@ -36,9 +36,12 @@ from flask import (
     url_for,
 )
 from flask_login import login_required
+from flask_restful import abort, Api
+
 from flask_wtf import CSRFProtect
 
 from changedetectionio import html_tools
+from changedetectionio.api import api_v1
 
 __version__ = '0.39.13.1'
 
@@ -78,6 +81,8 @@ csrf.init_app(app)
 
 notification_debug_log=[]
 
+watch_api = Api(app, decorators=[csrf.exempt])
+
 def init_app_secret(datastore_path):
     secret = ""
 
@@ -179,6 +184,25 @@ def changedetection_app(config=None, datastore_o=None):
     login_manager.login_view = 'login'
     app.secret_key = init_app_secret(config['datastore_path'])
 
+
+    watch_api.add_resource(api_v1.WatchSingleHistory,
+                           '/api/v1/watch/<string:uuid>/history/<string:timestamp>',
+                           resource_class_kwargs={'datastore': datastore, 'update_q': update_q})
+
+    watch_api.add_resource(api_v1.WatchHistory,
+                           '/api/v1/watch/<string:uuid>/history',
+                           resource_class_kwargs={'datastore': datastore})
+
+    watch_api.add_resource(api_v1.CreateWatch, '/api/v1/watch',
+                           resource_class_kwargs={'datastore': datastore, 'update_q': update_q})
+
+    watch_api.add_resource(api_v1.Watch, '/api/v1/watch/<string:uuid>',
+                           resource_class_kwargs={'datastore': datastore, 'update_q': update_q})
+
+
+
+
+
     # Setup cors headers to allow all domains
     # https://flask-cors.readthedocs.io/en/latest/
     #    CORS(app)
@@ -367,6 +391,8 @@ def changedetection_app(config=None, datastore_o=None):
 
             if limit_tag != None:
                 # Support for comma separated list of tags.
+                if watch['tag'] is None:
+                    continue
                 for tag_in_watch in watch['tag'].split(','):
                     tag_in_watch = tag_in_watch.strip()
                     if tag_in_watch == limit_tag:
@@ -520,16 +546,9 @@ def changedetection_app(config=None, datastore_o=None):
 
         # Defaults for proxy choice
         if datastore.proxy_list is not None:  # When enabled
-            system_proxy = datastore.data['settings']['requests']['proxy']
-            if default['proxy'] is None:
-                default['proxy'] = system_proxy
-            else:
-                # Does the chosen one exist?
-                if not any(default['proxy'] in tup for tup in datastore.proxy_list):
-                    default['proxy'] = datastore.proxy_list[0][0]
-
-            # Used by the form handler to keep or remove the proxy settings
-            default['proxy_list'] = datastore.proxy_list
+            # Radio needs '' not None, or incase that the chosen one no longer exists
+            if default['proxy'] is None or not any(default['proxy'] in tup for tup in datastore.proxy_list):
+                default['proxy'] = ''
 
         # proxy_override set to the json/text list of the items
         form = forms.watchForm(formdata=request.form if request.method == 'POST' else None,
@@ -540,9 +559,7 @@ def changedetection_app(config=None, datastore_o=None):
             # @todo - Couldn't get setattr() etc dynamic addition working, so remove it instead
             del form.proxy
         else:
-            form.proxy.choices = datastore.proxy_list
-            if default['proxy'] is None:
-                form.proxy.default='http://hello'
+            form.proxy.choices = [('', 'Default')] + datastore.proxy_list
 
         if request.method == 'POST' and form.validate():
             extra_update_obj = {}
@@ -579,6 +596,10 @@ def changedetection_app(config=None, datastore_o=None):
                 if len(datastore.data['watching'][uuid]['history']):
                     extra_update_obj['previous_md5'] = get_current_checksum_include_ignore_text(uuid=uuid)
 
+            # Be sure proxy value is None
+            if datastore.proxy_list is not None and form.data['proxy'] == '':
+                extra_update_obj['proxy'] = None
+
             datastore.data['watching'][uuid].update(form.data)
             datastore.data['watching'][uuid].update(extra_update_obj)
 
@@ -605,12 +626,12 @@ def changedetection_app(config=None, datastore_o=None):
             if request.method == 'POST' and not form.validate():
                 flash("An error occurred, please see below.", "error")
 
-
             output = render_template("edit.html",
                                      uuid=uuid,
                                      watch=datastore.data['watching'][uuid],
                                      form=form,
                                      has_empty_checktime=using_default_check_time,
+                                     using_global_webdriver_wait=default['webdriver_delay'] is None,
                                      current_base_url=datastore.data['settings']['application']['base_url'],
                                      emailprefix=os.getenv('NOTIFICATION_MAIL_BUTTON_PREFIX', False)
                                      )
@@ -676,6 +697,7 @@ def changedetection_app(config=None, datastore_o=None):
                                  form=form,
                                  current_base_url = datastore.data['settings']['application']['base_url'],
                                  hide_remove_pass=os.getenv("SALTED_PASS", False),
+                                 api_key=datastore.data['settings']['application'].get('api_access_token'),
                                  emailprefix=os.getenv('NOTIFICATION_MAIL_BUTTON_PREFIX', False))
 
         return output
@@ -874,27 +896,6 @@ def changedetection_app(config=None, datastore_o=None):
 
         return output
 
-    @app.route("/api/<string:uuid>/snapshot/current", methods=['GET'])
-    @login_required
-    def api_snapshot(uuid):
-
-        # More for testing, possible to return the first/only
-        if uuid == 'first':
-            uuid = list(datastore.data['watching'].keys()).pop()
-
-        try:
-            watch = datastore.data['watching'][uuid]
-        except KeyError:
-            return abort(400, "No history found for the specified link, bad link?")
-
-        newest = list(watch['history'].keys())[-1]
-        with open(watch['history'][newest], 'r') as f:
-            content = f.read()
-
-        resp = make_response(content)
-        resp.headers['Content-Type'] = 'text/plain'
-        return resp
-
     @app.route("/favicon.ico", methods=['GET'])
     def favicon():
         return send_from_directory("static/images", path="favicon.ico")
@@ -932,9 +933,6 @@ def changedetection_app(config=None, datastore_o=None):
             # Add the flask app secret
             zipObj.write(os.path.join(datastore_o.datastore_path, "secret.txt"), arcname="secret.txt")
 
-            # Add the sqlite3 db
-            zipObj.write(os.path.join(datastore_o.datastore_path, "watch.db"), arcname="watch.db")
-
             # Add any snapshot data we find, use the full path to access the file, but make the file 'relative' in the Zip.
             for txt_file_path in Path(datastore_o.datastore_path).rglob('*.txt'):
                 parent_p = txt_file_path.parent
@@ -1008,7 +1006,7 @@ def changedetection_app(config=None, datastore_o=None):
 
     @app.route("/api/add", methods=['POST'])
     @login_required
-    def api_watch_add():
+    def form_watch_add():
         from changedetectionio import forms
         form = forms.quickWatchForm(request.form)
 
@@ -1034,7 +1032,7 @@ def changedetection_app(config=None, datastore_o=None):
 
     @app.route("/api/delete", methods=['GET'])
     @login_required
-    def api_delete():
+    def form_delete():
         uuid = request.args.get('uuid')
 
         if uuid != 'all' and not uuid in datastore.data['watching'].keys():
@@ -1051,7 +1049,7 @@ def changedetection_app(config=None, datastore_o=None):
 
     @app.route("/api/clone", methods=['GET'])
     @login_required
-    def api_clone():
+    def form_clone():
         uuid = request.args.get('uuid')
         # More for testing, possible to return the first/only
         if uuid == 'first':
@@ -1065,7 +1063,7 @@ def changedetection_app(config=None, datastore_o=None):
 
     @app.route("/api/checknow", methods=['GET'])
     @login_required
-    def api_watch_checknow():
+    def form_watch_checknow():
 
         tag = request.args.get('tag')
         uuid = request.args.get('uuid')
@@ -1102,7 +1100,7 @@ def changedetection_app(config=None, datastore_o=None):
 
     @app.route("/api/share-url", methods=['GET'])
     @login_required
-    def api_share_put_watch():
+    def form_share_put_watch():
         """Given a watch UUID, upload the info and return a share-link
            the share-link can be imported/added"""
         import requests

changedetectionio/api/api_v1.py

@@ -0,0 +1,125 @@
+from flask_restful import abort, Resource
+from flask import request, make_response
+import validators
+from . import auth
+
+
+
+# https://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html
+
+class Watch(Resource):
+    def __init__(self, **kwargs):
+        # datastore is a black box dependency
+        self.datastore = kwargs['datastore']
+        self.update_q = kwargs['update_q']
+
+    # Get information about a single watch, excluding the history list (can be large)
+    # curl http://localhost:4000/api/v1/watch/<string:uuid>
+    # ?recheck=true
+    @auth.check_token
+    def get(self, uuid):
+        from copy import deepcopy
+        watch = deepcopy(self.datastore.data['watching'].get(uuid))
+        if not watch:
+            abort(404, message='No watch exists with the UUID of {}'.format(uuid))
+
+        if request.args.get('recheck'):
+            self.update_q.put(uuid)
+            return "OK", 200
+
+        # Return without history, get that via another API call
+        watch['history_n'] = len(watch['history'])
+        del (watch['history'])
+        return watch
+
+    @auth.check_token
+    def delete(self, uuid):
+        if not self.datastore.data['watching'].get(uuid):
+            abort(400, message='No watch exists with the UUID of {}'.format(uuid))
+
+        self.datastore.delete(uuid)
+        return 'OK', 204
+
+
+class WatchHistory(Resource):
+    def __init__(self, **kwargs):
+        # datastore is a black box dependency
+        self.datastore = kwargs['datastore']
+
+    # Get a list of available history for a watch by UUID
+    # curl http://localhost:4000/api/v1/watch/<string:uuid>/history
+    def get(self, uuid):
+        watch = self.datastore.data['watching'].get(uuid)
+        if not watch:
+            abort(404, message='No watch exists with the UUID of {}'.format(uuid))
+        return watch['history'], 200
+
+
+class WatchSingleHistory(Resource):
+    def __init__(self, **kwargs):
+        # datastore is a black box dependency
+        self.datastore = kwargs['datastore']
+
+    # Read a given history snapshot and return its content
+    # <string:timestamp> or "latest"
+    # curl http://localhost:4000/api/v1/watch/<string:uuid>/history/<int:timestamp>
+    @auth.check_token
+    def get(self, uuid, timestamp):
+        watch = self.datastore.data['watching'].get(uuid)
+        if not watch:
+            abort(404, message='No watch exists with the UUID of {}'.format(uuid))
+
+        if not len(watch['history']):
+            abort(404, message='Watch found but no history exists for the UUID {}'.format(uuid))
+
+        if timestamp == 'latest':
+            timestamp = list(watch['history'].keys())[-1]
+
+        with open(watch['history'][timestamp], 'r') as f:
+            content = f.read()
+
+        response = make_response(content, 200)
+        response.mimetype = "text/plain"
+        return response
+
+
+class CreateWatch(Resource):
+    def __init__(self, **kwargs):
+        # datastore is a black box dependency
+        self.datastore = kwargs['datastore']
+        self.update_q = kwargs['update_q']
+
+    @auth.check_token
+    def post(self):
+        # curl http://localhost:4000/api/v1/watch -H "Content-Type: application/json" -d '{"url": "https://my-nice.com", "tag": "one, two" }'
+        json_data = request.get_json()
+        tag = json_data['tag'].strip() if json_data.get('tag') else ''
+
+        if not validators.url(json_data['url'].strip()):
+            return "Invalid or unsupported URL", 400
+
+        extras = {'title': json_data['title'].strip()} if json_data.get('title') else {}
+
+        new_uuid = self.datastore.add_watch(url=json_data['url'].strip(), tag=tag, extras=extras)
+        self.update_q.put(new_uuid)
+        return {'uuid': new_uuid}, 201
+
+    # Return concise list of available watches and some very basic info
+    # curl http://localhost:4000/api/v1/watch|python -mjson.tool
+    # ?recheck_all=1 to recheck all
+    @auth.check_token
+    def get(self):
+        list = {}
+        for k, v in self.datastore.data['watching'].items():
+            list[k] = {'url': v['url'],
+                       'title': v['title'],
+                       'last_checked': v['last_checked'],
+                       'last_changed': v['last_changed'],
+                       'last_error': v['last_error']}
+
+        if request.args.get('recheck_all'):
+            for uuid in self.datastore.data['watching'].keys():
+                self.update_q.put(uuid)
+            return {'status': "OK"}, 200
+
+        return list, 200

changedetectionio/api/auth.py

@@ -0,0 +1,33 @@
+from flask import request, make_response, jsonify
+from functools import wraps
+
+
+# Simple API auth key comparison
+# @todo - Maybe short lived token in the future?
+
+def check_token(f):
+    @wraps(f)
+    def decorated(*args, **kwargs):
+        datastore = args[0].datastore
+
+        config_api_token_enabled = datastore.data['settings']['application'].get('api_access_token_enabled')
+        if not config_api_token_enabled:
+            return
+
+        try:
+            api_key_header = request.headers['x-api-key']
+        except KeyError:
+            return make_response(
+                jsonify("No authorization x-api-key header."), 403
+            )
+
+        config_api_token = datastore.data['settings']['application'].get('api_access_token')
+
+        if api_key_header != config_api_token:
+            return make_response(
+                jsonify("Invalid access - API key invalid."), 403
+            )
+
+        return f(*args, **kwargs)
+
+    return decorated

changedetectionio/content_fetcher.py

@@ -3,17 +3,22 @@ import chardet
 import os
 import requests
 import time
-import urllib3.exceptions
 import sys
 
-
 class EmptyReply(Exception):
     def __init__(self, status_code, url):
         # Set this so we can use it in other parts of the app
         self.status_code = status_code
         self.url = url
         return
+    pass
 
+class ReplyWithContentButNoText(Exception):
+    def __init__(self, status_code, url):
+        # Set this so we can use it in other parts of the app
+        self.status_code = status_code
+        self.url = url
+        return
     pass
 
 
@@ -28,6 +33,9 @@ class Fetcher():
     system_http_proxy = os.getenv('HTTP_PROXY')
     system_https_proxy = os.getenv('HTTPS_PROXY')
 
+    # Time ONTOP of the system defined env minimum time
+    render_extract_delay=0
+
     @abstractmethod
     def get_error(self):
         return self.error
@@ -97,7 +105,7 @@ class base_html_playwright(Fetcher):
         self.browser_type = os.getenv("PLAYWRIGHT_BROWSER_TYPE", 'chromium').strip('"')
         self.command_executor = os.getenv(
             "PLAYWRIGHT_DRIVER_URL",
-            'ws://playwright-chrome:3000/playwright'
+            'ws://playwright-chrome:3000'
         ).strip('"')
 
         # If any proxy settings are enabled, then we should setup the proxy object
@@ -147,7 +155,7 @@ class base_html_playwright(Fetcher):
                 # - `'commit'` - consider operation to be finished when network response is received and the document started loading.
                 # Better to not use any smarts from Playwright and just wait an arbitrary number of seconds
                 # This seemed to solve nearly all 'TimeoutErrors'
-                extra_wait = int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5))
+                extra_wait = int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)) + self.render_extract_delay
                 page.wait_for_timeout(extra_wait * 1000)
             except playwright._impl._api_types.TimeoutError as e:
                 raise EmptyReply(url=url, status_code=None)
@@ -155,6 +163,9 @@ class base_html_playwright(Fetcher):
             if response is None:
                 raise EmptyReply(url=url, status_code=None)
 
+            if len(page.content().strip()) == 0:
+                raise EmptyReply(url=url, status_code=None)
+
             self.status_code = response.status
             self.content = page.content()
             self.headers = response.all_headers()
@@ -240,7 +251,7 @@ class base_html_webdriver(Fetcher):
         # raise EmptyReply(url=url, status_code=r.status_code)
 
         # @todo - dom wait loaded?
-        time.sleep(int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)))
+        time.sleep(int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)) + self.render_extract_delay)
         self.content = self.driver.page_source
         self.headers = {}
         self.screenshot = self.driver.get_screenshot_as_png()

changedetectionio/fetch_site_status.py

@@ -97,7 +97,13 @@ class perform_site_check():
         proxy_args = self.set_proxy_from_list(watch)
         fetcher = klass(proxy_override=proxy_args)
 
-        # Proxy List support
+        # Configurable per-watch or global extra delay before extracting text (for webDriver types)
+        system_webdriver_delay = self.datastore.data['settings']['application'].get('webdriver_delay', None)
+        if watch['webdriver_delay'] is not None:
+            fetcher.render_extract_delay = watch['webdriver_delay']
+        elif system_webdriver_delay is not None:
+            fetcher.render_extract_delay = system_webdriver_delay
+
         fetcher.run(url, timeout, request_headers, request_body, request_method, ignore_status_code)
 
         # Fetching complete, now filters
@@ -178,6 +184,11 @@ class perform_site_check():
         # Re #340 - return the content before the 'ignore text' was applied
         text_content_before_ignored_filter = stripped_text_from_html.encode('utf-8')
 
+        # Treat pages with no renderable text content as a change? No by default
+        empty_pages_are_a_change = self.datastore.data['settings']['application'].get('empty_pages_are_a_change', False)
+        if not is_json and not empty_pages_are_a_change and len(stripped_text_from_html.strip()) == 0:
+            raise content_fetcher.ReplyWithContentButNoText(url=url, status_code=200)
+
         # We rely on the actual text in the html output.. many sites have random script vars etc,
         # in the future we'll implement other mechanisms.
 

changedetectionio/forms.py

@@ -318,6 +318,7 @@ class commonSettingsForm(Form):
     notification_format = SelectField('Notification format', choices=valid_notification_formats.keys(), default=default_notification_format)
     fetch_backend = RadioField(u'Fetch method', choices=content_fetcher.available_fetchers(), validators=[ValidateContentFetcherIsReady()])
     extract_title_as_title = BooleanField('Extract <title> from document and use as watch title', default=False)
+    webdriver_delay = IntegerField('Wait seconds before extracting text', validators=[validators.Optional(), validators.NumberRange(min=1, message="Should contain one or more seconds")] )
 
 class watchForm(commonSettingsForm):
 
@@ -370,8 +371,10 @@ class globalSettingsApplicationForm(commonSettingsForm):
     ignore_whitespace = BooleanField('Ignore whitespace')
     real_browser_save_screenshot = BooleanField('Save last screenshot when using Chrome?')
     removepassword_button = SubmitField('Remove password', render_kw={"class": "pure-button pure-button-primary"})
+    empty_pages_are_a_change =  BooleanField('Treat empty pages as a change?', default=False)
     render_anchor_tag_content = BooleanField('Render anchor tag content', default=False)
     fetch_backend = RadioField('Fetch Method', default="html_requests", choices=content_fetcher.available_fetchers(), validators=[ValidateContentFetcherIsReady()])
+    api_access_token_enabled = BooleanField('API access token security check enabled', default=True, validators=[validators.Optional()])
     password = SaltyPasswordField()
 
 

changedetectionio/model/App.py

@@ -27,9 +27,11 @@ class model(dict):
                     'proxy': None # Preferred proxy connection
                 },
                 'application': {
+                    'api_access_token_enabled': True,
                     'password': False,
                     'base_url' : None,
                     'extract_title_as_title': False,
+                    'empty_pages_are_a_change': False,
                     'fetch_backend': os.getenv("DEFAULT_FETCH_BACKEND", "html_requests"),
                     'global_ignore_text': [], # List of text to ignore when calculating the comparison checksum
                     'global_subtractive_selectors': [],
@@ -41,7 +43,8 @@ class model(dict):
                     'notification_body': default_notification_body,
                     'notification_format': default_notification_format,
                     'real_browser_save_screenshot': True,
-                    'schema_version' : 0
+                    'schema_version' : 0,
+                    'webdriver_delay': None  # Extra delay in seconds before extracting text
                 }
             }
         }

changedetectionio/model/Watch.py

@@ -27,8 +27,7 @@ class model(dict):
             'headers': {},  # Extra headers to send
             'body': None,
             'method': 'GET',
-            # now stored in a sqlite3 db to reduce memory usage
-            #'history': {},  # Dict of timestamp and output stripped filename
+            'history': {},  # Dict of timestamp and output stripped filename
             'ignore_text': [],  # List of text to ignore when calculating the comparison checksum
             # Custom notification content
             'notification_urls': [],  # List of URLs to add to the notification Queue (Usually AppRise)
@@ -44,7 +43,8 @@ class model(dict):
             # Re #110, so then if this is set to None, we know to use the default value instead
             # Requires setting to None on submit if it's the same as the default
             # Should be all None by default, so we use the system default in this case.
-            'time_between_check': {'weeks': None, 'days': None, 'hours': None, 'minutes': None, 'seconds': None}
+            'time_between_check': {'weeks': None, 'days': None, 'hours': None, 'minutes': None, 'seconds': None},
+            'webdriver_delay': None
         }
 
     def __init__(self, *arg, **kw):

changedetectionio/static/js/global-settings.js

@@ -0,0 +1,36 @@
+$(document).ready(function () {
+    function toggle() {
+        if ($('input[name="application-fetch_backend"]:checked').val() != 'html_requests') {
+            $('#requests-override-options').hide();
+            $('#webdriver-override-options').show();
+        } else {
+            $('#requests-override-options').show();
+            $('#webdriver-override-options').hide();
+        }
+    }
+
+    $('input[name="application-fetch_backend"]').click(function (e) {
+        toggle();
+    });
+    toggle();
+
+    $("#api-key").hover(
+        function () {
+            $("#api-key-copy").html('copy').fadeIn();
+        },
+        function () {
+            $("#api-key-copy").hide();
+        }
+    ).click(function (e) {
+        $("#api-key-copy").html('copied');
+        var range = document.createRange();
+        var n = $("#api-key")[0];
+        range.selectNode(n);
+        window.getSelection().removeAllRanges();
+        window.getSelection().addRange(range);
+        document.execCommand("copy");
+        window.getSelection().removeAllRanges();
+
+    });
+});
+

changedetectionio/static/js/watch-settings.js

@@ -2,8 +2,10 @@ $(document).ready(function() {
     function toggle() {
         if ($('input[name="fetch_backend"]:checked').val() != 'html_requests') {
             $('#requests-override-options').hide();
+            $('#webdriver-override-options').show();
         } else {
             $('#requests-override-options').show();
+            $('#webdriver-override-options').hide();
         }
     }
     $('input[name="fetch_backend"]').click(function (e) {

changedetectionio/static/styles/styles.css

@@ -284,6 +284,11 @@ footer {
 .pure-form {
   /* The input fields with errors */
   /* The list of errors */ }
+  .pure-form fieldset {
+    padding-top: 0px; }
+    .pure-form fieldset ul {
+      padding-bottom: 0px;
+      margin-bottom: 0px; }
   .pure-form .pure-control-group, .pure-form .pure-group, .pure-form .pure-controls {
     padding-bottom: 1em; }
     .pure-form .pure-control-group div, .pure-form .pure-group div, .pure-form .pure-controls div {
@@ -447,4 +452,13 @@ ul {
 .time-check-widget tr {
   display: inline; }
   .time-check-widget tr input[type="number"] {
-    width: 4em; }
+    width: 5em; }
+
+#webdriver-override-options input[type="number"] {
+  width: 5em; }
+
+#api-key:hover {
+  cursor: pointer; }
+
+#api-key-copy {
+  color: #0078e7; }

changedetectionio/static/styles/styles.scss

@@ -375,6 +375,13 @@ footer {
 
 
 .pure-form {
+    fieldset {
+        padding-top: 0px;
+        ul {
+            padding-bottom: 0px;
+            margin-bottom: 0px;
+        }
+    }
     .pure-control-group, .pure-group, .pure-controls {
         padding-bottom: 1em;
         div {
@@ -637,7 +644,23 @@ ul {
     tr {
         display: inline;
         input[type="number"] {
-            width: 4em;
+            width: 5em;
         }
     }
-}
+}
+
+#webdriver-override-options {
+        input[type="number"] {
+            width: 5em;
+        }
+}
+
+#api-key {
+  &:hover {
+    cursor: pointer;
+  }
+}
+
+#api-key-copy {
+  color: #0078e7;
+}

changedetectionio/store.py

@@ -12,11 +12,10 @@ from os import mkdir, path, unlink
 from threading import Lock
 import re
 import requests
-import sqlite3
+import secrets
 
 from . model import App, Watch
 
-
 # Is there an existing library to ensure some data store (JSON etc) is in sync with CRUD methods?
 # Open a github issue if you know something :)
 # https://stackoverflow.com/questions/6190468/how-to-trigger-function-on-value-change
@@ -34,11 +33,6 @@ class ChangeDetectionStore:
         self.needs_write = False
         self.datastore_path = datastore_path
         self.json_store_path = "{}/url-watches.json".format(self.datastore_path)
-        self.datastore_path = datastore_path
-
-        #@todo - check for better options
-        self.__history_db_connection = sqlite3.connect("{}/watch.db".format(self.datastore_path))
-
         self.proxy_list = None
         self.stop_thread = False
 
@@ -77,9 +71,6 @@ class ChangeDetectionStore:
                     if 'application' in from_disk['settings']:
                         self.__data['settings']['application'].update(from_disk['settings']['application'])
 
-                # Bump the update version by running updates
-                self.run_updates()
-
                 # Reinitialise each `watching` with our generic_definition in the case that we add a new var in the future.
                 # @todo pretty sure theres a python we todo this with an abstracted(?) object!
                 for uuid, watch in self.__data['watching'].items():
@@ -89,7 +80,6 @@ class ChangeDetectionStore:
                     self.__data['watching'][uuid]['newest_history_key'] = self.get_newest_history_key(uuid)
                     print("Watching:", uuid, self.__data['watching'][uuid]['url'])
 
-
         # First time ran, doesnt exist.
         except (FileNotFoundError, json.decoder.JSONDecodeError):
             if include_default_watches:
@@ -118,10 +108,14 @@ class ChangeDetectionStore:
 
         # Generate the URL access token for RSS feeds
         if not 'rss_access_token' in self.__data['settings']['application']:
-            import secrets
             secret = secrets.token_hex(16)
             self.__data['settings']['application']['rss_access_token'] = secret
 
+        # Generate the API access token
+        if not 'api_access_token' in self.__data['settings']['application']:
+            secret = secrets.token_hex(16)
+            self.__data['settings']['application']['api_access_token'] = secret
+
         # Proxy list support - available as a selection in settings when text file is imported
         # CSV list
         # "name, address", or just "name"
@@ -129,6 +123,8 @@ class ChangeDetectionStore:
         if path.isfile(proxy_list_file):
             self.import_proxy_list(proxy_list_file)
 
+        # Bump the update version by running updates
+        self.run_updates()
 
         self.needs_write = True
 
@@ -137,20 +133,19 @@ class ChangeDetectionStore:
 
     # Returns the newest key, but if theres only 1 record, then it's counted as not being new, so return 0.
     def get_newest_history_key(self, uuid):
-
-        cur = self.__history_db_connection.cursor()
-
-        c = cur.execute("SELECT COUNT(*) FROM watch_history WHERE watch_uuid = :uuid", {"uuid": uuid}).fetchone()
-        if c and c[0] <= 1:
+        if len(self.__data['watching'][uuid]['history']) == 1:
             return 0
 
-        max = cur.execute("SELECT MAX(timestamp) FROM watch_history WHERE  watch_uuid  = :uuid", {"uuid": uuid}).fetchone()
-        return max[0]
+        dates = list(self.__data['watching'][uuid]['history'].keys())
+        # Convert to int, sort and back to str again
+        # @todo replace datastore getter that does this automatically
+        dates = [int(i) for i in dates]
+        dates.sort(reverse=True)
+        if len(dates):
+            # always keyed as str
+            return str(dates[0])
 
-    def __refresh_history_max_timestamp(self):
-        # select watch_uuid, max(timestamp) from watch_history group by watch_uuid;
-        # could be way faster
-        x=1
+        return 0
 
     def set_last_viewed(self, uuid, timestamp):
         self.data['watching'][uuid].update({'last_viewed': int(timestamp)})
@@ -195,13 +190,13 @@ class ChangeDetectionStore:
     def data(self):
         has_unviewed = False
         for uuid, v in self.__data['watching'].items():
-#            self.__data['watching'][uuid]['newest_history_key'] = self.get_newest_history_key(uuid)
-#            if int(v['newest_history_key']) <= int(v['last_viewed']):
-#                self.__data['watching'][uuid]['viewed'] = True
+            self.__data['watching'][uuid]['newest_history_key'] = self.get_newest_history_key(uuid)
+            if int(v['newest_history_key']) <= int(v['last_viewed']):
+                self.__data['watching'][uuid]['viewed'] = True
 
-#            else:
-#                self.__data['watching'][uuid]['viewed'] = False
-#                has_unviewed = True
+            else:
+                self.__data['watching'][uuid]['viewed'] = False
+                has_unviewed = True
 
             # #106 - Be sure this is None on empty string, False, None, etc
             # Default var for fetch_backend
@@ -220,7 +215,8 @@ class ChangeDetectionStore:
     def get_all_tags(self):
         tags = []
         for uuid, watch in self.data['watching'].items():
-
+            if watch['tag'] is None:
+                continue
             # Support for comma separated list of tags.
             for tag in watch['tag'].split(','):
                 tag = tag.strip()
@@ -289,6 +285,10 @@ class ChangeDetectionStore:
     def add_watch(self, url, tag="", extras=None, write_to_disk_now=True):
         if extras is None:
             extras = {}
+        # should always be str
+        if tag is None or not tag:
+            tag=''
+
         # Incase these are copied across, assume it's a reference and deepcopy()
         apply_extras = deepcopy(extras)
 
@@ -504,31 +504,3 @@ class ChangeDetectionStore:
                 # Only upgrade individual watch time if it was set
                 if watch.get('minutes_between_check', False):
                     self.data['watching'][uuid]['time_between_check']['minutes'] = watch['minutes_between_check']
-
-    def update_3(self):
-        """Migrate storage of history data to SQLite
-        - No need to store the history list in memory and re-write it everytime
-        - I've seen memory usage grow exponentially due to having large lists of watches with long histories
-        - Data about 'last changed' still stored in the main JSON struct which is fine
-        - We don't really need this data until we query against it (like for listing other available snapshots in the diff page etc)
-        """
-
-        if self.__history_db_connection:
-            # Create the table
-            self.__history_db_connection.execute("CREATE TABLE IF NOT EXISTS  watch_history(id INTEGER PRIMARY KEY, watch_uuid VARCHAR(36), timestamp INT, path TEXT, snapshot_type VARCHAR(10))")
-            self.__history_db_connection.execute("CREATE INDEX IF NOT EXISTS `uuid` ON `watch_history` (`watch_uuid`)")
-            self.__history_db_connection.execute("CREATE INDEX IF NOT EXISTS `uuid_timestamp` ON `watch_history` (`watch_uuid`, `timestamp`)")
-
-            # Insert each watch history list as executemany() for faster migration
-            for uuid, watch in self.data['watching'].items():
-                history = []
-
-                if watch.get('history', False):
-                    for d, p in watch['history'].items():
-                        d = int(d) # Used to be keyed as str, we'll fix this now too
-                        history.append((uuid, d, p, 'text'))
-
-                    if len(history):
-                        self.__history_db_connection.executemany("INSERT INTO watch_history (watch_uuid, timestamp, path, snapshot_type) VALUES (?,?,?,?)", history)
-                        self.__history_db_connection.commit()
-                        del(self.data['watching'][uuid]['history'])

changedetectionio/templates/edit.html

@@ -73,6 +73,21 @@
                         </span>
                     </div>
                 {% endif %}
+                <fieldset id="webdriver-override-options">
+                    <div class="pure-form-message-inline">
+                        <strong>If you're having trouble waiting for the page to be fully rendered (text missing etc), try increasing the 'wait' time here.</strong>
+                        <br/>
+                        This will wait <i>n</i> seconds before extracting the text.
+                    </div>
+                    <div class="pure-control-group">
+                        {{ render_field(form.webdriver_delay) }}
+                    </div>
+                    {% if using_global_webdriver_wait %}
+                    <div class="pure-form-message-inline">
+                        <strong>Using the current global default settings</strong>
+                    </div>
+                    {% endif %}
+                </fieldset>
                 <fieldset class="pure-group" id="requests-override-options">
                     <div class="pure-form-message-inline">
                         <strong>Request override is currently only used by the <i>Basic fast Plaintext/HTTP Client</i> method.</strong>
@@ -110,8 +125,7 @@ User-Agent: wonderbra 1.0") }}
             </div>
 
             <div class="tab-pane-inner" id="filters-and-triggers">
-                <fieldset>
-                        <div class="pure-control-group">
+                    <div class="pure-control-group">
                             <strong>Pro-tips:</strong><br/>
                             <ul>
                                 <li>
@@ -122,7 +136,6 @@ User-Agent: wonderbra 1.0") }}
                                 </li>
                             </ul>
                     </div>
-
                     <div class="pure-control-group">
                         {{ render_field(form.css_filter, placeholder=".class-name or #some-id, or other CSS selector rule.",
                         class="m-d") }}
@@ -138,7 +151,7 @@ User-Agent: wonderbra 1.0") }}
                                 href="https://github.com/dgtlmoon/changedetection.io/wiki/CSS-Selector-help">here for more CSS selector help</a>.<br/>
                 </span>
                     </div>
-                    <fieldset class="pure-group">
+                    <div class="pure-control-group">
                       {{ render_field(form.subtractive_selectors, rows=5, placeholder="header
 footer
 nav
@@ -149,8 +162,7 @@ nav
                           <li> Add multiple elements or CSS selectors per line to ignore multiple parts of the HTML. </li>
                         </ul>
                       </span>
-                    </fieldset>
-                </fieldset>
+                    </div>
                 <fieldset class="pure-group">
                     {{ render_field(form.ignore_text, rows=5, placeholder="Some text to ignore in a line
 /some.regex\d{2}/ for case-INsensitive regex
@@ -187,9 +199,9 @@ nav
 
                       {{ render_button(form.save_button) }} {{ render_button(form.save_and_preview_button) }}
 
-                    <a href="{{url_for('api_delete', uuid=uuid)}}"
+                    <a href="{{url_for('form_delete', uuid=uuid)}}"
                        class="pure-button button-small button-error ">Delete</a>
-                    <a href="{{url_for('api_clone', uuid=uuid)}}"
+                    <a href="{{url_for('form_clone', uuid=uuid)}}"
                        class="pure-button button-small ">Create Copy</a>
                 </div>
             </div>

changedetectionio/templates/settings.html

@@ -12,6 +12,7 @@
 <script type="text/javascript" src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
 <script type="text/javascript" src="{{url_for('static_content', group='js', filename='notifications.js')}}" defer></script>
 
+<script type="text/javascript" src="{{url_for('static_content', group='js', filename='global-settings.js')}}" defer></script>
 <div class="edit-form">
     <div class="tabs collapsable">
         <ul>
@@ -19,6 +20,7 @@
             <li class="tab"><a href="#notifications">Notifications</a></li>
             <li class="tab"><a href="#fetching">Fetching</a></li>
             <li class="tab"><a href="#filters">Global Filters</a></li>
+            <li class="tab"><a href="#api">API</a></li>
         </ul>
     </div>
     <div class="box-wrap inner">
@@ -42,6 +44,7 @@
                             <span class="pure-form-message-inline">Password is locked.</span>
                         {% endif %}
                     </div>
+
                     <div class="pure-control-group">
                         {{ render_field(form.application.form.base_url, placeholder="http://yoursite.com:5000/",
                         class="m-d") }}
@@ -60,6 +63,11 @@
                         {{ render_checkbox_field(form.application.form.real_browser_save_screenshot) }}
                         <span class="pure-form-message-inline">When using a Chrome browser, a screenshot from the last check will be available on the Diff page</span>
                     </div>
+
+                    <div class="pure-control-group">
+                        {{ render_checkbox_field(form.application.form.empty_pages_are_a_change) }}
+                        <span class="pure-form-message-inline">When a page contains HTML, but no renderable text appears (empty page), is this considered a change?</span>
+                    </div>
                 {% if form.requests.proxy %}
                     <div class="pure-control-group inline-radio">
                         {{ render_field(form.requests.form.proxy, class="fetch-backend-proxy") }}
@@ -87,9 +95,18 @@
                         <p>The <strong>Chrome/Javascript</strong> method requires a network connection to a running WebDriver+Chrome server, set by the ENV var 'WEBDRIVER_URL'. </p>
                     </span>
                 </div>
+                <fieldset class="pure-group" id="webdriver-override-options">
+                    <div class="pure-form-message-inline">
+                        <strong>If you're having trouble waiting for the page to be fully rendered (text missing etc), try increasing the 'wait' time here.</strong>
+                        <br/>
+                        This will wait <i>n</i> seconds before extracting the text.
+                    </div>
+                    <div class="pure-control-group">
+                        {{ render_field(form.application.form.webdriver_delay) }}
+                    </div>
+                </fieldset>
             </div>
 
-
             <div class="tab-pane-inner" id="filters">
 
                     <fieldset class="pure-group">
@@ -134,12 +151,26 @@ nav
                     </fieldset>
            </div>
 
+            <div class="tab-pane-inner" id="api">
+
+                <p>Drive your changedetection.io via API, More about <a href="https://github.com/dgtlmoon/changedetection.io/wiki/API-Reference">API access here</a></p>
+
+                <div class="pure-control-group">
+                    {{ render_checkbox_field(form.application.form.api_access_token_enabled) }}
+                    <div class="pure-form-message-inline">Restrict API access limit by using <code>x-api-key</code> header</div><br/>
+                    <div class="pure-form-message-inline"><br/>API Key <span id="api-key">{{api_key}}</span>
+                        <span style="display:none;" id="api-key-copy" >copy</span>
+                    </div>
+                </div>
+            </div>
+
             <div id="actions">
                 <div class="pure-control-group">
                     {{ render_button(form.save_button) }}
                     <a href="{{url_for('index')}}" class="pure-button button-small button-cancel">Back</a>
                     <a href="{{url_for('scrub_page')}}" class="pure-button button-small button-cancel">Delete History Snapshot Data</a>
                 </div>
+
             </div>
         </form>
     </div>

changedetectionio/templates/watch-overview.html

@@ -5,7 +5,7 @@
 <script type="text/javascript" src="{{url_for('static_content', group='js', filename='watch-overview.js')}}" defer></script>
 <div class="box">
 
-    <form class="pure-form" action="{{ url_for('api_watch_add') }}" method="POST" id="new-watch-form">
+    <form class="pure-form" action="{{ url_for('form_watch_add') }}" method="POST" id="new-watch-form">
         <input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
         <fieldset>
             <legend>Add a new change detection watch</legend>
@@ -13,7 +13,7 @@
                 {{ render_simple_field(form.tag, value=active_tag if active_tag else '', placeholder="watch group") }}
             <button type="submit" class="pure-button pure-button-primary">Watch</button>
         </fieldset>
-        <span style="color:#eee; font-size: 80%;"><img style="height: 1em;display:inline-block;" src="{{url_for('static_content', group='images', filename='spread.svg')}}" /> Tip: You can also add 'shared' watches. <a href="#">More info</a></a></span>
+        <span style="color:#eee; font-size: 80%;"><img style="height: 1em;display:inline-block;" src="{{url_for('static_content', group='images', filename='spread.svg')}}" /> Tip: You can also add 'shared' watches. <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Sharing-a-Watch">More info</a></a></span>
     </form>
     <div>
         <a href="{{url_for('index')}}" class="pure-button button-tag {{'active' if not active_tag }}">All</a>
@@ -52,7 +52,7 @@
 
                 <td class="title-col inline">{{watch.title if watch.title is not none and watch.title|length > 0 else watch.url}}
                     <a class="external" target="_blank" rel="noopener" href="{{ watch.url.replace('source:','') }}"></a>
-                    <a href="{{url_for('api_share_put_watch', uuid=watch.uuid)}}"><img style="height: 1em;display:inline-block;" src="{{url_for('static_content', group='images', filename='spread.svg')}}" /></a>
+                    <a href="{{url_for('form_share_put_watch', uuid=watch.uuid)}}"><img style="height: 1em;display:inline-block;" src="{{url_for('static_content', group='images', filename='spread.svg')}}" /></a>
 
                     {%if watch.fetch_backend == "html_webdriver" %}<img style="height: 1em; display:inline-block;" src="{{url_for('static_content', group='images', filename='Google-Chrome-icon.png')}}" />{% endif %}
 
@@ -74,7 +74,7 @@
                     {% endif %}
                 </td>
                 <td>
-                    <a {% if watch.uuid in queued_uuids %}disabled="true"{% endif %} href="{{ url_for('api_watch_checknow', uuid=watch.uuid, tag=request.args.get('tag')) }}"
+                    <a {% if watch.uuid in queued_uuids %}disabled="true"{% endif %} href="{{ url_for('form_watch_checknow', uuid=watch.uuid, tag=request.args.get('tag')) }}"
                        class="recheck pure-button button-small pure-button-primary">{% if watch.uuid in queued_uuids %}Queued{% else %}Recheck{% endif %}</a>
                     <a href="{{ url_for('edit_page', uuid=watch.uuid)}}" class="pure-button button-small pure-button-primary">Edit</a>
                     {% if watch.history|length >= 2 %}
@@ -96,7 +96,7 @@
             </li>
             {% endif %}
             <li>
-               <a href="{{ url_for('api_watch_checknow', tag=active_tag) }}" class="pure-button button-tag ">Recheck
+               <a href="{{ url_for('form_watch_checknow', tag=active_tag) }}" class="pure-button button-tag ">Recheck
                 all {% if active_tag%}in "{{active_tag}}"{%endif%}</a>
             </li>
             <li>

changedetectionio/tests/test_api.py

@@ -2,73 +2,205 @@
 
 import time
 from flask import url_for
-from . util import live_server_setup
+from .util import live_server_setup
 
-def test_setup(live_server):
-    live_server_setup(live_server)
+import json
+import uuid
 
 
-def set_response_data(test_return_data):
+def set_original_response():
+    test_return_data = """<html>
+       <body>
+     Some initial text</br>
+     <p>Which is across multiple lines</p>
+     </br>
+     So let's see what happens.  </br>
+     <div id="sametext">Some text thats the same</div>
+     <div id="changetext">Some text that will change</div>
+     </body>
+     </html>
+    """
+
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write(test_return_data)
+    return None
+
+
+def set_modified_response():
+    test_return_data = """<html>
+       <body>
+     Some initial text</br>
+     <p>which has this one new line</p>
+     </br>
+     So let's see what happens.  </br>
+     <div id="sametext">Some text thats the same</div>
+     <div id="changetext">Some text that changes</div>
+     </body>
+     </html>
+    """
+
     with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write(test_return_data)
 
+    return None
 
-def test_snapshot_api_detects_change(client, live_server):
-    test_return_data = "Some initial text"
 
-    test_return_data_modified = "Some NEW nice initial text"
+def is_valid_uuid(val):
+    try:
+        uuid.UUID(str(val))
+        return True
+    except ValueError:
+        return False
 
-    sleep_time_for_fetch_thread = 3
 
-    set_response_data(test_return_data)
+# kinda funky, but works for now
+def _extract_api_key_from_UI(client):
+    import re
+    res = client.get(
+        url_for("settings_page"),
+    )
+    # <span id="api-key">{{api_key}}</span>
 
-    # Give the endpoint time to spin up
-    time.sleep(1)
+    m = re.search('<span id="api-key">(.+?)</span>', str(res.data))
+    api_key = m.group(1)
+    return api_key.strip()
 
-    # Add our URL to the import page
-    test_url = url_for('test_endpoint', content_type="text/plain",
-                       _external=True)
+
+def test_api_simple(client, live_server):
+    live_server_setup(live_server)
+
+    api_key = _extract_api_key_from_UI(client)
+
+    # Create a watch
+    set_original_response()
+    watch_uuid = None
+
+    # Validate bad URL
+    test_url = url_for('test_endpoint', _external=True,
+                       headers={'x-api-key': api_key}, )
     res = client.post(
-        url_for("import_page"),
-        data={"urls": test_url},
+        url_for("createwatch"),
+        data=json.dumps({"url": "h://xxxxxxxxxom"}),
+        headers={'content-type': 'application/json', 'x-api-key': api_key},
         follow_redirects=True
     )
-    assert b"1 Imported" in res.data
-
-    # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
-
-    # Give the thread time to pick it up
-    time.sleep(sleep_time_for_fetch_thread)
-
-    res = client.get(
-        url_for("api_snapshot", uuid="first"),
-        follow_redirects=True
-    )
-
-    assert test_return_data.encode() == res.data
-
-    #  Make a change
-    set_response_data(test_return_data_modified)
-
-    # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
-    # Give the thread time to pick it up
-    time.sleep(sleep_time_for_fetch_thread)
-
-    res = client.get(
-        url_for("api_snapshot", uuid="first"),
-        follow_redirects=True
-    )
-
-    assert test_return_data_modified.encode() == res.data
-
-def test_snapshot_api_invalid_uuid(client, live_server):
-
-    res = client.get(
-        url_for("api_snapshot", uuid="invalid"),
-        follow_redirects=True
-    )
-
     assert res.status_code == 400
 
+    # Create new
+    res = client.post(
+        url_for("createwatch"),
+        data=json.dumps({"url": test_url, 'tag': "One, Two", "title": "My test URL"}),
+        headers={'content-type': 'application/json', 'x-api-key': api_key},
+        follow_redirects=True
+    )
+    s = json.loads(res.data)
+    assert is_valid_uuid(s['uuid'])
+    watch_uuid = s['uuid']
+    assert res.status_code == 201
+
+    time.sleep(3)
+
+    # Verify its in the list and that recheck worked
+    res = client.get(
+        url_for("createwatch"),
+        headers={'x-api-key': api_key}
+    )
+    assert watch_uuid in json.loads(res.data).keys()
+    before_recheck_info = json.loads(res.data)[watch_uuid]
+    assert before_recheck_info['last_checked'] != 0
+    assert before_recheck_info['title'] == 'My test URL'
+
+    set_modified_response()
+    # Trigger recheck of all ?recheck_all=1
+    client.get(
+        url_for("createwatch", recheck_all='1'),
+        headers={'x-api-key': api_key},
+    )
+    time.sleep(3)
+
+    # Did the recheck fire?
+    res = client.get(
+        url_for("createwatch"),
+        headers={'x-api-key': api_key},
+    )
+    after_recheck_info = json.loads(res.data)[watch_uuid]
+    assert after_recheck_info['last_checked'] != before_recheck_info['last_checked']
+    assert after_recheck_info['last_changed'] != 0
+
+    # Check history index list
+    res = client.get(
+        url_for("watchhistory", uuid=watch_uuid),
+        headers={'x-api-key': api_key},
+    )
+    history = json.loads(res.data)
+    assert len(history) == 2, "Should have two history entries (the original and the changed)"
+
+    # Fetch a snapshot by timestamp, check the right one was found
+    res = client.get(
+        url_for("watchsinglehistory", uuid=watch_uuid, timestamp=list(history.keys())[-1]),
+        headers={'x-api-key': api_key},
+    )
+    assert b'which has this one new line' in res.data
+
+    # Fetch a snapshot by 'latest'', check the right one was found
+    res = client.get(
+        url_for("watchsinglehistory", uuid=watch_uuid, timestamp='latest'),
+        headers={'x-api-key': api_key},
+    )
+    assert b'which has this one new line' in res.data
+
+    # Fetch the whole watch
+    res = client.get(
+        url_for("watch", uuid=watch_uuid),
+        headers={'x-api-key': api_key}
+    )
+    watch = json.loads(res.data)
+    # @todo how to handle None/default global values?
+    assert watch['history_n'] == 2, "Found replacement history section, which is in its own API"
+
+    # Finally delete the watch
+    res = client.delete(
+        url_for("watch", uuid=watch_uuid),
+        headers={'x-api-key': api_key},
+    )
+    assert res.status_code == 204
+
+    # Check via a relist
+    res = client.get(
+        url_for("createwatch"),
+        headers={'x-api-key': api_key}
+    )
+    watch_list = json.loads(res.data)
+    assert len(watch_list) == 0, "Watch list should be empty"
+
+
+def test_access_denied(client, live_server):
+    # `config_api_token_enabled` Should be On by default
+    res = client.get(
+        url_for("createwatch")
+    )
+    assert res.status_code == 403
+
+    res = client.get(
+        url_for("createwatch"),
+        headers={'x-api-key': "something horrible"}
+    )
+    assert res.status_code == 403
+
+    # Disable config_api_token_enabled and it should work
+    res = client.post(
+        url_for("settings_page"),
+        data={
+            "requests-time_between_check-minutes": 180,
+            "application-fetch_backend": "html_requests",
+            "application-api_access_token_enabled": ""
+        },
+        follow_redirects=True
+    )
+
+    assert b"Settings updated." in res.data
+
+    res = client.get(
+        url_for("createwatch")
+    )
+    assert res.status_code == 200

changedetectionio/tests/test_auth.py

@@ -29,7 +29,7 @@ def test_basic_auth(client, live_server):
     assert b"Updated watch." in res.data
 
     # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
     time.sleep(1)
     res = client.get(
         url_for("preview_page", uuid="first"),

changedetectionio/tests/test_backend.py

@@ -32,7 +32,7 @@ def test_check_basic_change_detection_functionality(client, live_server):
 
     # Do this a few times.. ensures we dont accidently set the status
     for n in range(3):
-        client.get(url_for("api_watch_checknow"), follow_redirects=True)
+        client.get(url_for("form_watch_checknow"), follow_redirects=True)
 
         # Give the thread time to pick it up
         time.sleep(sleep_time_for_fetch_thread)
@@ -65,7 +65,7 @@ def test_check_basic_change_detection_functionality(client, live_server):
     assert b'which has this one new line' in res.read()
 
     # Force recheck
-    res = client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    res = client.get(url_for("form_watch_checknow"), follow_redirects=True)
     assert b'1 watches are queued for rechecking.' in res.data
 
     time.sleep(sleep_time_for_fetch_thread)
@@ -93,7 +93,7 @@ def test_check_basic_change_detection_functionality(client, live_server):
 
     # Do this a few times.. ensures we dont accidently set the status
     for n in range(2):
-        client.get(url_for("api_watch_checknow"), follow_redirects=True)
+        client.get(url_for("form_watch_checknow"), follow_redirects=True)
 
         # Give the thread time to pick it up
         time.sleep(sleep_time_for_fetch_thread)
@@ -113,7 +113,7 @@ def test_check_basic_change_detection_functionality(client, live_server):
         follow_redirects=True
     )
 
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
     time.sleep(sleep_time_for_fetch_thread)
 
     res = client.get(url_for("index"))
@@ -123,6 +123,6 @@ def test_check_basic_change_detection_functionality(client, live_server):
 
     #
     # Cleanup everything
-    res = client.get(url_for("api_delete", uuid="all"), follow_redirects=True)
+    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
 

changedetectionio/tests/test_clone.py

@@ -23,7 +23,7 @@ def test_trigger_functionality(client, live_server):
 
 
     res = client.get(
-        url_for("api_clone", uuid="first"),
+        url_for("form_clone", uuid="first"),
         follow_redirects=True
     )
 

changedetectionio/tests/test_css_selector.py

@@ -89,7 +89,7 @@ def test_check_markup_css_filter_restriction(client, live_server):
     assert b"1 Imported" in res.data
 
     # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
@@ -110,7 +110,7 @@ def test_check_markup_css_filter_restriction(client, live_server):
     assert bytes(css_filter.encode('utf-8')) in res.data
 
     # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
@@ -118,7 +118,7 @@ def test_check_markup_css_filter_restriction(client, live_server):
     set_modified_response()
 
     # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
 

changedetectionio/tests/test_element_removal.py

@@ -145,7 +145,7 @@ def test_element_removal_full(client, live_server):
     assert bytes(subtractive_selectors_data.encode("utf-8")) in res.data
 
     # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
@@ -158,7 +158,7 @@ def test_element_removal_full(client, live_server):
     set_modified_response()
 
     # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)

changedetectionio/tests/test_encoding.py

@@ -39,7 +39,7 @@ def test_check_encoding_detection(client, live_server):
     )
 
     # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(2)
@@ -71,7 +71,7 @@ def test_check_encoding_detection_missing_content_type_header(client, live_serve
     )
 
     # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(2)

changedetectionio/tests/test_errorhandling.py

@@ -29,7 +29,7 @@ def test_error_handler(client, live_server):
     assert b"1 Imported" in res.data
 
     # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(3)
@@ -54,7 +54,7 @@ def test_error_text_handler(client, live_server):
     assert b"1 Imported" in res.data
 
     # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(3)

changedetectionio/tests/test_ignore_text.py

@@ -102,7 +102,7 @@ def test_check_ignore_text_functionality(client, live_server):
     assert b"1 Imported" in res.data
 
     # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
@@ -123,7 +123,7 @@ def test_check_ignore_text_functionality(client, live_server):
     assert bytes(ignore_text.encode('utf-8')) in res.data
 
     # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
@@ -137,7 +137,7 @@ def test_check_ignore_text_functionality(client, live_server):
     set_modified_ignore_response()
 
     # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
 
@@ -152,7 +152,7 @@ def test_check_ignore_text_functionality(client, live_server):
 
     # Just to be sure.. set a regular modified change..
     set_modified_original_ignore_response()
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
     time.sleep(sleep_time_for_fetch_thread)
 
     res = client.get(url_for("index"))
@@ -165,7 +165,7 @@ def test_check_ignore_text_functionality(client, live_server):
     # We should be able to see what we ignored
     assert b'<div class="ignored">new ignore stuff' in res.data
 
-    res = client.get(url_for("api_delete", uuid="all"), follow_redirects=True)
+    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
 
 def test_check_global_ignore_text_functionality(client, live_server):
@@ -200,7 +200,7 @@ def test_check_global_ignore_text_functionality(client, live_server):
     assert b"1 Imported" in res.data
 
     # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
@@ -222,7 +222,7 @@ def test_check_global_ignore_text_functionality(client, live_server):
     assert bytes(ignore_text.encode('utf-8')) in res.data
 
     # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
@@ -240,7 +240,7 @@ def test_check_global_ignore_text_functionality(client, live_server):
     set_modified_ignore_response()
 
     # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
 
@@ -251,10 +251,10 @@ def test_check_global_ignore_text_functionality(client, live_server):
 
     # Just to be sure.. set a regular modified change that will trigger it
     set_modified_original_ignore_response()
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
     time.sleep(sleep_time_for_fetch_thread)
     res = client.get(url_for("index"))
     assert b'unviewed' in res.data
 
-    res = client.get(url_for("api_delete", uuid="all"), follow_redirects=True)
+    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data

changedetectionio/tests/test_ignorehyperlinks.py

@@ -72,14 +72,14 @@ def test_render_anchor_tag_content_true(client, live_server):
 
     time.sleep(sleep_time_for_fetch_thread)
     # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
 
     # set a new html text with a modified link
     set_modified_ignore_response()
     time.sleep(sleep_time_for_fetch_thread)
 
     # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
@@ -101,7 +101,7 @@ def test_render_anchor_tag_content_true(client, live_server):
     assert b"Settings updated." in res.data
 
     # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
@@ -119,7 +119,7 @@ def test_render_anchor_tag_content_true(client, live_server):
     assert b"/test-endpoint" in res.data
 
     # Cleanup everything
-    res = client.get(url_for("api_delete", uuid="all"),
+    res = client.get(url_for("form_delete", uuid="all"),
                      follow_redirects=True)
     assert b'Deleted' in res.data
 

changedetectionio/tests/test_ignorestatuscode.py

@@ -70,12 +70,12 @@ def test_normal_page_check_works_with_ignore_status_code(client, live_server):
 
     time.sleep(sleep_time_for_fetch_thread)
     # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
 
     set_some_changed_response()
     time.sleep(sleep_time_for_fetch_thread)
     # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
@@ -105,7 +105,7 @@ def test_403_page_check_works_with_ignore_status_code(client, live_server):
     assert b"1 Imported" in res.data
 
     # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
@@ -120,7 +120,7 @@ def test_403_page_check_works_with_ignore_status_code(client, live_server):
     assert b"Updated watch." in res.data
 
     # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
@@ -128,7 +128,7 @@ def test_403_page_check_works_with_ignore_status_code(client, live_server):
     set_some_changed_response()
 
     # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
 
@@ -157,7 +157,7 @@ def test_403_page_check_fails_without_ignore_status_code(client, live_server):
     assert b"1 Imported" in res.data
 
     # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
@@ -172,7 +172,7 @@ def test_403_page_check_fails_without_ignore_status_code(client, live_server):
     assert b"Updated watch." in res.data
 
     # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
@@ -180,7 +180,7 @@ def test_403_page_check_fails_without_ignore_status_code(client, live_server):
     set_some_changed_response()
 
     # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
 

changedetectionio/tests/test_ignorewhitespace.py

@@ -80,12 +80,12 @@ def test_check_ignore_whitespace(client, live_server):
 
     time.sleep(sleep_time_for_fetch_thread)
     # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
 
     set_original_ignore_response_but_with_whitespace()
     time.sleep(sleep_time_for_fetch_thread)
     # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)

changedetectionio/tests/test_import.py

@@ -25,7 +25,7 @@ https://example.com tag1, other tag"""
     assert b"3 Imported" in res.data
     assert b"tag1" in res.data
     assert b"other tag" in res.data
-    res = client.get(url_for("api_delete", uuid="all"), follow_redirects=True)
+    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
 
     # Clear flask alerts
     res = client.get( url_for("index"))
@@ -50,7 +50,7 @@ def xtest_import_skip_url(client, live_server):
     assert b"1 Imported" in res.data
     assert b"ht000000broken" in res.data
     assert b"1 Skipped" in res.data
-    res = client.get(url_for("api_delete", uuid="all"), follow_redirects=True)
+    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
     # Clear flask alerts
     res = client.get( url_for("index"))
 
@@ -79,7 +79,7 @@ def test_import_distillio(client, live_server):
 
     # Give the endpoint time to spin up
     time.sleep(1)
-    client.get(url_for("api_delete", uuid="all"), follow_redirects=True)
+    client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
     res = client.post(
         url_for("import_page"),
         data={
@@ -115,6 +115,6 @@ def test_import_distillio(client, live_server):
     assert b"nice stuff" in res.data
     assert b"nerd-news" in res.data
 
-    res = client.get(url_for("api_delete", uuid="all"), follow_redirects=True)
+    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
     # Clear flask alerts
     res = client.get(url_for("index"))

changedetectionio/tests/test_jsonpath_selector.py

@@ -171,7 +171,7 @@ def test_check_json_without_filter(client, live_server):
     )
 
     # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(3)
@@ -203,7 +203,7 @@ def test_check_json_filter(client, live_server):
     assert b"1 Imported" in res.data
 
     # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(3)
@@ -229,7 +229,7 @@ def test_check_json_filter(client, live_server):
     assert bytes(json_filter.encode('utf-8')) in res.data
 
     # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(3)
@@ -237,7 +237,7 @@ def test_check_json_filter(client, live_server):
     set_modified_response()
 
     # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
     # Give the thread time to pick it up
     time.sleep(4)
 
@@ -288,7 +288,7 @@ def test_check_json_filter_bool_val(client, live_server):
     time.sleep(3)
 
     # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(3)
@@ -296,7 +296,7 @@ def test_check_json_filter_bool_val(client, live_server):
     set_modified_response()
 
     # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
     # Give the thread time to pick it up
     time.sleep(3)
 
@@ -327,7 +327,7 @@ def test_check_json_ext_filter(client, live_server):
     assert b"1 Imported" in res.data
 
     # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(3)
@@ -353,7 +353,7 @@ def test_check_json_ext_filter(client, live_server):
     assert bytes(json_filter.encode('utf-8')) in res.data
 
     # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(3)
@@ -361,7 +361,7 @@ def test_check_json_ext_filter(client, live_server):
     set_modified_ext_response()
 
     # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
     # Give the thread time to pick it up
     time.sleep(4)
 

changedetectionio/tests/test_nonrenderable_pages.py

@@ -0,0 +1,102 @@
+#!/usr/bin/python3
+
+import time
+from flask import url_for
+from urllib.request import urlopen
+from .util import set_original_response, set_modified_response, live_server_setup
+
+sleep_time_for_fetch_thread = 3
+
+
+def set_nonrenderable_response():
+    test_return_data = """<html>
+    <head><title>modified head title</title></head>
+    <!-- like when some angular app was broken and doesnt render or whatever -->
+    <body>
+     </body>
+     </html>
+    """
+
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write(test_return_data)
+
+    return None
+
+def test_check_basic_change_detection_functionality(client, live_server):
+    set_original_response()
+    live_server_setup(live_server)
+
+    # Add our URL to the import page
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": url_for('test_endpoint', _external=True)},
+        follow_redirects=True
+    )
+
+    assert b"1 Imported" in res.data
+
+    time.sleep(sleep_time_for_fetch_thread)
+
+    # Do this a few times.. ensures we dont accidently set the status
+    for n in range(3):
+        client.get(url_for("form_watch_checknow"), follow_redirects=True)
+
+        # Give the thread time to pick it up
+        time.sleep(sleep_time_for_fetch_thread)
+
+        # It should report nothing found (no new 'unviewed' class)
+        res = client.get(url_for("index"))
+        assert b'unviewed' not in res.data
+
+
+    #####################
+    client.post(
+        url_for("settings_page"),
+        data={"application-empty_pages_are_a_change": "",
+              "requests-time_between_check-minutes": 180,
+              'application-fetch_backend': "html_requests"},
+        follow_redirects=True
+    )
+
+    # this should not trigger a change, because no good text could be converted from the HTML
+    set_nonrenderable_response()
+
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
+
+    # It should report nothing found (no new 'unviewed' class)
+    res = client.get(url_for("index"))
+    assert b'unviewed' not in res.data
+
+
+    # ok now do the opposite
+
+    client.post(
+        url_for("settings_page"),
+        data={"application-empty_pages_are_a_change": "y",
+              "requests-time_between_check-minutes": 180,
+              'application-fetch_backend': "html_requests"},
+        follow_redirects=True
+    )
+    set_modified_response()
+
+
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
+
+    # It should report nothing found (no new 'unviewed' class)
+    res = client.get(url_for("index"))
+    assert b'unviewed' in res.data
+
+
+
+
+    #
+    # Cleanup everything
+    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    assert b'Deleted' in res.data
+

changedetectionio/tests/test_notification.py

@@ -36,7 +36,7 @@ def test_check_notification(client, live_server):
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
-        url_for("api_watch_add"),
+        url_for("form_watch_add"),
         data={"url": test_url, "tag": ''},
         follow_redirects=True
     )
@@ -98,7 +98,7 @@ def test_check_notification(client, live_server):
     notification_submission = None
 
     # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
     time.sleep(3)
     # Verify what was sent as a notification, this file should exist
     with open("test-datastore/notification.txt", "r") as f:
@@ -133,7 +133,7 @@ def test_check_notification(client, live_server):
 
     # This should insert the {current_snapshot}
     set_more_modified_response()
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
     time.sleep(3)
     # Verify what was sent as a notification, this file should exist
     with open("test-datastore/notification.txt", "r") as f:
@@ -146,17 +146,17 @@ def test_check_notification(client, live_server):
     os.unlink("test-datastore/notification.txt")
 
     # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
     time.sleep(1)
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
     time.sleep(1)
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
     time.sleep(1)
     assert os.path.exists("test-datastore/notification.txt") == False
 
     # cleanup for the next
     client.get(
-        url_for("api_delete", uuid="all"),
+        url_for("form_delete", uuid="all"),
         follow_redirects=True
     )
 
@@ -168,7 +168,7 @@ def test_notification_validation(client, live_server):
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
-        url_for("api_watch_add"),
+        url_for("form_watch_add"),
         data={"url": test_url, "tag": 'nice one'},
         follow_redirects=True
     )
@@ -208,6 +208,6 @@ def test_notification_validation(client, live_server):
 
     # cleanup for the next
     client.get(
-        url_for("api_delete", uuid="all"),
+        url_for("form_delete", uuid="all"),
         follow_redirects=True
     )

changedetectionio/tests/test_notification_errors.py

@@ -16,7 +16,7 @@ def test_check_notification_error_handling(client, live_server):
     # use a different URL so that it doesnt interfere with the actual check until we are ready
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
-        url_for("api_watch_add"),
+        url_for("form_watch_add"),
         data={"url": "https://changedetection.io/CHANGELOG.txt", "tag": ''},
         follow_redirects=True
     )

changedetectionio/tests/test_share_watch.py

@@ -41,7 +41,7 @@ def test_share_watch(client, live_server):
 
     # click share the link
     res = client.get(
-        url_for("api_share_put_watch", uuid="first"),
+        url_for("form_share_put_watch", uuid="first"),
         follow_redirects=True
     )
 
@@ -54,7 +54,7 @@ def test_share_watch(client, live_server):
 
     # Now delete what we have, we will try to re-import it
     # Cleanup everything
-    res = client.get(url_for("api_delete", uuid="all"), follow_redirects=True)
+    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
 
     # Add our URL to the import page

changedetectionio/tests/test_source.py

@@ -39,7 +39,7 @@ def test_check_basic_change_detection_functionality_source(client, live_server):
     set_modified_response()
 
     # Force recheck
-    res = client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    res = client.get(url_for("form_watch_checknow"), follow_redirects=True)
     assert b'1 watches are queued for rechecking.' in res.data
 
     time.sleep(5)

changedetectionio/tests/test_trigger.py

@@ -76,7 +76,7 @@ def test_trigger_functionality(client, live_server):
     assert b"1 Imported" in res.data
 
     # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
@@ -99,7 +99,7 @@ def test_trigger_functionality(client, live_server):
     assert bytes(trigger_text.encode('utf-8')) in res.data
 
     # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
@@ -113,7 +113,7 @@ def test_trigger_functionality(client, live_server):
     set_modified_original_ignore_response()
 
     # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
 
@@ -125,7 +125,7 @@ def test_trigger_functionality(client, live_server):
     time.sleep(sleep_time_for_fetch_thread)
     set_modified_with_trigger_text_response()
 
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
     time.sleep(sleep_time_for_fetch_thread)
     res = client.get(url_for("index"))
     assert b'unviewed' in res.data

changedetectionio/tests/test_trigger_regex.py

@@ -43,7 +43,7 @@ def test_trigger_regex_functionality(client, live_server):
     assert b"1 Imported" in res.data
 
     # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
@@ -65,7 +65,7 @@ def test_trigger_regex_functionality(client, live_server):
     with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write("some new noise")
 
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
     time.sleep(sleep_time_for_fetch_thread)
 
     # It should report nothing found (nothing should match the regex)
@@ -75,7 +75,7 @@ def test_trigger_regex_functionality(client, live_server):
     with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write("regex test123<br/>\nsomething 123")
 
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
     time.sleep(sleep_time_for_fetch_thread)
     res = client.get(url_for("index"))
     assert b'unviewed' in res.data

changedetectionio/tests/test_trigger_regex_with_filter.py

@@ -43,7 +43,7 @@ def test_trigger_regex_functionality(client, live_server):
     assert b"1 Imported" in res.data
 
     # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
@@ -66,7 +66,7 @@ def test_trigger_regex_functionality(client, live_server):
     with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write("<html>some new noise with cool stuff2 ok</html>")
 
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
     time.sleep(sleep_time_for_fetch_thread)
 
     # It should report nothing found (nothing should match the regex and filter)
@@ -76,7 +76,7 @@ def test_trigger_regex_functionality(client, live_server):
     with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write("<html>some new noise with <span id=in-here>cool stuff6</span> ok</html>")
 
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
     time.sleep(sleep_time_for_fetch_thread)
     res = client.get(url_for("index"))
     assert b'unviewed' in res.data

changedetectionio/tests/test_xpath_selector.py

@@ -65,7 +65,7 @@ def test_check_markup_xpath_filter_restriction(client, live_server):
     assert b"1 Imported" in res.data
 
     # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
@@ -89,7 +89,7 @@ def test_check_markup_xpath_filter_restriction(client, live_server):
     set_modified_response()
 
     # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
 
@@ -121,7 +121,7 @@ def test_xpath_validation(client, live_server):
 
 # actually only really used by the distll.io importer, but could be handy too
 def test_check_with_prefix_css_filter(client, live_server):
-    res = client.get(url_for("api_delete", uuid="all"), follow_redirects=True)
+    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
 
     # Give the endpoint time to spin up
@@ -158,4 +158,4 @@ def test_check_with_prefix_css_filter(client, live_server):
     assert b"Some text thats the same" in res.data #in selector
     assert b"Some text that will change" not in res.data #not in selector
 
-    client.get(url_for("api_delete", uuid="all"), follow_redirects=True)
+    client.get(url_for("form_delete", uuid="all"), follow_redirects=True)

changedetectionio/update_worker.py

@@ -52,6 +52,10 @@ class update_worker(threading.Thread):
                             raise Exception("Error - returned data from the fetch handler SHOULD be bytes")
                     except PermissionError as e:
                         self.app.logger.error("File permission error updating", uuid, str(e))
+                    except content_fetcher.ReplyWithContentButNoText as e:
+                        # Totally fine, it's by choice - just continue on, nothing more to care about
+                        # Page had elements/content but no renderable text
+                        pass
                     except content_fetcher.EmptyReply as e:
                         # Some kind of custom to-str handler in the exception handler that does this?
                         err_text = "EmptyReply: Status Code {}".format(e.status_code)

requirements.txt

@@ -6,6 +6,7 @@ timeago ~=1.0
 inscriptis ~= 2.2
 feedgen ~= 0.9
 flask-login ~= 0.5
+flask_restful
 pytz
 
 # Set these versions together to avoid a RequestsDependencyWarning