Handle disabled situation

README-pip.md

@@ -16,6 +16,13 @@ Live your data-life *pro-actively* instead of *re-actively*, do not rely on mani
 
 <img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/screenshot.png" style="max-width:100%;" alt="Self-hosted web page change monitoring"  title="Self-hosted web page change monitoring"  />
 
+
+**Get your own private instance now! Let us host it for you!**
+
+[**Try our $6.99/month subscription - unlimited checks, watches and notifications!**](https://lemonade.changedetection.io/start), choose from different geographical locations, let us handle everything for you. 
+
+
+
 #### Example use cases
 
 Know when ...
@@ -58,14 +65,3 @@ Then visit http://127.0.0.1:5000 , You should now be able to access the UI.
 
 See https://github.com/dgtlmoon/changedetection.io for more information.
 
-
-
-### Support us
-
-Do you use changedetection.io to make money? does it save you time or money? Does it make your life easier? less stressful? Remember, we write this software when we should be doing actual paid work, we have to buy food and pay rent just like you.
-
-Please support us, even small amounts help a LOT.
-
-BTC `1PLFN327GyUarpJd7nVe7Reqg9qHx5frNn`
-
-<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/btc-support.png" style="max-width:50%;" alt="Support us!"  />

changedetectionio/__init__.py

@@ -36,9 +36,12 @@ from flask import (
     url_for,
 )
 from flask_login import login_required
+from flask_restful import abort, Api
+
 from flask_wtf import CSRFProtect
 
 from changedetectionio import html_tools
+from changedetectionio.api import api_v1
 
 __version__ = '0.39.13.1'
 
@@ -78,6 +81,8 @@ csrf.init_app(app)
 
 notification_debug_log=[]
 
+watch_api = Api(app, decorators=[csrf.exempt])
+
 def init_app_secret(datastore_path):
     secret = ""
 
@@ -179,6 +184,25 @@ def changedetection_app(config=None, datastore_o=None):
     login_manager.login_view = 'login'
     app.secret_key = init_app_secret(config['datastore_path'])
 
+
+    watch_api.add_resource(api_v1.WatchSingleHistory,
+                           '/api/v1/watch/<string:uuid>/history/<string:timestamp>',
+                           resource_class_kwargs={'datastore': datastore, 'update_q': update_q})
+
+    watch_api.add_resource(api_v1.WatchHistory,
+                           '/api/v1/watch/<string:uuid>/history',
+                           resource_class_kwargs={'datastore': datastore})
+
+    watch_api.add_resource(api_v1.CreateWatch, '/api/v1/watch',
+                           resource_class_kwargs={'datastore': datastore, 'update_q': update_q})
+
+    watch_api.add_resource(api_v1.Watch, '/api/v1/watch/<string:uuid>',
+                           resource_class_kwargs={'datastore': datastore, 'update_q': update_q})
+
+
+
+
+
     # Setup cors headers to allow all domains
     # https://flask-cors.readthedocs.io/en/latest/
     #    CORS(app)
@@ -367,6 +391,8 @@ def changedetection_app(config=None, datastore_o=None):
 
             if limit_tag != None:
                 # Support for comma separated list of tags.
+                if watch['tag'] is None:
+                    continue
                 for tag_in_watch in watch['tag'].split(','):
                     tag_in_watch = tag_in_watch.strip()
                     if tag_in_watch == limit_tag:
@@ -520,16 +546,9 @@ def changedetection_app(config=None, datastore_o=None):
 
         # Defaults for proxy choice
         if datastore.proxy_list is not None:  # When enabled
-            system_proxy = datastore.data['settings']['requests']['proxy']
-            if default['proxy'] is None:
-                default['proxy'] = system_proxy
-            else:
-                # Does the chosen one exist?
-                if not any(default['proxy'] in tup for tup in datastore.proxy_list):
-                    default['proxy'] = datastore.proxy_list[0][0]
-
-            # Used by the form handler to keep or remove the proxy settings
-            default['proxy_list'] = datastore.proxy_list
+            # Radio needs '' not None, or incase that the chosen one no longer exists
+            if default['proxy'] is None or not any(default['proxy'] in tup for tup in datastore.proxy_list):
+                default['proxy'] = ''
 
         # proxy_override set to the json/text list of the items
         form = forms.watchForm(formdata=request.form if request.method == 'POST' else None,
@@ -540,9 +559,7 @@ def changedetection_app(config=None, datastore_o=None):
             # @todo - Couldn't get setattr() etc dynamic addition working, so remove it instead
             del form.proxy
         else:
-            form.proxy.choices = datastore.proxy_list
-            if default['proxy'] is None:
-                form.proxy.default='http://hello'
+            form.proxy.choices = [('', 'Default')] + datastore.proxy_list
 
         if request.method == 'POST' and form.validate():
             extra_update_obj = {}
@@ -579,6 +596,10 @@ def changedetection_app(config=None, datastore_o=None):
                 if len(datastore.data['watching'][uuid]['history']):
                     extra_update_obj['previous_md5'] = get_current_checksum_include_ignore_text(uuid=uuid)
 
+            # Be sure proxy value is None
+            if datastore.proxy_list is not None and form.data['proxy'] == '':
+                extra_update_obj['proxy'] = None
+
             datastore.data['watching'][uuid].update(form.data)
             datastore.data['watching'][uuid].update(extra_update_obj)
 
@@ -605,12 +626,12 @@ def changedetection_app(config=None, datastore_o=None):
             if request.method == 'POST' and not form.validate():
                 flash("An error occurred, please see below.", "error")
 
-
             output = render_template("edit.html",
                                      uuid=uuid,
                                      watch=datastore.data['watching'][uuid],
                                      form=form,
                                      has_empty_checktime=using_default_check_time,
+                                     using_global_webdriver_wait=default['webdriver_delay'] is None,
                                      current_base_url=datastore.data['settings']['application']['base_url'],
                                      emailprefix=os.getenv('NOTIFICATION_MAIL_BUTTON_PREFIX', False)
                                      )
@@ -676,6 +697,7 @@ def changedetection_app(config=None, datastore_o=None):
                                  form=form,
                                  current_base_url = datastore.data['settings']['application']['base_url'],
                                  hide_remove_pass=os.getenv("SALTED_PASS", False),
+                                 api_key=datastore.data['settings']['application'].get('api_access_token'),
                                  emailprefix=os.getenv('NOTIFICATION_MAIL_BUTTON_PREFIX', False))
 
         return output
@@ -874,27 +896,6 @@ def changedetection_app(config=None, datastore_o=None):
 
         return output
 
-    @app.route("/api/<string:uuid>/snapshot/current", methods=['GET'])
-    @login_required
-    def api_snapshot(uuid):
-
-        # More for testing, possible to return the first/only
-        if uuid == 'first':
-            uuid = list(datastore.data['watching'].keys()).pop()
-
-        try:
-            watch = datastore.data['watching'][uuid]
-        except KeyError:
-            return abort(400, "No history found for the specified link, bad link?")
-
-        newest = list(watch['history'].keys())[-1]
-        with open(watch['history'][newest], 'r') as f:
-            content = f.read()
-
-        resp = make_response(content)
-        resp.headers['Content-Type'] = 'text/plain'
-        return resp
-
     @app.route("/favicon.ico", methods=['GET'])
     def favicon():
         return send_from_directory("static/images", path="favicon.ico")
@@ -1005,7 +1006,7 @@ def changedetection_app(config=None, datastore_o=None):
 
     @app.route("/api/add", methods=['POST'])
     @login_required
-    def api_watch_add():
+    def form_watch_add():
         from changedetectionio import forms
         form = forms.quickWatchForm(request.form)
 
@@ -1031,7 +1032,7 @@ def changedetection_app(config=None, datastore_o=None):
 
     @app.route("/api/delete", methods=['GET'])
     @login_required
-    def api_delete():
+    def form_delete():
         uuid = request.args.get('uuid')
 
         if uuid != 'all' and not uuid in datastore.data['watching'].keys():
@@ -1048,7 +1049,7 @@ def changedetection_app(config=None, datastore_o=None):
 
     @app.route("/api/clone", methods=['GET'])
     @login_required
-    def api_clone():
+    def form_clone():
         uuid = request.args.get('uuid')
         # More for testing, possible to return the first/only
         if uuid == 'first':
@@ -1062,7 +1063,7 @@ def changedetection_app(config=None, datastore_o=None):
 
     @app.route("/api/checknow", methods=['GET'])
     @login_required
-    def api_watch_checknow():
+    def form_watch_checknow():
 
         tag = request.args.get('tag')
         uuid = request.args.get('uuid')
@@ -1099,7 +1100,7 @@ def changedetection_app(config=None, datastore_o=None):
 
     @app.route("/api/share-url", methods=['GET'])
     @login_required
-    def api_share_put_watch():
+    def form_share_put_watch():
         """Given a watch UUID, upload the info and return a share-link
            the share-link can be imported/added"""
         import requests

changedetectionio/api/api_v1.py

@@ -0,0 +1,125 @@
+from flask_restful import abort, Resource
+from flask import request, make_response
+import validators
+from . import auth
+
+
+
+# https://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html
+
+class Watch(Resource):
+    def __init__(self, **kwargs):
+        # datastore is a black box dependency
+        self.datastore = kwargs['datastore']
+        self.update_q = kwargs['update_q']
+
+    # Get information about a single watch, excluding the history list (can be large)
+    # curl http://localhost:4000/api/v1/watch/<string:uuid>
+    # ?recheck=true
+    @auth.check_token
+    def get(self, uuid):
+        from copy import deepcopy
+        watch = deepcopy(self.datastore.data['watching'].get(uuid))
+        if not watch:
+            abort(404, message='No watch exists with the UUID of {}'.format(uuid))
+
+        if request.args.get('recheck'):
+            self.update_q.put(uuid)
+            return "OK", 200
+
+        # Return without history, get that via another API call
+        watch['history_n'] = len(watch['history'])
+        del (watch['history'])
+        return watch
+
+    @auth.check_token
+    def delete(self, uuid):
+        if not self.datastore.data['watching'].get(uuid):
+            abort(400, message='No watch exists with the UUID of {}'.format(uuid))
+
+        self.datastore.delete(uuid)
+        return 'OK', 204
+
+
+class WatchHistory(Resource):
+    def __init__(self, **kwargs):
+        # datastore is a black box dependency
+        self.datastore = kwargs['datastore']
+
+    # Get a list of available history for a watch by UUID
+    # curl http://localhost:4000/api/v1/watch/<string:uuid>/history
+    def get(self, uuid):
+        watch = self.datastore.data['watching'].get(uuid)
+        if not watch:
+            abort(404, message='No watch exists with the UUID of {}'.format(uuid))
+        return watch['history'], 200
+
+
+class WatchSingleHistory(Resource):
+    def __init__(self, **kwargs):
+        # datastore is a black box dependency
+        self.datastore = kwargs['datastore']
+
+    # Read a given history snapshot and return its content
+    # <string:timestamp> or "latest"
+    # curl http://localhost:4000/api/v1/watch/<string:uuid>/history/<int:timestamp>
+    @auth.check_token
+    def get(self, uuid, timestamp):
+        watch = self.datastore.data['watching'].get(uuid)
+        if not watch:
+            abort(404, message='No watch exists with the UUID of {}'.format(uuid))
+
+        if not len(watch['history']):
+            abort(404, message='Watch found but no history exists for the UUID {}'.format(uuid))
+
+        if timestamp == 'latest':
+            timestamp = list(watch['history'].keys())[-1]
+
+        with open(watch['history'][timestamp], 'r') as f:
+            content = f.read()
+
+        response = make_response(content, 200)
+        response.mimetype = "text/plain"
+        return response
+
+
+class CreateWatch(Resource):
+    def __init__(self, **kwargs):
+        # datastore is a black box dependency
+        self.datastore = kwargs['datastore']
+        self.update_q = kwargs['update_q']
+
+    @auth.check_token
+    def post(self):
+        # curl http://localhost:4000/api/v1/watch -H "Content-Type: application/json" -d '{"url": "https://my-nice.com", "tag": "one, two" }'
+        json_data = request.get_json()
+        tag = json_data['tag'].strip() if json_data.get('tag') else ''
+
+        if not validators.url(json_data['url'].strip()):
+            return "Invalid or unsupported URL", 400
+
+        extras = {'title': json_data['title'].strip()} if json_data.get('title') else {}
+
+        new_uuid = self.datastore.add_watch(url=json_data['url'].strip(), tag=tag, extras=extras)
+        self.update_q.put(new_uuid)
+        return {'uuid': new_uuid}, 201
+
+    # Return concise list of available watches and some very basic info
+    # curl http://localhost:4000/api/v1/watch|python -mjson.tool
+    # ?recheck_all=1 to recheck all
+    @auth.check_token
+    def get(self):
+        list = {}
+        for k, v in self.datastore.data['watching'].items():
+            list[k] = {'url': v['url'],
+                       'title': v['title'],
+                       'last_checked': v['last_checked'],
+                       'last_changed': v['last_changed'],
+                       'last_error': v['last_error']}
+
+        if request.args.get('recheck_all'):
+            for uuid in self.datastore.data['watching'].keys():
+                self.update_q.put(uuid)
+            return {'status': "OK"}, 200
+
+        return list, 200

changedetectionio/api/auth.py

@@ -0,0 +1,33 @@
+from flask import request, make_response, jsonify
+from functools import wraps
+
+
+# Simple API auth key comparison
+# @todo - Maybe short lived token in the future?
+
+def check_token(f):
+    @wraps(f)
+    def decorated(*args, **kwargs):
+        datastore = args[0].datastore
+
+        config_api_token_enabled = datastore.data['settings']['application'].get('api_access_token_enabled')
+        if not config_api_token_enabled:
+            return
+
+        try:
+            api_key_header = request.headers['x-api-key']
+        except KeyError:
+            return make_response(
+                jsonify("No authorization x-api-key header."), 403
+            )
+
+        config_api_token = datastore.data['settings']['application'].get('api_access_token')
+
+        if api_key_header != config_api_token:
+            return make_response(
+                jsonify("Invalid access - API key invalid."), 403
+            )
+
+        return f(*args, **kwargs)
+
+    return decorated

changedetectionio/content_fetcher.py

@@ -3,17 +3,22 @@ import chardet
 import os
 import requests
 import time
-import urllib3.exceptions
 import sys
 
-
 class EmptyReply(Exception):
     def __init__(self, status_code, url):
         # Set this so we can use it in other parts of the app
         self.status_code = status_code
         self.url = url
         return
+    pass
 
+class ReplyWithContentButNoText(Exception):
+    def __init__(self, status_code, url):
+        # Set this so we can use it in other parts of the app
+        self.status_code = status_code
+        self.url = url
+        return
     pass
 
 
@@ -28,6 +33,9 @@ class Fetcher():
     system_http_proxy = os.getenv('HTTP_PROXY')
     system_https_proxy = os.getenv('HTTPS_PROXY')
 
+    # Time ONTOP of the system defined env minimum time
+    render_extract_delay=0
+
     @abstractmethod
     def get_error(self):
         return self.error
@@ -97,7 +105,7 @@ class base_html_playwright(Fetcher):
         self.browser_type = os.getenv("PLAYWRIGHT_BROWSER_TYPE", 'chromium').strip('"')
         self.command_executor = os.getenv(
             "PLAYWRIGHT_DRIVER_URL",
-            'ws://playwright-chrome:3000/playwright'
+            'ws://playwright-chrome:3000'
         ).strip('"')
 
         # If any proxy settings are enabled, then we should setup the proxy object
@@ -147,7 +155,7 @@ class base_html_playwright(Fetcher):
                 # - `'commit'` - consider operation to be finished when network response is received and the document started loading.
                 # Better to not use any smarts from Playwright and just wait an arbitrary number of seconds
                 # This seemed to solve nearly all 'TimeoutErrors'
-                extra_wait = int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5))
+                extra_wait = int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)) + self.render_extract_delay
                 page.wait_for_timeout(extra_wait * 1000)
             except playwright._impl._api_types.TimeoutError as e:
                 raise EmptyReply(url=url, status_code=None)
@@ -155,6 +163,9 @@ class base_html_playwright(Fetcher):
             if response is None:
                 raise EmptyReply(url=url, status_code=None)
 
+            if len(page.content().strip()) == 0:
+                raise EmptyReply(url=url, status_code=None)
+
             self.status_code = response.status
             self.content = page.content()
             self.headers = response.all_headers()
@@ -240,7 +251,7 @@ class base_html_webdriver(Fetcher):
         # raise EmptyReply(url=url, status_code=r.status_code)
 
         # @todo - dom wait loaded?
-        time.sleep(int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)))
+        time.sleep(int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)) + self.render_extract_delay)
         self.content = self.driver.page_source
         self.headers = {}
         self.screenshot = self.driver.get_screenshot_as_png()

changedetectionio/fetch_site_status.py

@@ -97,7 +97,13 @@ class perform_site_check():
         proxy_args = self.set_proxy_from_list(watch)
         fetcher = klass(proxy_override=proxy_args)
 
-        # Proxy List support
+        # Configurable per-watch or global extra delay before extracting text (for webDriver types)
+        system_webdriver_delay = self.datastore.data['settings']['application'].get('webdriver_delay', None)
+        if watch['webdriver_delay'] is not None:
+            fetcher.render_extract_delay = watch['webdriver_delay']
+        elif system_webdriver_delay is not None:
+            fetcher.render_extract_delay = system_webdriver_delay
+
         fetcher.run(url, timeout, request_headers, request_body, request_method, ignore_status_code)
 
         # Fetching complete, now filters
@@ -178,6 +184,11 @@ class perform_site_check():
         # Re #340 - return the content before the 'ignore text' was applied
         text_content_before_ignored_filter = stripped_text_from_html.encode('utf-8')
 
+        # Treat pages with no renderable text content as a change? No by default
+        empty_pages_are_a_change = self.datastore.data['settings']['application'].get('empty_pages_are_a_change', False)
+        if not is_json and not empty_pages_are_a_change and len(stripped_text_from_html.strip()) == 0:
+            raise content_fetcher.ReplyWithContentButNoText(url=url, status_code=200)
+
         # We rely on the actual text in the html output.. many sites have random script vars etc,
         # in the future we'll implement other mechanisms.
 

changedetectionio/forms.py

@@ -318,6 +318,7 @@ class commonSettingsForm(Form):
     notification_format = SelectField('Notification format', choices=valid_notification_formats.keys(), default=default_notification_format)
     fetch_backend = RadioField(u'Fetch method', choices=content_fetcher.available_fetchers(), validators=[ValidateContentFetcherIsReady()])
     extract_title_as_title = BooleanField('Extract <title> from document and use as watch title', default=False)
+    webdriver_delay = IntegerField('Wait seconds before extracting text', validators=[validators.Optional(), validators.NumberRange(min=1, message="Should contain one or more seconds")] )
 
 class watchForm(commonSettingsForm):
 
@@ -370,8 +371,10 @@ class globalSettingsApplicationForm(commonSettingsForm):
     ignore_whitespace = BooleanField('Ignore whitespace')
     real_browser_save_screenshot = BooleanField('Save last screenshot when using Chrome?')
     removepassword_button = SubmitField('Remove password', render_kw={"class": "pure-button pure-button-primary"})
+    empty_pages_are_a_change =  BooleanField('Treat empty pages as a change?', default=False)
     render_anchor_tag_content = BooleanField('Render anchor tag content', default=False)
     fetch_backend = RadioField('Fetch Method', default="html_requests", choices=content_fetcher.available_fetchers(), validators=[ValidateContentFetcherIsReady()])
+    api_access_token_enabled = BooleanField('API access token security check enabled', default=True, validators=[validators.Optional()])
     password = SaltyPasswordField()
 
 

changedetectionio/model/App.py

@@ -27,9 +27,11 @@ class model(dict):
                     'proxy': None # Preferred proxy connection
                 },
                 'application': {
+                    'api_access_token_enabled': True,
                     'password': False,
                     'base_url' : None,
                     'extract_title_as_title': False,
+                    'empty_pages_are_a_change': False,
                     'fetch_backend': os.getenv("DEFAULT_FETCH_BACKEND", "html_requests"),
                     'global_ignore_text': [], # List of text to ignore when calculating the comparison checksum
                     'global_subtractive_selectors': [],
@@ -41,7 +43,8 @@ class model(dict):
                     'notification_body': default_notification_body,
                     'notification_format': default_notification_format,
                     'real_browser_save_screenshot': True,
-                    'schema_version' : 0
+                    'schema_version' : 0,
+                    'webdriver_delay': None  # Extra delay in seconds before extracting text
                 }
             }
         }

changedetectionio/model/Watch.py

@@ -43,7 +43,8 @@ class model(dict):
             # Re #110, so then if this is set to None, we know to use the default value instead
             # Requires setting to None on submit if it's the same as the default
             # Should be all None by default, so we use the system default in this case.
-            'time_between_check': {'weeks': None, 'days': None, 'hours': None, 'minutes': None, 'seconds': None}
+            'time_between_check': {'weeks': None, 'days': None, 'hours': None, 'minutes': None, 'seconds': None},
+            'webdriver_delay': None
         }
 
     def __init__(self, *arg, **kw):

changedetectionio/static/js/global-settings.js

@@ -0,0 +1,36 @@
+$(document).ready(function () {
+    function toggle() {
+        if ($('input[name="application-fetch_backend"]:checked').val() != 'html_requests') {
+            $('#requests-override-options').hide();
+            $('#webdriver-override-options').show();
+        } else {
+            $('#requests-override-options').show();
+            $('#webdriver-override-options').hide();
+        }
+    }
+
+    $('input[name="application-fetch_backend"]').click(function (e) {
+        toggle();
+    });
+    toggle();
+
+    $("#api-key").hover(
+        function () {
+            $("#api-key-copy").html('copy').fadeIn();
+        },
+        function () {
+            $("#api-key-copy").hide();
+        }
+    ).click(function (e) {
+        $("#api-key-copy").html('copied');
+        var range = document.createRange();
+        var n = $("#api-key")[0];
+        range.selectNode(n);
+        window.getSelection().removeAllRanges();
+        window.getSelection().addRange(range);
+        document.execCommand("copy");
+        window.getSelection().removeAllRanges();
+
+    });
+});
+

changedetectionio/static/js/watch-settings.js

@@ -2,8 +2,10 @@ $(document).ready(function() {
     function toggle() {
         if ($('input[name="fetch_backend"]:checked').val() != 'html_requests') {
             $('#requests-override-options').hide();
+            $('#webdriver-override-options').show();
         } else {
             $('#requests-override-options').show();
+            $('#webdriver-override-options').hide();
         }
     }
     $('input[name="fetch_backend"]').click(function (e) {

changedetectionio/static/styles/styles.css

@@ -284,6 +284,11 @@ footer {
 .pure-form {
   /* The input fields with errors */
   /* The list of errors */ }
+  .pure-form fieldset {
+    padding-top: 0px; }
+    .pure-form fieldset ul {
+      padding-bottom: 0px;
+      margin-bottom: 0px; }
   .pure-form .pure-control-group, .pure-form .pure-group, .pure-form .pure-controls {
     padding-bottom: 1em; }
     .pure-form .pure-control-group div, .pure-form .pure-group div, .pure-form .pure-controls div {
@@ -447,4 +452,13 @@ ul {
 .time-check-widget tr {
   display: inline; }
   .time-check-widget tr input[type="number"] {
-    width: 4em; }
+    width: 5em; }
+
+#webdriver-override-options input[type="number"] {
+  width: 5em; }
+
+#api-key:hover {
+  cursor: pointer; }
+
+#api-key-copy {
+  color: #0078e7; }

changedetectionio/static/styles/styles.scss

@@ -375,6 +375,13 @@ footer {
 
 
 .pure-form {
+    fieldset {
+        padding-top: 0px;
+        ul {
+            padding-bottom: 0px;
+            margin-bottom: 0px;
+        }
+    }
     .pure-control-group, .pure-group, .pure-controls {
         padding-bottom: 1em;
         div {
@@ -637,7 +644,23 @@ ul {
     tr {
         display: inline;
         input[type="number"] {
-            width: 4em;
+            width: 5em;
         }
     }
-}
+}
+
+#webdriver-override-options {
+        input[type="number"] {
+            width: 5em;
+        }
+}
+
+#api-key {
+  &:hover {
+    cursor: pointer;
+  }
+}
+
+#api-key-copy {
+  color: #0078e7;
+}

changedetectionio/store.py

@@ -12,6 +12,7 @@ from os import mkdir, path, unlink
 from threading import Lock
 import re
 import requests
+import secrets
 
 from . model import App, Watch
 
@@ -107,10 +108,13 @@ class ChangeDetectionStore:
 
         # Generate the URL access token for RSS feeds
         if not 'rss_access_token' in self.__data['settings']['application']:
-            import secrets
             secret = secrets.token_hex(16)
             self.__data['settings']['application']['rss_access_token'] = secret
 
+        # Generate the API access token
+        if not 'api_access_token' in self.__data['settings']['application']:
+            secret = secrets.token_hex(16)
+            self.__data['settings']['application']['api_access_token'] = secret
 
         # Proxy list support - available as a selection in settings when text file is imported
         # CSV list
@@ -211,7 +215,8 @@ class ChangeDetectionStore:
     def get_all_tags(self):
         tags = []
         for uuid, watch in self.data['watching'].items():
-
+            if watch['tag'] is None:
+                continue
             # Support for comma separated list of tags.
             for tag in watch['tag'].split(','):
                 tag = tag.strip()
@@ -280,6 +285,10 @@ class ChangeDetectionStore:
     def add_watch(self, url, tag="", extras=None, write_to_disk_now=True):
         if extras is None:
             extras = {}
+        # should always be str
+        if tag is None or not tag:
+            tag=''
+
         # Incase these are copied across, assume it's a reference and deepcopy()
         apply_extras = deepcopy(extras)
 

changedetectionio/templates/edit.html

@@ -73,6 +73,21 @@
                         </span>
                     </div>
                 {% endif %}
+                <fieldset id="webdriver-override-options">
+                    <div class="pure-form-message-inline">
+                        <strong>If you're having trouble waiting for the page to be fully rendered (text missing etc), try increasing the 'wait' time here.</strong>
+                        <br/>
+                        This will wait <i>n</i> seconds before extracting the text.
+                    </div>
+                    <div class="pure-control-group">
+                        {{ render_field(form.webdriver_delay) }}
+                    </div>
+                    {% if using_global_webdriver_wait %}
+                    <div class="pure-form-message-inline">
+                        <strong>Using the current global default settings</strong>
+                    </div>
+                    {% endif %}
+                </fieldset>
                 <fieldset class="pure-group" id="requests-override-options">
                     <div class="pure-form-message-inline">
                         <strong>Request override is currently only used by the <i>Basic fast Plaintext/HTTP Client</i> method.</strong>
@@ -110,8 +125,7 @@ User-Agent: wonderbra 1.0") }}
             </div>
 
             <div class="tab-pane-inner" id="filters-and-triggers">
-                <fieldset>
-                        <div class="pure-control-group">
+                    <div class="pure-control-group">
                             <strong>Pro-tips:</strong><br/>
                             <ul>
                                 <li>
@@ -122,7 +136,6 @@ User-Agent: wonderbra 1.0") }}
                                 </li>
                             </ul>
                     </div>
-
                     <div class="pure-control-group">
                         {{ render_field(form.css_filter, placeholder=".class-name or #some-id, or other CSS selector rule.",
                         class="m-d") }}
@@ -138,7 +151,7 @@ User-Agent: wonderbra 1.0") }}
                                 href="https://github.com/dgtlmoon/changedetection.io/wiki/CSS-Selector-help">here for more CSS selector help</a>.<br/>
                 </span>
                     </div>
-                    <fieldset class="pure-group">
+                    <div class="pure-control-group">
                       {{ render_field(form.subtractive_selectors, rows=5, placeholder="header
 footer
 nav
@@ -149,8 +162,7 @@ nav
                           <li> Add multiple elements or CSS selectors per line to ignore multiple parts of the HTML. </li>
                         </ul>
                       </span>
-                    </fieldset>
-                </fieldset>
+                    </div>
                 <fieldset class="pure-group">
                     {{ render_field(form.ignore_text, rows=5, placeholder="Some text to ignore in a line
 /some.regex\d{2}/ for case-INsensitive regex
@@ -187,9 +199,9 @@ nav
 
                       {{ render_button(form.save_button) }} {{ render_button(form.save_and_preview_button) }}
 
-                    <a href="{{url_for('api_delete', uuid=uuid)}}"
+                    <a href="{{url_for('form_delete', uuid=uuid)}}"
                        class="pure-button button-small button-error ">Delete</a>
-                    <a href="{{url_for('api_clone', uuid=uuid)}}"
+                    <a href="{{url_for('form_clone', uuid=uuid)}}"
                        class="pure-button button-small ">Create Copy</a>
                 </div>
             </div>

changedetectionio/templates/settings.html

@@ -12,6 +12,7 @@
 <script type="text/javascript" src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
 <script type="text/javascript" src="{{url_for('static_content', group='js', filename='notifications.js')}}" defer></script>
 
+<script type="text/javascript" src="{{url_for('static_content', group='js', filename='global-settings.js')}}" defer></script>
 <div class="edit-form">
     <div class="tabs collapsable">
         <ul>
@@ -19,6 +20,7 @@
             <li class="tab"><a href="#notifications">Notifications</a></li>
             <li class="tab"><a href="#fetching">Fetching</a></li>
             <li class="tab"><a href="#filters">Global Filters</a></li>
+            <li class="tab"><a href="#api">API</a></li>
         </ul>
     </div>
     <div class="box-wrap inner">
@@ -42,6 +44,7 @@
                             <span class="pure-form-message-inline">Password is locked.</span>
                         {% endif %}
                     </div>
+
                     <div class="pure-control-group">
                         {{ render_field(form.application.form.base_url, placeholder="http://yoursite.com:5000/",
                         class="m-d") }}
@@ -60,6 +63,11 @@
                         {{ render_checkbox_field(form.application.form.real_browser_save_screenshot) }}
                         <span class="pure-form-message-inline">When using a Chrome browser, a screenshot from the last check will be available on the Diff page</span>
                     </div>
+
+                    <div class="pure-control-group">
+                        {{ render_checkbox_field(form.application.form.empty_pages_are_a_change) }}
+                        <span class="pure-form-message-inline">When a page contains HTML, but no renderable text appears (empty page), is this considered a change?</span>
+                    </div>
                 {% if form.requests.proxy %}
                     <div class="pure-control-group inline-radio">
                         {{ render_field(form.requests.form.proxy, class="fetch-backend-proxy") }}
@@ -87,9 +95,18 @@
                         <p>The <strong>Chrome/Javascript</strong> method requires a network connection to a running WebDriver+Chrome server, set by the ENV var 'WEBDRIVER_URL'. </p>
                     </span>
                 </div>
+                <fieldset class="pure-group" id="webdriver-override-options">
+                    <div class="pure-form-message-inline">
+                        <strong>If you're having trouble waiting for the page to be fully rendered (text missing etc), try increasing the 'wait' time here.</strong>
+                        <br/>
+                        This will wait <i>n</i> seconds before extracting the text.
+                    </div>
+                    <div class="pure-control-group">
+                        {{ render_field(form.application.form.webdriver_delay) }}
+                    </div>
+                </fieldset>
             </div>
 
-
             <div class="tab-pane-inner" id="filters">
 
                     <fieldset class="pure-group">
@@ -134,12 +151,26 @@ nav
                     </fieldset>
            </div>
 
+            <div class="tab-pane-inner" id="api">
+
+                <p>Drive your changedetection.io via API, More about <a href="https://github.com/dgtlmoon/changedetection.io/wiki/API-Reference">API access here</a></p>
+
+                <div class="pure-control-group">
+                    {{ render_checkbox_field(form.application.form.api_access_token_enabled) }}
+                    <div class="pure-form-message-inline">Restrict API access limit by using <code>x-api-key</code> header</div><br/>
+                    <div class="pure-form-message-inline"><br/>API Key <span id="api-key">{{api_key}}</span>
+                        <span style="display:none;" id="api-key-copy" >copy</span>
+                    </div>
+                </div>
+            </div>
+
             <div id="actions">
                 <div class="pure-control-group">
                     {{ render_button(form.save_button) }}
                     <a href="{{url_for('index')}}" class="pure-button button-small button-cancel">Back</a>
                     <a href="{{url_for('scrub_page')}}" class="pure-button button-small button-cancel">Delete History Snapshot Data</a>
                 </div>
+
             </div>
         </form>
     </div>

changedetectionio/templates/watch-overview.html

@@ -5,7 +5,7 @@
 <script type="text/javascript" src="{{url_for('static_content', group='js', filename='watch-overview.js')}}" defer></script>
 <div class="box">
 
-    <form class="pure-form" action="{{ url_for('api_watch_add') }}" method="POST" id="new-watch-form">
+    <form class="pure-form" action="{{ url_for('form_watch_add') }}" method="POST" id="new-watch-form">
         <input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
         <fieldset>
             <legend>Add a new change detection watch</legend>
@@ -13,7 +13,7 @@
                 {{ render_simple_field(form.tag, value=active_tag if active_tag else '', placeholder="watch group") }}
             <button type="submit" class="pure-button pure-button-primary">Watch</button>
         </fieldset>
-        <span style="color:#eee; font-size: 80%;"><img style="height: 1em;display:inline-block;" src="{{url_for('static_content', group='images', filename='spread.svg')}}" /> Tip: You can also add 'shared' watches. <a href="#">More info</a></a></span>
+        <span style="color:#eee; font-size: 80%;"><img style="height: 1em;display:inline-block;" src="{{url_for('static_content', group='images', filename='spread.svg')}}" /> Tip: You can also add 'shared' watches. <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Sharing-a-Watch">More info</a></a></span>
     </form>
     <div>
         <a href="{{url_for('index')}}" class="pure-button button-tag {{'active' if not active_tag }}">All</a>
@@ -52,7 +52,7 @@
 
                 <td class="title-col inline">{{watch.title if watch.title is not none and watch.title|length > 0 else watch.url}}
                     <a class="external" target="_blank" rel="noopener" href="{{ watch.url.replace('source:','') }}"></a>
-                    <a href="{{url_for('api_share_put_watch', uuid=watch.uuid)}}"><img style="height: 1em;display:inline-block;" src="{{url_for('static_content', group='images', filename='spread.svg')}}" /></a>
+                    <a href="{{url_for('form_share_put_watch', uuid=watch.uuid)}}"><img style="height: 1em;display:inline-block;" src="{{url_for('static_content', group='images', filename='spread.svg')}}" /></a>
 
                     {%if watch.fetch_backend == "html_webdriver" %}<img style="height: 1em; display:inline-block;" src="{{url_for('static_content', group='images', filename='Google-Chrome-icon.png')}}" />{% endif %}
 
@@ -74,7 +74,7 @@
                     {% endif %}
                 </td>
                 <td>
-                    <a {% if watch.uuid in queued_uuids %}disabled="true"{% endif %} href="{{ url_for('api_watch_checknow', uuid=watch.uuid, tag=request.args.get('tag')) }}"
+                    <a {% if watch.uuid in queued_uuids %}disabled="true"{% endif %} href="{{ url_for('form_watch_checknow', uuid=watch.uuid, tag=request.args.get('tag')) }}"
                        class="recheck pure-button button-small pure-button-primary">{% if watch.uuid in queued_uuids %}Queued{% else %}Recheck{% endif %}</a>
                     <a href="{{ url_for('edit_page', uuid=watch.uuid)}}" class="pure-button button-small pure-button-primary">Edit</a>
                     {% if watch.history|length >= 2 %}
@@ -96,7 +96,7 @@
             </li>
             {% endif %}
             <li>
-               <a href="{{ url_for('api_watch_checknow', tag=active_tag) }}" class="pure-button button-tag ">Recheck
+               <a href="{{ url_for('form_watch_checknow', tag=active_tag) }}" class="pure-button button-tag ">Recheck
                 all {% if active_tag%}in "{{active_tag}}"{%endif%}</a>
             </li>
             <li>

changedetectionio/tests/test_api.py

@@ -2,73 +2,205 @@
 
 import time
 from flask import url_for
-from . util import live_server_setup
+from .util import live_server_setup
 
-def test_setup(live_server):
-    live_server_setup(live_server)
+import json
+import uuid
 
 
-def set_response_data(test_return_data):
+def set_original_response():
+    test_return_data = """<html>
+       <body>
+     Some initial text</br>
+     <p>Which is across multiple lines</p>
+     </br>
+     So let's see what happens.  </br>
+     <div id="sametext">Some text thats the same</div>
+     <div id="changetext">Some text that will change</div>
+     </body>
+     </html>
+    """
+
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write(test_return_data)
+    return None
+
+
+def set_modified_response():
+    test_return_data = """<html>
+       <body>
+     Some initial text</br>
+     <p>which has this one new line</p>
+     </br>
+     So let's see what happens.  </br>
+     <div id="sametext">Some text thats the same</div>
+     <div id="changetext">Some text that changes</div>
+     </body>
+     </html>
+    """
+
     with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write(test_return_data)
 
+    return None
 
-def test_snapshot_api_detects_change(client, live_server):
-    test_return_data = "Some initial text"
 
-    test_return_data_modified = "Some NEW nice initial text"
+def is_valid_uuid(val):
+    try:
+        uuid.UUID(str(val))
+        return True
+    except ValueError:
+        return False
 
-    sleep_time_for_fetch_thread = 3
 
-    set_response_data(test_return_data)
+# kinda funky, but works for now
+def _extract_api_key_from_UI(client):
+    import re
+    res = client.get(
+        url_for("settings_page"),
+    )
+    # <span id="api-key">{{api_key}}</span>
 
-    # Give the endpoint time to spin up
-    time.sleep(1)
+    m = re.search('<span id="api-key">(.+?)</span>', str(res.data))
+    api_key = m.group(1)
+    return api_key.strip()
 
-    # Add our URL to the import page
-    test_url = url_for('test_endpoint', content_type="text/plain",
-                       _external=True)
+
+def test_api_simple(client, live_server):
+    live_server_setup(live_server)
+
+    api_key = _extract_api_key_from_UI(client)
+
+    # Create a watch
+    set_original_response()
+    watch_uuid = None
+
+    # Validate bad URL
+    test_url = url_for('test_endpoint', _external=True,
+                       headers={'x-api-key': api_key}, )
     res = client.post(
-        url_for("import_page"),
-        data={"urls": test_url},
+        url_for("createwatch"),
+        data=json.dumps({"url": "h://xxxxxxxxxom"}),
+        headers={'content-type': 'application/json', 'x-api-key': api_key},
         follow_redirects=True
     )
-    assert b"1 Imported" in res.data
-
-    # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
-
-    # Give the thread time to pick it up
-    time.sleep(sleep_time_for_fetch_thread)
-
-    res = client.get(
-        url_for("api_snapshot", uuid="first"),
-        follow_redirects=True
-    )
-
-    assert test_return_data.encode() == res.data
-
-    #  Make a change
-    set_response_data(test_return_data_modified)
-
-    # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
-    # Give the thread time to pick it up
-    time.sleep(sleep_time_for_fetch_thread)
-
-    res = client.get(
-        url_for("api_snapshot", uuid="first"),
-        follow_redirects=True
-    )
-
-    assert test_return_data_modified.encode() == res.data
-
-def test_snapshot_api_invalid_uuid(client, live_server):
-
-    res = client.get(
-        url_for("api_snapshot", uuid="invalid"),
-        follow_redirects=True
-    )
-
     assert res.status_code == 400
 
+    # Create new
+    res = client.post(
+        url_for("createwatch"),
+        data=json.dumps({"url": test_url, 'tag': "One, Two", "title": "My test URL"}),
+        headers={'content-type': 'application/json', 'x-api-key': api_key},
+        follow_redirects=True
+    )
+    s = json.loads(res.data)
+    assert is_valid_uuid(s['uuid'])
+    watch_uuid = s['uuid']
+    assert res.status_code == 201
+
+    time.sleep(3)
+
+    # Verify its in the list and that recheck worked
+    res = client.get(
+        url_for("createwatch"),
+        headers={'x-api-key': api_key}
+    )
+    assert watch_uuid in json.loads(res.data).keys()
+    before_recheck_info = json.loads(res.data)[watch_uuid]
+    assert before_recheck_info['last_checked'] != 0
+    assert before_recheck_info['title'] == 'My test URL'
+
+    set_modified_response()
+    # Trigger recheck of all ?recheck_all=1
+    client.get(
+        url_for("createwatch", recheck_all='1'),
+        headers={'x-api-key': api_key},
+    )
+    time.sleep(3)
+
+    # Did the recheck fire?
+    res = client.get(
+        url_for("createwatch"),
+        headers={'x-api-key': api_key},
+    )
+    after_recheck_info = json.loads(res.data)[watch_uuid]
+    assert after_recheck_info['last_checked'] != before_recheck_info['last_checked']
+    assert after_recheck_info['last_changed'] != 0
+
+    # Check history index list
+    res = client.get(
+        url_for("watchhistory", uuid=watch_uuid),
+        headers={'x-api-key': api_key},
+    )
+    history = json.loads(res.data)
+    assert len(history) == 2, "Should have two history entries (the original and the changed)"
+
+    # Fetch a snapshot by timestamp, check the right one was found
+    res = client.get(
+        url_for("watchsinglehistory", uuid=watch_uuid, timestamp=list(history.keys())[-1]),
+        headers={'x-api-key': api_key},
+    )
+    assert b'which has this one new line' in res.data
+
+    # Fetch a snapshot by 'latest'', check the right one was found
+    res = client.get(
+        url_for("watchsinglehistory", uuid=watch_uuid, timestamp='latest'),
+        headers={'x-api-key': api_key},
+    )
+    assert b'which has this one new line' in res.data
+
+    # Fetch the whole watch
+    res = client.get(
+        url_for("watch", uuid=watch_uuid),
+        headers={'x-api-key': api_key}
+    )
+    watch = json.loads(res.data)
+    # @todo how to handle None/default global values?
+    assert watch['history_n'] == 2, "Found replacement history section, which is in its own API"
+
+    # Finally delete the watch
+    res = client.delete(
+        url_for("watch", uuid=watch_uuid),
+        headers={'x-api-key': api_key},
+    )
+    assert res.status_code == 204
+
+    # Check via a relist
+    res = client.get(
+        url_for("createwatch"),
+        headers={'x-api-key': api_key}
+    )
+    watch_list = json.loads(res.data)
+    assert len(watch_list) == 0, "Watch list should be empty"
+
+
+def test_access_denied(client, live_server):
+    # `config_api_token_enabled` Should be On by default
+    res = client.get(
+        url_for("createwatch")
+    )
+    assert res.status_code == 403
+
+    res = client.get(
+        url_for("createwatch"),
+        headers={'x-api-key': "something horrible"}
+    )
+    assert res.status_code == 403
+
+    # Disable config_api_token_enabled and it should work
+    res = client.post(
+        url_for("settings_page"),
+        data={
+            "requests-time_between_check-minutes": 180,
+            "application-fetch_backend": "html_requests",
+            "application-api_access_token_enabled": ""
+        },
+        follow_redirects=True
+    )
+
+    assert b"Settings updated." in res.data
+
+    res = client.get(
+        url_for("createwatch")
+    )
+    assert res.status_code == 200

changedetectionio/tests/test_auth.py

@@ -29,7 +29,7 @@ def test_basic_auth(client, live_server):
     assert b"Updated watch." in res.data
 
     # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
     time.sleep(1)
     res = client.get(
         url_for("preview_page", uuid="first"),

changedetectionio/tests/test_backend.py

@@ -32,7 +32,7 @@ def test_check_basic_change_detection_functionality(client, live_server):
 
     # Do this a few times.. ensures we dont accidently set the status
     for n in range(3):
-        client.get(url_for("api_watch_checknow"), follow_redirects=True)
+        client.get(url_for("form_watch_checknow"), follow_redirects=True)
 
         # Give the thread time to pick it up
         time.sleep(sleep_time_for_fetch_thread)
@@ -65,7 +65,7 @@ def test_check_basic_change_detection_functionality(client, live_server):
     assert b'which has this one new line' in res.read()
 
     # Force recheck
-    res = client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    res = client.get(url_for("form_watch_checknow"), follow_redirects=True)
     assert b'1 watches are queued for rechecking.' in res.data
 
     time.sleep(sleep_time_for_fetch_thread)
@@ -93,7 +93,7 @@ def test_check_basic_change_detection_functionality(client, live_server):
 
     # Do this a few times.. ensures we dont accidently set the status
     for n in range(2):
-        client.get(url_for("api_watch_checknow"), follow_redirects=True)
+        client.get(url_for("form_watch_checknow"), follow_redirects=True)
 
         # Give the thread time to pick it up
         time.sleep(sleep_time_for_fetch_thread)
@@ -113,7 +113,7 @@ def test_check_basic_change_detection_functionality(client, live_server):
         follow_redirects=True
     )
 
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
     time.sleep(sleep_time_for_fetch_thread)
 
     res = client.get(url_for("index"))
@@ -123,6 +123,6 @@ def test_check_basic_change_detection_functionality(client, live_server):
 
     #
     # Cleanup everything
-    res = client.get(url_for("api_delete", uuid="all"), follow_redirects=True)
+    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
 

changedetectionio/tests/test_clone.py

@@ -23,7 +23,7 @@ def test_trigger_functionality(client, live_server):
 
 
     res = client.get(
-        url_for("api_clone", uuid="first"),
+        url_for("form_clone", uuid="first"),
         follow_redirects=True
     )
 

changedetectionio/tests/test_css_selector.py

@@ -89,7 +89,7 @@ def test_check_markup_css_filter_restriction(client, live_server):
     assert b"1 Imported" in res.data
 
     # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
@@ -110,7 +110,7 @@ def test_check_markup_css_filter_restriction(client, live_server):
     assert bytes(css_filter.encode('utf-8')) in res.data
 
     # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
@@ -118,7 +118,7 @@ def test_check_markup_css_filter_restriction(client, live_server):
     set_modified_response()
 
     # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
 

changedetectionio/tests/test_element_removal.py

@@ -145,7 +145,7 @@ def test_element_removal_full(client, live_server):
     assert bytes(subtractive_selectors_data.encode("utf-8")) in res.data
 
     # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
@@ -158,7 +158,7 @@ def test_element_removal_full(client, live_server):
     set_modified_response()
 
     # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)

changedetectionio/tests/test_encoding.py

@@ -39,7 +39,7 @@ def test_check_encoding_detection(client, live_server):
     )
 
     # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(2)
@@ -71,7 +71,7 @@ def test_check_encoding_detection_missing_content_type_header(client, live_serve
     )
 
     # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(2)

changedetectionio/tests/test_errorhandling.py

@@ -29,7 +29,7 @@ def test_error_handler(client, live_server):
     assert b"1 Imported" in res.data
 
     # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(3)
@@ -54,7 +54,7 @@ def test_error_text_handler(client, live_server):
     assert b"1 Imported" in res.data
 
     # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(3)

changedetectionio/tests/test_ignore_text.py

@@ -102,7 +102,7 @@ def test_check_ignore_text_functionality(client, live_server):
     assert b"1 Imported" in res.data
 
     # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
@@ -123,7 +123,7 @@ def test_check_ignore_text_functionality(client, live_server):
     assert bytes(ignore_text.encode('utf-8')) in res.data
 
     # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
@@ -137,7 +137,7 @@ def test_check_ignore_text_functionality(client, live_server):
     set_modified_ignore_response()
 
     # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
 
@@ -152,7 +152,7 @@ def test_check_ignore_text_functionality(client, live_server):
 
     # Just to be sure.. set a regular modified change..
     set_modified_original_ignore_response()
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
     time.sleep(sleep_time_for_fetch_thread)
 
     res = client.get(url_for("index"))
@@ -165,7 +165,7 @@ def test_check_ignore_text_functionality(client, live_server):
     # We should be able to see what we ignored
     assert b'<div class="ignored">new ignore stuff' in res.data
 
-    res = client.get(url_for("api_delete", uuid="all"), follow_redirects=True)
+    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
 
 def test_check_global_ignore_text_functionality(client, live_server):
@@ -200,7 +200,7 @@ def test_check_global_ignore_text_functionality(client, live_server):
     assert b"1 Imported" in res.data
 
     # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
@@ -222,7 +222,7 @@ def test_check_global_ignore_text_functionality(client, live_server):
     assert bytes(ignore_text.encode('utf-8')) in res.data
 
     # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
@@ -240,7 +240,7 @@ def test_check_global_ignore_text_functionality(client, live_server):
     set_modified_ignore_response()
 
     # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
 
@@ -251,10 +251,10 @@ def test_check_global_ignore_text_functionality(client, live_server):
 
     # Just to be sure.. set a regular modified change that will trigger it
     set_modified_original_ignore_response()
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
     time.sleep(sleep_time_for_fetch_thread)
     res = client.get(url_for("index"))
     assert b'unviewed' in res.data
 
-    res = client.get(url_for("api_delete", uuid="all"), follow_redirects=True)
+    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data

changedetectionio/tests/test_ignorehyperlinks.py

@@ -72,14 +72,14 @@ def test_render_anchor_tag_content_true(client, live_server):
 
     time.sleep(sleep_time_for_fetch_thread)
     # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
 
     # set a new html text with a modified link
     set_modified_ignore_response()
     time.sleep(sleep_time_for_fetch_thread)
 
     # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
@@ -101,7 +101,7 @@ def test_render_anchor_tag_content_true(client, live_server):
     assert b"Settings updated." in res.data
 
     # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
@@ -119,7 +119,7 @@ def test_render_anchor_tag_content_true(client, live_server):
     assert b"/test-endpoint" in res.data
 
     # Cleanup everything
-    res = client.get(url_for("api_delete", uuid="all"),
+    res = client.get(url_for("form_delete", uuid="all"),
                      follow_redirects=True)
     assert b'Deleted' in res.data
 

changedetectionio/tests/test_ignorestatuscode.py

@@ -70,12 +70,12 @@ def test_normal_page_check_works_with_ignore_status_code(client, live_server):
 
     time.sleep(sleep_time_for_fetch_thread)
     # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
 
     set_some_changed_response()
     time.sleep(sleep_time_for_fetch_thread)
     # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
@@ -105,7 +105,7 @@ def test_403_page_check_works_with_ignore_status_code(client, live_server):
     assert b"1 Imported" in res.data
 
     # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
@@ -120,7 +120,7 @@ def test_403_page_check_works_with_ignore_status_code(client, live_server):
     assert b"Updated watch." in res.data
 
     # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
@@ -128,7 +128,7 @@ def test_403_page_check_works_with_ignore_status_code(client, live_server):
     set_some_changed_response()
 
     # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
 
@@ -157,7 +157,7 @@ def test_403_page_check_fails_without_ignore_status_code(client, live_server):
     assert b"1 Imported" in res.data
 
     # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
@@ -172,7 +172,7 @@ def test_403_page_check_fails_without_ignore_status_code(client, live_server):
     assert b"Updated watch." in res.data
 
     # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
@@ -180,7 +180,7 @@ def test_403_page_check_fails_without_ignore_status_code(client, live_server):
     set_some_changed_response()
 
     # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
 

changedetectionio/tests/test_ignorewhitespace.py

@@ -80,12 +80,12 @@ def test_check_ignore_whitespace(client, live_server):
 
     time.sleep(sleep_time_for_fetch_thread)
     # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
 
     set_original_ignore_response_but_with_whitespace()
     time.sleep(sleep_time_for_fetch_thread)
     # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)

changedetectionio/tests/test_import.py

@@ -25,7 +25,7 @@ https://example.com tag1, other tag"""
     assert b"3 Imported" in res.data
     assert b"tag1" in res.data
     assert b"other tag" in res.data
-    res = client.get(url_for("api_delete", uuid="all"), follow_redirects=True)
+    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
 
     # Clear flask alerts
     res = client.get( url_for("index"))
@@ -50,7 +50,7 @@ def xtest_import_skip_url(client, live_server):
     assert b"1 Imported" in res.data
     assert b"ht000000broken" in res.data
     assert b"1 Skipped" in res.data
-    res = client.get(url_for("api_delete", uuid="all"), follow_redirects=True)
+    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
     # Clear flask alerts
     res = client.get( url_for("index"))
 
@@ -79,7 +79,7 @@ def test_import_distillio(client, live_server):
 
     # Give the endpoint time to spin up
     time.sleep(1)
-    client.get(url_for("api_delete", uuid="all"), follow_redirects=True)
+    client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
     res = client.post(
         url_for("import_page"),
         data={
@@ -115,6 +115,6 @@ def test_import_distillio(client, live_server):
     assert b"nice stuff" in res.data
     assert b"nerd-news" in res.data
 
-    res = client.get(url_for("api_delete", uuid="all"), follow_redirects=True)
+    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
     # Clear flask alerts
     res = client.get(url_for("index"))

changedetectionio/tests/test_jsonpath_selector.py

@@ -171,7 +171,7 @@ def test_check_json_without_filter(client, live_server):
     )
 
     # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(3)
@@ -203,7 +203,7 @@ def test_check_json_filter(client, live_server):
     assert b"1 Imported" in res.data
 
     # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(3)
@@ -229,7 +229,7 @@ def test_check_json_filter(client, live_server):
     assert bytes(json_filter.encode('utf-8')) in res.data
 
     # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(3)
@@ -237,7 +237,7 @@ def test_check_json_filter(client, live_server):
     set_modified_response()
 
     # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
     # Give the thread time to pick it up
     time.sleep(4)
 
@@ -288,7 +288,7 @@ def test_check_json_filter_bool_val(client, live_server):
     time.sleep(3)
 
     # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(3)
@@ -296,7 +296,7 @@ def test_check_json_filter_bool_val(client, live_server):
     set_modified_response()
 
     # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
     # Give the thread time to pick it up
     time.sleep(3)
 
@@ -327,7 +327,7 @@ def test_check_json_ext_filter(client, live_server):
     assert b"1 Imported" in res.data
 
     # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(3)
@@ -353,7 +353,7 @@ def test_check_json_ext_filter(client, live_server):
     assert bytes(json_filter.encode('utf-8')) in res.data
 
     # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(3)
@@ -361,7 +361,7 @@ def test_check_json_ext_filter(client, live_server):
     set_modified_ext_response()
 
     # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
     # Give the thread time to pick it up
     time.sleep(4)
 

changedetectionio/tests/test_nonrenderable_pages.py

@@ -0,0 +1,102 @@
+#!/usr/bin/python3
+
+import time
+from flask import url_for
+from urllib.request import urlopen
+from .util import set_original_response, set_modified_response, live_server_setup
+
+sleep_time_for_fetch_thread = 3
+
+
+def set_nonrenderable_response():
+    test_return_data = """<html>
+    <head><title>modified head title</title></head>
+    <!-- like when some angular app was broken and doesnt render or whatever -->
+    <body>
+     </body>
+     </html>
+    """
+
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write(test_return_data)
+
+    return None
+
+def test_check_basic_change_detection_functionality(client, live_server):
+    set_original_response()
+    live_server_setup(live_server)
+
+    # Add our URL to the import page
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": url_for('test_endpoint', _external=True)},
+        follow_redirects=True
+    )
+
+    assert b"1 Imported" in res.data
+
+    time.sleep(sleep_time_for_fetch_thread)
+
+    # Do this a few times.. ensures we dont accidently set the status
+    for n in range(3):
+        client.get(url_for("form_watch_checknow"), follow_redirects=True)
+
+        # Give the thread time to pick it up
+        time.sleep(sleep_time_for_fetch_thread)
+
+        # It should report nothing found (no new 'unviewed' class)
+        res = client.get(url_for("index"))
+        assert b'unviewed' not in res.data
+
+
+    #####################
+    client.post(
+        url_for("settings_page"),
+        data={"application-empty_pages_are_a_change": "",
+              "requests-time_between_check-minutes": 180,
+              'application-fetch_backend': "html_requests"},
+        follow_redirects=True
+    )
+
+    # this should not trigger a change, because no good text could be converted from the HTML
+    set_nonrenderable_response()
+
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
+
+    # It should report nothing found (no new 'unviewed' class)
+    res = client.get(url_for("index"))
+    assert b'unviewed' not in res.data
+
+
+    # ok now do the opposite
+
+    client.post(
+        url_for("settings_page"),
+        data={"application-empty_pages_are_a_change": "y",
+              "requests-time_between_check-minutes": 180,
+              'application-fetch_backend': "html_requests"},
+        follow_redirects=True
+    )
+    set_modified_response()
+
+
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
+
+    # It should report nothing found (no new 'unviewed' class)
+    res = client.get(url_for("index"))
+    assert b'unviewed' in res.data
+
+
+
+
+    #
+    # Cleanup everything
+    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    assert b'Deleted' in res.data
+

changedetectionio/tests/test_notification.py

@@ -36,7 +36,7 @@ def test_check_notification(client, live_server):
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
-        url_for("api_watch_add"),
+        url_for("form_watch_add"),
         data={"url": test_url, "tag": ''},
         follow_redirects=True
     )
@@ -98,7 +98,7 @@ def test_check_notification(client, live_server):
     notification_submission = None
 
     # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
     time.sleep(3)
     # Verify what was sent as a notification, this file should exist
     with open("test-datastore/notification.txt", "r") as f:
@@ -133,7 +133,7 @@ def test_check_notification(client, live_server):
 
     # This should insert the {current_snapshot}
     set_more_modified_response()
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
     time.sleep(3)
     # Verify what was sent as a notification, this file should exist
     with open("test-datastore/notification.txt", "r") as f:
@@ -146,17 +146,17 @@ def test_check_notification(client, live_server):
     os.unlink("test-datastore/notification.txt")
 
     # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
     time.sleep(1)
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
     time.sleep(1)
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
     time.sleep(1)
     assert os.path.exists("test-datastore/notification.txt") == False
 
     # cleanup for the next
     client.get(
-        url_for("api_delete", uuid="all"),
+        url_for("form_delete", uuid="all"),
         follow_redirects=True
     )
 
@@ -168,7 +168,7 @@ def test_notification_validation(client, live_server):
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
-        url_for("api_watch_add"),
+        url_for("form_watch_add"),
         data={"url": test_url, "tag": 'nice one'},
         follow_redirects=True
     )
@@ -208,6 +208,6 @@ def test_notification_validation(client, live_server):
 
     # cleanup for the next
     client.get(
-        url_for("api_delete", uuid="all"),
+        url_for("form_delete", uuid="all"),
         follow_redirects=True
     )

changedetectionio/tests/test_notification_errors.py

@@ -16,7 +16,7 @@ def test_check_notification_error_handling(client, live_server):
     # use a different URL so that it doesnt interfere with the actual check until we are ready
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
-        url_for("api_watch_add"),
+        url_for("form_watch_add"),
         data={"url": "https://changedetection.io/CHANGELOG.txt", "tag": ''},
         follow_redirects=True
     )

changedetectionio/tests/test_share_watch.py

@@ -41,7 +41,7 @@ def test_share_watch(client, live_server):
 
     # click share the link
     res = client.get(
-        url_for("api_share_put_watch", uuid="first"),
+        url_for("form_share_put_watch", uuid="first"),
         follow_redirects=True
     )
 
@@ -54,7 +54,7 @@ def test_share_watch(client, live_server):
 
     # Now delete what we have, we will try to re-import it
     # Cleanup everything
-    res = client.get(url_for("api_delete", uuid="all"), follow_redirects=True)
+    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
 
     # Add our URL to the import page

changedetectionio/tests/test_source.py

@@ -39,7 +39,7 @@ def test_check_basic_change_detection_functionality_source(client, live_server):
     set_modified_response()
 
     # Force recheck
-    res = client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    res = client.get(url_for("form_watch_checknow"), follow_redirects=True)
     assert b'1 watches are queued for rechecking.' in res.data
 
     time.sleep(5)

changedetectionio/tests/test_trigger.py

@@ -76,7 +76,7 @@ def test_trigger_functionality(client, live_server):
     assert b"1 Imported" in res.data
 
     # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
@@ -99,7 +99,7 @@ def test_trigger_functionality(client, live_server):
     assert bytes(trigger_text.encode('utf-8')) in res.data
 
     # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
@@ -113,7 +113,7 @@ def test_trigger_functionality(client, live_server):
     set_modified_original_ignore_response()
 
     # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
 
@@ -125,7 +125,7 @@ def test_trigger_functionality(client, live_server):
     time.sleep(sleep_time_for_fetch_thread)
     set_modified_with_trigger_text_response()
 
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
     time.sleep(sleep_time_for_fetch_thread)
     res = client.get(url_for("index"))
     assert b'unviewed' in res.data

changedetectionio/tests/test_trigger_regex.py

@@ -43,7 +43,7 @@ def test_trigger_regex_functionality(client, live_server):
     assert b"1 Imported" in res.data
 
     # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
@@ -65,7 +65,7 @@ def test_trigger_regex_functionality(client, live_server):
     with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write("some new noise")
 
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
     time.sleep(sleep_time_for_fetch_thread)
 
     # It should report nothing found (nothing should match the regex)
@@ -75,7 +75,7 @@ def test_trigger_regex_functionality(client, live_server):
     with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write("regex test123<br/>\nsomething 123")
 
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
     time.sleep(sleep_time_for_fetch_thread)
     res = client.get(url_for("index"))
     assert b'unviewed' in res.data

changedetectionio/tests/test_trigger_regex_with_filter.py

@@ -43,7 +43,7 @@ def test_trigger_regex_functionality(client, live_server):
     assert b"1 Imported" in res.data
 
     # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
@@ -66,7 +66,7 @@ def test_trigger_regex_functionality(client, live_server):
     with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write("<html>some new noise with cool stuff2 ok</html>")
 
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
     time.sleep(sleep_time_for_fetch_thread)
 
     # It should report nothing found (nothing should match the regex and filter)
@@ -76,7 +76,7 @@ def test_trigger_regex_functionality(client, live_server):
     with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write("<html>some new noise with <span id=in-here>cool stuff6</span> ok</html>")
 
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
     time.sleep(sleep_time_for_fetch_thread)
     res = client.get(url_for("index"))
     assert b'unviewed' in res.data

changedetectionio/tests/test_xpath_selector.py

@@ -65,7 +65,7 @@ def test_check_markup_xpath_filter_restriction(client, live_server):
     assert b"1 Imported" in res.data
 
     # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
@@ -89,7 +89,7 @@ def test_check_markup_xpath_filter_restriction(client, live_server):
     set_modified_response()
 
     # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
 
@@ -121,7 +121,7 @@ def test_xpath_validation(client, live_server):
 
 # actually only really used by the distll.io importer, but could be handy too
 def test_check_with_prefix_css_filter(client, live_server):
-    res = client.get(url_for("api_delete", uuid="all"), follow_redirects=True)
+    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
 
     # Give the endpoint time to spin up
@@ -158,4 +158,4 @@ def test_check_with_prefix_css_filter(client, live_server):
     assert b"Some text thats the same" in res.data #in selector
     assert b"Some text that will change" not in res.data #not in selector
 
-    client.get(url_for("api_delete", uuid="all"), follow_redirects=True)
+    client.get(url_for("form_delete", uuid="all"), follow_redirects=True)

changedetectionio/update_worker.py

@@ -52,6 +52,10 @@ class update_worker(threading.Thread):
                             raise Exception("Error - returned data from the fetch handler SHOULD be bytes")
                     except PermissionError as e:
                         self.app.logger.error("File permission error updating", uuid, str(e))
+                    except content_fetcher.ReplyWithContentButNoText as e:
+                        # Totally fine, it's by choice - just continue on, nothing more to care about
+                        # Page had elements/content but no renderable text
+                        pass
                     except content_fetcher.EmptyReply as e:
                         # Some kind of custom to-str handler in the exception handler that does this?
                         err_text = "EmptyReply: Status Code {}".format(e.status_code)

requirements.txt

@@ -6,6 +6,7 @@ timeago ~=1.0
 inscriptis ~= 2.2
 feedgen ~= 0.9
 flask-login ~= 0.5
+flask_restful
 pytz
 
 # Set these versions together to avoid a RequestsDependencyWarning