WIP

changedetectionio/__init__.py

@@ -36,12 +36,9 @@ from flask import (
     url_for,
 )
 from flask_login import login_required
-from flask_restful import abort, Api
-
 from flask_wtf import CSRFProtect
 
 from changedetectionio import html_tools
-from changedetectionio.api import api_v1
 
 __version__ = '0.39.13.1'
 
@@ -81,8 +78,6 @@ csrf.init_app(app)
 
 notification_debug_log=[]
 
-watch_api = Api(app, decorators=[csrf.exempt])
-
 def init_app_secret(datastore_path):
     secret = ""
 
@@ -184,25 +179,6 @@ def changedetection_app(config=None, datastore_o=None):
     login_manager.login_view = 'login'
     app.secret_key = init_app_secret(config['datastore_path'])
 
-
-    watch_api.add_resource(api_v1.WatchSingleHistory,
-                           '/api/v1/watch/<string:uuid>/history/<string:timestamp>',
-                           resource_class_kwargs={'datastore': datastore, 'update_q': update_q})
-
-    watch_api.add_resource(api_v1.WatchHistory,
-                           '/api/v1/watch/<string:uuid>/history',
-                           resource_class_kwargs={'datastore': datastore})
-
-    watch_api.add_resource(api_v1.CreateWatch, '/api/v1/watch',
-                           resource_class_kwargs={'datastore': datastore, 'update_q': update_q})
-
-    watch_api.add_resource(api_v1.Watch, '/api/v1/watch/<string:uuid>',
-                           resource_class_kwargs={'datastore': datastore, 'update_q': update_q})
-
-
-
-
-
     # Setup cors headers to allow all domains
     # https://flask-cors.readthedocs.io/en/latest/
     #    CORS(app)
@@ -391,8 +367,6 @@ def changedetection_app(config=None, datastore_o=None):
 
             if limit_tag != None:
                 # Support for comma separated list of tags.
-                if watch['tag'] is None:
-                    continue
                 for tag_in_watch in watch['tag'].split(','):
                     tag_in_watch = tag_in_watch.strip()
                     if tag_in_watch == limit_tag:
@@ -546,9 +520,16 @@ def changedetection_app(config=None, datastore_o=None):
 
         # Defaults for proxy choice
         if datastore.proxy_list is not None:  # When enabled
-            # Radio needs '' not None, or incase that the chosen one no longer exists
+            system_proxy = datastore.data['settings']['requests']['proxy']
-            if default['proxy'] is None or not any(default['proxy'] in tup for tup in datastore.proxy_list):
+            if default['proxy'] is None:
-                default['proxy'] = ''
+                default['proxy'] = system_proxy
+            else:
+                # Does the chosen one exist?
+                if not any(default['proxy'] in tup for tup in datastore.proxy_list):
+                    default['proxy'] = datastore.proxy_list[0][0]
+
+            # Used by the form handler to keep or remove the proxy settings
+            default['proxy_list'] = datastore.proxy_list
 
         # proxy_override set to the json/text list of the items
         form = forms.watchForm(formdata=request.form if request.method == 'POST' else None,
@@ -559,7 +540,9 @@ def changedetection_app(config=None, datastore_o=None):
             # @todo - Couldn't get setattr() etc dynamic addition working, so remove it instead
             del form.proxy
         else:
-            form.proxy.choices = [('', 'Default')] + datastore.proxy_list
+            form.proxy.choices = datastore.proxy_list
+            if default['proxy'] is None:
+                form.proxy.default='http://hello'
 
         if request.method == 'POST' and form.validate():
             extra_update_obj = {}
@@ -596,10 +579,6 @@ def changedetection_app(config=None, datastore_o=None):
                 if len(datastore.data['watching'][uuid]['history']):
                     extra_update_obj['previous_md5'] = get_current_checksum_include_ignore_text(uuid=uuid)
 
-            # Be sure proxy value is None
-            if datastore.proxy_list is not None and form.data['proxy'] == '':
-                extra_update_obj['proxy'] = None
-
             datastore.data['watching'][uuid].update(form.data)
             datastore.data['watching'][uuid].update(extra_update_obj)
 
@@ -626,12 +605,12 @@ def changedetection_app(config=None, datastore_o=None):
             if request.method == 'POST' and not form.validate():
                 flash("An error occurred, please see below.", "error")
 
+
             output = render_template("edit.html",
                                      uuid=uuid,
                                      watch=datastore.data['watching'][uuid],
                                      form=form,
                                      has_empty_checktime=using_default_check_time,
-                                     using_global_webdriver_wait=default['webdriver_delay'] is None,
                                      current_base_url=datastore.data['settings']['application']['base_url'],
                                      emailprefix=os.getenv('NOTIFICATION_MAIL_BUTTON_PREFIX', False)
                                      )
@@ -697,7 +676,6 @@ def changedetection_app(config=None, datastore_o=None):
                                  form=form,
                                  current_base_url = datastore.data['settings']['application']['base_url'],
                                  hide_remove_pass=os.getenv("SALTED_PASS", False),
-                                 api_key=datastore.data['settings']['application'].get('api_access_token'),
                                  emailprefix=os.getenv('NOTIFICATION_MAIL_BUTTON_PREFIX', False))
 
         return output
@@ -896,6 +874,27 @@ def changedetection_app(config=None, datastore_o=None):
 
         return output
 
+    @app.route("/api/<string:uuid>/snapshot/current", methods=['GET'])
+    @login_required
+    def api_snapshot(uuid):
+
+        # More for testing, possible to return the first/only
+        if uuid == 'first':
+            uuid = list(datastore.data['watching'].keys()).pop()
+
+        try:
+            watch = datastore.data['watching'][uuid]
+        except KeyError:
+            return abort(400, "No history found for the specified link, bad link?")
+
+        newest = list(watch['history'].keys())[-1]
+        with open(watch['history'][newest], 'r') as f:
+            content = f.read()
+
+        resp = make_response(content)
+        resp.headers['Content-Type'] = 'text/plain'
+        return resp
+
     @app.route("/favicon.ico", methods=['GET'])
     def favicon():
         return send_from_directory("static/images", path="favicon.ico")
@@ -933,6 +932,9 @@ def changedetection_app(config=None, datastore_o=None):
             # Add the flask app secret
             zipObj.write(os.path.join(datastore_o.datastore_path, "secret.txt"), arcname="secret.txt")
 
+            # Add the sqlite3 db
+            zipObj.write(os.path.join(datastore_o.datastore_path, "watch.db"), arcname="watch.db")
+
             # Add any snapshot data we find, use the full path to access the file, but make the file 'relative' in the Zip.
             for txt_file_path in Path(datastore_o.datastore_path).rglob('*.txt'):
                 parent_p = txt_file_path.parent
@@ -1006,7 +1008,7 @@ def changedetection_app(config=None, datastore_o=None):
 
     @app.route("/api/add", methods=['POST'])
     @login_required
-    def form_watch_add():
+    def api_watch_add():
         from changedetectionio import forms
         form = forms.quickWatchForm(request.form)
 
@@ -1032,7 +1034,7 @@ def changedetection_app(config=None, datastore_o=None):
 
     @app.route("/api/delete", methods=['GET'])
     @login_required
-    def form_delete():
+    def api_delete():
         uuid = request.args.get('uuid')
 
         if uuid != 'all' and not uuid in datastore.data['watching'].keys():
@@ -1049,7 +1051,7 @@ def changedetection_app(config=None, datastore_o=None):
 
     @app.route("/api/clone", methods=['GET'])
     @login_required
-    def form_clone():
+    def api_clone():
         uuid = request.args.get('uuid')
         # More for testing, possible to return the first/only
         if uuid == 'first':
@@ -1063,7 +1065,7 @@ def changedetection_app(config=None, datastore_o=None):
 
     @app.route("/api/checknow", methods=['GET'])
     @login_required
-    def form_watch_checknow():
+    def api_watch_checknow():
 
         tag = request.args.get('tag')
         uuid = request.args.get('uuid')
@@ -1100,7 +1102,7 @@ def changedetection_app(config=None, datastore_o=None):
 
     @app.route("/api/share-url", methods=['GET'])
     @login_required
-    def form_share_put_watch():
+    def api_share_put_watch():
         """Given a watch UUID, upload the info and return a share-link
            the share-link can be imported/added"""
         import requests

changedetectionio/api/api_v1.py

@@ -1,125 +0,0 @@
-from flask_restful import abort, Resource
-from flask import request, make_response
-import validators
-from . import auth
-
-
-
-# https://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html
-
-class Watch(Resource):
-    def __init__(self, **kwargs):
-        # datastore is a black box dependency
-        self.datastore = kwargs['datastore']
-        self.update_q = kwargs['update_q']
-
-    # Get information about a single watch, excluding the history list (can be large)
-    # curl http://localhost:4000/api/v1/watch/<string:uuid>
-    # ?recheck=true
-    @auth.check_token
-    def get(self, uuid):
-        from copy import deepcopy
-        watch = deepcopy(self.datastore.data['watching'].get(uuid))
-        if not watch:
-            abort(404, message='No watch exists with the UUID of {}'.format(uuid))
-
-        if request.args.get('recheck'):
-            self.update_q.put(uuid)
-            return "OK", 200
-
-        # Return without history, get that via another API call
-        watch['history_n'] = len(watch['history'])
-        del (watch['history'])
-        return watch
-
-    @auth.check_token
-    def delete(self, uuid):
-        if not self.datastore.data['watching'].get(uuid):
-            abort(400, message='No watch exists with the UUID of {}'.format(uuid))
-
-        self.datastore.delete(uuid)
-        return 'OK', 204
-
-
-class WatchHistory(Resource):
-    def __init__(self, **kwargs):
-        # datastore is a black box dependency
-        self.datastore = kwargs['datastore']
-
-    # Get a list of available history for a watch by UUID
-    # curl http://localhost:4000/api/v1/watch/<string:uuid>/history
-    def get(self, uuid):
-        watch = self.datastore.data['watching'].get(uuid)
-        if not watch:
-            abort(404, message='No watch exists with the UUID of {}'.format(uuid))
-        return watch['history'], 200
-
-
-class WatchSingleHistory(Resource):
-    def __init__(self, **kwargs):
-        # datastore is a black box dependency
-        self.datastore = kwargs['datastore']
-
-    # Read a given history snapshot and return its content
-    # <string:timestamp> or "latest"
-    # curl http://localhost:4000/api/v1/watch/<string:uuid>/history/<int:timestamp>
-    @auth.check_token
-    def get(self, uuid, timestamp):
-        watch = self.datastore.data['watching'].get(uuid)
-        if not watch:
-            abort(404, message='No watch exists with the UUID of {}'.format(uuid))
-
-        if not len(watch['history']):
-            abort(404, message='Watch found but no history exists for the UUID {}'.format(uuid))
-
-        if timestamp == 'latest':
-            timestamp = list(watch['history'].keys())[-1]
-
-        with open(watch['history'][timestamp], 'r') as f:
-            content = f.read()
-
-        response = make_response(content, 200)
-        response.mimetype = "text/plain"
-        return response
-
-
-class CreateWatch(Resource):
-    def __init__(self, **kwargs):
-        # datastore is a black box dependency
-        self.datastore = kwargs['datastore']
-        self.update_q = kwargs['update_q']
-
-    @auth.check_token
-    def post(self):
-        # curl http://localhost:4000/api/v1/watch -H "Content-Type: application/json" -d '{"url": "https://my-nice.com", "tag": "one, two" }'
-        json_data = request.get_json()
-        tag = json_data['tag'].strip() if json_data.get('tag') else ''
-
-        if not validators.url(json_data['url'].strip()):
-            return "Invalid or unsupported URL", 400
-
-        extras = {'title': json_data['title'].strip()} if json_data.get('title') else {}
-
-        new_uuid = self.datastore.add_watch(url=json_data['url'].strip(), tag=tag, extras=extras)
-        self.update_q.put(new_uuid)
-        return {'uuid': new_uuid}, 201
-
-    # Return concise list of available watches and some very basic info
-    # curl http://localhost:4000/api/v1/watch|python -mjson.tool
-    # ?recheck_all=1 to recheck all
-    @auth.check_token
-    def get(self):
-        list = {}
-        for k, v in self.datastore.data['watching'].items():
-            list[k] = {'url': v['url'],
-                       'title': v['title'],
-                       'last_checked': v['last_checked'],
-                       'last_changed': v['last_changed'],
-                       'last_error': v['last_error']}
-
-        if request.args.get('recheck_all'):
-            for uuid in self.datastore.data['watching'].keys():
-                self.update_q.put(uuid)
-            return {'status': "OK"}, 200
-
-        return list, 200

changedetectionio/api/auth.py

@@ -1,33 +0,0 @@
-from flask import request, make_response, jsonify
-from functools import wraps
-
-
-# Simple API auth key comparison
-# @todo - Maybe short lived token in the future?
-
-def check_token(f):
-    @wraps(f)
-    def decorated(*args, **kwargs):
-        datastore = args[0].datastore
-
-        config_api_token_enabled = datastore.data['settings']['application'].get('api_access_token_enabled')
-        if not config_api_token_enabled:
-            return
-
-        try:
-            api_key_header = request.headers['x-api-key']
-        except KeyError:
-            return make_response(
-                jsonify("No authorization x-api-key header."), 403
-            )
-
-        config_api_token = datastore.data['settings']['application'].get('api_access_token')
-
-        if api_key_header != config_api_token:
-            return make_response(
-                jsonify("Invalid access - API key invalid."), 403
-            )
-
-        return f(*args, **kwargs)
-
-    return decorated

changedetectionio/content_fetcher.py

@@ -3,22 +3,17 @@ import chardet
 import os
 import requests
 import time
+import urllib3.exceptions
 import sys
 
+
 class EmptyReply(Exception):
     def __init__(self, status_code, url):
         # Set this so we can use it in other parts of the app
         self.status_code = status_code
         self.url = url
         return
-    pass
 
-class ReplyWithContentButNoText(Exception):
-    def __init__(self, status_code, url):
-        # Set this so we can use it in other parts of the app
-        self.status_code = status_code
-        self.url = url
-        return
     pass
 
 
@@ -33,9 +28,6 @@ class Fetcher():
     system_http_proxy = os.getenv('HTTP_PROXY')
     system_https_proxy = os.getenv('HTTPS_PROXY')
 
-    # Time ONTOP of the system defined env minimum time
-    render_extract_delay=0
-
     @abstractmethod
     def get_error(self):
         return self.error
@@ -105,7 +97,7 @@ class base_html_playwright(Fetcher):
         self.browser_type = os.getenv("PLAYWRIGHT_BROWSER_TYPE", 'chromium').strip('"')
         self.command_executor = os.getenv(
             "PLAYWRIGHT_DRIVER_URL",
-            'ws://playwright-chrome:3000'
+            'ws://playwright-chrome:3000/playwright'
         ).strip('"')
 
         # If any proxy settings are enabled, then we should setup the proxy object
@@ -155,7 +147,7 @@ class base_html_playwright(Fetcher):
                 # - `'commit'` - consider operation to be finished when network response is received and the document started loading.
                 # Better to not use any smarts from Playwright and just wait an arbitrary number of seconds
                 # This seemed to solve nearly all 'TimeoutErrors'
-                extra_wait = int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)) + self.render_extract_delay
+                extra_wait = int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5))
                 page.wait_for_timeout(extra_wait * 1000)
             except playwright._impl._api_types.TimeoutError as e:
                 raise EmptyReply(url=url, status_code=None)
@@ -163,9 +155,6 @@ class base_html_playwright(Fetcher):
             if response is None:
                 raise EmptyReply(url=url, status_code=None)
 
-            if len(page.content().strip()) == 0:
-                raise EmptyReply(url=url, status_code=None)
-
             self.status_code = response.status
             self.content = page.content()
             self.headers = response.all_headers()
@@ -251,7 +240,7 @@ class base_html_webdriver(Fetcher):
         # raise EmptyReply(url=url, status_code=r.status_code)
 
         # @todo - dom wait loaded?
-        time.sleep(int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)) + self.render_extract_delay)
+        time.sleep(int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)))
         self.content = self.driver.page_source
         self.headers = {}
         self.screenshot = self.driver.get_screenshot_as_png()

changedetectionio/fetch_site_status.py

@@ -97,13 +97,7 @@ class perform_site_check():
         proxy_args = self.set_proxy_from_list(watch)
         fetcher = klass(proxy_override=proxy_args)
 
-        # Configurable per-watch or global extra delay before extracting text (for webDriver types)
+        # Proxy List support
-        system_webdriver_delay = self.datastore.data['settings']['application'].get('webdriver_delay', None)
-        if watch['webdriver_delay'] is not None:
-            fetcher.render_extract_delay = watch['webdriver_delay']
-        elif system_webdriver_delay is not None:
-            fetcher.render_extract_delay = system_webdriver_delay
-
         fetcher.run(url, timeout, request_headers, request_body, request_method, ignore_status_code)
 
         # Fetching complete, now filters
@@ -184,11 +178,6 @@ class perform_site_check():
         # Re #340 - return the content before the 'ignore text' was applied
         text_content_before_ignored_filter = stripped_text_from_html.encode('utf-8')
 
-        # Treat pages with no renderable text content as a change? No by default
-        empty_pages_are_a_change = self.datastore.data['settings']['application'].get('empty_pages_are_a_change', False)
-        if not is_json and not empty_pages_are_a_change and len(stripped_text_from_html.strip()) == 0:
-            raise content_fetcher.ReplyWithContentButNoText(url=url, status_code=200)
-
         # We rely on the actual text in the html output.. many sites have random script vars etc,
         # in the future we'll implement other mechanisms.
 

changedetectionio/forms.py

@@ -318,7 +318,6 @@ class commonSettingsForm(Form):
     notification_format = SelectField('Notification format', choices=valid_notification_formats.keys(), default=default_notification_format)
     fetch_backend = RadioField(u'Fetch method', choices=content_fetcher.available_fetchers(), validators=[ValidateContentFetcherIsReady()])
     extract_title_as_title = BooleanField('Extract <title> from document and use as watch title', default=False)
-    webdriver_delay = IntegerField('Wait seconds before extracting text', validators=[validators.Optional(), validators.NumberRange(min=1, message="Should contain one or more seconds")] )
 
 class watchForm(commonSettingsForm):
 
@@ -371,10 +370,8 @@ class globalSettingsApplicationForm(commonSettingsForm):
     ignore_whitespace = BooleanField('Ignore whitespace')
     real_browser_save_screenshot = BooleanField('Save last screenshot when using Chrome?')
     removepassword_button = SubmitField('Remove password', render_kw={"class": "pure-button pure-button-primary"})
-    empty_pages_are_a_change =  BooleanField('Treat empty pages as a change?', default=False)
     render_anchor_tag_content = BooleanField('Render anchor tag content', default=False)
     fetch_backend = RadioField('Fetch Method', default="html_requests", choices=content_fetcher.available_fetchers(), validators=[ValidateContentFetcherIsReady()])
-    api_access_token_enabled = BooleanField('API access token security check enabled', default=True, validators=[validators.Optional()])
     password = SaltyPasswordField()
 
 

changedetectionio/model/App.py

@@ -27,11 +27,9 @@ class model(dict):
                     'proxy': None # Preferred proxy connection
                 },
                 'application': {
-                    'api_access_token_enabled': True,
                     'password': False,
                     'base_url' : None,
                     'extract_title_as_title': False,
-                    'empty_pages_are_a_change': False,
                     'fetch_backend': os.getenv("DEFAULT_FETCH_BACKEND", "html_requests"),
                     'global_ignore_text': [], # List of text to ignore when calculating the comparison checksum
                     'global_subtractive_selectors': [],
@@ -43,8 +41,7 @@ class model(dict):
                     'notification_body': default_notification_body,
                     'notification_format': default_notification_format,
                     'real_browser_save_screenshot': True,
-                    'schema_version' : 0,
+                    'schema_version' : 0
-                    'webdriver_delay': None  # Extra delay in seconds before extracting text
                 }
             }
         }

changedetectionio/model/Watch.py

@@ -27,7 +27,8 @@ class model(dict):
             'headers': {},  # Extra headers to send
             'body': None,
             'method': 'GET',
-            'history': {},  # Dict of timestamp and output stripped filename
+            # now stored in a sqlite3 db to reduce memory usage
+            #'history': {},  # Dict of timestamp and output stripped filename
             'ignore_text': [],  # List of text to ignore when calculating the comparison checksum
             # Custom notification content
             'notification_urls': [],  # List of URLs to add to the notification Queue (Usually AppRise)
@@ -43,8 +44,7 @@ class model(dict):
             # Re #110, so then if this is set to None, we know to use the default value instead
             # Requires setting to None on submit if it's the same as the default
             # Should be all None by default, so we use the system default in this case.
-            'time_between_check': {'weeks': None, 'days': None, 'hours': None, 'minutes': None, 'seconds': None},
+            'time_between_check': {'weeks': None, 'days': None, 'hours': None, 'minutes': None, 'seconds': None}
-            'webdriver_delay': None
         }
 
     def __init__(self, *arg, **kw):

changedetectionio/static/js/global-settings.js

@@ -1,36 +0,0 @@
-$(document).ready(function () {
-    function toggle() {
-        if ($('input[name="application-fetch_backend"]:checked').val() != 'html_requests') {
-            $('#requests-override-options').hide();
-            $('#webdriver-override-options').show();
-        } else {
-            $('#requests-override-options').show();
-            $('#webdriver-override-options').hide();
-        }
-    }
-
-    $('input[name="application-fetch_backend"]').click(function (e) {
-        toggle();
-    });
-    toggle();
-
-    $("#api-key").hover(
-        function () {
-            $("#api-key-copy").html('copy').fadeIn();
-        },
-        function () {
-            $("#api-key-copy").hide();
-        }
-    ).click(function (e) {
-        $("#api-key-copy").html('copied');
-        var range = document.createRange();
-        var n = $("#api-key")[0];
-        range.selectNode(n);
-        window.getSelection().removeAllRanges();
-        window.getSelection().addRange(range);
-        document.execCommand("copy");
-        window.getSelection().removeAllRanges();
-
-    });
-});
-

changedetectionio/static/js/watch-settings.js

@@ -2,10 +2,8 @@ $(document).ready(function() {
     function toggle() {
         if ($('input[name="fetch_backend"]:checked').val() != 'html_requests') {
             $('#requests-override-options').hide();
-            $('#webdriver-override-options').show();
         } else {
             $('#requests-override-options').show();
-            $('#webdriver-override-options').hide();
         }
     }
     $('input[name="fetch_backend"]').click(function (e) {

changedetectionio/static/styles/styles.css

@@ -284,11 +284,6 @@ footer {
 .pure-form {
   /* The input fields with errors */
   /* The list of errors */ }
-  .pure-form fieldset {
-    padding-top: 0px; }
-    .pure-form fieldset ul {
-      padding-bottom: 0px;
-      margin-bottom: 0px; }
   .pure-form .pure-control-group, .pure-form .pure-group, .pure-form .pure-controls {
     padding-bottom: 1em; }
     .pure-form .pure-control-group div, .pure-form .pure-group div, .pure-form .pure-controls div {
@@ -452,13 +447,4 @@ ul {
 .time-check-widget tr {
   display: inline; }
   .time-check-widget tr input[type="number"] {
-    width: 5em; }
+    width: 4em; }
-
-#webdriver-override-options input[type="number"] {
-  width: 5em; }
-
-#api-key:hover {
-  cursor: pointer; }
-
-#api-key-copy {
-  color: #0078e7; }

changedetectionio/static/styles/styles.scss

@@ -375,13 +375,6 @@ footer {
 
 
 .pure-form {
-    fieldset {
-        padding-top: 0px;
-        ul {
-            padding-bottom: 0px;
-            margin-bottom: 0px;
-        }
-    }
     .pure-control-group, .pure-group, .pure-controls {
         padding-bottom: 1em;
         div {
@@ -644,23 +637,7 @@ ul {
     tr {
         display: inline;
         input[type="number"] {
-            width: 5em;
+            width: 4em;
         }
     }
 }
-
-#webdriver-override-options {
-        input[type="number"] {
-            width: 5em;
-        }
-}
-
-#api-key {
-  &:hover {
-    cursor: pointer;
-  }
-}
-
-#api-key-copy {
-  color: #0078e7;
-}

changedetectionio/store.py

@@ -12,10 +12,11 @@ from os import mkdir, path, unlink
 from threading import Lock
 import re
 import requests
-import secrets
+import sqlite3
 
 from . model import App, Watch
 
+
 # Is there an existing library to ensure some data store (JSON etc) is in sync with CRUD methods?
 # Open a github issue if you know something :)
 # https://stackoverflow.com/questions/6190468/how-to-trigger-function-on-value-change
@@ -33,6 +34,11 @@ class ChangeDetectionStore:
         self.needs_write = False
         self.datastore_path = datastore_path
         self.json_store_path = "{}/url-watches.json".format(self.datastore_path)
+        self.datastore_path = datastore_path
+
+        #@todo - check for better options
+        self.__history_db_connection = sqlite3.connect("{}/watch.db".format(self.datastore_path))
+
         self.proxy_list = None
         self.stop_thread = False
 
@@ -71,6 +77,9 @@ class ChangeDetectionStore:
                     if 'application' in from_disk['settings']:
                         self.__data['settings']['application'].update(from_disk['settings']['application'])
 
+                # Bump the update version by running updates
+                self.run_updates()
+
                 # Reinitialise each `watching` with our generic_definition in the case that we add a new var in the future.
                 # @todo pretty sure theres a python we todo this with an abstracted(?) object!
                 for uuid, watch in self.__data['watching'].items():
@@ -80,6 +89,7 @@ class ChangeDetectionStore:
                     self.__data['watching'][uuid]['newest_history_key'] = self.get_newest_history_key(uuid)
                     print("Watching:", uuid, self.__data['watching'][uuid]['url'])
 
+
         # First time ran, doesnt exist.
         except (FileNotFoundError, json.decoder.JSONDecodeError):
             if include_default_watches:
@@ -108,14 +118,10 @@ class ChangeDetectionStore:
 
         # Generate the URL access token for RSS feeds
         if not 'rss_access_token' in self.__data['settings']['application']:
+            import secrets
             secret = secrets.token_hex(16)
             self.__data['settings']['application']['rss_access_token'] = secret
 
-        # Generate the API access token
-        if not 'api_access_token' in self.__data['settings']['application']:
-            secret = secrets.token_hex(16)
-            self.__data['settings']['application']['api_access_token'] = secret
-
         # Proxy list support - available as a selection in settings when text file is imported
         # CSV list
         # "name, address", or just "name"
@@ -123,8 +129,6 @@ class ChangeDetectionStore:
         if path.isfile(proxy_list_file):
             self.import_proxy_list(proxy_list_file)
 
-        # Bump the update version by running updates
-        self.run_updates()
 
         self.needs_write = True
 
@@ -133,19 +137,20 @@ class ChangeDetectionStore:
 
     # Returns the newest key, but if theres only 1 record, then it's counted as not being new, so return 0.
     def get_newest_history_key(self, uuid):
-        if len(self.__data['watching'][uuid]['history']) == 1:
+
+        cur = self.__history_db_connection.cursor()
+
+        c = cur.execute("SELECT COUNT(*) FROM watch_history WHERE watch_uuid = :uuid", {"uuid": uuid}).fetchone()
+        if c and c[0] <= 1:
             return 0
 
-        dates = list(self.__data['watching'][uuid]['history'].keys())
+        max = cur.execute("SELECT MAX(timestamp) FROM watch_history WHERE  watch_uuid  = :uuid", {"uuid": uuid}).fetchone()
-        # Convert to int, sort and back to str again
+        return max[0]
-        # @todo replace datastore getter that does this automatically
-        dates = [int(i) for i in dates]
-        dates.sort(reverse=True)
-        if len(dates):
-            # always keyed as str
-            return str(dates[0])
 
-        return 0
+    def __refresh_history_max_timestamp(self):
+        # select watch_uuid, max(timestamp) from watch_history group by watch_uuid;
+        # could be way faster
+        x=1
 
     def set_last_viewed(self, uuid, timestamp):
         self.data['watching'][uuid].update({'last_viewed': int(timestamp)})
@@ -190,13 +195,13 @@ class ChangeDetectionStore:
     def data(self):
         has_unviewed = False
         for uuid, v in self.__data['watching'].items():
-            self.__data['watching'][uuid]['newest_history_key'] = self.get_newest_history_key(uuid)
+#            self.__data['watching'][uuid]['newest_history_key'] = self.get_newest_history_key(uuid)
-            if int(v['newest_history_key']) <= int(v['last_viewed']):
+#            if int(v['newest_history_key']) <= int(v['last_viewed']):
-                self.__data['watching'][uuid]['viewed'] = True
+#                self.__data['watching'][uuid]['viewed'] = True
 
-            else:
+#            else:
-                self.__data['watching'][uuid]['viewed'] = False
+#                self.__data['watching'][uuid]['viewed'] = False
-                has_unviewed = True
+#                has_unviewed = True
 
             # #106 - Be sure this is None on empty string, False, None, etc
             # Default var for fetch_backend
@@ -215,8 +220,7 @@ class ChangeDetectionStore:
     def get_all_tags(self):
         tags = []
         for uuid, watch in self.data['watching'].items():
-            if watch['tag'] is None:
+
-                continue
             # Support for comma separated list of tags.
             for tag in watch['tag'].split(','):
                 tag = tag.strip()
@@ -285,10 +289,6 @@ class ChangeDetectionStore:
     def add_watch(self, url, tag="", extras=None, write_to_disk_now=True):
         if extras is None:
             extras = {}
-        # should always be str
-        if tag is None or not tag:
-            tag=''
-
         # Incase these are copied across, assume it's a reference and deepcopy()
         apply_extras = deepcopy(extras)
 
@@ -504,3 +504,31 @@ class ChangeDetectionStore:
                 # Only upgrade individual watch time if it was set
                 if watch.get('minutes_between_check', False):
                     self.data['watching'][uuid]['time_between_check']['minutes'] = watch['minutes_between_check']
+
+    def update_3(self):
+        """Migrate storage of history data to SQLite
+        - No need to store the history list in memory and re-write it everytime
+        - I've seen memory usage grow exponentially due to having large lists of watches with long histories
+        - Data about 'last changed' still stored in the main JSON struct which is fine
+        - We don't really need this data until we query against it (like for listing other available snapshots in the diff page etc)
+        """
+
+        if self.__history_db_connection:
+            # Create the table
+            self.__history_db_connection.execute("CREATE TABLE IF NOT EXISTS  watch_history(id INTEGER PRIMARY KEY, watch_uuid VARCHAR(36), timestamp INT, path TEXT, snapshot_type VARCHAR(10))")
+            self.__history_db_connection.execute("CREATE INDEX IF NOT EXISTS `uuid` ON `watch_history` (`watch_uuid`)")
+            self.__history_db_connection.execute("CREATE INDEX IF NOT EXISTS `uuid_timestamp` ON `watch_history` (`watch_uuid`, `timestamp`)")
+
+            # Insert each watch history list as executemany() for faster migration
+            for uuid, watch in self.data['watching'].items():
+                history = []
+
+                if watch.get('history', False):
+                    for d, p in watch['history'].items():
+                        d = int(d) # Used to be keyed as str, we'll fix this now too
+                        history.append((uuid, d, p, 'text'))
+
+                    if len(history):
+                        self.__history_db_connection.executemany("INSERT INTO watch_history (watch_uuid, timestamp, path, snapshot_type) VALUES (?,?,?,?)", history)
+                        self.__history_db_connection.commit()
+                        del(self.data['watching'][uuid]['history'])

changedetectionio/templates/edit.html

@@ -73,21 +73,6 @@
                         </span>
                     </div>
                 {% endif %}
-                <fieldset id="webdriver-override-options">
-                    <div class="pure-form-message-inline">
-                        <strong>If you're having trouble waiting for the page to be fully rendered (text missing etc), try increasing the 'wait' time here.</strong>
-                        <br/>
-                        This will wait <i>n</i> seconds before extracting the text.
-                    </div>
-                    <div class="pure-control-group">
-                        {{ render_field(form.webdriver_delay) }}
-                    </div>
-                    {% if using_global_webdriver_wait %}
-                    <div class="pure-form-message-inline">
-                        <strong>Using the current global default settings</strong>
-                    </div>
-                    {% endif %}
-                </fieldset>
                 <fieldset class="pure-group" id="requests-override-options">
                     <div class="pure-form-message-inline">
                         <strong>Request override is currently only used by the <i>Basic fast Plaintext/HTTP Client</i> method.</strong>
@@ -125,7 +110,8 @@ User-Agent: wonderbra 1.0") }}
             </div>
 
             <div class="tab-pane-inner" id="filters-and-triggers">
-                    <div class="pure-control-group">
+                <fieldset>
+                        <div class="pure-control-group">
                             <strong>Pro-tips:</strong><br/>
                             <ul>
                                 <li>
@@ -136,6 +122,7 @@ User-Agent: wonderbra 1.0") }}
                                 </li>
                             </ul>
                     </div>
+
                     <div class="pure-control-group">
                         {{ render_field(form.css_filter, placeholder=".class-name or #some-id, or other CSS selector rule.",
                         class="m-d") }}
@@ -151,7 +138,7 @@ User-Agent: wonderbra 1.0") }}
                                 href="https://github.com/dgtlmoon/changedetection.io/wiki/CSS-Selector-help">here for more CSS selector help</a>.<br/>
                 </span>
                     </div>
-                    <div class="pure-control-group">
+                    <fieldset class="pure-group">
                       {{ render_field(form.subtractive_selectors, rows=5, placeholder="header
 footer
 nav
@@ -162,7 +149,8 @@ nav
                           <li> Add multiple elements or CSS selectors per line to ignore multiple parts of the HTML. </li>
                         </ul>
                       </span>
-                    </div>
+                    </fieldset>
+                </fieldset>
                 <fieldset class="pure-group">
                     {{ render_field(form.ignore_text, rows=5, placeholder="Some text to ignore in a line
 /some.regex\d{2}/ for case-INsensitive regex
@@ -199,9 +187,9 @@ nav
 
                       {{ render_button(form.save_button) }} {{ render_button(form.save_and_preview_button) }}
 
-                    <a href="{{url_for('form_delete', uuid=uuid)}}"
+                    <a href="{{url_for('api_delete', uuid=uuid)}}"
                        class="pure-button button-small button-error ">Delete</a>
-                    <a href="{{url_for('form_clone', uuid=uuid)}}"
+                    <a href="{{url_for('api_clone', uuid=uuid)}}"
                        class="pure-button button-small ">Create Copy</a>
                 </div>
             </div>

changedetectionio/templates/settings.html

@@ -12,7 +12,6 @@
 <script type="text/javascript" src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
 <script type="text/javascript" src="{{url_for('static_content', group='js', filename='notifications.js')}}" defer></script>
 
-<script type="text/javascript" src="{{url_for('static_content', group='js', filename='global-settings.js')}}" defer></script>
 <div class="edit-form">
     <div class="tabs collapsable">
         <ul>
@@ -20,7 +19,6 @@
             <li class="tab"><a href="#notifications">Notifications</a></li>
             <li class="tab"><a href="#fetching">Fetching</a></li>
             <li class="tab"><a href="#filters">Global Filters</a></li>
-            <li class="tab"><a href="#api">API</a></li>
         </ul>
     </div>
     <div class="box-wrap inner">
@@ -44,7 +42,6 @@
                             <span class="pure-form-message-inline">Password is locked.</span>
                         {% endif %}
                     </div>
-
                     <div class="pure-control-group">
                         {{ render_field(form.application.form.base_url, placeholder="http://yoursite.com:5000/",
                         class="m-d") }}
@@ -63,11 +60,6 @@
                         {{ render_checkbox_field(form.application.form.real_browser_save_screenshot) }}
                         <span class="pure-form-message-inline">When using a Chrome browser, a screenshot from the last check will be available on the Diff page</span>
                     </div>
-
-                    <div class="pure-control-group">
-                        {{ render_checkbox_field(form.application.form.empty_pages_are_a_change) }}
-                        <span class="pure-form-message-inline">When a page contains HTML, but no renderable text appears (empty page), is this considered a change?</span>
-                    </div>
                 {% if form.requests.proxy %}
                     <div class="pure-control-group inline-radio">
                         {{ render_field(form.requests.form.proxy, class="fetch-backend-proxy") }}
@@ -95,18 +87,9 @@
                         <p>The <strong>Chrome/Javascript</strong> method requires a network connection to a running WebDriver+Chrome server, set by the ENV var 'WEBDRIVER_URL'. </p>
                     </span>
                 </div>
-                <fieldset class="pure-group" id="webdriver-override-options">
-                    <div class="pure-form-message-inline">
-                        <strong>If you're having trouble waiting for the page to be fully rendered (text missing etc), try increasing the 'wait' time here.</strong>
-                        <br/>
-                        This will wait <i>n</i> seconds before extracting the text.
-                    </div>
-                    <div class="pure-control-group">
-                        {{ render_field(form.application.form.webdriver_delay) }}
-                    </div>
-                </fieldset>
             </div>
 
+
             <div class="tab-pane-inner" id="filters">
 
                     <fieldset class="pure-group">
@@ -151,26 +134,12 @@ nav
                     </fieldset>
            </div>
 
-            <div class="tab-pane-inner" id="api">
-
-                <p>Drive your changedetection.io via API, More about <a href="https://github.com/dgtlmoon/changedetection.io/wiki/API-Reference">API access here</a></p>
-
-                <div class="pure-control-group">
-                    {{ render_checkbox_field(form.application.form.api_access_token_enabled) }}
-                    <div class="pure-form-message-inline">Restrict API access limit by using <code>x-api-key</code> header</div><br/>
-                    <div class="pure-form-message-inline"><br/>API Key <span id="api-key">{{api_key}}</span>
-                        <span style="display:none;" id="api-key-copy" >copy</span>
-                    </div>
-                </div>
-            </div>
-
             <div id="actions">
                 <div class="pure-control-group">
                     {{ render_button(form.save_button) }}
                     <a href="{{url_for('index')}}" class="pure-button button-small button-cancel">Back</a>
                     <a href="{{url_for('scrub_page')}}" class="pure-button button-small button-cancel">Delete History Snapshot Data</a>
                 </div>
-
             </div>
         </form>
     </div>

changedetectionio/templates/watch-overview.html

@@ -5,7 +5,7 @@
 <script type="text/javascript" src="{{url_for('static_content', group='js', filename='watch-overview.js')}}" defer></script>
 <div class="box">
 
-    <form class="pure-form" action="{{ url_for('form_watch_add') }}" method="POST" id="new-watch-form">
+    <form class="pure-form" action="{{ url_for('api_watch_add') }}" method="POST" id="new-watch-form">
         <input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
         <fieldset>
             <legend>Add a new change detection watch</legend>
@@ -13,7 +13,7 @@
                 {{ render_simple_field(form.tag, value=active_tag if active_tag else '', placeholder="watch group") }}
             <button type="submit" class="pure-button pure-button-primary">Watch</button>
         </fieldset>
-        <span style="color:#eee; font-size: 80%;"><img style="height: 1em;display:inline-block;" src="{{url_for('static_content', group='images', filename='spread.svg')}}" /> Tip: You can also add 'shared' watches. <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Sharing-a-Watch">More info</a></a></span>
+        <span style="color:#eee; font-size: 80%;"><img style="height: 1em;display:inline-block;" src="{{url_for('static_content', group='images', filename='spread.svg')}}" /> Tip: You can also add 'shared' watches. <a href="#">More info</a></a></span>
     </form>
     <div>
         <a href="{{url_for('index')}}" class="pure-button button-tag {{'active' if not active_tag }}">All</a>
@@ -52,7 +52,7 @@
 
                 <td class="title-col inline">{{watch.title if watch.title is not none and watch.title|length > 0 else watch.url}}
                     <a class="external" target="_blank" rel="noopener" href="{{ watch.url.replace('source:','') }}"></a>
-                    <a href="{{url_for('form_share_put_watch', uuid=watch.uuid)}}"><img style="height: 1em;display:inline-block;" src="{{url_for('static_content', group='images', filename='spread.svg')}}" /></a>
+                    <a href="{{url_for('api_share_put_watch', uuid=watch.uuid)}}"><img style="height: 1em;display:inline-block;" src="{{url_for('static_content', group='images', filename='spread.svg')}}" /></a>
 
                     {%if watch.fetch_backend == "html_webdriver" %}<img style="height: 1em; display:inline-block;" src="{{url_for('static_content', group='images', filename='Google-Chrome-icon.png')}}" />{% endif %}
 
@@ -74,7 +74,7 @@
                     {% endif %}
                 </td>
                 <td>
-                    <a {% if watch.uuid in queued_uuids %}disabled="true"{% endif %} href="{{ url_for('form_watch_checknow', uuid=watch.uuid, tag=request.args.get('tag')) }}"
+                    <a {% if watch.uuid in queued_uuids %}disabled="true"{% endif %} href="{{ url_for('api_watch_checknow', uuid=watch.uuid, tag=request.args.get('tag')) }}"
                        class="recheck pure-button button-small pure-button-primary">{% if watch.uuid in queued_uuids %}Queued{% else %}Recheck{% endif %}</a>
                     <a href="{{ url_for('edit_page', uuid=watch.uuid)}}" class="pure-button button-small pure-button-primary">Edit</a>
                     {% if watch.history|length >= 2 %}
@@ -96,7 +96,7 @@
             </li>
             {% endif %}
             <li>
-               <a href="{{ url_for('form_watch_checknow', tag=active_tag) }}" class="pure-button button-tag ">Recheck
+               <a href="{{ url_for('api_watch_checknow', tag=active_tag) }}" class="pure-button button-tag ">Recheck
                 all {% if active_tag%}in "{{active_tag}}"{%endif%}</a>
             </li>
             <li>

changedetectionio/tests/test_api.py

@@ -2,205 +2,73 @@
 
 import time
 from flask import url_for
-from .util import live_server_setup
+from . util import live_server_setup
 
-import json
+def test_setup(live_server):
-import uuid
-
-
-def set_original_response():
-    test_return_data = """<html>
-       <body>
-     Some initial text</br>
-     <p>Which is across multiple lines</p>
-     </br>
-     So let's see what happens.  </br>
-     <div id="sametext">Some text thats the same</div>
-     <div id="changetext">Some text that will change</div>
-     </body>
-     </html>
-    """
-
-    with open("test-datastore/endpoint-content.txt", "w") as f:
-        f.write(test_return_data)
-    return None
-
-
-def set_modified_response():
-    test_return_data = """<html>
-       <body>
-     Some initial text</br>
-     <p>which has this one new line</p>
-     </br>
-     So let's see what happens.  </br>
-     <div id="sametext">Some text thats the same</div>
-     <div id="changetext">Some text that changes</div>
-     </body>
-     </html>
-    """
-
-    with open("test-datastore/endpoint-content.txt", "w") as f:
-        f.write(test_return_data)
-
-    return None
-
-
-def is_valid_uuid(val):
-    try:
-        uuid.UUID(str(val))
-        return True
-    except ValueError:
-        return False
-
-
-# kinda funky, but works for now
-def _extract_api_key_from_UI(client):
-    import re
-    res = client.get(
-        url_for("settings_page"),
-    )
-    # <span id="api-key">{{api_key}}</span>
-
-    m = re.search('<span id="api-key">(.+?)</span>', str(res.data))
-    api_key = m.group(1)
-    return api_key.strip()
-
-
-def test_api_simple(client, live_server):
     live_server_setup(live_server)
 
-    api_key = _extract_api_key_from_UI(client)
 
-    # Create a watch
+def set_response_data(test_return_data):
-    set_original_response()
+    with open("test-datastore/endpoint-content.txt", "w") as f:
-    watch_uuid = None
+        f.write(test_return_data)
 
-    # Validate bad URL
+
-    test_url = url_for('test_endpoint', _external=True,
+def test_snapshot_api_detects_change(client, live_server):
-                       headers={'x-api-key': api_key}, )
+    test_return_data = "Some initial text"
+
+    test_return_data_modified = "Some NEW nice initial text"
+
+    sleep_time_for_fetch_thread = 3
+
+    set_response_data(test_return_data)
+
+    # Give the endpoint time to spin up
+    time.sleep(1)
+
+    # Add our URL to the import page
+    test_url = url_for('test_endpoint', content_type="text/plain",
+                       _external=True)
     res = client.post(
-        url_for("createwatch"),
+        url_for("import_page"),
-        data=json.dumps({"url": "h://xxxxxxxxxom"}),
+        data={"urls": test_url},
-        headers={'content-type': 'application/json', 'x-api-key': api_key},
         follow_redirects=True
     )
+    assert b"1 Imported" in res.data
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
+
+    res = client.get(
+        url_for("api_snapshot", uuid="first"),
+        follow_redirects=True
+    )
+
+    assert test_return_data.encode() == res.data
+
+    #  Make a change
+    set_response_data(test_return_data_modified)
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
+
+    res = client.get(
+        url_for("api_snapshot", uuid="first"),
+        follow_redirects=True
+    )
+
+    assert test_return_data_modified.encode() == res.data
+
+def test_snapshot_api_invalid_uuid(client, live_server):
+
+    res = client.get(
+        url_for("api_snapshot", uuid="invalid"),
+        follow_redirects=True
+    )
+
     assert res.status_code == 400
 
-    # Create new
-    res = client.post(
-        url_for("createwatch"),
-        data=json.dumps({"url": test_url, 'tag': "One, Two", "title": "My test URL"}),
-        headers={'content-type': 'application/json', 'x-api-key': api_key},
-        follow_redirects=True
-    )
-    s = json.loads(res.data)
-    assert is_valid_uuid(s['uuid'])
-    watch_uuid = s['uuid']
-    assert res.status_code == 201
-
-    time.sleep(3)
-
-    # Verify its in the list and that recheck worked
-    res = client.get(
-        url_for("createwatch"),
-        headers={'x-api-key': api_key}
-    )
-    assert watch_uuid in json.loads(res.data).keys()
-    before_recheck_info = json.loads(res.data)[watch_uuid]
-    assert before_recheck_info['last_checked'] != 0
-    assert before_recheck_info['title'] == 'My test URL'
-
-    set_modified_response()
-    # Trigger recheck of all ?recheck_all=1
-    client.get(
-        url_for("createwatch", recheck_all='1'),
-        headers={'x-api-key': api_key},
-    )
-    time.sleep(3)
-
-    # Did the recheck fire?
-    res = client.get(
-        url_for("createwatch"),
-        headers={'x-api-key': api_key},
-    )
-    after_recheck_info = json.loads(res.data)[watch_uuid]
-    assert after_recheck_info['last_checked'] != before_recheck_info['last_checked']
-    assert after_recheck_info['last_changed'] != 0
-
-    # Check history index list
-    res = client.get(
-        url_for("watchhistory", uuid=watch_uuid),
-        headers={'x-api-key': api_key},
-    )
-    history = json.loads(res.data)
-    assert len(history) == 2, "Should have two history entries (the original and the changed)"
-
-    # Fetch a snapshot by timestamp, check the right one was found
-    res = client.get(
-        url_for("watchsinglehistory", uuid=watch_uuid, timestamp=list(history.keys())[-1]),
-        headers={'x-api-key': api_key},
-    )
-    assert b'which has this one new line' in res.data
-
-    # Fetch a snapshot by 'latest'', check the right one was found
-    res = client.get(
-        url_for("watchsinglehistory", uuid=watch_uuid, timestamp='latest'),
-        headers={'x-api-key': api_key},
-    )
-    assert b'which has this one new line' in res.data
-
-    # Fetch the whole watch
-    res = client.get(
-        url_for("watch", uuid=watch_uuid),
-        headers={'x-api-key': api_key}
-    )
-    watch = json.loads(res.data)
-    # @todo how to handle None/default global values?
-    assert watch['history_n'] == 2, "Found replacement history section, which is in its own API"
-
-    # Finally delete the watch
-    res = client.delete(
-        url_for("watch", uuid=watch_uuid),
-        headers={'x-api-key': api_key},
-    )
-    assert res.status_code == 204
-
-    # Check via a relist
-    res = client.get(
-        url_for("createwatch"),
-        headers={'x-api-key': api_key}
-    )
-    watch_list = json.loads(res.data)
-    assert len(watch_list) == 0, "Watch list should be empty"
-
-
-def test_access_denied(client, live_server):
-    # `config_api_token_enabled` Should be On by default
-    res = client.get(
-        url_for("createwatch")
-    )
-    assert res.status_code == 403
-
-    res = client.get(
-        url_for("createwatch"),
-        headers={'x-api-key': "something horrible"}
-    )
-    assert res.status_code == 403
-
-    # Disable config_api_token_enabled and it should work
-    res = client.post(
-        url_for("settings_page"),
-        data={
-            "requests-time_between_check-minutes": 180,
-            "application-fetch_backend": "html_requests",
-            "application-api_access_token_enabled": ""
-        },
-        follow_redirects=True
-    )
-
-    assert b"Settings updated." in res.data
-
-    res = client.get(
-        url_for("createwatch")
-    )
-    assert res.status_code == 200

changedetectionio/tests/test_auth.py

@@ -29,7 +29,7 @@ def test_basic_auth(client, live_server):
     assert b"Updated watch." in res.data
 
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
     time.sleep(1)
     res = client.get(
         url_for("preview_page", uuid="first"),

changedetectionio/tests/test_backend.py

@@ -32,7 +32,7 @@ def test_check_basic_change_detection_functionality(client, live_server):
 
     # Do this a few times.. ensures we dont accidently set the status
     for n in range(3):
-        client.get(url_for("form_watch_checknow"), follow_redirects=True)
+        client.get(url_for("api_watch_checknow"), follow_redirects=True)
 
         # Give the thread time to pick it up
         time.sleep(sleep_time_for_fetch_thread)
@@ -65,7 +65,7 @@ def test_check_basic_change_detection_functionality(client, live_server):
     assert b'which has this one new line' in res.read()
 
     # Force recheck
-    res = client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    res = client.get(url_for("api_watch_checknow"), follow_redirects=True)
     assert b'1 watches are queued for rechecking.' in res.data
 
     time.sleep(sleep_time_for_fetch_thread)
@@ -93,7 +93,7 @@ def test_check_basic_change_detection_functionality(client, live_server):
 
     # Do this a few times.. ensures we dont accidently set the status
     for n in range(2):
-        client.get(url_for("form_watch_checknow"), follow_redirects=True)
+        client.get(url_for("api_watch_checknow"), follow_redirects=True)
 
         # Give the thread time to pick it up
         time.sleep(sleep_time_for_fetch_thread)
@@ -113,7 +113,7 @@ def test_check_basic_change_detection_functionality(client, live_server):
         follow_redirects=True
     )
 
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
     time.sleep(sleep_time_for_fetch_thread)
 
     res = client.get(url_for("index"))
@@ -123,6 +123,6 @@ def test_check_basic_change_detection_functionality(client, live_server):
 
     #
     # Cleanup everything
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    res = client.get(url_for("api_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
 

changedetectionio/tests/test_clone.py

@@ -23,7 +23,7 @@ def test_trigger_functionality(client, live_server):
 
 
     res = client.get(
-        url_for("form_clone", uuid="first"),
+        url_for("api_clone", uuid="first"),
         follow_redirects=True
     )
 

changedetectionio/tests/test_css_selector.py

@@ -89,7 +89,7 @@ def test_check_markup_css_filter_restriction(client, live_server):
     assert b"1 Imported" in res.data
 
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
@@ -110,7 +110,7 @@ def test_check_markup_css_filter_restriction(client, live_server):
     assert bytes(css_filter.encode('utf-8')) in res.data
 
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
@@ -118,7 +118,7 @@ def test_check_markup_css_filter_restriction(client, live_server):
     set_modified_response()
 
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
 

changedetectionio/tests/test_element_removal.py

@@ -145,7 +145,7 @@ def test_element_removal_full(client, live_server):
     assert bytes(subtractive_selectors_data.encode("utf-8")) in res.data
 
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
@@ -158,7 +158,7 @@ def test_element_removal_full(client, live_server):
     set_modified_response()
 
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)

changedetectionio/tests/test_encoding.py

@@ -39,7 +39,7 @@ def test_check_encoding_detection(client, live_server):
     )
 
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(2)
@@ -71,7 +71,7 @@ def test_check_encoding_detection_missing_content_type_header(client, live_serve
     )
 
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(2)

changedetectionio/tests/test_errorhandling.py

@@ -29,7 +29,7 @@ def test_error_handler(client, live_server):
     assert b"1 Imported" in res.data
 
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(3)
@@ -54,7 +54,7 @@ def test_error_text_handler(client, live_server):
     assert b"1 Imported" in res.data
 
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(3)

changedetectionio/tests/test_ignore_text.py

@@ -102,7 +102,7 @@ def test_check_ignore_text_functionality(client, live_server):
     assert b"1 Imported" in res.data
 
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
@@ -123,7 +123,7 @@ def test_check_ignore_text_functionality(client, live_server):
     assert bytes(ignore_text.encode('utf-8')) in res.data
 
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
@@ -137,7 +137,7 @@ def test_check_ignore_text_functionality(client, live_server):
     set_modified_ignore_response()
 
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
 
@@ -152,7 +152,7 @@ def test_check_ignore_text_functionality(client, live_server):
 
     # Just to be sure.. set a regular modified change..
     set_modified_original_ignore_response()
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
     time.sleep(sleep_time_for_fetch_thread)
 
     res = client.get(url_for("index"))
@@ -165,7 +165,7 @@ def test_check_ignore_text_functionality(client, live_server):
     # We should be able to see what we ignored
     assert b'<div class="ignored">new ignore stuff' in res.data
 
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    res = client.get(url_for("api_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
 
 def test_check_global_ignore_text_functionality(client, live_server):
@@ -200,7 +200,7 @@ def test_check_global_ignore_text_functionality(client, live_server):
     assert b"1 Imported" in res.data
 
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
@@ -222,7 +222,7 @@ def test_check_global_ignore_text_functionality(client, live_server):
     assert bytes(ignore_text.encode('utf-8')) in res.data
 
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
@@ -240,7 +240,7 @@ def test_check_global_ignore_text_functionality(client, live_server):
     set_modified_ignore_response()
 
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
 
@@ -251,10 +251,10 @@ def test_check_global_ignore_text_functionality(client, live_server):
 
     # Just to be sure.. set a regular modified change that will trigger it
     set_modified_original_ignore_response()
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
     time.sleep(sleep_time_for_fetch_thread)
     res = client.get(url_for("index"))
     assert b'unviewed' in res.data
 
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    res = client.get(url_for("api_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data

changedetectionio/tests/test_ignorehyperlinks.py

@@ -72,14 +72,14 @@ def test_render_anchor_tag_content_true(client, live_server):
 
     time.sleep(sleep_time_for_fetch_thread)
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
 
     # set a new html text with a modified link
     set_modified_ignore_response()
     time.sleep(sleep_time_for_fetch_thread)
 
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
@@ -101,7 +101,7 @@ def test_render_anchor_tag_content_true(client, live_server):
     assert b"Settings updated." in res.data
 
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
@@ -119,7 +119,7 @@ def test_render_anchor_tag_content_true(client, live_server):
     assert b"/test-endpoint" in res.data
 
     # Cleanup everything
-    res = client.get(url_for("form_delete", uuid="all"),
+    res = client.get(url_for("api_delete", uuid="all"),
                      follow_redirects=True)
     assert b'Deleted' in res.data
 

changedetectionio/tests/test_ignorestatuscode.py

@@ -70,12 +70,12 @@ def test_normal_page_check_works_with_ignore_status_code(client, live_server):
 
     time.sleep(sleep_time_for_fetch_thread)
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
 
     set_some_changed_response()
     time.sleep(sleep_time_for_fetch_thread)
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
@@ -105,7 +105,7 @@ def test_403_page_check_works_with_ignore_status_code(client, live_server):
     assert b"1 Imported" in res.data
 
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
@@ -120,7 +120,7 @@ def test_403_page_check_works_with_ignore_status_code(client, live_server):
     assert b"Updated watch." in res.data
 
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
@@ -128,7 +128,7 @@ def test_403_page_check_works_with_ignore_status_code(client, live_server):
     set_some_changed_response()
 
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
 
@@ -157,7 +157,7 @@ def test_403_page_check_fails_without_ignore_status_code(client, live_server):
     assert b"1 Imported" in res.data
 
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
@@ -172,7 +172,7 @@ def test_403_page_check_fails_without_ignore_status_code(client, live_server):
     assert b"Updated watch." in res.data
 
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
@@ -180,7 +180,7 @@ def test_403_page_check_fails_without_ignore_status_code(client, live_server):
     set_some_changed_response()
 
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
 

changedetectionio/tests/test_ignorewhitespace.py

@@ -80,12 +80,12 @@ def test_check_ignore_whitespace(client, live_server):
 
     time.sleep(sleep_time_for_fetch_thread)
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
 
     set_original_ignore_response_but_with_whitespace()
     time.sleep(sleep_time_for_fetch_thread)
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)

changedetectionio/tests/test_import.py

@@ -25,7 +25,7 @@ https://example.com tag1, other tag"""
     assert b"3 Imported" in res.data
     assert b"tag1" in res.data
     assert b"other tag" in res.data
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    res = client.get(url_for("api_delete", uuid="all"), follow_redirects=True)
 
     # Clear flask alerts
     res = client.get( url_for("index"))
@@ -50,7 +50,7 @@ def xtest_import_skip_url(client, live_server):
     assert b"1 Imported" in res.data
     assert b"ht000000broken" in res.data
     assert b"1 Skipped" in res.data
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    res = client.get(url_for("api_delete", uuid="all"), follow_redirects=True)
     # Clear flask alerts
     res = client.get( url_for("index"))
 
@@ -79,7 +79,7 @@ def test_import_distillio(client, live_server):
 
     # Give the endpoint time to spin up
     time.sleep(1)
-    client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    client.get(url_for("api_delete", uuid="all"), follow_redirects=True)
     res = client.post(
         url_for("import_page"),
         data={
@@ -115,6 +115,6 @@ def test_import_distillio(client, live_server):
     assert b"nice stuff" in res.data
     assert b"nerd-news" in res.data
 
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    res = client.get(url_for("api_delete", uuid="all"), follow_redirects=True)
     # Clear flask alerts
     res = client.get(url_for("index"))

changedetectionio/tests/test_jsonpath_selector.py

@@ -171,7 +171,7 @@ def test_check_json_without_filter(client, live_server):
     )
 
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(3)
@@ -203,7 +203,7 @@ def test_check_json_filter(client, live_server):
     assert b"1 Imported" in res.data
 
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(3)
@@ -229,7 +229,7 @@ def test_check_json_filter(client, live_server):
     assert bytes(json_filter.encode('utf-8')) in res.data
 
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(3)
@@ -237,7 +237,7 @@ def test_check_json_filter(client, live_server):
     set_modified_response()
 
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
     # Give the thread time to pick it up
     time.sleep(4)
 
@@ -288,7 +288,7 @@ def test_check_json_filter_bool_val(client, live_server):
     time.sleep(3)
 
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(3)
@@ -296,7 +296,7 @@ def test_check_json_filter_bool_val(client, live_server):
     set_modified_response()
 
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
     # Give the thread time to pick it up
     time.sleep(3)
 
@@ -327,7 +327,7 @@ def test_check_json_ext_filter(client, live_server):
     assert b"1 Imported" in res.data
 
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(3)
@@ -353,7 +353,7 @@ def test_check_json_ext_filter(client, live_server):
     assert bytes(json_filter.encode('utf-8')) in res.data
 
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(3)
@@ -361,7 +361,7 @@ def test_check_json_ext_filter(client, live_server):
     set_modified_ext_response()
 
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
     # Give the thread time to pick it up
     time.sleep(4)
 

changedetectionio/tests/test_nonrenderable_pages.py

@@ -1,102 +0,0 @@
-#!/usr/bin/python3
-
-import time
-from flask import url_for
-from urllib.request import urlopen
-from .util import set_original_response, set_modified_response, live_server_setup
-
-sleep_time_for_fetch_thread = 3
-
-
-def set_nonrenderable_response():
-    test_return_data = """<html>
-    <head><title>modified head title</title></head>
-    <!-- like when some angular app was broken and doesnt render or whatever -->
-    <body>
-     </body>
-     </html>
-    """
-
-    with open("test-datastore/endpoint-content.txt", "w") as f:
-        f.write(test_return_data)
-
-    return None
-
-def test_check_basic_change_detection_functionality(client, live_server):
-    set_original_response()
-    live_server_setup(live_server)
-
-    # Add our URL to the import page
-    res = client.post(
-        url_for("import_page"),
-        data={"urls": url_for('test_endpoint', _external=True)},
-        follow_redirects=True
-    )
-
-    assert b"1 Imported" in res.data
-
-    time.sleep(sleep_time_for_fetch_thread)
-
-    # Do this a few times.. ensures we dont accidently set the status
-    for n in range(3):
-        client.get(url_for("form_watch_checknow"), follow_redirects=True)
-
-        # Give the thread time to pick it up
-        time.sleep(sleep_time_for_fetch_thread)
-
-        # It should report nothing found (no new 'unviewed' class)
-        res = client.get(url_for("index"))
-        assert b'unviewed' not in res.data
-
-
-    #####################
-    client.post(
-        url_for("settings_page"),
-        data={"application-empty_pages_are_a_change": "",
-              "requests-time_between_check-minutes": 180,
-              'application-fetch_backend': "html_requests"},
-        follow_redirects=True
-    )
-
-    # this should not trigger a change, because no good text could be converted from the HTML
-    set_nonrenderable_response()
-
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
-
-    # Give the thread time to pick it up
-    time.sleep(sleep_time_for_fetch_thread)
-
-    # It should report nothing found (no new 'unviewed' class)
-    res = client.get(url_for("index"))
-    assert b'unviewed' not in res.data
-
-
-    # ok now do the opposite
-
-    client.post(
-        url_for("settings_page"),
-        data={"application-empty_pages_are_a_change": "y",
-              "requests-time_between_check-minutes": 180,
-              'application-fetch_backend': "html_requests"},
-        follow_redirects=True
-    )
-    set_modified_response()
-
-
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
-
-    # Give the thread time to pick it up
-    time.sleep(sleep_time_for_fetch_thread)
-
-    # It should report nothing found (no new 'unviewed' class)
-    res = client.get(url_for("index"))
-    assert b'unviewed' in res.data
-
-
-
-
-    #
-    # Cleanup everything
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
-    assert b'Deleted' in res.data
-

changedetectionio/tests/test_notification.py

@@ -36,7 +36,7 @@ def test_check_notification(client, live_server):
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
-        url_for("form_watch_add"),
+        url_for("api_watch_add"),
         data={"url": test_url, "tag": ''},
         follow_redirects=True
     )
@@ -98,7 +98,7 @@ def test_check_notification(client, live_server):
     notification_submission = None
 
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
     time.sleep(3)
     # Verify what was sent as a notification, this file should exist
     with open("test-datastore/notification.txt", "r") as f:
@@ -133,7 +133,7 @@ def test_check_notification(client, live_server):
 
     # This should insert the {current_snapshot}
     set_more_modified_response()
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
     time.sleep(3)
     # Verify what was sent as a notification, this file should exist
     with open("test-datastore/notification.txt", "r") as f:
@@ -146,17 +146,17 @@ def test_check_notification(client, live_server):
     os.unlink("test-datastore/notification.txt")
 
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
     time.sleep(1)
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
     time.sleep(1)
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
     time.sleep(1)
     assert os.path.exists("test-datastore/notification.txt") == False
 
     # cleanup for the next
     client.get(
-        url_for("form_delete", uuid="all"),
+        url_for("api_delete", uuid="all"),
         follow_redirects=True
     )
 
@@ -168,7 +168,7 @@ def test_notification_validation(client, live_server):
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
-        url_for("form_watch_add"),
+        url_for("api_watch_add"),
         data={"url": test_url, "tag": 'nice one'},
         follow_redirects=True
     )
@@ -208,6 +208,6 @@ def test_notification_validation(client, live_server):
 
     # cleanup for the next
     client.get(
-        url_for("form_delete", uuid="all"),
+        url_for("api_delete", uuid="all"),
         follow_redirects=True
     )

changedetectionio/tests/test_notification_errors.py

@@ -16,7 +16,7 @@ def test_check_notification_error_handling(client, live_server):
     # use a different URL so that it doesnt interfere with the actual check until we are ready
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
-        url_for("form_watch_add"),
+        url_for("api_watch_add"),
         data={"url": "https://changedetection.io/CHANGELOG.txt", "tag": ''},
         follow_redirects=True
     )

changedetectionio/tests/test_share_watch.py

@@ -41,7 +41,7 @@ def test_share_watch(client, live_server):
 
     # click share the link
     res = client.get(
-        url_for("form_share_put_watch", uuid="first"),
+        url_for("api_share_put_watch", uuid="first"),
         follow_redirects=True
     )
 
@@ -54,7 +54,7 @@ def test_share_watch(client, live_server):
 
     # Now delete what we have, we will try to re-import it
     # Cleanup everything
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    res = client.get(url_for("api_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
 
     # Add our URL to the import page

changedetectionio/tests/test_source.py

@@ -39,7 +39,7 @@ def test_check_basic_change_detection_functionality_source(client, live_server):
     set_modified_response()
 
     # Force recheck
-    res = client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    res = client.get(url_for("api_watch_checknow"), follow_redirects=True)
     assert b'1 watches are queued for rechecking.' in res.data
 
     time.sleep(5)

changedetectionio/tests/test_trigger.py

@@ -76,7 +76,7 @@ def test_trigger_functionality(client, live_server):
     assert b"1 Imported" in res.data
 
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
@@ -99,7 +99,7 @@ def test_trigger_functionality(client, live_server):
     assert bytes(trigger_text.encode('utf-8')) in res.data
 
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
@@ -113,7 +113,7 @@ def test_trigger_functionality(client, live_server):
     set_modified_original_ignore_response()
 
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
 
@@ -125,7 +125,7 @@ def test_trigger_functionality(client, live_server):
     time.sleep(sleep_time_for_fetch_thread)
     set_modified_with_trigger_text_response()
 
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
     time.sleep(sleep_time_for_fetch_thread)
     res = client.get(url_for("index"))
     assert b'unviewed' in res.data

changedetectionio/tests/test_trigger_regex.py

@@ -43,7 +43,7 @@ def test_trigger_regex_functionality(client, live_server):
     assert b"1 Imported" in res.data
 
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
@@ -65,7 +65,7 @@ def test_trigger_regex_functionality(client, live_server):
     with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write("some new noise")
 
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
     time.sleep(sleep_time_for_fetch_thread)
 
     # It should report nothing found (nothing should match the regex)
@@ -75,7 +75,7 @@ def test_trigger_regex_functionality(client, live_server):
     with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write("regex test123<br/>\nsomething 123")
 
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
     time.sleep(sleep_time_for_fetch_thread)
     res = client.get(url_for("index"))
     assert b'unviewed' in res.data

changedetectionio/tests/test_trigger_regex_with_filter.py

@@ -43,7 +43,7 @@ def test_trigger_regex_functionality(client, live_server):
     assert b"1 Imported" in res.data
 
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
@@ -66,7 +66,7 @@ def test_trigger_regex_functionality(client, live_server):
     with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write("<html>some new noise with cool stuff2 ok</html>")
 
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
     time.sleep(sleep_time_for_fetch_thread)
 
     # It should report nothing found (nothing should match the regex and filter)
@@ -76,7 +76,7 @@ def test_trigger_regex_functionality(client, live_server):
     with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write("<html>some new noise with <span id=in-here>cool stuff6</span> ok</html>")
 
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
     time.sleep(sleep_time_for_fetch_thread)
     res = client.get(url_for("index"))
     assert b'unviewed' in res.data

changedetectionio/tests/test_xpath_selector.py

@@ -65,7 +65,7 @@ def test_check_markup_xpath_filter_restriction(client, live_server):
     assert b"1 Imported" in res.data
 
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
@@ -89,7 +89,7 @@ def test_check_markup_xpath_filter_restriction(client, live_server):
     set_modified_response()
 
     # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
 
@@ -121,7 +121,7 @@ def test_xpath_validation(client, live_server):
 
 # actually only really used by the distll.io importer, but could be handy too
 def test_check_with_prefix_css_filter(client, live_server):
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    res = client.get(url_for("api_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
 
     # Give the endpoint time to spin up
@@ -158,4 +158,4 @@ def test_check_with_prefix_css_filter(client, live_server):
     assert b"Some text thats the same" in res.data #in selector
     assert b"Some text that will change" not in res.data #not in selector
 
-    client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    client.get(url_for("api_delete", uuid="all"), follow_redirects=True)

changedetectionio/update_worker.py

@@ -52,10 +52,6 @@ class update_worker(threading.Thread):
                             raise Exception("Error - returned data from the fetch handler SHOULD be bytes")
                     except PermissionError as e:
                         self.app.logger.error("File permission error updating", uuid, str(e))
-                    except content_fetcher.ReplyWithContentButNoText as e:
-                        # Totally fine, it's by choice - just continue on, nothing more to care about
-                        # Page had elements/content but no renderable text
-                        pass
                     except content_fetcher.EmptyReply as e:
                         # Some kind of custom to-str handler in the exception handler that does this?
                         err_text = "EmptyReply: Status Code {}".format(e.status_code)

requirements.txt

@@ -6,7 +6,6 @@ timeago ~=1.0
 inscriptis ~= 2.2
 feedgen ~= 0.9
 flask-login ~= 0.5
-flask_restful
 pytz
 
 # Set these versions together to avoid a RequestsDependencyWarning