Also test for 204 - with empty reply

Re #962
Re #1371

.github/workflows/test-only.yml

@@ -67,10 +67,10 @@ jobs:
           sleep 3
           # Should return 0 (no error) when grep finds it
           curl -s http://localhost:5556 |grep -q checkbox-uuid
-          curl -s http://localhost:5556/rss|grep -q rss-specification
+          
           # and IPv6
           curl -s -g -6 "http://[::1]:5556"|grep -q checkbox-uuid
-          curl -s -g -6 "http://[::1]:5556/rss"|grep -q rss-specification
+          
 
 #export WEBDRIVER_URL=http://localhost:4444/wd/hub
 #pytest tests/fetchers/test_content.py

changedetectionio/__init__.py

@@ -1,5 +1,15 @@
 #!/usr/bin/python3
 
+from changedetectionio import queuedWatchMetaData
+from copy import deepcopy
+from distutils.util import strtobool
+from feedgen.feed import FeedGenerator
+from flask_compress import Compress as FlaskCompress
+from flask_login import current_user
+from flask_restful import abort, Api
+from flask_wtf import CSRFProtect
+from functools import wraps
+from threading import Event
 import datetime
 import flask_login
 import logging
@@ -10,12 +20,6 @@ import threading
 import time
 import timeago
 
-from changedetectionio import queuedWatchMetaData
-from copy import deepcopy
-from distutils.util import strtobool
-from feedgen.feed import FeedGenerator
-from threading import Event
-
 from flask import (
     Flask,
     abort,
@@ -28,15 +32,11 @@ from flask import (
     session,
     url_for,
 )
-from flask_compress import Compress as FlaskCompress
-from flask_login import login_required
-from flask_restful import abort, Api
-from flask_wtf import CSRFProtect
 
 from changedetectionio import html_tools
 from changedetectionio.api import api_v1
 
-__version__ = '0.40.1.1'
+__version__ = '0.40.2'
 
 datastore = None
 
@@ -53,7 +53,6 @@ app = Flask(__name__,
             static_url_path="",
             static_folder="static",
             template_folder="templates")
-from flask_compress import Compress
 
 # Super handy for compressing large BrowserSteps responses and others
 FlaskCompress(app)
@@ -65,8 +64,6 @@ app.config.exit = Event()
 
 app.config['NEW_VERSION_AVAILABLE'] = False
 
-app.config['LOGIN_DISABLED'] = False
-
 #app.config["EXPLAIN_TEMPLATE_LOADING"] = True
 
 # Disables caching of the templates
@@ -148,7 +145,6 @@ class User(flask_login.UserMixin):
 
     # Compare given password against JSON store or Env var
     def check_password(self, password):
-
         import base64
         import hashlib
 
@@ -156,11 +152,9 @@ class User(flask_login.UserMixin):
         raw_salt_pass = os.getenv("SALTED_PASS", False)
 
         if not raw_salt_pass:
-            raw_salt_pass = datastore.data['settings']['application']['password']
+            raw_salt_pass = datastore.data['settings']['application'].get('password')
 
         raw_salt_pass = base64.b64decode(raw_salt_pass)
-
-
         salt_from_storage = raw_salt_pass[:32]  # 32 is the length of the salt
 
         # Use the exact same setup you used to generate the key, but this time put in the password to check
@@ -170,21 +164,44 @@ class User(flask_login.UserMixin):
             salt_from_storage,
             100000
         )
-        new_key =  salt_from_storage + new_key
+        new_key = salt_from_storage + new_key
 
         return new_key == raw_salt_pass
 
     pass
 
+
+def login_optionally_required(func):
+    @wraps(func)
+    def decorated_view(*args, **kwargs):
+
+        has_password_enabled = datastore.data['settings']['application'].get('password') or os.getenv("SALTED_PASS", False)
+
+        # Permitted
+        if request.endpoint == 'static_content' and request.view_args['group'] == 'styles':
+            return func(*args, **kwargs)
+        # Permitted
+        elif request.endpoint == 'diff_history_page' and datastore.data['settings']['application'].get('shared_diff_access'):
+            return func(*args, **kwargs)
+
+        elif request.method in flask_login.config.EXEMPT_METHODS:
+            return func(*args, **kwargs)
+        elif app.config.get('LOGIN_DISABLED'):
+            return func(*args, **kwargs)
+        elif has_password_enabled and not current_user.is_authenticated:
+            return app.login_manager.unauthorized()
+
+        return func(*args, **kwargs)
+
+    return decorated_view
+
 def changedetection_app(config=None, datastore_o=None):
     global datastore
     datastore = datastore_o
 
     # so far just for read-only via tests, but this will be moved eventually to be the main source
     # (instead of the global var)
-    app.config['DATASTORE']=datastore_o
-
-    #app.config.update(config or {})
+    app.config['DATASTORE'] = datastore_o
 
     login_manager = flask_login.LoginManager(app)
     login_manager.login_view = 'login'
@@ -212,6 +229,8 @@ def changedetection_app(config=None, datastore_o=None):
     # https://flask-cors.readthedocs.io/en/latest/
     #    CORS(app)
 
+
+
     @login_manager.user_loader
     def user_loader(email):
         user = User()
@@ -220,7 +239,7 @@ def changedetection_app(config=None, datastore_o=None):
 
     @login_manager.unauthorized_handler
     def unauthorized_handler():
-        # @todo validate its a URL of this host and use that
+        flash("You must be logged in, please log in.", 'error')
         return redirect(url_for('login', next=url_for('index')))
 
     @app.route('/logout')
@@ -233,10 +252,6 @@ def changedetection_app(config=None, datastore_o=None):
     @app.route('/login', methods=['GET', 'POST'])
     def login():
 
-        if not datastore.data['settings']['application']['password'] and not os.getenv("SALTED_PASS", False):
-            flash("Login not required, no password enabled.", "notice")
-            return redirect(url_for('index'))
-
         if request.method == 'GET':
             if flask_login.current_user.is_authenticated:
                 flash("Already logged in")
@@ -271,27 +286,22 @@ def changedetection_app(config=None, datastore_o=None):
         return redirect(url_for('login'))
 
     @app.before_request
-    def do_something_whenever_a_request_comes_in():
-
-        # Disable password login if there is not one set
-        # (No password in settings or env var)
-        app.config['LOGIN_DISABLED'] = datastore.data['settings']['application']['password'] == False and os.getenv("SALTED_PASS", False) == False
-
+    def before_request_handle_cookie_x_settings():
         # Set the auth cookie path if we're running as X-settings/X-Forwarded-Prefix
         if os.getenv('USE_X_SETTINGS') and 'X-Forwarded-Prefix' in request.headers:
             app.config['REMEMBER_COOKIE_PATH'] = request.headers['X-Forwarded-Prefix']
             app.config['SESSION_COOKIE_PATH'] = request.headers['X-Forwarded-Prefix']
 
-        # For the RSS path, allow access via a token
-        if request.path == '/rss' and request.args.get('token'):
-            app_rss_token = datastore.data['settings']['application']['rss_access_token']
-            rss_url_token = request.args.get('token')
-            if app_rss_token == rss_url_token:
-                app.config['LOGIN_DISABLED'] = True
+        return None
 
     @app.route("/rss", methods=['GET'])
-    @login_required
     def rss():
+        # Always requires token set
+        app_rss_token = datastore.data['settings']['application'].get('rss_access_token')
+        rss_url_token = request.args.get('token')
+        if rss_url_token != app_rss_token:
+            return "Access denied, bad token", 403
+
         from . import diff
         limit_tag = request.args.get('tag')
 
@@ -365,7 +375,7 @@ def changedetection_app(config=None, datastore_o=None):
         return response
 
     @app.route("/", methods=['GET'])
-    @login_required
+    @login_optionally_required
     def index():
         from changedetectionio import forms
 
@@ -429,7 +439,7 @@ def changedetection_app(config=None, datastore_o=None):
 
     # AJAX endpoint for sending a test
     @app.route("/notification/send-test", methods=['POST'])
-    @login_required
+    @login_optionally_required
     def ajax_callback_send_notification_test():
 
         import apprise
@@ -462,7 +472,7 @@ def changedetection_app(config=None, datastore_o=None):
 
 
     @app.route("/clear_history/<string:uuid>", methods=['GET'])
-    @login_required
+    @login_optionally_required
     def clear_watch_history(uuid):
         try:
             datastore.clear_watch_history(uuid)
@@ -474,7 +484,7 @@ def changedetection_app(config=None, datastore_o=None):
         return redirect(url_for('index'))
 
     @app.route("/clear_history", methods=['GET', 'POST'])
-    @login_required
+    @login_optionally_required
     def clear_all_history():
 
         if request.method == 'POST':
@@ -531,7 +541,7 @@ def changedetection_app(config=None, datastore_o=None):
 
 
     @app.route("/edit/<string:uuid>", methods=['GET', 'POST'])
-    @login_required
+    @login_optionally_required
     # https://stackoverflow.com/questions/42984453/wtforms-populate-form-with-data-if-data-exists
     # https://wtforms.readthedocs.io/en/3.0.x/forms/#wtforms.form.Form.populate_obj ?
 
@@ -677,7 +687,7 @@ def changedetection_app(config=None, datastore_o=None):
         return output
 
     @app.route("/settings", methods=['GET', "POST"])
-    @login_required
+    @login_optionally_required
     def settings_page():
         from changedetectionio import content_fetcher, forms
 
@@ -757,7 +767,7 @@ def changedetection_app(config=None, datastore_o=None):
         return output
 
     @app.route("/import", methods=['GET', "POST"])
-    @login_required
+    @login_optionally_required
     def import_page():
         remaining_urls = []
         if request.method == 'POST':
@@ -795,7 +805,7 @@ def changedetection_app(config=None, datastore_o=None):
 
     # Clear all statuses, so we do not see the 'unviewed' class
     @app.route("/form/mark-all-viewed", methods=['GET'])
-    @login_required
+    @login_optionally_required
     def mark_all_viewed():
 
         # Save the current newest history as the most recently viewed
@@ -805,7 +815,7 @@ def changedetection_app(config=None, datastore_o=None):
         return redirect(url_for('index'))
 
     @app.route("/diff/<string:uuid>", methods=['GET', 'POST'])
-    @login_required
+    @login_optionally_required
     def diff_history_page(uuid):
 
         from changedetectionio import forms
@@ -881,8 +891,13 @@ def changedetection_app(config=None, datastore_o=None):
 
         system_uses_webdriver = datastore.data['settings']['application']['fetch_backend'] == 'html_webdriver'
 
-        is_html_webdriver = True if watch.get('fetch_backend') == 'html_webdriver' or (
-                    watch.get('fetch_backend', None) is None and system_uses_webdriver) else False
+        is_html_webdriver = False
+        if (watch.get('fetch_backend') == 'system' and system_uses_webdriver) or watch.get('fetch_backend') == 'html_webdriver':
+            is_html_webdriver = True
+
+        password_enabled_and_share_is_off = False
+        if datastore.data['settings']['application'].get('password') or os.getenv("SALTED_PASS", False):
+            password_enabled_and_share_is_off = not datastore.data['settings']['application'].get('shared_diff_access')
 
         output = render_template("diff.html",
                                  current_diff_url=watch['url'],
@@ -897,6 +912,7 @@ def changedetection_app(config=None, datastore_o=None):
                                  left_sticky=True,
                                  newest=newest_version_file_contents,
                                  newest_version_timestamp=dates[-1],
+                                 password_enabled_and_share_is_off=password_enabled_and_share_is_off,
                                  previous=previous_version_file_contents,
                                  screenshot=screenshot_url,
                                  uuid=uuid,
@@ -907,7 +923,7 @@ def changedetection_app(config=None, datastore_o=None):
         return output
 
     @app.route("/preview/<string:uuid>", methods=['GET'])
-    @login_required
+    @login_optionally_required
     def preview_page(uuid):
         content = []
         ignored_line_numbers = []
@@ -997,7 +1013,7 @@ def changedetection_app(config=None, datastore_o=None):
         return output
 
     @app.route("/settings/notification-logs", methods=['GET'])
-    @login_required
+    @login_optionally_required
     def notification_logs():
         global notification_debug_log
         output = render_template("notification-log.html",
@@ -1007,7 +1023,7 @@ def changedetection_app(config=None, datastore_o=None):
 
     # We're good but backups are even better!
     @app.route("/backup", methods=['GET'])
-    @login_required
+    @login_optionally_required
     def get_backup():
 
         import zipfile
@@ -1127,7 +1143,7 @@ def changedetection_app(config=None, datastore_o=None):
             abort(404)
 
     @app.route("/form/add/quickwatch", methods=['POST'])
-    @login_required
+    @login_optionally_required
     def form_quick_watch_add():
         from changedetectionio import forms
         form = forms.quickWatchForm(request.form)
@@ -1159,7 +1175,7 @@ def changedetection_app(config=None, datastore_o=None):
 
 
     @app.route("/api/delete", methods=['GET'])
-    @login_required
+    @login_optionally_required
     def form_delete():
         uuid = request.args.get('uuid')
 
@@ -1176,7 +1192,7 @@ def changedetection_app(config=None, datastore_o=None):
         return redirect(url_for('index'))
 
     @app.route("/api/clone", methods=['GET'])
-    @login_required
+    @login_optionally_required
     def form_clone():
         uuid = request.args.get('uuid')
         # More for testing, possible to return the first/only
@@ -1191,7 +1207,7 @@ def changedetection_app(config=None, datastore_o=None):
         return redirect(url_for('index'))
 
     @app.route("/api/checknow", methods=['GET'])
-    @login_required
+    @login_optionally_required
     def form_watch_checknow():
         # Forced recheck will skip the 'skip if content is the same' rule (, 'reprocess_existing_data': True})))
         tag = request.args.get('tag')
@@ -1225,7 +1241,7 @@ def changedetection_app(config=None, datastore_o=None):
         return redirect(url_for('index', tag=tag))
 
     @app.route("/form/checkbox-operations", methods=['POST'])
-    @login_required
+    @login_optionally_required
     def form_watch_list_checkbox_operations():
         op = request.form['op']
         uuids = request.form.getlist('uuids')
@@ -1289,7 +1305,7 @@ def changedetection_app(config=None, datastore_o=None):
         return redirect(url_for('index'))
 
     @app.route("/api/share-url", methods=['GET'])
-    @login_required
+    @login_optionally_required
     def form_share_put_watch():
         """Given a watch UUID, upload the info and return a share-link
            the share-link can be imported/added"""

changedetectionio/blueprint/browser_steps/__init__.py

@@ -23,11 +23,10 @@
 
 from distutils.util import strtobool
 from flask import Blueprint, request, make_response
-from flask_login import login_required
 import os
 import logging
 from changedetectionio.store import ChangeDetectionStore
-
+from changedetectionio import login_optionally_required
 browsersteps_live_ui_o = {}
 browsersteps_playwright_browser_interface = None
 browsersteps_playwright_browser_interface_browser = None
@@ -65,7 +64,7 @@ def construct_blueprint(datastore: ChangeDetectionStore):
 
     browser_steps_blueprint = Blueprint('browser_steps', __name__, template_folder="templates")
 
-    @login_required
+    @login_optionally_required
     @browser_steps_blueprint.route("/browsersteps_update", methods=['GET', 'POST'])
     def browsersteps_ui_update():
         import base64

changedetectionio/content_fetcher.py

@@ -297,8 +297,8 @@ class base_html_playwright(Fetcher):
                 proxy=self.proxy,
                 # This is needed to enable JavaScript execution on GitHub and others
                 bypass_csp=True,
-                # Can't think why we need the service workers for our use case?
-                service_workers='block',
+                # Should be `allow` or `block` - sites like YouTube can transmit large amounts of data via Service Workers
+                service_workers=os.getenv('PLAYWRIGHT_SERVICE_WORKERS', 'allow'),
                 # Should never be needed
                 accept_downloads=False
             )
@@ -568,7 +568,6 @@ class html_requests(Fetcher):
         if not r.content or not len(r.content):
             raise EmptyReply(url=url, status_code=r.status_code)
 
-        # @todo test this
         # @todo maybe you really want to test zero-byte return pages?
         if r.status_code != 200 and not ignore_status_codes:
             # maybe check with content works?

changedetectionio/forms.py

@@ -459,17 +459,17 @@ class globalSettingsRequestForm(Form):
 # datastore.data['settings']['application']..
 class globalSettingsApplicationForm(commonSettingsForm):
 
-    base_url = StringField('Base URL', validators=[validators.Optional()])
-    global_subtractive_selectors = StringListField('Remove elements', [ValidateCSSJSONXPATHInput(allow_xpath=False, allow_json=False)])
-    global_ignore_text = StringListField('Ignore Text', [ValidateListRegex()])
-    ignore_whitespace = BooleanField('Ignore whitespace')
-    removepassword_button = SubmitField('Remove password', render_kw={"class": "pure-button pure-button-primary"})
-    empty_pages_are_a_change =  BooleanField('Treat empty pages as a change?', default=False)
-    render_anchor_tag_content = BooleanField('Render anchor tag content', default=False)
-    fetch_backend = RadioField('Fetch Method', default="html_requests", choices=content_fetcher.available_fetchers(), validators=[ValidateContentFetcherIsReady()])
     api_access_token_enabled = BooleanField('API access token security check enabled', default=True, validators=[validators.Optional()])
+    base_url = StringField('Base URL', validators=[validators.Optional()])
+    empty_pages_are_a_change =  BooleanField('Treat empty pages as a change?', default=False)
+    fetch_backend = RadioField('Fetch Method', default="html_requests", choices=content_fetcher.available_fetchers(), validators=[ValidateContentFetcherIsReady()])
+    global_ignore_text = StringListField('Ignore Text', [ValidateListRegex()])
+    global_subtractive_selectors = StringListField('Remove elements', [ValidateCSSJSONXPATHInput(allow_xpath=False, allow_json=False)])
+    ignore_whitespace = BooleanField('Ignore whitespace')
     password = SaltyPasswordField()
-
+    removepassword_button = SubmitField('Remove password', render_kw={"class": "pure-button pure-button-primary"})
+    render_anchor_tag_content = BooleanField('Render anchor tag content', default=False)
+    shared_diff_access = BooleanField('Allow access to view diff page when password is enabled', default=False, validators=[validators.Optional()])
     filter_failure_notification_threshold_attempts = IntegerField('Number of times the filter can be missing before sending a notification',
                                                                   render_kw={"style": "width: 5em;"},
                                                                   validators=[validators.NumberRange(min=0,

changedetectionio/model/App.py

@@ -40,6 +40,7 @@ class model(dict):
                     'notification_body': default_notification_body,
                     'notification_format': default_notification_format,
                     'schema_version' : 0,
+                    'shared_diff_access': False,
                     'webdriver_delay': None  # Extra delay in seconds before extracting text
                 }
             }

changedetectionio/templates/diff.html

@@ -76,8 +76,12 @@
     </div>
 
      <div class="tab-pane-inner" id="text">
-         <div class="tip">Pro-tip: Use <strong>show current snapshot</strong> tab to visualise what will be ignored.
-         </div>
+         <div class="tip">Pro-tip: Use <strong>show current snapshot</strong> tab to visualise what will be ignored.</div>
+
+         {% if password_enabled_and_share_is_off %}
+           <div class="tip">Pro-tip: You can enable <strong>"share access when password is enabled"</strong> from settings</div>
+         {% endif %}
+
          <div class="snapshot-age">{{watch_a.snapshot_text_ctime|format_timestamp_timeago}}</div>
 
          <table>

changedetectionio/templates/settings.html

@@ -57,6 +57,11 @@
                         {% endif %}
                     </div>
 
+                    <div class="pure-control-group">
+                        {{ render_checkbox_field(form.application.form.shared_diff_access, class="shared_diff_access") }}
+                        <span class="pure-form-message-inline">Allow access to view watch diff page when password is enabled (Good for sharing the diff page)
+                        </span>
+                    </div>
                     <div class="pure-control-group">
                         {{ render_field(form.application.form.base_url, placeholder="http://yoursite.com:5000/",
                         class="m-d") }}

changedetectionio/tests/test_access_control.py

@@ -1,18 +1,34 @@
+from . util import live_server_setup, extract_UUID_from_client
 from flask import url_for
-from . util import live_server_setup
+import time
 
-def test_check_access_control(app, client):
+def test_check_access_control(app, client, live_server):
     # Still doesnt work, but this is closer.
+    live_server_setup(live_server)
 
     with app.test_client(use_cookies=True) as c:
         # Check we don't have any password protection enabled yet.
         res = c.get(url_for("settings_page"))
         assert b"Remove password" not in res.data
 
-        # Enable password check.
+        # add something that we can hit via diff page later
+        res = c.post(
+            url_for("import_page"),
+            data={"urls": url_for('test_random_content_endpoint', _external=True)},
+            follow_redirects=True
+        )
+
+        assert b"1 Imported" in res.data
+        time.sleep(2)
+        res = client.get(url_for("form_watch_checknow"), follow_redirects=True)
+        assert b'1 watches queued for rechecking.' in res.data
+        time.sleep(2)
+
+        # Enable password check and diff page access bypass
         res = c.post(
             url_for("settings_page"),
             data={"application-password": "foobar",
+                  "application-shared_diff_access": "True",
                   "requests-time_between_check-minutes": 180,
                   'application-fetch_backend': "html_requests"},
             follow_redirects=True
@@ -22,9 +38,15 @@ def test_check_access_control(app, client):
 
         # Check we hit the login
         res = c.get(url_for("index"), follow_redirects=True)
-
+        # Should be logged out
         assert b"Login" in res.data
 
+        # The diff page should return something valid when logged out
+        res = client.get(url_for("diff_history_page", uuid="first"))
+        assert b'Random content' in res.data
+
+
+
         # Menu should not be available yet
         #        assert b"SETTINGS" not in res.data
         #        assert b"BACKUP" not in res.data
@@ -109,3 +131,25 @@ def test_check_access_control(app, client):
 
         assert b"Password protection enabled" not in res.data
 
+        # Now checking the diff access
+        # Enable password check and diff page access bypass
+        res = c.post(
+            url_for("settings_page"),
+            data={"application-password": "foobar",
+                  # Should be disabled
+#                  "application-shared_diff_access": "True",
+                  "requests-time_between_check-minutes": 180,
+                  'application-fetch_backend': "html_requests"},
+            follow_redirects=True
+        )
+
+        assert b"Password protection enabled." in res.data
+
+        # Check we hit the login
+        res = c.get(url_for("index"), follow_redirects=True)
+        # Should be logged out
+        assert b"Login" in res.data
+
+        # The diff page should return something valid when logged out
+        res = client.get(url_for("diff_history_page", uuid="first"))
+        assert b'Random content' not in res.data

changedetectionio/tests/test_backend.py

@@ -3,7 +3,7 @@
 import time
 from flask import url_for
 from urllib.request import urlopen
-from .util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks
+from .util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks, extract_rss_token_from_UI
 
 sleep_time_for_fetch_thread = 3
 
@@ -15,10 +15,78 @@ def test_inscriptus():
     stripped_text_from_html = get_text(html_content)
     assert stripped_text_from_html == 'test!\nok man'
 
+def test_setup(client, live_server):
+    live_server_setup(live_server)
+
+# Assert that non-200's dont give notifications or register as a change
+def test_non_200_doesnt_trigger_change(client, live_server):
+  #  live_server_setup(live_server)
+
+    set_original_response()
+    url = url_for('test_changing_status_code_endpoint', _external=True)
+
+    # Add our URL to the import page
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": url},
+        follow_redirects=True
+    )
+
+    assert b"1 Imported" in res.data
+
+    time.sleep(sleep_time_for_fetch_thread)
+
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={
+            "include_filters": ".foobar-detection",
+            "fetch_backend": "html_requests",
+            "headers": "",
+            "tag": "",
+            "url": url
+        },
+        follow_redirects=True
+    )
+
+    # A recheck will happen here automatically
+    time.sleep(sleep_time_for_fetch_thread)
+
+    # hit the mark all viewed link
+    res = client.get(url_for("mark_all_viewed"), follow_redirects=True)
+
+    # Now be sure the filter is missing and then recheck it
+    set_modified_response()
+
+    # https://github.com/dgtlmoon/changedetection.io/issues/962#issuecomment-1416807742
+    for ecode in ['429', '400', '204', '429', '403', '404', '500']:
+        with open("test-endpoint-status-code.txt", 'w') as f:
+            f.write(ecode)
+
+        res = client.get(url_for("form_watch_checknow"), follow_redirects=True)
+        assert b'1 watches queued for rechecking.' in res.data
+        time.sleep(sleep_time_for_fetch_thread)
+
+        # No change should be seen/no trigger of change
+        res = client.get(url_for("index"))
+        assert b'unviewed' not in res.data
+
+        # load preview page so we can see what was returned
+        res = client.get(url_for("preview_page", uuid="first"))
+        #        with open('/tmp/debug-'+ecode+'.html', 'wb') as f:
+        #            f.write(res.data)
+
+        # Should still say the original 200, because "ignore_status_codes" should be off by default
+        # commented out - this will fail because we also show what the error was
+        # assert b'code: '+ecode.encode('utf-8') not in res.data
+
+        assert b'code: 200' in res.data
+
+    # Cleanup everything
+    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    assert b'Deleted' in res.data
 
 def test_check_basic_change_detection_functionality(client, live_server):
     set_original_response()
-    live_server_setup(live_server)
 
     # Add our URL to the import page
     res = client.post(
@@ -76,7 +144,8 @@ def test_check_basic_change_detection_functionality(client, live_server):
     assert b'unviewed' in res.data
 
     # #75, and it should be in the RSS feed
-    res = client.get(url_for("rss"))
+    rss_token = extract_rss_token_from_UI(client)
+    res = client.get(url_for("rss", token=rss_token, _external=True))
     expected_url = url_for('test_endpoint', _external=True)
     assert b'<rss' in res.data
 

changedetectionio/tests/test_rss.py

@@ -0,0 +1,39 @@
+#!/usr/bin/python3
+
+import time
+from flask import url_for
+from .util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks, extract_rss_token_from_UI
+
+
+def test_rss_and_token(client, live_server):
+    set_original_response()
+    live_server_setup(live_server)
+
+    # Add our URL to the import page
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": url_for('test_random_content_endpoint', _external=True)},
+        follow_redirects=True
+    )
+
+    assert b"1 Imported" in res.data
+    rss_token = extract_rss_token_from_UI(client)
+
+    time.sleep(2)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    time.sleep(2)
+
+    # Add our URL to the import page
+    res = client.get(
+        url_for("rss", token="bad token", _external=True),
+        follow_redirects=True
+    )
+
+    assert b"Access denied, bad token" in res.data
+
+    res = client.get(
+        url_for("rss", token=rss_token, _external=True),
+        follow_redirects=True
+    )
+    assert b"Access denied, bad token" not in res.data
+    assert b"Random content" in res.data

changedetectionio/tests/util.py

@@ -1,4 +1,5 @@
 #!/usr/bin/python3
+import os.path
 
 from flask import make_response, request
 from flask import url_for
@@ -70,6 +71,15 @@ def extract_api_key_from_UI(client):
     api_key = m.group(1)
     return api_key.strip()
 
+# kinda funky, but works for now
+def extract_rss_token_from_UI(client):
+    import re
+    res = client.get(
+        url_for("index"),
+    )
+    m = re.search('token=(.+?)"', str(res.data))
+    token_key = m.group(1)
+    return token_key.strip()
 
 # kinda funky, but works for now
 def extract_UUID_from_client(client):
@@ -98,6 +108,31 @@ def wait_for_all_checks(client):
 
 def live_server_setup(live_server):
 
+    @live_server.app.route('/test-random-content-endpoint')
+    def test_random_content_endpoint():
+        import secrets
+        return "Random content - {}\n".format(secrets.token_hex(64))
+
+    @live_server.app.route('/test-changing-status-code-endpoint')
+    def test_changing_status_code_endpoint():
+        # status_code can also be overriden in a file, used for doing things that it wouldnt normally expect
+        # (test_non_200_doesnt_trigger_change)
+        status_code = '200'
+        if os.path.isfile("test-endpoint-status-code.txt"):
+            with open("test-endpoint-status-code.txt", 'r') as f:
+                status_code = f.read().strip()
+            os.unlink("test-endpoint-status-code.txt")
+
+        # Contents includes the status code, which will change and should not trigger a change
+        # (Non-200 should get ignored)
+        with open("test-datastore/endpoint-content.txt", "r") as f:
+            contents ="{} code: {} ".format(f.read(), status_code)
+            if status_code == '204':
+                contents=''
+            resp = make_response(contents, status_code)
+            resp.headers['Content-Type'] = 'text/html'
+            return resp, status_code
+
     @live_server.app.route('/test-endpoint')
     def test_endpoint():
         ctype = request.args.get('content_type')