Also test for 204 - with empty reply

Re #962
Re #1371

changedetectionio/content_fetcher.py

@@ -568,7 +568,6 @@ class html_requests(Fetcher):
         if not r.content or not len(r.content):
             raise EmptyReply(url=url, status_code=r.status_code)
 
-        # @todo test this
         # @todo maybe you really want to test zero-byte return pages?
         if r.status_code != 200 and not ignore_status_codes:
             # maybe check with content works?

changedetectionio/tests/test_backend.py

@@ -15,10 +15,78 @@ def test_inscriptus():
     stripped_text_from_html = get_text(html_content)
     assert stripped_text_from_html == 'test!\nok man'
 
+def test_setup(client, live_server):
+    live_server_setup(live_server)
+
+# Assert that non-200's dont give notifications or register as a change
+def test_non_200_doesnt_trigger_change(client, live_server):
+  #  live_server_setup(live_server)
+
+    set_original_response()
+    url = url_for('test_changing_status_code_endpoint', _external=True)
+
+    # Add our URL to the import page
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": url},
+        follow_redirects=True
+    )
+
+    assert b"1 Imported" in res.data
+
+    time.sleep(sleep_time_for_fetch_thread)
+
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={
+            "include_filters": ".foobar-detection",
+            "fetch_backend": "html_requests",
+            "headers": "",
+            "tag": "",
+            "url": url
+        },
+        follow_redirects=True
+    )
+
+    # A recheck will happen here automatically
+    time.sleep(sleep_time_for_fetch_thread)
+
+    # hit the mark all viewed link
+    res = client.get(url_for("mark_all_viewed"), follow_redirects=True)
+
+    # Now be sure the filter is missing and then recheck it
+    set_modified_response()
+
+    # https://github.com/dgtlmoon/changedetection.io/issues/962#issuecomment-1416807742
+    for ecode in ['429', '400', '204', '429', '403', '404', '500']:
+        with open("test-endpoint-status-code.txt", 'w') as f:
+            f.write(ecode)
+
+        res = client.get(url_for("form_watch_checknow"), follow_redirects=True)
+        assert b'1 watches queued for rechecking.' in res.data
+        time.sleep(sleep_time_for_fetch_thread)
+
+        # No change should be seen/no trigger of change
+        res = client.get(url_for("index"))
+        assert b'unviewed' not in res.data
+
+        # load preview page so we can see what was returned
+        res = client.get(url_for("preview_page", uuid="first"))
+        #        with open('/tmp/debug-'+ecode+'.html', 'wb') as f:
+        #            f.write(res.data)
+
+        # Should still say the original 200, because "ignore_status_codes" should be off by default
+        # commented out - this will fail because we also show what the error was
+        # assert b'code: '+ecode.encode('utf-8') not in res.data
+
+        assert b'code: 200' in res.data
+
+    # Cleanup everything
+    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    assert b'Deleted' in res.data
 
 def test_check_basic_change_detection_functionality(client, live_server):
     set_original_response()
-    live_server_setup(live_server)
 
     # Add our URL to the import page
     res = client.post(

changedetectionio/tests/util.py

@@ -1,4 +1,5 @@
 #!/usr/bin/python3
+import os.path
 
 from flask import make_response, request
 from flask import url_for
@@ -112,6 +113,25 @@ def live_server_setup(live_server):
         import secrets
         return "Random content - {}\n".format(secrets.token_hex(64))
 
+    @live_server.app.route('/test-changing-status-code-endpoint')
+    def test_changing_status_code_endpoint():
+        # status_code can also be overriden in a file, used for doing things that it wouldnt normally expect
+        # (test_non_200_doesnt_trigger_change)
+        status_code = '200'
+        if os.path.isfile("test-endpoint-status-code.txt"):
+            with open("test-endpoint-status-code.txt", 'r') as f:
+                status_code = f.read().strip()
+            os.unlink("test-endpoint-status-code.txt")
+
+        # Contents includes the status code, which will change and should not trigger a change
+        # (Non-200 should get ignored)
+        with open("test-datastore/endpoint-content.txt", "r") as f:
+            contents ="{} code: {} ".format(f.read(), status_code)
+            if status_code == '204':
+                contents=''
+            resp = make_response(contents, status_code)
+            resp.headers['Content-Type'] = 'text/html'
+            return resp, status_code
 
     @live_server.app.route('/test-endpoint')
     def test_endpoint():