Also test for 204 - with empty reply

Re #962
Re #1371

changedetectionio/__init__.py

@@ -505,6 +505,41 @@ def changedetection_app(config=None, datastore_o=None):
         output = render_template("clear_all_history.html")
         return output
 
+
+    # If they edited an existing watch, we need to know to reset the current/previous md5 to include
+    # the excluded text.
+    def get_current_checksum_include_ignore_text(uuid):
+
+        import hashlib
+
+        from changedetectionio import fetch_site_status
+
+        # Get the most recent one
+        newest_history_key = datastore.data['watching'][uuid].get('newest_history_key')
+
+        # 0 means that theres only one, so that there should be no 'unviewed' history available
+        if newest_history_key == 0:
+            newest_history_key = list(datastore.data['watching'][uuid].history.keys())[0]
+
+        if newest_history_key:
+            with open(datastore.data['watching'][uuid].history[newest_history_key],
+                      encoding='utf-8') as file:
+                raw_content = file.read()
+
+                handler = fetch_site_status.perform_site_check(datastore=datastore)
+                stripped_content = html_tools.strip_ignore_text(raw_content,
+                                                             datastore.data['watching'][uuid]['ignore_text'])
+
+                if datastore.data['settings']['application'].get('ignore_whitespace', False):
+                    checksum = hashlib.md5(stripped_content.translate(None, b'\r\n\t ')).hexdigest()
+                else:
+                    checksum = hashlib.md5(stripped_content).hexdigest()
+
+                return checksum
+
+        return datastore.data['watching'][uuid]['previous_md5']
+
+
     @app.route("/edit/<string:uuid>", methods=['GET', 'POST'])
     @login_optionally_required
     # https://stackoverflow.com/questions/42984453/wtforms-populate-form-with-data-if-data-exists
@@ -908,9 +943,8 @@ def changedetection_app(config=None, datastore_o=None):
         extra_stylesheets = [url_for('static_content', group='styles', filename='diff.css')]
 
 
-        is_html_webdriver = False
-        if (watch.get('fetch_backend') == 'system' and system_uses_webdriver) or watch.get('fetch_backend') == 'html_webdriver':
-            is_html_webdriver = True
+        is_html_webdriver = True if watch.get('fetch_backend') == 'html_webdriver' or (
+                watch.get('fetch_backend', None) is None and system_uses_webdriver) else False
 
         # Never requested successfully, but we detected a fetch error
         if datastore.data['watching'][uuid].history_n == 0 and (watch.get_error_text() or watch.get_error_snapshot()):

changedetectionio/content_fetcher.py

@@ -568,7 +568,6 @@ class html_requests(Fetcher):
         if not r.content or not len(r.content):
             raise EmptyReply(url=url, status_code=r.status_code)
 
-        # @todo test this
         # @todo maybe you really want to test zero-byte return pages?
         if r.status_code != 200 and not ignore_status_codes:
             # maybe check with content works?

changedetectionio/tests/test_backend.py

@@ -15,10 +15,78 @@ def test_inscriptus():
     stripped_text_from_html = get_text(html_content)
     assert stripped_text_from_html == 'test!\nok man'
 
+def test_setup(client, live_server):
+    live_server_setup(live_server)
+
+# Assert that non-200's dont give notifications or register as a change
+def test_non_200_doesnt_trigger_change(client, live_server):
+  #  live_server_setup(live_server)
+
+    set_original_response()
+    url = url_for('test_changing_status_code_endpoint', _external=True)
+
+    # Add our URL to the import page
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": url},
+        follow_redirects=True
+    )
+
+    assert b"1 Imported" in res.data
+
+    time.sleep(sleep_time_for_fetch_thread)
+
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={
+            "include_filters": ".foobar-detection",
+            "fetch_backend": "html_requests",
+            "headers": "",
+            "tag": "",
+            "url": url
+        },
+        follow_redirects=True
+    )
+
+    # A recheck will happen here automatically
+    time.sleep(sleep_time_for_fetch_thread)
+
+    # hit the mark all viewed link
+    res = client.get(url_for("mark_all_viewed"), follow_redirects=True)
+
+    # Now be sure the filter is missing and then recheck it
+    set_modified_response()
+
+    # https://github.com/dgtlmoon/changedetection.io/issues/962#issuecomment-1416807742
+    for ecode in ['429', '400', '204', '429', '403', '404', '500']:
+        with open("test-endpoint-status-code.txt", 'w') as f:
+            f.write(ecode)
+
+        res = client.get(url_for("form_watch_checknow"), follow_redirects=True)
+        assert b'1 watches queued for rechecking.' in res.data
+        time.sleep(sleep_time_for_fetch_thread)
+
+        # No change should be seen/no trigger of change
+        res = client.get(url_for("index"))
+        assert b'unviewed' not in res.data
+
+        # load preview page so we can see what was returned
+        res = client.get(url_for("preview_page", uuid="first"))
+        #        with open('/tmp/debug-'+ecode+'.html', 'wb') as f:
+        #            f.write(res.data)
+
+        # Should still say the original 200, because "ignore_status_codes" should be off by default
+        # commented out - this will fail because we also show what the error was
+        # assert b'code: '+ecode.encode('utf-8') not in res.data
+
+        assert b'code: 200' in res.data
+
+    # Cleanup everything
+    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    assert b'Deleted' in res.data
 
 def test_check_basic_change_detection_functionality(client, live_server):
     set_original_response()
-    live_server_setup(live_server)
 
     # Add our URL to the import page
     res = client.post(

changedetectionio/tests/util.py

@@ -1,4 +1,5 @@
 #!/usr/bin/python3
+import os.path
 
 from flask import make_response, request
 from flask import url_for
@@ -112,6 +113,25 @@ def live_server_setup(live_server):
         import secrets
         return "Random content - {}\n".format(secrets.token_hex(64))
 
+    @live_server.app.route('/test-changing-status-code-endpoint')
+    def test_changing_status_code_endpoint():
+        # status_code can also be overriden in a file, used for doing things that it wouldnt normally expect
+        # (test_non_200_doesnt_trigger_change)
+        status_code = '200'
+        if os.path.isfile("test-endpoint-status-code.txt"):
+            with open("test-endpoint-status-code.txt", 'r') as f:
+                status_code = f.read().strip()
+            os.unlink("test-endpoint-status-code.txt")
+
+        # Contents includes the status code, which will change and should not trigger a change
+        # (Non-200 should get ignored)
+        with open("test-datastore/endpoint-content.txt", "r") as f:
+            contents ="{} code: {} ".format(f.read(), status_code)
+            if status_code == '204':
+                contents=''
+            resp = make_response(contents, status_code)
+            resp.headers['Content-Type'] = 'text/html'
+            return resp, status_code
 
     @live_server.app.route('/test-endpoint')
     def test_endpoint():

docker-compose.yml

@@ -41,6 +41,7 @@ services:
   #
   #        Base URL of your changedetection.io install (Added to the notification alert)
   #      - BASE_URL=https://mysite.com
+
   #        Respect proxy_pass type settings, `proxy_set_header Host "localhost";` and `proxy_set_header X-Forwarded-Prefix /app;`
   #        More here https://github.com/dgtlmoon/changedetection.io/wiki/Running-changedetection.io-behind-a-reverse-proxy-sub-directory
   #      - USE_X_SETTINGS=1
@@ -94,10 +95,7 @@ services:
 #            - CHROME_REFRESH_TIME=600000
 #            - DEFAULT_BLOCK_ADS=true
 #            - DEFAULT_STEALTH=true
-#
-#             Ignore HTTPS errors, like for self-signed certs
-#            - DEFAULT_IGNORE_HTTPS_ERRORS=true
-#
+
 volumes:
   changedetection-data: