Also test for 204 - with empty reply

Re #962
Re #1371

changedetectionio/__init__.py

@@ -943,10 +943,9 @@ def changedetection_app(config=None, datastore_o=None):
         extra_stylesheets = [url_for('static_content', group='styles', filename='diff.css')]
 
 
-        is_html_webdriver = False
-        if (watch.get('fetch_backend') == 'system' and system_uses_webdriver) or watch.get('fetch_backend') == 'html_webdriver':
-            is_html_webdriver = True
-            
+        is_html_webdriver = True if watch.get('fetch_backend') == 'html_webdriver' or (
+                watch.get('fetch_backend', None) is None and system_uses_webdriver) else False
+
         # Never requested successfully, but we detected a fetch error
         if datastore.data['watching'][uuid].history_n == 0 and (watch.get_error_text() or watch.get_error_snapshot()):
             flash("Preview unavailable - No fetch/check completed or triggers not reached", "error")
@@ -1036,8 +1035,7 @@ def changedetection_app(config=None, datastore_o=None):
             os.unlink(previous_backup_filename)
 
         # create a ZipFile object
-        timestamp = datetime.datetime.now().strftime("%Y%m%d%H%M%S")
-        backupname = "changedetection-backup-{}.zip".format(timestamp)
+        backupname = "changedetection-backup-{}.zip".format(int(time.time()))
         backup_filepath = os.path.join(datastore_o.datastore_path, backupname)
 
         with zipfile.ZipFile(backup_filepath, "w",

changedetectionio/content_fetcher.py

@@ -568,7 +568,6 @@ class html_requests(Fetcher):
         if not r.content or not len(r.content):
             raise EmptyReply(url=url, status_code=r.status_code)
 
-        # @todo test this
         # @todo maybe you really want to test zero-byte return pages?
         if r.status_code != 200 and not ignore_status_codes:
             # maybe check with content works?

changedetectionio/tests/test_backend.py

@@ -15,10 +15,78 @@ def test_inscriptus():
     stripped_text_from_html = get_text(html_content)
     assert stripped_text_from_html == 'test!\nok man'
 
+def test_setup(client, live_server):
+    live_server_setup(live_server)
+
+# Assert that non-200's dont give notifications or register as a change
+def test_non_200_doesnt_trigger_change(client, live_server):
+  #  live_server_setup(live_server)
+
+    set_original_response()
+    url = url_for('test_changing_status_code_endpoint', _external=True)
+
+    # Add our URL to the import page
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": url},
+        follow_redirects=True
+    )
+
+    assert b"1 Imported" in res.data
+
+    time.sleep(sleep_time_for_fetch_thread)
+
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={
+            "include_filters": ".foobar-detection",
+            "fetch_backend": "html_requests",
+            "headers": "",
+            "tag": "",
+            "url": url
+        },
+        follow_redirects=True
+    )
+
+    # A recheck will happen here automatically
+    time.sleep(sleep_time_for_fetch_thread)
+
+    # hit the mark all viewed link
+    res = client.get(url_for("mark_all_viewed"), follow_redirects=True)
+
+    # Now be sure the filter is missing and then recheck it
+    set_modified_response()
+
+    # https://github.com/dgtlmoon/changedetection.io/issues/962#issuecomment-1416807742
+    for ecode in ['429', '400', '204', '429', '403', '404', '500']:
+        with open("test-endpoint-status-code.txt", 'w') as f:
+            f.write(ecode)
+
+        res = client.get(url_for("form_watch_checknow"), follow_redirects=True)
+        assert b'1 watches queued for rechecking.' in res.data
+        time.sleep(sleep_time_for_fetch_thread)
+
+        # No change should be seen/no trigger of change
+        res = client.get(url_for("index"))
+        assert b'unviewed' not in res.data
+
+        # load preview page so we can see what was returned
+        res = client.get(url_for("preview_page", uuid="first"))
+        #        with open('/tmp/debug-'+ecode+'.html', 'wb') as f:
+        #            f.write(res.data)
+
+        # Should still say the original 200, because "ignore_status_codes" should be off by default
+        # commented out - this will fail because we also show what the error was
+        # assert b'code: '+ecode.encode('utf-8') not in res.data
+
+        assert b'code: 200' in res.data
+
+    # Cleanup everything
+    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    assert b'Deleted' in res.data
 
 def test_check_basic_change_detection_functionality(client, live_server):
     set_original_response()
-    live_server_setup(live_server)
 
     # Add our URL to the import page
     res = client.post(

changedetectionio/tests/util.py

@@ -1,4 +1,5 @@
 #!/usr/bin/python3
+import os.path
 
 from flask import make_response, request
 from flask import url_for
@@ -112,6 +113,25 @@ def live_server_setup(live_server):
         import secrets
         return "Random content - {}\n".format(secrets.token_hex(64))
 
+    @live_server.app.route('/test-changing-status-code-endpoint')
+    def test_changing_status_code_endpoint():
+        # status_code can also be overriden in a file, used for doing things that it wouldnt normally expect
+        # (test_non_200_doesnt_trigger_change)
+        status_code = '200'
+        if os.path.isfile("test-endpoint-status-code.txt"):
+            with open("test-endpoint-status-code.txt", 'r') as f:
+                status_code = f.read().strip()
+            os.unlink("test-endpoint-status-code.txt")
+
+        # Contents includes the status code, which will change and should not trigger a change
+        # (Non-200 should get ignored)
+        with open("test-datastore/endpoint-content.txt", "r") as f:
+            contents ="{} code: {} ".format(f.read(), status_code)
+            if status_code == '204':
+                contents=''
+            resp = make_response(contents, status_code)
+            resp.headers['Content-Type'] = 'text/html'
+            return resp, status_code
 
     @live_server.app.route('/test-endpoint')
     def test_endpoint():

docker-compose.yml

@@ -41,6 +41,7 @@ services:
   #
   #        Base URL of your changedetection.io install (Added to the notification alert)
   #      - BASE_URL=https://mysite.com
+
   #        Respect proxy_pass type settings, `proxy_set_header Host "localhost";` and `proxy_set_header X-Forwarded-Prefix /app;`
   #        More here https://github.com/dgtlmoon/changedetection.io/wiki/Running-changedetection.io-behind-a-reverse-proxy-sub-directory
   #      - USE_X_SETTINGS=1
@@ -94,10 +95,7 @@ services:
 #            - CHROME_REFRESH_TIME=600000
 #            - DEFAULT_BLOCK_ADS=true
 #            - DEFAULT_STEALTH=true
-#
-#             Ignore HTTPS errors, like for self-signed certs
-#            - DEFAULT_IGNORE_HTTPS_ERRORS=true
-#
+
 volumes:
   changedetection-data:
 

price-finder

@@ -1,55 +0,0 @@
-diff --git a/changedetectionio/content_fetcher.py b/changedetectionio/content_fetcher.py
-index 475e90c5..843a273e 100644
---- a/changedetectionio/content_fetcher.py
-+++ b/changedetectionio/content_fetcher.py
-@@ -122,6 +122,15 @@ class Fetcher():
-         # Should set self.error, self.status_code and self.content
-         pass
- 
-+    def restock_status(self):
-+        x=1
-+
-+    def extracted_price(self):
-+        x=1
-+
-+    def run_all_js_extractions(self):
-+        x=1
-+
-     @abstractmethod
-     def quit(self):
-         return
-diff --git a/changedetectionio/fetch_site_status.py b/changedetectionio/fetch_site_status.py
-index 04b57367..0ef63fbf 100644
---- a/changedetectionio/fetch_site_status.py
-+++ b/changedetectionio/fetch_site_status.py
-@@ -199,6 +199,13 @@ class perform_site_check():
-         if watch.get('track_ldjson_price_data', '') == PRICE_DATA_TRACK_ACCEPT:
-             include_filters_rule.append(html_tools.LD_JSON_PRODUCT_OFFER_SELECTOR)
- 
-+        # maybe run them all in playwright, return it in the `fetcher` object
-+        if watch.get('pluggable_logic_rules', []):
-+            # Run some JS and parse it back through Python
-+            # I think we want to empty the content and append a joined list of results from these processors
-+            #
-+            x=1
-+
-         has_filter_rule = include_filters_rule and len("".join(include_filters_rule).strip())
-         has_subtractive_selectors = subtractive_selectors and len(subtractive_selectors[0].strip())
- 
-diff --git a/changedetectionio/pluggable_filters/__init__.py b/changedetectionio/pluggable_filters/__init__.py
-index e69de29b..e6f0d975 100644
---- a/changedetectionio/pluggable_filters/__init__.py
-+++ b/changedetectionio/pluggable_filters/__init__.py
-@@ -0,0 +1,11 @@
-+class pluggable_js_filter():
-+    # Test it against the page, could be JS? means its possible to enable
-+    def plugin_name(self):
-+        return 'restock check'
-+
-+    def could_be_active(self):
-+        return False
-+
-+    def get_value(self):
-+        # ie "In-stock" "out-of-stock"
-+        return ''
-\ No newline at end of file