Number of fetching workers can be overriden by Env "FETCH_WORKERS"

changedetectionio/__init__.py

@@ -36,7 +36,7 @@ from flask import (
 )
 from flask_login import login_required
 
-__version__ = '0.39.8'
+__version__ = '0.39.7'
 
 datastore = None
 
@@ -224,18 +224,10 @@ def changedetection_app(config=None, datastore_o=None):
 
         if (user.check_password(password)):
             flask_login.login_user(user, remember=True)
-
-            # For now there's nothing else interesting here other than the index/list page
-            # It's more reliable and safe to ignore the 'next' redirect
-            # When we used...
-            # next = request.args.get('next')
-            # return redirect(next or url_for('index'))
-            # We would sometimes get login loop errors on sites hosted in sub-paths
-
-            # note for the future:
+            next = request.args.get('next')
             #            if not is_safe_url(next):
             #                return flask.abort(400)
-            return redirect(url_for('index'))
+            return redirect(next or url_for('index'))
 
         else:
             flash('Incorrect password', 'error')
@@ -695,10 +687,6 @@ def changedetection_app(config=None, datastore_o=None):
     @app.route("/diff/<string:uuid>", methods=['GET'])
     @login_required
     def diff_history_page(uuid):
-        from changedetectionio import content_fetcher
-
-        newest_version_file_contents = ""
-        previous_version_file_contents = ""
 
         # More for testing, possible to return the first/only
         if uuid == 'first':
@@ -724,28 +712,21 @@ def changedetection_app(config=None, datastore_o=None):
 
         # Save the current newest history as the most recently viewed
         datastore.set_last_viewed(uuid, dates[0])
+        newest_file = watch['history'][dates[0]]
+        with open(newest_file, 'r') as f:
+            newest_version_file_contents = f.read()
 
         previous_version = request.args.get('previous_version')
-        if ('content-type' in watch and content_fetcher.supported_binary_type(watch['content-type'])):
-            template = "diff-image.html"
-        else:
-            newest_file = watch['history'][dates[0]]
-            with open(newest_file, 'r') as f:
-                newest_version_file_contents = f.read()
+        try:
+            previous_file = watch['history'][previous_version]
+        except KeyError:
+            # Not present, use a default value, the second one in the sorted list.
+            previous_file = watch['history'][dates[1]]
 
-            try:
-                previous_file = watch['history'][previous_version]
-            except KeyError:
-                # Not present, use a default value, the second one in the sorted list.
-                previous_file = watch['history'][dates[1]]
+        with open(previous_file, 'r') as f:
+            previous_version_file_contents = f.read()
 
-            with open(previous_file, 'r') as f:
-                previous_version_file_contents = f.read()
-
-            template = "diff.html"
-
-        output = render_template(template,
-                                 watch_a=watch,
+        output = render_template("diff.html", watch_a=watch,
                                  newest=newest_version_file_contents,
                                  previous=previous_version_file_contents,
                                  extra_stylesheets=extra_stylesheets,
@@ -762,7 +743,6 @@ def changedetection_app(config=None, datastore_o=None):
     @app.route("/preview/<string:uuid>", methods=['GET'])
     @login_required
     def preview_page(uuid):
-        from changedetectionio import content_fetcher
 
         # More for testing, possible to return the first/only
         if uuid == 'first':
@@ -777,25 +757,14 @@ def changedetection_app(config=None, datastore_o=None):
             return redirect(url_for('index'))
 
         newest = list(watch['history'].keys())[-1]
-        fname = watch['history'][newest]
-
-        if ('content-type' in watch and content_fetcher.supported_binary_type(watch['content-type'])):
-            template = "preview-image.html"
-            content = fname
-        else:
-            template = "preview.html"
-            try:
-                with open(fname, 'r') as f:
-                    content = f.read()
-            except:
-                content = "Cant read {}".format(fname)
+        with open(watch['history'][newest], 'r') as f:
+            content = f.readlines()
 
         output = render_template("preview.html",
                                  content=content,
                                  extra_stylesheets=extra_stylesheets,
                                  current_diff_url=watch['url'],
-                                 uuid=uuid,
-                                 watch=watch)
+                                 uuid=uuid)
         return output
 
     @app.route("/settings/notification-logs", methods=['GET'])
@@ -806,50 +775,6 @@ def changedetection_app(config=None, datastore_o=None):
                                  logs=notification_debug_log if len(notification_debug_log) else ["No errors or warnings detected"])
 
         return output
-
-
-    # render an image which contains the diff of two images
-    # We always compare the newest against whatever compare_date we are given
-    @app.route("/diff/show-image/<string:uuid>/<string:datestr>")
-    def show_single_image(uuid, datestr):
-
-        from flask import make_response
-        watch = datastore.data['watching'][uuid]
-
-        if datestr == 'None' or datestr is None:
-            datestr = list(watch['history'].keys())[0]
-
-        fname = watch['history'][datestr]
-        with open(fname, 'rb') as f:
-            resp = make_response(f.read())
-            
-        # @todo assumption here about the type, re-encode? detect?
-        resp.headers['Content-Type'] = 'image/jpeg'
-        return resp
-
-    # render an image which contains the diff of two images
-    # We always compare the newest against whatever compare_date we are given
-    @app.route("/diff/image/<string:uuid>/<string:compare_date>")
-    def render_diff_image(uuid, compare_date):
-        from changedetectionio import image_diff
-
-        from flask import make_response
-        watch = datastore.data['watching'][uuid]
-        newest = list(watch['history'].keys())[-1]
-
-        # @todo this is weird
-        if compare_date == 'None' or compare_date is None:
-            compare_date = list(watch['history'].keys())[0]
-
-        new_img = watch['history'][newest]
-        prev_img = watch['history'][compare_date]
-        img = image_diff.render_diff(new_img, prev_img)
-
-        resp = make_response(img)
-        resp.headers['Content-Type'] = 'image/jpeg'
-        return resp
-
-
     @app.route("/api/<string:uuid>/snapshot/current", methods=['GET'])
     @login_required
     def api_snapshot(uuid):
@@ -1118,7 +1043,7 @@ def ticker_thread_check_time_launch_checks():
 
     # Spin up Workers that do the fetching
     # Can be overriden by ENV or use the default settings
-    n_workers = int(os.getenv("FETCH_WORKERS", datastore.data['settings']['requests']['workers']))
+    n_workers = os.getenv("FETCH_WORKERS", datastore.data['settings']['requests']['workers'])
     for _ in range(n_workers):
         new_worker = update_worker.update_worker(update_q, notification_q, app, datastore)
         running_update_threads.append(new_worker)

changedetectionio/content_fetcher.py

@@ -5,9 +5,8 @@ from selenium import webdriver
 from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
 from selenium.webdriver.common.proxy import Proxy as SeleniumProxy
 from selenium.common.exceptions import WebDriverException
+import urllib3.exceptions
 
-# image/jpeg etc
-supported_binary_types = ['image']
 
 class EmptyReply(Exception):
     def __init__(self, status_code, url):
@@ -52,15 +51,6 @@ class Fetcher():
 #    def return_diff(self, stream_a, stream_b):
 #        return
 
-# Assume we dont support it as binary if its not in our list
-def supported_binary_type(content_type):
-    # Not a binary thing we support? then use text (also used for JSON/XML etc)
-    # @todo - future - use regex for matching
-    if content_type and content_type.lower().strip().split('/')[0] not in (string.lower() for string in supported_binary_types):
-        return False
-
-    return True
-
 def available_fetchers():
         import inspect
         from changedetectionio import content_fetcher
@@ -130,7 +120,7 @@ class html_webdriver(Fetcher):
         # raise EmptyReply(url=url, status_code=r.status_code)
 
         # @todo - dom wait loaded?
-        time.sleep(int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)))
+        time.sleep(5)
         self.content = driver.page_source
         self.headers = {}
 
@@ -166,18 +156,15 @@ class html_requests(Fetcher):
                          verify=False)
 
         # https://stackoverflow.com/questions/44203397/python-requests-get-returns-improperly-decoded-text-instead-of-utf-8
-
-        if not supported_binary_type(r.headers.get('Content-Type', '')):
-            content = r.text
-        else:
-            content = r.content
+        # Return bytes here
+        html = r.text
 
         # @todo test this
         # @todo maybe you really want to test zero-byte return pages?
-        if not r or not content or not len(content):
+        if not r or not html or not len(html):
             raise EmptyReply(url=url, status_code=r.status_code)
 
         self.status_code = r.status_code
-        self.content = content
+        self.content = html
         self.headers = r.headers
 

changedetectionio/fetch_site_status.py

@@ -55,13 +55,10 @@ class perform_site_check():
 
         changed_detected = False
         stripped_text_from_html = ""
-        fetched_md5 = ""
-
-        original_content_before_filters = False
 
         watch = self.datastore.data['watching'][uuid]
-
         # Unset any existing notification error
+
         update_obj = {'last_notification_error': False, 'last_error': False}
 
         extra_headers = self.datastore.get_val(uuid, 'headers')
@@ -95,7 +92,6 @@ class perform_site_check():
 
             fetcher = klass()
             fetcher.run(url, timeout, request_headers, request_body, request_method)
-
             # Fetching complete, now filters
             # @todo move to class / maybe inside of fetcher abstract base?
 
@@ -105,39 +101,26 @@ class perform_site_check():
             #  - Do we convert to JSON?
             # https://stackoverflow.com/questions/41817578/basic-method-chaining ?
             # return content().textfilter().jsonextract().checksumcompare() ?
-            update_obj['content-type'] = fetcher.headers.get('Content-Type', '').lower().strip()
-            
-            # Could be 'application/json; charset=utf-8' etc
-            is_json = 'application/json' in update_obj['content-type']
-            is_text_or_html = 'text/' in update_obj['content-type'] # text/plain , text/html etc
-            is_binary = not is_text_or_html and content_fetcher.supported_binary_type(update_obj['content-type'])
-            css_filter_rule = watch['css_filter']
-            has_filter_rule = css_filter_rule and len(css_filter_rule.strip())
 
-            # Auto-detect application/json, make it reformat the JSON to something nice
+            is_json = fetcher.headers.get('Content-Type', '') == 'application/json'
+            is_html = not is_json
+            css_filter_rule = watch['css_filter']
+
+            has_filter_rule = css_filter_rule and len(css_filter_rule.strip())
             if is_json and not has_filter_rule:
                 css_filter_rule = "json:$"
                 has_filter_rule = True
 
-            ##### CONVERT THE INPUT TO TEXT, EXTRACT THE PARTS THAT NEED TO BE FILTERED
+            if has_filter_rule:
+                if 'json:' in css_filter_rule:
+                    stripped_text_from_html = html_tools.extract_json_as_string(content=fetcher.content, jsonpath_filter=css_filter_rule)
+                    is_html = False
 
-            # Dont depend on the content-type header here, maybe it's not present
-            if 'json:' in css_filter_rule:
-                is_json = True
-                rule = css_filter_rule.replace('json:', '')
-                stripped_text_from_html = html_tools.extract_json_as_string(content=fetcher.content,
-                                                                            jsonpath_filter=rule).encode('utf-8')
-                is_text_or_html = False
-                original_content_before_filters = stripped_text_from_html
-
-            if is_text_or_html:
+            if is_html:
                 # CSS Filter, extract the HTML that matches and feed that into the existing inscriptis::get_text
                 html_content = fetcher.content
-                if 'text/plain' in update_obj['content-type']:
-                    stripped_text_from_html = html_content
+                if not fetcher.headers.get('Content-Type', '') == 'text/plain':
 
-                # Assume it's HTML if it's not text/plain
-                if not 'text/plain' in update_obj['content-type']:
                     if has_filter_rule:
                         # For HTML/XML we offer xpath as an option, just start a regular xPath "/.."
                         if css_filter_rule[0] == '/':
@@ -145,52 +128,35 @@ class perform_site_check():
                         else:
                             # CSS Filter, extract the HTML that matches and feed that into the existing inscriptis::get_text
                             html_content = html_tools.css_filter(css_filter=css_filter_rule, html_content=fetcher.content)
+
                     # get_text() via inscriptis
                     stripped_text_from_html = get_text(html_content)
+                else:
+                    # Don't run get_text or xpath/css filters on plaintext
+                    stripped_text_from_html = html_content
 
-                    # Extract title as title
-                    if self.datastore.data['settings']['application']['extract_title_as_title'] or watch['extract_title_as_title']:
-                        if not watch['title'] or not len(watch['title']):
-                            update_obj['title'] = html_tools.extract_element(find='title', html_content=fetcher.content)
-
-                # Re #340 - return the content before the 'ignore text' was applied
-                original_content_before_filters = stripped_text_from_html.encode('utf-8')
-
+            # Re #340 - return the content before the 'ignore text' was applied
+            text_content_before_ignored_filter = stripped_text_from_html.encode('utf-8')
 
             # We rely on the actual text in the html output.. many sites have random script vars etc,
             # in the future we'll implement other mechanisms.
 
             update_obj["last_check_status"] = fetcher.get_last_status_code()
 
-            ######## AFTER FILTERING, STRIP OUT IGNORE TEXT
-            if is_text_or_html:
-                text_to_ignore = watch.get('ignore_text', []) + self.datastore.data['settings']['application'].get('global_ignore_text', [])
-                if len(text_to_ignore):
-                    stripped_text_from_html = self.strip_ignore_text(stripped_text_from_html, text_to_ignore)
-                else:
-                    stripped_text_from_html = stripped_text_from_html.encode('utf8')
+            # If there's text to skip
+            # @todo we could abstract out the get_text() to handle this cleaner
+            text_to_ignore = watch.get('ignore_text', []) + self.datastore.data['settings']['application'].get('global_ignore_text', [])
+            if len(text_to_ignore):
+                stripped_text_from_html = self.strip_ignore_text(stripped_text_from_html, text_to_ignore)
+            else:
+                stripped_text_from_html = stripped_text_from_html.encode('utf8')
 
-
-            ######## CALCULATE CHECKSUM FOR DIFF DETECTION
             # Re #133 - if we should strip whitespaces from triggering the change detected comparison
-            if is_text_or_html:
-                if self.datastore.data['settings']['application'].get('ignore_whitespace', False):
-                    fetched_md5 = hashlib.md5(stripped_text_from_html.translate(None, b'\r\n\t ')).hexdigest()
-                else:
-                    fetched_md5 = hashlib.md5(stripped_text_from_html).hexdigest()
-
-            if is_json:
+            if self.datastore.data['settings']['application'].get('ignore_whitespace', False):
+                fetched_md5 = hashlib.md5(stripped_text_from_html.translate(None, b'\r\n\t ')).hexdigest()
+            else:
                 fetched_md5 = hashlib.md5(stripped_text_from_html).hexdigest()
 
-            # Goal here in the future is to be able to abstract out different content type checks into their own class
-
-            if is_binary:
-                # @todo - use some actual image hash here where possible, audio hash, etc etc
-                m = hashlib.sha256()
-                m.update(fetcher.content)
-                fetched_md5 = m.hexdigest()
-                original_content_before_filters = fetcher.content
-
             # On the first run of a site, watch['previous_md5'] will be an empty string, set it the current one.
             if not len(watch['previous_md5']):
                 watch['previous_md5'] = fetched_md5
@@ -198,30 +164,36 @@ class perform_site_check():
 
             blocked_by_not_found_trigger_text = False
 
-            # Trigger text can apply to JSON parsed documents too
-            if is_text_or_html or is_json:
-                if len(watch['trigger_text']):
-                    blocked_by_not_found_trigger_text = True
-                    for line in watch['trigger_text']:
-                        # Because JSON wont serialize a re.compile object
-                        if line[0] == '/' and line[-1] == '/':
-                            regex = re.compile(line.strip('/'), re.IGNORECASE)
-                            # Found it? so we don't wait for it anymore
-                            r = re.search(regex, str(stripped_text_from_html))
-                            if r:
-                                blocked_by_not_found_trigger_text = False
-                                break
-
-                        elif line.lower() in str(stripped_text_from_html).lower():
-                            # We found it don't wait for it.
+            if len(watch['trigger_text']):
+                blocked_by_not_found_trigger_text = True
+                for line in watch['trigger_text']:
+                    # Because JSON wont serialize a re.compile object
+                    if line[0] == '/' and line[-1] == '/':
+                        regex = re.compile(line.strip('/'), re.IGNORECASE)
+                        # Found it? so we don't wait for it anymore
+                        r = re.search(regex, str(stripped_text_from_html))
+                        if r:
                             blocked_by_not_found_trigger_text = False
                             break
 
+                    elif line.lower() in str(stripped_text_from_html).lower():
+                        # We found it don't wait for it.
+                        blocked_by_not_found_trigger_text = False
+                        break
+
+
+
             if not blocked_by_not_found_trigger_text and watch['previous_md5'] != fetched_md5:
                 changed_detected = True
                 update_obj["previous_md5"] = fetched_md5
                 update_obj["last_changed"] = timestamp
 
 
-        # original_content_before_filters is returned for saving the data to disk
-        return changed_detected, update_obj, original_content_before_filters
+            # Extract title as title
+            if is_html:
+                if self.datastore.data['settings']['application']['extract_title_as_title'] or watch['extract_title_as_title']:
+                    if not watch['title'] or not len(watch['title']):
+                        update_obj['title'] = html_tools.extract_element(find='title', html_content=fetcher.content)
+
+
+        return changed_detected, update_obj, text_content_before_ignored_filter

changedetectionio/image_diff.py

@@ -1,41 +0,0 @@
-# import the necessary packages
-from skimage.metrics import structural_similarity as compare_ssim
-import argparse
-import imutils
-import cv2
-
-# From https://www.pyimagesearch.com/2017/06/19/image-difference-with-opencv-and-python/
-def render_diff(fpath_imageA, fpath_imageB):
-
-	imageA = cv2.imread(fpath_imageA)
-	imageB = cv2.imread(fpath_imageB)
-
-	# convert the images to grayscale
-	grayA = cv2.cvtColor(imageA, cv2.COLOR_BGR2GRAY)
-	grayB = cv2.cvtColor(imageB, cv2.COLOR_BGR2GRAY)
-
-	# compute the Structural Similarity Index (SSIM) between the two
-	# images, ensuring that the difference image is returned
-	(score, diff) = compare_ssim(grayA, grayB, full=True)
-	diff = (diff * 255).astype("uint8")
-	print("SSIM: {}".format(score))
-
-	# threshold the difference image, followed by finding contours to
-	# obtain the regions of the two input images that differ
-	thresh = cv2.threshold(diff, 0, 255,
-		cv2.THRESH_BINARY_INV | cv2.THRESH_OTSU)[1]
-	cnts = cv2.findContours(thresh.copy(), cv2.RETR_EXTERNAL,
-		cv2.CHAIN_APPROX_SIMPLE)
-	cnts = imutils.grab_contours(cnts)
-
-	# loop over the contours
-	for c in cnts:
-		# compute the bounding box of the contour and then draw the
-		# bounding box on both input images to represent where the two
-		# images differ
-		(x, y, w, h) = cv2.boundingRect(c)
-		cv2.rectangle(imageA, (x, y), (x + w, y + h), (0, 0, 255), 2)
-		cv2.rectangle(imageB, (x, y), (x + w, y + h), (0, 0, 255), 2)
-
-	#return cv2.imencode('.jpg', imageB)[1].tobytes()
-	return cv2.imencode('.jpg', imageA)[1].tobytes()

changedetectionio/static/styles/package-lock.json

@@ -9,7 +9,7 @@
       "version": "0.0.3",
       "license": "ISC",
       "dependencies": {
-        "node-sass": "^7.0.0",
+        "node-sass": "^6.0.1",
         "tar": "^6.1.9",
         "trim-newlines": "^3.0.1"
       }
@@ -128,35 +128,13 @@
       }
     },
     "node_modules/ansi-styles": {
-      "version": "4.3.0",
-      "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
-      "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
-      "dependencies": {
-        "color-convert": "^2.0.1"
-      },
+      "version": "2.2.1",
+      "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.2.1.tgz",
+      "integrity": "sha1-tDLdM1i2NM914eRmQ2gkBTPB3b4=",
       "engines": {
-        "node": ">=8"
-      },
-      "funding": {
-        "url": "https://github.com/chalk/ansi-styles?sponsor=1"
+        "node": ">=0.10.0"
       }
     },
-    "node_modules/ansi-styles/node_modules/color-convert": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
-      "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
-      "dependencies": {
-        "color-name": "~1.1.4"
-      },
-      "engines": {
-        "node": ">=7.0.0"
-      }
-    },
-    "node_modules/ansi-styles/node_modules/color-name": {
-      "version": "1.1.4",
-      "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
-      "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA=="
-    },
     "node_modules/aproba": {
       "version": "1.2.0",
       "resolved": "https://registry.npmjs.org/aproba/-/aproba-1.2.0.tgz",
@@ -273,18 +251,18 @@
       "integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw="
     },
     "node_modules/chalk": {
-      "version": "4.1.2",
-      "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
-      "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==",
+      "version": "1.1.3",
+      "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz",
+      "integrity": "sha1-qBFcVeSnAv5NFQq9OHKCKn4J/Jg=",
       "dependencies": {
-        "ansi-styles": "^4.1.0",
-        "supports-color": "^7.1.0"
+        "ansi-styles": "^2.2.1",
+        "escape-string-regexp": "^1.0.2",
+        "has-ansi": "^2.0.0",
+        "strip-ansi": "^3.0.0",
+        "supports-color": "^2.0.0"
       },
       "engines": {
-        "node": ">=10"
-      },
-      "funding": {
-        "url": "https://github.com/chalk/chalk?sponsor=1"
+        "node": ">=0.10.0"
       }
     },
     "node_modules/chownr": {
@@ -366,14 +344,6 @@
       "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz",
       "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU="
     },
-    "node_modules/color-support": {
-      "version": "1.1.3",
-      "resolved": "https://registry.npmjs.org/color-support/-/color-support-1.1.3.tgz",
-      "integrity": "sha512-qiBjkpbMLO/HL68y+lh4q0/O1MZFj2RX6X/KmMa3+gJD3z+WwI1ZzDHysvqHGS3mP6mznPckpXmw1nI9cJjyRg==",
-      "bin": {
-        "color-support": "bin.js"
-      }
-    },
     "node_modules/combined-stream": {
       "version": "1.0.8",
       "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz",
@@ -707,6 +677,17 @@
         "node": ">= 0.4.0"
       }
     },
+    "node_modules/has-ansi": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/has-ansi/-/has-ansi-2.0.0.tgz",
+      "integrity": "sha1-NPUEnOHs3ysGSa8+8k5F7TVBbZE=",
+      "dependencies": {
+        "ansi-regex": "^2.0.0"
+      },
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
     "node_modules/has-flag": {
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz",
@@ -1061,13 +1042,13 @@
       }
     },
     "node_modules/node-sass": {
-      "version": "7.0.0",
-      "resolved": "https://registry.npmjs.org/node-sass/-/node-sass-7.0.0.tgz",
-      "integrity": "sha512-6yUnsD3L8fVbgMX6nKQqZkjRcG7a/PpmF0pEyeWf+BgbTj2ToJlCYrnUifL2KbjV5gIY22I3oppahBWA3B+jUg==",
+      "version": "6.0.1",
+      "resolved": "https://registry.npmjs.org/node-sass/-/node-sass-6.0.1.tgz",
+      "integrity": "sha512-f+Rbqt92Ful9gX0cGtdYwjTrWAaGURgaK5rZCWOgCNyGWusFYHhbqCCBoFBeat+HKETOU02AyTxNhJV0YZf2jQ==",
       "hasInstallScript": true,
       "dependencies": {
         "async-foreach": "^0.1.3",
-        "chalk": "^4.1.2",
+        "chalk": "^1.1.1",
         "cross-spawn": "^7.0.3",
         "gaze": "^1.0.0",
         "get-stdin": "^4.0.1",
@@ -1076,7 +1057,7 @@
         "meow": "^9.0.0",
         "nan": "^2.13.2",
         "node-gyp": "^7.1.0",
-        "npmlog": "^5.0.0",
+        "npmlog": "^4.0.0",
         "request": "^2.88.0",
         "sass-graph": "2.2.5",
         "stdout-stream": "^1.4.0",
@@ -1089,106 +1070,6 @@
         "node": ">=12"
       }
     },
-    "node_modules/node-sass/node_modules/ansi-regex": {
-      "version": "5.0.1",
-      "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz",
-      "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==",
-      "engines": {
-        "node": ">=8"
-      }
-    },
-    "node_modules/node-sass/node_modules/are-we-there-yet": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/are-we-there-yet/-/are-we-there-yet-2.0.0.tgz",
-      "integrity": "sha512-Ci/qENmwHnsYo9xKIcUJN5LeDKdJ6R1Z1j9V/J5wyq8nh/mYPEpIKJbBZXtZjG04HiK7zV/p6Vs9952MrMeUIw==",
-      "dependencies": {
-        "delegates": "^1.0.0",
-        "readable-stream": "^3.6.0"
-      },
-      "engines": {
-        "node": ">=10"
-      }
-    },
-    "node_modules/node-sass/node_modules/emoji-regex": {
-      "version": "8.0.0",
-      "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz",
-      "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A=="
-    },
-    "node_modules/node-sass/node_modules/gauge": {
-      "version": "3.0.2",
-      "resolved": "https://registry.npmjs.org/gauge/-/gauge-3.0.2.tgz",
-      "integrity": "sha512-+5J6MS/5XksCuXq++uFRsnUd7Ovu1XenbeuIuNRJxYWjgQbPuFhT14lAvsWfqfAmnwluf1OwMjz39HjfLPci0Q==",
-      "dependencies": {
-        "aproba": "^1.0.3 || ^2.0.0",
-        "color-support": "^1.1.2",
-        "console-control-strings": "^1.0.0",
-        "has-unicode": "^2.0.1",
-        "object-assign": "^4.1.1",
-        "signal-exit": "^3.0.0",
-        "string-width": "^4.2.3",
-        "strip-ansi": "^6.0.1",
-        "wide-align": "^1.1.2"
-      },
-      "engines": {
-        "node": ">=10"
-      }
-    },
-    "node_modules/node-sass/node_modules/is-fullwidth-code-point": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz",
-      "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==",
-      "engines": {
-        "node": ">=8"
-      }
-    },
-    "node_modules/node-sass/node_modules/npmlog": {
-      "version": "5.0.1",
-      "resolved": "https://registry.npmjs.org/npmlog/-/npmlog-5.0.1.tgz",
-      "integrity": "sha512-AqZtDUWOMKs1G/8lwylVjrdYgqA4d9nu8hc+0gzRxlDb1I10+FHBGMXs6aiQHFdCUUlqH99MUMuLfzWDNDtfxw==",
-      "dependencies": {
-        "are-we-there-yet": "^2.0.0",
-        "console-control-strings": "^1.1.0",
-        "gauge": "^3.0.0",
-        "set-blocking": "^2.0.0"
-      }
-    },
-    "node_modules/node-sass/node_modules/readable-stream": {
-      "version": "3.6.0",
-      "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz",
-      "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==",
-      "dependencies": {
-        "inherits": "^2.0.3",
-        "string_decoder": "^1.1.1",
-        "util-deprecate": "^1.0.1"
-      },
-      "engines": {
-        "node": ">= 6"
-      }
-    },
-    "node_modules/node-sass/node_modules/string-width": {
-      "version": "4.2.3",
-      "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz",
-      "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==",
-      "dependencies": {
-        "emoji-regex": "^8.0.0",
-        "is-fullwidth-code-point": "^3.0.0",
-        "strip-ansi": "^6.0.1"
-      },
-      "engines": {
-        "node": ">=8"
-      }
-    },
-    "node_modules/node-sass/node_modules/strip-ansi": {
-      "version": "6.0.1",
-      "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz",
-      "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==",
-      "dependencies": {
-        "ansi-regex": "^5.0.1"
-      },
-      "engines": {
-        "node": ">=8"
-      }
-    },
     "node_modules/nopt": {
       "version": "5.0.0",
       "resolved": "https://registry.npmjs.org/nopt/-/nopt-5.0.0.tgz",
@@ -1735,22 +1616,11 @@
       }
     },
     "node_modules/supports-color": {
-      "version": "7.2.0",
-      "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
-      "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
-      "dependencies": {
-        "has-flag": "^4.0.0"
-      },
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz",
+      "integrity": "sha1-U10EXOa2Nj+kARcIRimZXp3zJMc=",
       "engines": {
-        "node": ">=8"
-      }
-    },
-    "node_modules/supports-color/node_modules/has-flag": {
-      "version": "4.0.0",
-      "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
-      "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
-      "engines": {
-        "node": ">=8"
+        "node": ">=0.8.0"
       }
     },
     "node_modules/tar": {
@@ -2180,27 +2050,9 @@
       "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8="
     },
     "ansi-styles": {
-      "version": "4.3.0",
-      "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
-      "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
-      "requires": {
-        "color-convert": "^2.0.1"
-      },
-      "dependencies": {
-        "color-convert": {
-          "version": "2.0.1",
-          "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
-          "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
-          "requires": {
-            "color-name": "~1.1.4"
-          }
-        },
-        "color-name": {
-          "version": "1.1.4",
-          "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
-          "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA=="
-        }
-      }
+      "version": "2.2.1",
+      "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.2.1.tgz",
+      "integrity": "sha1-tDLdM1i2NM914eRmQ2gkBTPB3b4="
     },
     "aproba": {
       "version": "1.2.0",
@@ -2297,12 +2149,15 @@
       "integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw="
     },
     "chalk": {
-      "version": "4.1.2",
-      "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
-      "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==",
+      "version": "1.1.3",
+      "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz",
+      "integrity": "sha1-qBFcVeSnAv5NFQq9OHKCKn4J/Jg=",
       "requires": {
-        "ansi-styles": "^4.1.0",
-        "supports-color": "^7.1.0"
+        "ansi-styles": "^2.2.1",
+        "escape-string-regexp": "^1.0.2",
+        "has-ansi": "^2.0.0",
+        "strip-ansi": "^3.0.0",
+        "supports-color": "^2.0.0"
       }
     },
     "chownr": {
@@ -2368,11 +2223,6 @@
       "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz",
       "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU="
     },
-    "color-support": {
-      "version": "1.1.3",
-      "resolved": "https://registry.npmjs.org/color-support/-/color-support-1.1.3.tgz",
-      "integrity": "sha512-qiBjkpbMLO/HL68y+lh4q0/O1MZFj2RX6X/KmMa3+gJD3z+WwI1ZzDHysvqHGS3mP6mznPckpXmw1nI9cJjyRg=="
-    },
     "combined-stream": {
       "version": "1.0.8",
       "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz",
@@ -2635,6 +2485,14 @@
         "function-bind": "^1.1.1"
       }
     },
+    "has-ansi": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/has-ansi/-/has-ansi-2.0.0.tgz",
+      "integrity": "sha1-NPUEnOHs3ysGSa8+8k5F7TVBbZE=",
+      "requires": {
+        "ansi-regex": "^2.0.0"
+      }
+    },
     "has-flag": {
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz",
@@ -2910,12 +2768,12 @@
       }
     },
     "node-sass": {
-      "version": "7.0.0",
-      "resolved": "https://registry.npmjs.org/node-sass/-/node-sass-7.0.0.tgz",
-      "integrity": "sha512-6yUnsD3L8fVbgMX6nKQqZkjRcG7a/PpmF0pEyeWf+BgbTj2ToJlCYrnUifL2KbjV5gIY22I3oppahBWA3B+jUg==",
+      "version": "6.0.1",
+      "resolved": "https://registry.npmjs.org/node-sass/-/node-sass-6.0.1.tgz",
+      "integrity": "sha512-f+Rbqt92Ful9gX0cGtdYwjTrWAaGURgaK5rZCWOgCNyGWusFYHhbqCCBoFBeat+HKETOU02AyTxNhJV0YZf2jQ==",
       "requires": {
         "async-foreach": "^0.1.3",
-        "chalk": "^4.1.2",
+        "chalk": "^1.1.1",
         "cross-spawn": "^7.0.3",
         "gaze": "^1.0.0",
         "get-stdin": "^4.0.1",
@@ -2924,92 +2782,11 @@
         "meow": "^9.0.0",
         "nan": "^2.13.2",
         "node-gyp": "^7.1.0",
-        "npmlog": "^5.0.0",
+        "npmlog": "^4.0.0",
         "request": "^2.88.0",
         "sass-graph": "2.2.5",
         "stdout-stream": "^1.4.0",
         "true-case-path": "^1.0.2"
-      },
-      "dependencies": {
-        "ansi-regex": {
-          "version": "5.0.1",
-          "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz",
-          "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ=="
-        },
-        "are-we-there-yet": {
-          "version": "2.0.0",
-          "resolved": "https://registry.npmjs.org/are-we-there-yet/-/are-we-there-yet-2.0.0.tgz",
-          "integrity": "sha512-Ci/qENmwHnsYo9xKIcUJN5LeDKdJ6R1Z1j9V/J5wyq8nh/mYPEpIKJbBZXtZjG04HiK7zV/p6Vs9952MrMeUIw==",
-          "requires": {
-            "delegates": "^1.0.0",
-            "readable-stream": "^3.6.0"
-          }
-        },
-        "emoji-regex": {
-          "version": "8.0.0",
-          "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz",
-          "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A=="
-        },
-        "gauge": {
-          "version": "3.0.2",
-          "resolved": "https://registry.npmjs.org/gauge/-/gauge-3.0.2.tgz",
-          "integrity": "sha512-+5J6MS/5XksCuXq++uFRsnUd7Ovu1XenbeuIuNRJxYWjgQbPuFhT14lAvsWfqfAmnwluf1OwMjz39HjfLPci0Q==",
-          "requires": {
-            "aproba": "^1.0.3 || ^2.0.0",
-            "color-support": "^1.1.2",
-            "console-control-strings": "^1.0.0",
-            "has-unicode": "^2.0.1",
-            "object-assign": "^4.1.1",
-            "signal-exit": "^3.0.0",
-            "string-width": "^4.2.3",
-            "strip-ansi": "^6.0.1",
-            "wide-align": "^1.1.2"
-          }
-        },
-        "is-fullwidth-code-point": {
-          "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz",
-          "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg=="
-        },
-        "npmlog": {
-          "version": "5.0.1",
-          "resolved": "https://registry.npmjs.org/npmlog/-/npmlog-5.0.1.tgz",
-          "integrity": "sha512-AqZtDUWOMKs1G/8lwylVjrdYgqA4d9nu8hc+0gzRxlDb1I10+FHBGMXs6aiQHFdCUUlqH99MUMuLfzWDNDtfxw==",
-          "requires": {
-            "are-we-there-yet": "^2.0.0",
-            "console-control-strings": "^1.1.0",
-            "gauge": "^3.0.0",
-            "set-blocking": "^2.0.0"
-          }
-        },
-        "readable-stream": {
-          "version": "3.6.0",
-          "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz",
-          "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==",
-          "requires": {
-            "inherits": "^2.0.3",
-            "string_decoder": "^1.1.1",
-            "util-deprecate": "^1.0.1"
-          }
-        },
-        "string-width": {
-          "version": "4.2.3",
-          "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz",
-          "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==",
-          "requires": {
-            "emoji-regex": "^8.0.0",
-            "is-fullwidth-code-point": "^3.0.0",
-            "strip-ansi": "^6.0.1"
-          }
-        },
-        "strip-ansi": {
-          "version": "6.0.1",
-          "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz",
-          "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==",
-          "requires": {
-            "ansi-regex": "^5.0.1"
-          }
-        }
       }
     },
     "nopt": {
@@ -3436,19 +3213,9 @@
       }
     },
     "supports-color": {
-      "version": "7.2.0",
-      "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
-      "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
-      "requires": {
-        "has-flag": "^4.0.0"
-      },
-      "dependencies": {
-        "has-flag": {
-          "version": "4.0.0",
-          "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
-          "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ=="
-        }
-      }
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz",
+      "integrity": "sha1-U10EXOa2Nj+kARcIRimZXp3zJMc="
     },
     "tar": {
       "version": "6.1.9",

changedetectionio/static/styles/package.json

@@ -10,7 +10,7 @@
   "author": "",
   "license": "ISC",
   "dependencies": {
-    "node-sass": "^7.0.0",
+    "node-sass": "^6.0.1",
     "tar": "^6.1.9",
     "trim-newlines": "^3.0.1"
   }

changedetectionio/store.py

@@ -372,9 +372,7 @@ class ChangeDetectionStore:
         if not os.path.isdir(output_path):
             mkdir(output_path)
 
-        suffix = "stripped.txt"
-
-        fname = "{}/{}.{}".format(output_path, uuid.uuid4(), suffix)
+        fname = "{}/{}.stripped.txt".format(output_path, uuid.uuid4())
         with open(fname, 'wb') as f:
             f.write(contents)
             f.close()

changedetectionio/templates/diff-image.html

@@ -1,59 +0,0 @@
-{% extends 'base.html' %}
-
-{% block content %}
-
-<div id="settings">
-    <h1>Differences</h1>
-    <form class="pure-form " action="" method="GET">
-        <fieldset>
-            {% if versions|length >= 1 %}
-            <label for="diff-version">Compare newest (<span id="current-v-date"></span>) with</label>
-            <select id="diff-version" name="previous_version">
-                {% for version in versions %}
-                <option value="{{version}}" {% if version== current_previous_version %} selected="" {% endif %}>
-                    {{version}}
-                </option>
-                {% endfor %}
-            </select>
-            <button type="submit" class="pure-button pure-button-primary">Go</button>
-            {% endif %}
-        </fieldset>
-    </form>
-
-</div>
-
-<div id="diff-ui">
-  <img style="max-width: 100%" src="{{ url_for('render_diff_image', uuid=uuid, compare_date=current_previous_version) }}" />
-
-    <div>
-        <span style="width: 50%">
-            <img style="max-width: 100%" src="{{ url_for('show_single_image', uuid=uuid, datestr=newest_version_timestamp) }}" />
-        </span>
-        <span style="width: 50%">
-            <img style="max-width: 100%" src="{{ url_for('show_single_image', uuid=uuid, datestr=current_previous_version) }}" />
-        </span>
-    </div>
-</div>
-
-
-<script type="text/javascript" src="{{url_for('static_content', group='js', filename='diff.js')}}"></script>
-
-<script defer="">
-window.onload = function() {
-    /* Set current version date as local time in the browser also */
-    var current_v = document.getElementById("current-v-date");
-    var dateObject = new Date({{ newest_version_timestamp }}*1000);
-    current_v.innerHTML=dateObject.toLocaleString();
-
-    /* Convert what is options from UTC time.time() to local browser time */
-    var diffList=document.getElementById("diff-version");
-    if (typeof(diffList) != 'undefined' && diffList != null) {
-        for (var option of diffList.options) {
-          var dateObject = new Date(option.value*1000);
-          option.label=dateObject.toLocaleString();
-        }
-    }
-}
-</script>
-
-{% endblock %}

changedetectionio/templates/preview-image.html

@@ -1,13 +0,0 @@
-{% extends 'base.html' %}
-
-{% block content %}
-
-<div id="settings">
-    <h1>Current</h1>
-</div>
-
-<div id="diff-ui">
-    image goes here
-</div>
-
-{% endblock %}

changedetectionio/templates/preview.html

@@ -11,7 +11,7 @@
         <tbody>
         <tr>
             <td id="diff-col">
-                <span id="result">{{content}}</span>
+                <span id="result">{% for row in content %}{{row}}{% endfor %}</span>
             </td>
         </tr>
         </tbody>

changedetectionio/tests/test_backend.py

@@ -100,14 +100,6 @@ def test_check_basic_change_detection_functionality(client, live_server):
     # It should have picked up the <title>
     assert b'head title' in res.data
 
-
-    # be sure the HTML converter worked
-    res = client.get(url_for("preview_page", uuid="first"))
-    assert b'<html>' not in res.data
-
-    res = client.get(url_for("preview_page", uuid="first"))
-    assert b'Some initial text' in res.data
-
     #
     # Cleanup everything
     res = client.get(url_for("api_delete", uuid="all"), follow_redirects=True)

changedetectionio/tests/test_binary_fetch.py

@@ -1,56 +0,0 @@
-#!/usr/bin/python3
-
-import time
-import secrets
-from flask import url_for
-from . util import live_server_setup
-
-
-def test_binary_file_change(client, live_server):
-    with open("test-datastore/test.bin", "wb") as f:
-        f.write(secrets.token_bytes())
-
-    live_server_setup(live_server)
-
-    sleep_time_for_fetch_thread = 3
-
-    # Give the endpoint time to spin up
-    time.sleep(1)
-
-    # Add our URL to the import page
-    test_url = url_for('test_binaryfile_endpoint', _external=True)
-    res = client.post(
-        url_for("import_page"),
-        data={"urls": test_url},
-        follow_redirects=True
-    )
-    assert b"1 Imported" in res.data
-
-    # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
-
-    # Give the thread time to pick it up
-    time.sleep(sleep_time_for_fetch_thread)
-
-    # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
-
-    # It should report nothing found (no new 'unviewed' class)
-    res = client.get(url_for("index"))
-    assert b'unviewed' not in res.data
-    assert b'/test-binary-endpoint' in res.data
-
-    #  Make a change
-    with open("test-datastore/test.bin", "wb") as f:
-        f.write(secrets.token_bytes())
-
-
-    # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
-
-    # Give the thread time to pick it up
-    time.sleep(sleep_time_for_fetch_thread)
-
-    # It should report nothing found (no new 'unviewed' class)
-    res = client.get(url_for("index"))
-    assert b'unviewed' in res.data

changedetectionio/tests/util.py

@@ -37,16 +37,6 @@ def set_modified_response():
 
 def live_server_setup(live_server):
 
-    @live_server.app.route('/test-binary-endpoint')
-    def test_binaryfile_endpoint():
-
-        from flask import make_response
-
-        # Tried using a global var here but didn't seem to work, so reading from a file instead.
-        with open("test-datastore/test.bin", "rb") as f:
-            resp = make_response(f.read())
-            resp.headers['Content-Type'] = 'image/jpeg'
-            return resp
 
     @live_server.app.route('/test-endpoint')
     def test_endpoint():

changedetectionio/update_worker.py

@@ -42,6 +42,7 @@ class update_worker(threading.Thread):
                     now = time.time()
 
                     try:
+
                         changed_detected, update_obj, contents = update_handler.run(uuid)
 
                         # Re #342
@@ -134,8 +135,8 @@ class update_worker(threading.Thread):
 
                         except Exception as e:
                             # Catch everything possible here, so that if a worker crashes, we don't lose it until restart!
-                            self.app.logger.error("Exception reached processing watch UUID: %s - %s", uuid, str(e))
-                            self.datastore.update_watch(uuid=uuid, update_obj={'last_error': str(e)})
+                            print("!!!! Exception in update_worker !!!\n", e)
+
                     finally:
                         # Always record that we atleast tried
                         self.datastore.update_watch(uuid=uuid, update_obj={'fetch_time': round(time.time() - now, 3),