Number of fetching workers can be overriden by Env "FETCH_WORKERS"

README.md

@@ -99,8 +99,6 @@ See the wiki for more information https://github.com/dgtlmoon/changedetection.io
 ## Filters
 XPath, JSONPath and CSS support comes baked in! You can be as specific as you need, use XPath exported from various XPath element query creation tools.
 
-(We support LXML re:test, re:math and re:replace.)
-
 ## Notifications
 
 ChangeDetection.io supports a massive amount of notifications (including email, office365, custom APIs, etc) when a web-page has a change detected thanks to the <a href="https://github.com/caronc/apprise">apprise</a> library.

changedetectionio/__init__.py

@@ -35,9 +35,8 @@ from flask import (
     url_for,
 )
 from flask_login import login_required
-from changedetectionio import html_tools
 
-__version__ = '0.39.8'
+__version__ = '0.39.7'
 
 datastore = None
 
@@ -225,18 +224,10 @@ def changedetection_app(config=None, datastore_o=None):
 
         if (user.check_password(password)):
             flask_login.login_user(user, remember=True)
-
-            # For now there's nothing else interesting here other than the index/list page
-            # It's more reliable and safe to ignore the 'next' redirect
-            # When we used...
-            # next = request.args.get('next')
-            # return redirect(next or url_for('index'))
-            # We would sometimes get login loop errors on sites hosted in sub-paths
-
-            # note for the future:
+            next = request.args.get('next')
             #            if not is_safe_url(next):
             #                return flask.abort(400)
-            return redirect(url_for('index'))
+            return redirect(next or url_for('index'))
 
         else:
             flash('Incorrect password', 'error')
@@ -442,7 +433,7 @@ def changedetection_app(config=None, datastore_o=None):
                 raw_content = file.read()
 
                 handler = fetch_site_status.perform_site_check(datastore=datastore)
-                stripped_content = html_tools.strip_ignore_text(raw_content,
+                stripped_content = handler.strip_ignore_text(raw_content,
                                                              datastore.data['watching'][uuid]['ignore_text'])
 
                 if datastore.data['settings']['application'].get('ignore_whitespace', False):
@@ -547,14 +538,10 @@ def changedetection_app(config=None, datastore_o=None):
                     flash('No notification URLs set, cannot send test.', 'error')
 
             # Diff page [edit] link should go back to diff page
-            if request.args.get("next") and request.args.get("next") == 'diff' and not form.save_and_preview_button.data:
+            if request.args.get("next") and request.args.get("next") == 'diff':
                 return redirect(url_for('diff_history_page', uuid=uuid))
             else:
-                if form.save_and_preview_button.data:
-                    flash('You may need to reload this page to see the new content.')
-                    return redirect(url_for('preview_page', uuid=uuid))
-                else:
-                    return redirect(url_for('index'))
+                return redirect(url_for('index'))
 
         else:
             if request.method == 'POST' and not form.validate():
@@ -726,12 +713,8 @@ def changedetection_app(config=None, datastore_o=None):
         # Save the current newest history as the most recently viewed
         datastore.set_last_viewed(uuid, dates[0])
         newest_file = watch['history'][dates[0]]
-
-        try:
-            with open(newest_file, 'r') as f:
-                newest_version_file_contents = f.read()
-        except Exception as e:
-            newest_version_file_contents = "Unable to read {}.\n".format(newest_file)
+        with open(newest_file, 'r') as f:
+            newest_version_file_contents = f.read()
 
         previous_version = request.args.get('previous_version')
         try:
@@ -740,11 +723,8 @@ def changedetection_app(config=None, datastore_o=None):
             # Not present, use a default value, the second one in the sorted list.
             previous_file = watch['history'][dates[1]]
 
-        try:
-            with open(previous_file, 'r') as f:
-                previous_version_file_contents = f.read()
-        except Exception as e:
-            previous_version_file_contents = "Unable to read {}.\n".format(previous_file)
+        with open(previous_file, 'r') as f:
+            previous_version_file_contents = f.read()
 
         output = render_template("diff.html", watch_a=watch,
                                  newest=newest_version_file_contents,
@@ -763,9 +743,6 @@ def changedetection_app(config=None, datastore_o=None):
     @app.route("/preview/<string:uuid>", methods=['GET'])
     @login_required
     def preview_page(uuid):
-        content = []
-        ignored_line_numbers = []
-        trigger_line_numbers = []
 
         # More for testing, possible to return the first/only
         if uuid == 'first':
@@ -779,51 +756,14 @@ def changedetection_app(config=None, datastore_o=None):
             flash("No history found for the specified link, bad link?", "error")
             return redirect(url_for('index'))
 
-        if len(watch['history']):
-            timestamps = sorted(watch['history'].keys(), key=lambda x: int(x))
-            filename = watch['history'][timestamps[-1]]
-            try:
-                with open(filename, 'r') as f:
-                    tmp = f.readlines()
-
-                    # Get what needs to be highlighted
-                    ignore_rules = watch.get('ignore_text', []) + datastore.data['settings']['application']['global_ignore_text']
-
-                    # .readlines will keep the \n, but we will parse it here again, in the future tidy this up
-                    ignored_line_numbers = html_tools.strip_ignore_text(content="".join(tmp),
-                                                                        wordlist=ignore_rules,
-                                                                        mode='line numbers'
-                                                                        )
-
-                    trigger_line_numbers = html_tools.strip_ignore_text(content="".join(tmp),
-                                                                        wordlist=watch['trigger_text'],
-                                                                        mode='line numbers'
-                                                                        )
-                    # Prepare the classes and lines used in the template
-                    i=0
-                    for l in tmp:
-                        classes=[]
-                        i+=1
-                        if i in ignored_line_numbers:
-                            classes.append('ignored')
-                        if i in trigger_line_numbers:
-                            classes.append('triggered')
-                        content.append({'line': l, 'classes': ' '.join(classes)})
-
-
-            except Exception as e:
-                content.append({'line': "File doesnt exist or unable to read file {}".format(filename), 'classes': ''})
-        else:
-            content.append({'line': "No history found", 'classes': ''})
-
+        newest = list(watch['history'].keys())[-1]
+        with open(watch['history'][newest], 'r') as f:
+            content = f.readlines()
 
         output = render_template("preview.html",
                                  content=content,
                                  extra_stylesheets=extra_stylesheets,
-                                 ignored_line_numbers=ignored_line_numbers,
-                                 triggered_line_numbers=trigger_line_numbers,
                                  current_diff_url=watch['url'],
-                                 watch=watch,
                                  uuid=uuid)
         return output
 
@@ -1103,7 +1043,7 @@ def ticker_thread_check_time_launch_checks():
 
     # Spin up Workers that do the fetching
     # Can be overriden by ENV or use the default settings
-    n_workers = int(os.getenv("FETCH_WORKERS", datastore.data['settings']['requests']['workers']))
+    n_workers = os.getenv("FETCH_WORKERS", datastore.data['settings']['requests']['workers'])
     for _ in range(n_workers):
         new_worker = update_worker.update_worker(update_q, notification_q, app, datastore)
         running_update_threads.append(new_worker)
@@ -1118,14 +1058,7 @@ def ticker_thread_check_time_launch_checks():
                 running_uuids.append(t.current_uuid)
 
         # Re #232 - Deepcopy the data incase it changes while we're iterating through it all
-        while True:
-            try:
-                copied_datastore = deepcopy(datastore)
-            except RuntimeError as e:
-                # RuntimeError: dictionary changed size during iteration
-                time.sleep(0.1)
-            else:
-                break
+        copied_datastore = deepcopy(datastore)
 
         # Check for watches outside of the time threshold to put in the thread queue.
         for uuid, watch in copied_datastore.data['watching'].items():

changedetectionio/content_fetcher.py

@@ -120,7 +120,7 @@ class html_webdriver(Fetcher):
         # raise EmptyReply(url=url, status_code=r.status_code)
 
         # @todo - dom wait loaded?
-        time.sleep(int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)))
+        time.sleep(5)
         self.content = driver.page_source
         self.headers = {}
 

changedetectionio/fetch_site_status.py

@@ -1,6 +1,5 @@
 import time
 from changedetectionio import content_fetcher
-from changedetectionio import html_tools
 import hashlib
 from inscriptis import get_text
 import urllib3
@@ -17,6 +16,40 @@ class perform_site_check():
         super().__init__(*args, **kwargs)
         self.datastore = datastore
 
+    def strip_ignore_text(self, content, list_ignore_text):
+        import re
+        ignore = []
+        ignore_regex = []
+        for k in list_ignore_text:
+
+            # Is it a regex?
+            if k[0] == '/':
+                ignore_regex.append(k.strip(" /"))
+            else:
+                ignore.append(k)
+
+        output = []
+        for line in content.splitlines():
+
+            # Always ignore blank lines in this mode. (when this function gets called)
+            if len(line.strip()):
+                regex_matches = False
+
+                # if any of these match, skip
+                for regex in ignore_regex:
+                    try:
+                        if re.search(regex, line, re.IGNORECASE):
+                            regex_matches = True
+                    except Exception as e:
+                        continue
+
+                if not regex_matches and not any(skip_text in line for skip_text in ignore):
+                    output.append(line.encode('utf8'))
+
+        return "\n".encode('utf8').join(output)
+
+
+
     def run(self, uuid):
         timestamp = int(time.time())  # used for storage etc too
 
@@ -69,7 +102,7 @@ class perform_site_check():
             # https://stackoverflow.com/questions/41817578/basic-method-chaining ?
             # return content().textfilter().jsonextract().checksumcompare() ?
 
-            is_json = 'application/json' in fetcher.headers.get('Content-Type', '')
+            is_json = fetcher.headers.get('Content-Type', '') == 'application/json'
             is_html = not is_json
             css_filter_rule = watch['css_filter']
 
@@ -114,7 +147,7 @@ class perform_site_check():
             # @todo we could abstract out the get_text() to handle this cleaner
             text_to_ignore = watch.get('ignore_text', []) + self.datastore.data['settings']['application'].get('global_ignore_text', [])
             if len(text_to_ignore):
-                stripped_text_from_html = html_tools.strip_ignore_text(stripped_text_from_html, text_to_ignore)
+                stripped_text_from_html = self.strip_ignore_text(stripped_text_from_html, text_to_ignore)
             else:
                 stripped_text_from_html = stripped_text_from_html.encode('utf8')
 
@@ -132,14 +165,22 @@ class perform_site_check():
             blocked_by_not_found_trigger_text = False
 
             if len(watch['trigger_text']):
-                # Yeah, lets block first until something matches
                 blocked_by_not_found_trigger_text = True
-                # Filter and trigger works the same, so reuse it
-                result = html_tools.strip_ignore_text(content=str(stripped_text_from_html),
-                                                      wordlist=watch['trigger_text'],
-                                                      mode="line numbers")
-                if result:
-                    blocked_by_not_found_trigger_text = False
+                for line in watch['trigger_text']:
+                    # Because JSON wont serialize a re.compile object
+                    if line[0] == '/' and line[-1] == '/':
+                        regex = re.compile(line.strip('/'), re.IGNORECASE)
+                        # Found it? so we don't wait for it anymore
+                        r = re.search(regex, str(stripped_text_from_html))
+                        if r:
+                            blocked_by_not_found_trigger_text = False
+                            break
+
+                    elif line.lower() in str(stripped_text_from_html).lower():
+                        # We found it don't wait for it.
+                        blocked_by_not_found_trigger_text = False
+                        break
+
 
 
             if not blocked_by_not_found_trigger_text and watch['previous_md5'] != fetched_md5:

changedetectionio/forms.py

@@ -1,7 +1,6 @@
 from wtforms import Form, SelectField, RadioField, BooleanField, StringField, PasswordField, validators, IntegerField, fields, TextAreaField, \
     Field
-
-from wtforms import widgets, SubmitField
+from wtforms import widgets
 from wtforms.validators import ValidationError
 from wtforms.fields import html5
 from changedetectionio import content_fetcher
@@ -291,9 +290,6 @@ class watchForm(commonSettingsForm):
     method = SelectField('Request Method', choices=valid_method, default=default_method)
     trigger_text = StringListField('Trigger/wait for text', [validators.Optional(), ValidateListRegex()])
 
-    save_button = SubmitField('Save', render_kw={"class": "pure-button pure-button-primary"})
-    save_and_preview_button = SubmitField('Save & Preview', render_kw={"class": "pure-button pure-button-primary"})
-
     def validate(self, **kwargs):
         if not super().validate():
             return False

changedetectionio/html_tools.py

@@ -1,7 +1,7 @@
 import json
 from bs4 import BeautifulSoup
 from jsonpath_ng.ext import parse
-import re
+
 
 class JSONNotFound(ValueError):
     def __init__(self, msg):
@@ -25,7 +25,7 @@ def xpath_filter(xpath_filter, html_content):
     tree = html.fromstring(html_content)
     html_block = ""
 
-    for item in tree.xpath(xpath_filter.strip(), namespaces={'re':'http://exslt.org/regular-expressions'}):
+    for item in tree.xpath(xpath_filter.strip()):
         html_block+= etree.tostring(item, pretty_print=True).decode('utf-8')+"<br/>"
 
     return html_block
@@ -105,50 +105,3 @@ def extract_json_as_string(content, jsonpath_filter):
         return ''
 
     return stripped_text_from_html
-
-# Mode     - "content" return the content without the matches (default)
-#          - "line numbers" return a list of line numbers that match (int list)
-#
-# wordlist - list of regex's (str) or words (str)
-def strip_ignore_text(content, wordlist, mode="content"):
-    ignore = []
-    ignore_regex = []
-
-    # @todo check this runs case insensitive
-    for k in wordlist:
-
-        # Is it a regex?
-        if k[0] == '/':
-            ignore_regex.append(k.strip(" /"))
-        else:
-            ignore.append(k)
-
-    i = 0
-    output = []
-    ignored_line_numbers = []
-    for line in content.splitlines():
-        i += 1
-        # Always ignore blank lines in this mode. (when this function gets called)
-        if len(line.strip()):
-            regex_matches = False
-
-            # if any of these match, skip
-            for regex in ignore_regex:
-                try:
-                    if re.search(regex, line, re.IGNORECASE):
-                        regex_matches = True
-                except Exception as e:
-                    continue
-
-            if not regex_matches and not any(skip_text.lower() in line.lower() for skip_text in ignore):
-                output.append(line.encode('utf8'))
-            else:
-                ignored_line_numbers.append(i)
-
-
-
-    # Used for finding out what to highlight
-    if mode == "line numbers":
-        return ignored_line_numbers
-
-    return "\n".encode('utf8').join(output)

changedetectionio/static/styles/diff.css

@@ -54,19 +54,3 @@ ins {
   body {
     height: 99%;
     /* Hide scroll bar in Firefox */ } }
-
-td#diff-col div {
-  text-align: justify;
-  white-space: pre-wrap; }
-
-.ignored {
-  background-color: #ccc;
-  /*  border: #0d91fa 1px solid; */
-  opacity: 0.7; }
-
-.triggered {
-  background-color: #1b98f8; }
-
-/* ignored and triggered? make it obvious error */
-.ignored.triggered {
-  background-color: #ff0000; }

changedetectionio/static/styles/diff.scss

@@ -66,23 +66,3 @@ ins {
 		height: 99%; /* Hide scroll bar in Firefox */
 	}
 }
-
-td#diff-col div {
-    text-align: justify;
-    white-space: pre-wrap;
-}
-
-.ignored {
-    background-color: #ccc;
-   /*  border: #0d91fa 1px solid; */
-    opacity: 0.7;
-}
-
-.triggered {
-    background-color: #1b98f8;
-}
-
-/* ignored and triggered? make it obvious error */
-.ignored.triggered {
-  background-color: #ff0000;
-}

changedetectionio/static/styles/package-lock.json

@@ -9,7 +9,7 @@
       "version": "0.0.3",
       "license": "ISC",
       "dependencies": {
-        "node-sass": "^7.0.0",
+        "node-sass": "^6.0.1",
         "tar": "^6.1.9",
         "trim-newlines": "^3.0.1"
       }
@@ -128,35 +128,13 @@
       }
     },
     "node_modules/ansi-styles": {
-      "version": "4.3.0",
-      "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
-      "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
-      "dependencies": {
-        "color-convert": "^2.0.1"
-      },
+      "version": "2.2.1",
+      "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.2.1.tgz",
+      "integrity": "sha1-tDLdM1i2NM914eRmQ2gkBTPB3b4=",
       "engines": {
-        "node": ">=8"
-      },
-      "funding": {
-        "url": "https://github.com/chalk/ansi-styles?sponsor=1"
+        "node": ">=0.10.0"
       }
     },
-    "node_modules/ansi-styles/node_modules/color-convert": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
-      "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
-      "dependencies": {
-        "color-name": "~1.1.4"
-      },
-      "engines": {
-        "node": ">=7.0.0"
-      }
-    },
-    "node_modules/ansi-styles/node_modules/color-name": {
-      "version": "1.1.4",
-      "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
-      "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA=="
-    },
     "node_modules/aproba": {
       "version": "1.2.0",
       "resolved": "https://registry.npmjs.org/aproba/-/aproba-1.2.0.tgz",
@@ -273,18 +251,18 @@
       "integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw="
     },
     "node_modules/chalk": {
-      "version": "4.1.2",
-      "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
-      "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==",
+      "version": "1.1.3",
+      "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz",
+      "integrity": "sha1-qBFcVeSnAv5NFQq9OHKCKn4J/Jg=",
       "dependencies": {
-        "ansi-styles": "^4.1.0",
-        "supports-color": "^7.1.0"
+        "ansi-styles": "^2.2.1",
+        "escape-string-regexp": "^1.0.2",
+        "has-ansi": "^2.0.0",
+        "strip-ansi": "^3.0.0",
+        "supports-color": "^2.0.0"
       },
       "engines": {
-        "node": ">=10"
-      },
-      "funding": {
-        "url": "https://github.com/chalk/chalk?sponsor=1"
+        "node": ">=0.10.0"
       }
     },
     "node_modules/chownr": {
@@ -366,14 +344,6 @@
       "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz",
       "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU="
     },
-    "node_modules/color-support": {
-      "version": "1.1.3",
-      "resolved": "https://registry.npmjs.org/color-support/-/color-support-1.1.3.tgz",
-      "integrity": "sha512-qiBjkpbMLO/HL68y+lh4q0/O1MZFj2RX6X/KmMa3+gJD3z+WwI1ZzDHysvqHGS3mP6mznPckpXmw1nI9cJjyRg==",
-      "bin": {
-        "color-support": "bin.js"
-      }
-    },
     "node_modules/combined-stream": {
       "version": "1.0.8",
       "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz",
@@ -707,6 +677,17 @@
         "node": ">= 0.4.0"
       }
     },
+    "node_modules/has-ansi": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/has-ansi/-/has-ansi-2.0.0.tgz",
+      "integrity": "sha1-NPUEnOHs3ysGSa8+8k5F7TVBbZE=",
+      "dependencies": {
+        "ansi-regex": "^2.0.0"
+      },
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
     "node_modules/has-flag": {
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz",
@@ -1061,13 +1042,13 @@
       }
     },
     "node_modules/node-sass": {
-      "version": "7.0.0",
-      "resolved": "https://registry.npmjs.org/node-sass/-/node-sass-7.0.0.tgz",
-      "integrity": "sha512-6yUnsD3L8fVbgMX6nKQqZkjRcG7a/PpmF0pEyeWf+BgbTj2ToJlCYrnUifL2KbjV5gIY22I3oppahBWA3B+jUg==",
+      "version": "6.0.1",
+      "resolved": "https://registry.npmjs.org/node-sass/-/node-sass-6.0.1.tgz",
+      "integrity": "sha512-f+Rbqt92Ful9gX0cGtdYwjTrWAaGURgaK5rZCWOgCNyGWusFYHhbqCCBoFBeat+HKETOU02AyTxNhJV0YZf2jQ==",
       "hasInstallScript": true,
       "dependencies": {
         "async-foreach": "^0.1.3",
-        "chalk": "^4.1.2",
+        "chalk": "^1.1.1",
         "cross-spawn": "^7.0.3",
         "gaze": "^1.0.0",
         "get-stdin": "^4.0.1",
@@ -1076,7 +1057,7 @@
         "meow": "^9.0.0",
         "nan": "^2.13.2",
         "node-gyp": "^7.1.0",
-        "npmlog": "^5.0.0",
+        "npmlog": "^4.0.0",
         "request": "^2.88.0",
         "sass-graph": "2.2.5",
         "stdout-stream": "^1.4.0",
@@ -1089,106 +1070,6 @@
         "node": ">=12"
       }
     },
-    "node_modules/node-sass/node_modules/ansi-regex": {
-      "version": "5.0.1",
-      "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz",
-      "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==",
-      "engines": {
-        "node": ">=8"
-      }
-    },
-    "node_modules/node-sass/node_modules/are-we-there-yet": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/are-we-there-yet/-/are-we-there-yet-2.0.0.tgz",
-      "integrity": "sha512-Ci/qENmwHnsYo9xKIcUJN5LeDKdJ6R1Z1j9V/J5wyq8nh/mYPEpIKJbBZXtZjG04HiK7zV/p6Vs9952MrMeUIw==",
-      "dependencies": {
-        "delegates": "^1.0.0",
-        "readable-stream": "^3.6.0"
-      },
-      "engines": {
-        "node": ">=10"
-      }
-    },
-    "node_modules/node-sass/node_modules/emoji-regex": {
-      "version": "8.0.0",
-      "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz",
-      "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A=="
-    },
-    "node_modules/node-sass/node_modules/gauge": {
-      "version": "3.0.2",
-      "resolved": "https://registry.npmjs.org/gauge/-/gauge-3.0.2.tgz",
-      "integrity": "sha512-+5J6MS/5XksCuXq++uFRsnUd7Ovu1XenbeuIuNRJxYWjgQbPuFhT14lAvsWfqfAmnwluf1OwMjz39HjfLPci0Q==",
-      "dependencies": {
-        "aproba": "^1.0.3 || ^2.0.0",
-        "color-support": "^1.1.2",
-        "console-control-strings": "^1.0.0",
-        "has-unicode": "^2.0.1",
-        "object-assign": "^4.1.1",
-        "signal-exit": "^3.0.0",
-        "string-width": "^4.2.3",
-        "strip-ansi": "^6.0.1",
-        "wide-align": "^1.1.2"
-      },
-      "engines": {
-        "node": ">=10"
-      }
-    },
-    "node_modules/node-sass/node_modules/is-fullwidth-code-point": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz",
-      "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==",
-      "engines": {
-        "node": ">=8"
-      }
-    },
-    "node_modules/node-sass/node_modules/npmlog": {
-      "version": "5.0.1",
-      "resolved": "https://registry.npmjs.org/npmlog/-/npmlog-5.0.1.tgz",
-      "integrity": "sha512-AqZtDUWOMKs1G/8lwylVjrdYgqA4d9nu8hc+0gzRxlDb1I10+FHBGMXs6aiQHFdCUUlqH99MUMuLfzWDNDtfxw==",
-      "dependencies": {
-        "are-we-there-yet": "^2.0.0",
-        "console-control-strings": "^1.1.0",
-        "gauge": "^3.0.0",
-        "set-blocking": "^2.0.0"
-      }
-    },
-    "node_modules/node-sass/node_modules/readable-stream": {
-      "version": "3.6.0",
-      "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz",
-      "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==",
-      "dependencies": {
-        "inherits": "^2.0.3",
-        "string_decoder": "^1.1.1",
-        "util-deprecate": "^1.0.1"
-      },
-      "engines": {
-        "node": ">= 6"
-      }
-    },
-    "node_modules/node-sass/node_modules/string-width": {
-      "version": "4.2.3",
-      "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz",
-      "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==",
-      "dependencies": {
-        "emoji-regex": "^8.0.0",
-        "is-fullwidth-code-point": "^3.0.0",
-        "strip-ansi": "^6.0.1"
-      },
-      "engines": {
-        "node": ">=8"
-      }
-    },
-    "node_modules/node-sass/node_modules/strip-ansi": {
-      "version": "6.0.1",
-      "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz",
-      "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==",
-      "dependencies": {
-        "ansi-regex": "^5.0.1"
-      },
-      "engines": {
-        "node": ">=8"
-      }
-    },
     "node_modules/nopt": {
       "version": "5.0.0",
       "resolved": "https://registry.npmjs.org/nopt/-/nopt-5.0.0.tgz",
@@ -1735,22 +1616,11 @@
       }
     },
     "node_modules/supports-color": {
-      "version": "7.2.0",
-      "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
-      "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
-      "dependencies": {
-        "has-flag": "^4.0.0"
-      },
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz",
+      "integrity": "sha1-U10EXOa2Nj+kARcIRimZXp3zJMc=",
       "engines": {
-        "node": ">=8"
-      }
-    },
-    "node_modules/supports-color/node_modules/has-flag": {
-      "version": "4.0.0",
-      "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
-      "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
-      "engines": {
-        "node": ">=8"
+        "node": ">=0.8.0"
       }
     },
     "node_modules/tar": {
@@ -2180,27 +2050,9 @@
       "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8="
     },
     "ansi-styles": {
-      "version": "4.3.0",
-      "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
-      "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
-      "requires": {
-        "color-convert": "^2.0.1"
-      },
-      "dependencies": {
-        "color-convert": {
-          "version": "2.0.1",
-          "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
-          "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
-          "requires": {
-            "color-name": "~1.1.4"
-          }
-        },
-        "color-name": {
-          "version": "1.1.4",
-          "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
-          "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA=="
-        }
-      }
+      "version": "2.2.1",
+      "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.2.1.tgz",
+      "integrity": "sha1-tDLdM1i2NM914eRmQ2gkBTPB3b4="
     },
     "aproba": {
       "version": "1.2.0",
@@ -2297,12 +2149,15 @@
       "integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw="
     },
     "chalk": {
-      "version": "4.1.2",
-      "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
-      "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==",
+      "version": "1.1.3",
+      "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz",
+      "integrity": "sha1-qBFcVeSnAv5NFQq9OHKCKn4J/Jg=",
       "requires": {
-        "ansi-styles": "^4.1.0",
-        "supports-color": "^7.1.0"
+        "ansi-styles": "^2.2.1",
+        "escape-string-regexp": "^1.0.2",
+        "has-ansi": "^2.0.0",
+        "strip-ansi": "^3.0.0",
+        "supports-color": "^2.0.0"
       }
     },
     "chownr": {
@@ -2368,11 +2223,6 @@
       "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz",
       "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU="
     },
-    "color-support": {
-      "version": "1.1.3",
-      "resolved": "https://registry.npmjs.org/color-support/-/color-support-1.1.3.tgz",
-      "integrity": "sha512-qiBjkpbMLO/HL68y+lh4q0/O1MZFj2RX6X/KmMa3+gJD3z+WwI1ZzDHysvqHGS3mP6mznPckpXmw1nI9cJjyRg=="
-    },
     "combined-stream": {
       "version": "1.0.8",
       "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz",
@@ -2635,6 +2485,14 @@
         "function-bind": "^1.1.1"
       }
     },
+    "has-ansi": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/has-ansi/-/has-ansi-2.0.0.tgz",
+      "integrity": "sha1-NPUEnOHs3ysGSa8+8k5F7TVBbZE=",
+      "requires": {
+        "ansi-regex": "^2.0.0"
+      }
+    },
     "has-flag": {
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz",
@@ -2910,12 +2768,12 @@
       }
     },
     "node-sass": {
-      "version": "7.0.0",
-      "resolved": "https://registry.npmjs.org/node-sass/-/node-sass-7.0.0.tgz",
-      "integrity": "sha512-6yUnsD3L8fVbgMX6nKQqZkjRcG7a/PpmF0pEyeWf+BgbTj2ToJlCYrnUifL2KbjV5gIY22I3oppahBWA3B+jUg==",
+      "version": "6.0.1",
+      "resolved": "https://registry.npmjs.org/node-sass/-/node-sass-6.0.1.tgz",
+      "integrity": "sha512-f+Rbqt92Ful9gX0cGtdYwjTrWAaGURgaK5rZCWOgCNyGWusFYHhbqCCBoFBeat+HKETOU02AyTxNhJV0YZf2jQ==",
       "requires": {
         "async-foreach": "^0.1.3",
-        "chalk": "^4.1.2",
+        "chalk": "^1.1.1",
         "cross-spawn": "^7.0.3",
         "gaze": "^1.0.0",
         "get-stdin": "^4.0.1",
@@ -2924,92 +2782,11 @@
         "meow": "^9.0.0",
         "nan": "^2.13.2",
         "node-gyp": "^7.1.0",
-        "npmlog": "^5.0.0",
+        "npmlog": "^4.0.0",
         "request": "^2.88.0",
         "sass-graph": "2.2.5",
         "stdout-stream": "^1.4.0",
         "true-case-path": "^1.0.2"
-      },
-      "dependencies": {
-        "ansi-regex": {
-          "version": "5.0.1",
-          "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz",
-          "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ=="
-        },
-        "are-we-there-yet": {
-          "version": "2.0.0",
-          "resolved": "https://registry.npmjs.org/are-we-there-yet/-/are-we-there-yet-2.0.0.tgz",
-          "integrity": "sha512-Ci/qENmwHnsYo9xKIcUJN5LeDKdJ6R1Z1j9V/J5wyq8nh/mYPEpIKJbBZXtZjG04HiK7zV/p6Vs9952MrMeUIw==",
-          "requires": {
-            "delegates": "^1.0.0",
-            "readable-stream": "^3.6.0"
-          }
-        },
-        "emoji-regex": {
-          "version": "8.0.0",
-          "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz",
-          "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A=="
-        },
-        "gauge": {
-          "version": "3.0.2",
-          "resolved": "https://registry.npmjs.org/gauge/-/gauge-3.0.2.tgz",
-          "integrity": "sha512-+5J6MS/5XksCuXq++uFRsnUd7Ovu1XenbeuIuNRJxYWjgQbPuFhT14lAvsWfqfAmnwluf1OwMjz39HjfLPci0Q==",
-          "requires": {
-            "aproba": "^1.0.3 || ^2.0.0",
-            "color-support": "^1.1.2",
-            "console-control-strings": "^1.0.0",
-            "has-unicode": "^2.0.1",
-            "object-assign": "^4.1.1",
-            "signal-exit": "^3.0.0",
-            "string-width": "^4.2.3",
-            "strip-ansi": "^6.0.1",
-            "wide-align": "^1.1.2"
-          }
-        },
-        "is-fullwidth-code-point": {
-          "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz",
-          "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg=="
-        },
-        "npmlog": {
-          "version": "5.0.1",
-          "resolved": "https://registry.npmjs.org/npmlog/-/npmlog-5.0.1.tgz",
-          "integrity": "sha512-AqZtDUWOMKs1G/8lwylVjrdYgqA4d9nu8hc+0gzRxlDb1I10+FHBGMXs6aiQHFdCUUlqH99MUMuLfzWDNDtfxw==",
-          "requires": {
-            "are-we-there-yet": "^2.0.0",
-            "console-control-strings": "^1.1.0",
-            "gauge": "^3.0.0",
-            "set-blocking": "^2.0.0"
-          }
-        },
-        "readable-stream": {
-          "version": "3.6.0",
-          "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz",
-          "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==",
-          "requires": {
-            "inherits": "^2.0.3",
-            "string_decoder": "^1.1.1",
-            "util-deprecate": "^1.0.1"
-          }
-        },
-        "string-width": {
-          "version": "4.2.3",
-          "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz",
-          "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==",
-          "requires": {
-            "emoji-regex": "^8.0.0",
-            "is-fullwidth-code-point": "^3.0.0",
-            "strip-ansi": "^6.0.1"
-          }
-        },
-        "strip-ansi": {
-          "version": "6.0.1",
-          "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz",
-          "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==",
-          "requires": {
-            "ansi-regex": "^5.0.1"
-          }
-        }
       }
     },
     "nopt": {
@@ -3436,19 +3213,9 @@
       }
     },
     "supports-color": {
-      "version": "7.2.0",
-      "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
-      "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
-      "requires": {
-        "has-flag": "^4.0.0"
-      },
-      "dependencies": {
-        "has-flag": {
-          "version": "4.0.0",
-          "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
-          "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ=="
-        }
-      }
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz",
+      "integrity": "sha1-U10EXOa2Nj+kARcIRimZXp3zJMc="
     },
     "tar": {
       "version": "6.1.9",

changedetectionio/static/styles/package.json

@@ -4,12 +4,13 @@
   "description": "",
   "main": "index.js",
   "scripts": {
-    "build": "node-sass styles.scss -o .;node-sass diff.scss -o ."
+    "build": "node-sass styles.scss diff.scss -o .",
+    "watch": "node-sass --watch styles.scss diff.scss -o ."
   },
   "author": "",
   "license": "ISC",
   "dependencies": {
-    "node-sass": "^7.0.0",
+    "node-sass": "^6.0.1",
     "tar": "^6.1.9",
     "trim-newlines": "^3.0.1"
   }

changedetectionio/static/styles/styles.scss

@@ -567,8 +567,3 @@ $form-edge-padding: 20px;
   }
 }
 
-ul {
-    padding-left: 1em;
-    padding-top: 0px;
-    margin-top: 4px;
-}

changedetectionio/store.py

@@ -398,10 +398,13 @@ class ChangeDetectionStore:
                 # system was out of memory, out of RAM etc
                 with open(self.json_store_path+".tmp", 'w') as json_file:
                     json.dump(data, json_file, indent=4)
-                os.rename(self.json_store_path+".tmp", self.json_store_path)
+
             except Exception as e:
                 logging.error("Error writing JSON!! (Main JSON file save was skipped) : %s", str(e))
 
+            else:
+                os.rename(self.json_store_path+".tmp", self.json_store_path)
+
             self.needs_write = False
 
     # Thread runner, this helps with thread/write issues when there are many operations that want to update the JSON

changedetectionio/templates/_helpers.jinja

@@ -25,6 +25,3 @@
 {% endmacro %}
 
 
-{% macro render_button(field) %}
-  {{ field(**kwargs)|safe }}
-{% endmacro %}

changedetectionio/templates/diff.html

@@ -36,7 +36,6 @@
     <a onclick="next_diff();">Jump</a>
 </div>
 <div id="diff-ui">
-    <div class="tip">Pro-tip: Use <strong>show current snapshot</strong> tab to visualise what will be ignored.</div>
     <table>
         <tbody>
         <tr>

changedetectionio/templates/edit.html

@@ -1,7 +1,6 @@
 {% extends 'base.html' %}
 {% block content %}
 {% from '_helpers.jinja' import render_field %}
-{% from '_helpers.jinja' import render_button %}
 {% from '_common_fields.jinja' import render_common_settings_form %}
 <script type="text/javascript" src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
 
@@ -89,18 +88,6 @@ User-Agent: wonderbra 1.0") }}
 
             <div class="tab-pane-inner" id="filters-and-triggers">
                 <fieldset>
-                        <div class="pure-control-group">
-                            <strong>Pro-tips:</strong><br/>
-                            <ul>
-                                <li>
-                                    Use the preview page to see your filters and triggers highlighted.
-                                </li>
-                                <li>
-                                    Some sites use JavaScript to create the content, for this you should <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Fetching-pages-with-WebDriver">use the Chrome/WebDriver Fetcher</a>
-                                </li>
-                            </ul>
-                    </div>
-
                     <div class="pure-control-group">
                         {{ render_field(form.css_filter, placeholder=".class-name or #some-id, or other CSS selector rule.",
                         class="m-d") }}
@@ -127,7 +114,6 @@ User-Agent: wonderbra 1.0") }}
                             <li>Each line processed separately, any line matching will be ignored (removed before creating the checksum)</li>
                             <li>Regular Expression support, wrap the line in forward slash <b>/regex/</b></li>
                             <li>Changing this will affect the comparison checksum which may trigger an alert</li>
-                            <li>Use the preview/show current tab to see ignores</li>
                         </ul>
                 </span>
 
@@ -152,8 +138,7 @@ User-Agent: wonderbra 1.0") }}
             <div id="actions">
                 <div class="pure-control-group">
 
-                      {{ render_button(form.save_button) }} {{ render_button(form.save_and_preview_button) }}
-
+                    <button type="submit" class="pure-button pure-button-primary">Save</button>
                     <a href="{{url_for('api_delete', uuid=uuid)}}"
                        class="pure-button button-small button-error ">Delete</a>
                     <a href="{{url_for('api_clone', uuid=uuid)}}"

changedetectionio/templates/preview.html

@@ -3,21 +3,19 @@
 {% block content %}
 
 <div id="settings">
-    <h1>Current - {{watch.last_checked|format_timestamp_timeago}}</h1>
+    <h1>Current</h1>
 </div>
 
 <div id="diff-ui">
-    <span class="ignored">Grey lines are ignored</span> <span class="triggered">Blue lines are triggers</span>
     <table>
         <tbody>
         <tr>
             <td id="diff-col">
-                    {% for row in content %}
-                    <div class="{{row.classes}}">{{row.line}}</div>
-                    {% endfor %}
+                <span id="result">{% for row in content %}{{row}}{% endfor %}</span>
             </td>
         </tr>
         </tbody>
     </table>
 </div>
+
 {% endblock %}

changedetectionio/templates/settings.html

@@ -95,7 +95,6 @@
                             <li>Each line processed separately, any line matching will be ignored (removed before creating the checksum)</li>
                             <li>Regular Expression support, wrap the line in forward slash <b>/regex/</b></li>
                             <li>Changing this will affect the comparison checksum which may trigger an alert</li>
-                            <li>Use the preview/show current tab to see ignores</li>
                         </ul>
                      </span>
                     </fieldset>

changedetectionio/tests/test_ignore_regex_text.py

@@ -3,7 +3,6 @@
 import time
 from flask import url_for
 from . util import live_server_setup
-from changedetectionio import html_tools
 
 def test_setup(live_server):
     live_server_setup(live_server)
@@ -24,7 +23,7 @@ def test_strip_regex_text_func():
     ignore_lines = ["sometimes", "/\s\d{2,3}\s/", "/ignore-case text/"]
 
     fetcher = fetch_site_status.perform_site_check(datastore=False)
-    stripped_content = html_tools.strip_ignore_text(test_content, ignore_lines)
+    stripped_content = fetcher.strip_ignore_text(test_content, ignore_lines)
 
     assert b"but 1 lines" in stripped_content
     assert b"igNORe-cAse text" not in stripped_content

changedetectionio/tests/test_ignore_text.py

@@ -3,7 +3,6 @@
 import time
 from flask import url_for
 from . util import live_server_setup
-from changedetectionio import html_tools
 
 def test_setup(live_server):
     live_server_setup(live_server)
@@ -24,7 +23,7 @@ def test_strip_text_func():
     ignore_lines = ["sometimes"]
 
     fetcher = fetch_site_status.perform_site_check(datastore=False)
-    stripped_content = html_tools.strip_ignore_text(test_content, ignore_lines)
+    stripped_content = fetcher.strip_ignore_text(test_content, ignore_lines)
 
     assert b"sometimes" not in stripped_content
     assert b"Some content" in stripped_content
@@ -53,8 +52,6 @@ def set_modified_original_ignore_response():
      <p>Which is across multiple lines</p>
      </br>
      So let's see what happens.  </br>
-     <p>new ignore stuff</p>
-     <p>blah</p>
      </body>
      </html>
 
@@ -70,7 +67,7 @@ def set_modified_ignore_response():
        <body>
      Some initial text</br>
      <p>Which is across multiple lines</p>
-     <P>ZZZZz</P>
+     <P>ZZZZZ</P>
      </br>
      So let's see what happens.  </br>
      </body>
@@ -85,8 +82,7 @@ def set_modified_ignore_response():
 def test_check_ignore_text_functionality(client, live_server):
     sleep_time_for_fetch_thread = 3
 
-    # Use a mix of case in ZzZ to prove it works case-insensitive.
-    ignore_text = "XXXXX\r\nYYYYY\r\nzZzZZ\r\nnew ignore stuff"
+    ignore_text = "XXXXX\r\nYYYYY\r\nZZZZZ"
     set_original_ignore_response()
 
     # Give the endpoint time to spin up
@@ -146,25 +142,13 @@ def test_check_ignore_text_functionality(client, live_server):
     assert b'unviewed' not in res.data
     assert b'/test-endpoint' in res.data
 
-
-
-
-
     # Just to be sure.. set a regular modified change..
     set_modified_original_ignore_response()
     client.get(url_for("api_watch_checknow"), follow_redirects=True)
     time.sleep(sleep_time_for_fetch_thread)
-
     res = client.get(url_for("index"))
     assert b'unviewed' in res.data
 
-    # Check the preview/highlighter, we should be able to see what we ignored, but it should be highlighted
-    # We only introduce the "modified" content that includes what we ignore so we can prove the newest version also displays
-    # at /preview
-    res = client.get(url_for("preview_page", uuid="first"))
-    # We should be able to see what we ignored
-    assert b'<div class="ignored">new ignore stuff' in res.data
-
     res = client.get(url_for("api_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
 

changedetectionio/tests/test_trigger.py

@@ -129,8 +129,3 @@ def test_trigger_functionality(client, live_server):
     time.sleep(sleep_time_for_fetch_thread)
     res = client.get(url_for("index"))
     assert b'unviewed' in res.data
-
-    # Check the preview/highlighter, we should be able to see what we triggered on, but it should be highlighted
-    res = client.get(url_for("preview_page", uuid="first"))
-    # We should be able to see what we ignored
-    assert b'<div class="triggered">foobar' in res.data

changedetectionio/tests/test_xpath_selector.py

@@ -96,7 +96,6 @@ def test_check_markup_xpath_filter_restriction(client, live_server):
     res = client.get(url_for("index"))
     assert b'unviewed' not in res.data
 
-
 def test_xpath_validation(client, live_server):
 
     # Give the endpoint time to spin up

changedetectionio/update_worker.py

@@ -136,8 +136,6 @@ class update_worker(threading.Thread):
                         except Exception as e:
                             # Catch everything possible here, so that if a worker crashes, we don't lose it until restart!
                             print("!!!! Exception in update_worker !!!\n", e)
-                            self.app.logger.error("Exception reached processing watch UUID: %s - %s", uuid, str(e))
-                            self.datastore.update_watch(uuid=uuid, update_obj={'last_error': str(e)})
 
                     finally:
                         # Always record that we atleast tried