Re #23 - always check value of interval time, not just on start

Prepare 0.27

.github/workflows/python-app.yml

@@ -4,11 +4,7 @@
 name: changedetection.io
 
 
-on:
-  push:
-    branches: [ master ]
-  pull_request:
-    branches: [ master ]
+on: [push, pull_request]
 
 jobs:
   build:

README.md

@@ -4,7 +4,7 @@
   <img src="https://img.shields.io/docker/pulls/dgtlmoon/changedetection.io" alt="Docker Pulls"/>
 </a>
 <a href="https://hub.docker.com/r/dgtlmoon/changedetection.io" target="_blank" title="Change detection docker hub">
-  <img src="https://img.shields.io/docker/v/dgtlmoon/changedetection.io" alt="Change detection latest tag version"/> 
+  <img src="https://img.shields.io/docker/v/dgtlmoon/changedetection.io/0.27" alt="Change detection latest tag version"/> 
 </a>
 
 ## Self-hosted change monitoring of web pages.

backend/__init__.py

@@ -2,10 +2,8 @@
 
 
 # @todo logging
-# @todo sort by last_changed
 # @todo extra options for url like , verify=False etc.
 # @todo enable https://urllib3.readthedocs.io/en/latest/user-guide.html#ssl as option?
-# @todo maybe a button to reset all 'last-changed'.. so you can see it clearly when something happens since your last visit
 # @todo option for interval day/6 hour/etc
 # @todo on change detected, config for calling some API
 # @todo make tables responsive!
@@ -19,9 +17,16 @@ import os
 import timeago
 
 import threading
+from threading import Event
+
 import queue
 
-from flask import Flask, render_template, request, send_file, send_from_directory,  abort, redirect, url_for
+from flask import Flask, render_template, request, send_file, send_from_directory, abort, redirect, url_for
+
+from feedgen.feed import FeedGenerator
+from flask import make_response
+import datetime
+import pytz
 
 datastore = None
 
@@ -39,7 +44,9 @@ app = Flask(__name__, static_url_path="/var/www/change-detection/backen/static")
 # Stop browser caching of assets
 app.config['SEND_FILE_MAX_AGE_DEFAULT'] = 0
 
-app.config['STOP_THREADS'] = False
+app.config.exit = Event()
+
+app.config['NEW_VERSION_AVAILABLE'] = False
 
 # Disables caching of the templates
 app.config['TEMPLATES_AUTO_RELOAD'] = True
@@ -76,7 +83,7 @@ def _jinja2_filter_datetimestamp(timestamp, format="%Y-%m-%d %H:%M:%S"):
 def changedetection_app(config=None, datastore_o=None):
     global datastore
     datastore = datastore_o
-    # Hmm
+
     app.config.update(dict(DEBUG=True))
     app.config.update(config or {})
 
@@ -92,6 +99,8 @@ def changedetection_app(config=None, datastore_o=None):
         global messages
 
         limit_tag = request.args.get('tag')
+        rss = request.args.get('rss')
+        mode = request.args.get('mode')
 
         # Sort by last_changed and add the uuid which is usually the key..
         sorted_watches = []
@@ -112,14 +121,97 @@ def changedetection_app(config=None, datastore_o=None):
         sorted_watches.sort(key=lambda x: x['last_changed'], reverse=True)
 
         existing_tags = datastore.get_all_tags()
-        output = render_template("watch-overview.html",
-                                 watches=sorted_watches,
-                                 messages=messages,
-                                 tags=existing_tags,
-                                 active_tag=limit_tag)
 
-        # Show messages but once.
-        messages = []
+        if mode == 'stream':
+            import difflib
+
+            import pprint
+            streams = []
+
+            extra_stylesheets = ['/static/css/diff.css']
+            for watch in sorted_watches:
+                if not watch['viewed']:
+
+                    # get last two date keys
+                    dates = list(watch['history'].keys())
+                    # Convert to int, sort and back to str again
+                    dates = [int(i) for i in dates]
+                    dates.sort(reverse=True)
+                    dates = [str(i) for i in dates]
+                    print ("OK", watch['uuid'])
+
+                    if len(dates) < 2:
+                        print ("Skipping", watch['url'])
+                        continue
+                    else:
+                        try:
+                            path = datastore.data['watching'][watch['uuid']]['history'][str(dates[1])]
+                            with open(path,
+                                      encoding='utf-8') as file:
+                                txt1=[line.rstrip() for line in file.readlines()]
+
+                            path = datastore.data['watching'][watch['uuid']]['history'][str(dates[0])]
+                            with open(path,
+                                      encoding='utf-8') as file:
+                                txt2 = [line.rstrip() for line in file.readlines()]
+                        except FileNotFoundError:
+                            print ("Skipping", watch['url'])
+                            continue
+
+                        df = list(difflib.unified_diff(txt1, txt2,n=1))
+                        diff_entry=[]
+                        for line in df:
+                            if line[0] == '-' or line[0] == '+':
+                                diff_entry.append(line)
+
+
+                        # pprint(df)
+                        #s = pprint.pformat(df)
+                        streams.append(diff_entry)
+
+
+            print ("###########", len(streams))
+
+            output = render_template("watch-diff-stream.html",
+                                     streams=streams,
+                                     extra_stylesheets=extra_stylesheets
+                                     )
+            return output
+
+
+        if rss:
+            fg = FeedGenerator()
+            fg.title('changedetection.io')
+            fg.description('Feed description')
+            fg.link(href='https://changedetection.io')
+
+            for watch in sorted_watches:
+                if not watch['viewed']:
+                    fe = fg.add_entry()
+                    fe.title(watch['url'])
+                    fe.link(href=watch['url'])
+                    fe.description(watch['url'])
+                    fe.guid(watch['uuid'], permalink=False)
+                    dt = datetime.datetime.fromtimestamp(int(watch['newest_history_key']))
+                    dt = dt.replace(tzinfo=pytz.UTC)
+                    fe.pubDate(dt)
+
+            response = make_response(fg.rss_str())
+            response.headers.set('Content-Type', 'application/rss+xml')
+            return response
+
+        else:
+            #table = render_template('watch-table.html', watches=sorted_watches)
+            output = render_template("watch-table.html",
+                                     watches=sorted_watches,
+                                     messages=messages,
+                                     tags=existing_tags,
+                                     active_tag=limit_tag,
+                                     has_unviewed=datastore.data['has_unviewed'])
+
+            # Show messages but once.
+            messages = []
+
         return output
 
     @app.route("/scrub", methods=['GET', 'POST'])
@@ -151,29 +243,80 @@ def changedetection_app(config=None, datastore_o=None):
 
         return render_template("scrub.html")
 
-    @app.route("/edit", methods=['GET', 'POST'])
-    def edit_page():
+    # If they edited an existing watch, we need to know to reset the current/previous md5 to include
+    # the excluded text.
+    def get_current_checksum_include_ignore_text(uuid):
+
+        import hashlib
+        from backend import fetch_site_status
+
+        # Get the most recent one
+        newest_history_key = datastore.get_val(uuid, 'newest_history_key')
+
+        # 0 means that theres only one, so that there should be no 'unviewed' history availabe
+        if newest_history_key == 0:
+            newest_history_key = list(datastore.data['watching'][uuid]['history'].keys())[0]
+
+        if newest_history_key:
+            with open(datastore.data['watching'][uuid]['history'][newest_history_key],
+                      encoding='utf-8') as file:
+                raw_content = file.read()
+
+                handler = fetch_site_status.perform_site_check(datastore=datastore)
+                stripped_content = handler.strip_ignore_text(raw_content,
+                                                             datastore.data['watching'][uuid]['ignore_text'])
+
+                checksum = hashlib.md5(stripped_content).hexdigest()
+                return checksum
+
+        return datastore.data['watching'][uuid]['previous_md5']
+
+    @app.route("/edit/<string:uuid>", methods=['GET', 'POST'])
+    def edit_page(uuid):
         global messages
         import validators
 
+        # More for testing, possible to return the first/only
+        if uuid == 'first':
+            uuid = list(datastore.data['watching'].keys()).pop()
+
         if request.method == 'POST':
-            uuid = request.args.get('uuid')
 
             url = request.form.get('url').strip()
             tag = request.form.get('tag').strip()
 
+            # Extra headers
             form_headers = request.form.get('headers').strip().split("\n")
             extra_headers = {}
             if form_headers:
                 for header in form_headers:
                     if len(header):
                         parts = header.split(':', 1)
-                        extra_headers.update({parts[0].strip(): parts[1].strip()})
+                        if len(parts) == 2:
+                            extra_headers.update({parts[0].strip(): parts[1].strip()})
+
+            update_obj = {'url': url,
+                          'tag': tag,
+                          'headers': extra_headers
+                          }
+
+            # Ignore text
+            form_ignore_text = request.form.get('ignore-text').strip()
+            ignore_text = []
+            if len(form_ignore_text):
+                for text in form_ignore_text.split("\n"):
+                    text = text.strip()
+                    if len(text):
+                        ignore_text.append(text)
+
+                datastore.data['watching'][uuid]['ignore_text'] = ignore_text
+
+                # Reset the previous_md5 so we process a new snapshot including stripping ignore text.
+                if len(datastore.data['watching'][uuid]['history']):
+                    update_obj['previous_md5'] = get_current_checksum_include_ignore_text(uuid=uuid)
 
             validators.url(url)  # @todo switch to prop/attr/observer
-            datastore.data['watching'][uuid].update({'url': url,
-                                                     'tag': tag,
-                                                     'headers': extra_headers})
+            datastore.data['watching'][uuid].update(update_obj)
             datastore.needs_write = True
 
             messages.append({'class': 'ok', 'message': 'Updated watch.'})
@@ -181,8 +324,6 @@ def changedetection_app(config=None, datastore_o=None):
             return redirect(url_for('index'))
 
         else:
-
-            uuid = request.args.get('uuid')
             output = render_template("edit.html", uuid=uuid, watch=datastore.data['watching'][uuid], messages=messages)
 
         return output
@@ -235,23 +376,37 @@ def changedetection_app(config=None, datastore_o=None):
 
             messages.append({'class': 'ok', 'message': "{} Imported, {} Skipped.".format(good, len(remaining_urls))})
 
-        if len(remaining_urls) == 0:
-            return redirect(url_for('index'))
-        else:
-            output = render_template("import.html",
-                                     messages=messages,
-                                     remaining="\n".join(remaining_urls)
-                                     )
-            messages = []
+            if len(remaining_urls) == 0:
+                # Looking good, redirect to index.
+                return redirect(url_for('index'))
+
+        # Could be some remaining, or we could be on GET
+        output = render_template("import.html",
+                                 messages=messages,
+                                 remaining="\n".join(remaining_urls)
+                                 )
+        messages = []
+
         return output
 
+    # Clear all statuses, so we do not see the 'unviewed' class
+    @app.route("/api/mark-all-viewed", methods=['GET'])
+    def mark_all_viewed():
+
+        # Save the current newest history as the most recently viewed
+        for watch_uuid, watch in datastore.data['watching'].items():
+            datastore.set_last_viewed(watch_uuid, watch['newest_history_key'])
+
+        messages.append({'class': 'ok', 'message': "Cleared all statuses."})
+        return redirect(url_for('index'))
+
     @app.route("/diff/<string:uuid>", methods=['GET'])
     def diff_history_page(uuid):
         global messages
 
         # More for testing, possible to return the first/only
         if uuid == 'first':
-            uuid= list(datastore.data['watching'].keys()).pop()
+            uuid = list(datastore.data['watching'].keys()).pop()
 
         extra_stylesheets = ['/static/css/diff.css']
         try:
@@ -266,9 +421,9 @@ def changedetection_app(config=None, datastore_o=None):
         dates.sort(reverse=True)
         dates = [str(i) for i in dates]
 
-
         if len(dates) < 2:
-            messages.append({'class': 'error', 'message': "Not enough saved change detection snapshots to produce a report."})
+            messages.append(
+                {'class': 'error', 'message': "Not enough saved change detection snapshots to produce a report."})
             return redirect(url_for('index'))
 
         # Save the current newest history as the most recently viewed
@@ -409,9 +564,38 @@ def changedetection_app(config=None, datastore_o=None):
     # @todo handle ctrl break
     ticker_thread = threading.Thread(target=ticker_thread_check_time_launch_checks).start()
 
+    # Check for new release version
+    threading.Thread(target=check_for_new_version).start()
     return app
 
 
+# Check for new version and anonymous stats
+def check_for_new_version():
+    import requests
+
+    import urllib3
+    urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+
+    while not app.config.exit.is_set():
+        try:
+            r = requests.post("https://changedetection.io/check-ver.php",
+                              data={'version': datastore.data['version_tag'],
+                                    'app_guid': datastore.data['app_guid']},
+
+                              verify=False)
+        except:
+            pass
+
+        try:
+            if "new_version" in r.text:
+                app.config['NEW_VERSION_AVAILABLE'] = True
+        except:
+            pass
+
+        # Check daily
+        app.config.exit.wait(86400)
+
+
 # Requests for checking on the site use a pool of thread Workers managed by a Queue.
 class Worker(threading.Thread):
     current_uuid = None
@@ -425,16 +609,13 @@ class Worker(threading.Thread):
 
         update_handler = fetch_site_status.perform_site_check(datastore=datastore)
 
-        while True:
+        while not app.config.exit.is_set():
 
             try:
-                uuid = self.q.get(block=True, timeout=1)
+                uuid = self.q.get(block=False)
             except queue.Empty:
-                # We have a chance to kill this thread that needs to monitor for new jobs..
-                # Delays here would be caused by a current response object pending
-                # @todo switch to threaded response handler
-                if app.config['STOP_THREADS']:
-                    return
+                pass
+
             else:
                 self.current_uuid = uuid
 
@@ -453,10 +634,11 @@ class Worker(threading.Thread):
                                 # A change was detected
                                 datastore.save_history_text(uuid=uuid, contents=contents, result_obj=result)
 
-
                 self.current_uuid = None  # Done
                 self.q.task_done()
 
+            app.config.exit.wait(1)
+
 
 # Thread runner to check every minute, look for new watches to feed into the Queue.
 def ticker_thread_check_time_launch_checks():
@@ -466,24 +648,21 @@ def ticker_thread_check_time_launch_checks():
         running_update_threads.append(new_worker)
         new_worker.start()
 
-    # Every minute check for new UUIDs to follow up on
-    while True:
-
-        if app.config['STOP_THREADS']:
-            return
-
+    while not app.config.exit.is_set():
         running_uuids = []
         for t in running_update_threads:
             running_uuids.append(t.current_uuid)
 
         # Look at the dataset, find a stale watch to process
-        minutes = datastore.data['settings']['requests']['minutes_between_check']
-        for uuid, watch in datastore.data['watching'].items():
-            if watch['last_checked'] <= time.time() - (minutes * 60):
 
-                # @todo maybe update_q.queue is enough?
+        # Every minute check for new UUIDs to follow up on, should be inside the loop incase it changes.
+        minutes = datastore.data['settings']['requests']['minutes_between_check']
+
+        threshold = time.time() - (minutes * 60)
+        for uuid, watch in datastore.data['watching'].items():
+            if watch['last_checked'] <= threshold:
                 if not uuid in running_uuids and uuid not in update_q.queue:
                     update_q.put(uuid)
 
         # Should be low so we can break this out in testing
-        time.sleep(1)
+        app.config.exit.wait(1)

backend/dev-docker/Dockerfile

@@ -3,9 +3,6 @@ FROM python:3.8-slim
 # https://stackoverflow.com/questions/58701233/docker-logs-erroneously-appears-empty-until-container-stops
 ENV PYTHONUNBUFFERED=1
 
-# Should be mounted from docker-compose-development.yml
-RUN pip3 install -r /requirements.txt
-
 WORKDIR /app
 
 RUN [ ! -d "/datastore" ] && mkdir /datastore

backend/fetch_site_status.py

@@ -2,7 +2,8 @@ import time
 import requests
 import hashlib
 from inscriptis import get_text
-
+import urllib3
+urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 
 # Some common stuff here that can be moved to a base class
 class perform_site_check():
@@ -11,6 +12,24 @@ class perform_site_check():
         super().__init__(*args, **kwargs)
         self.datastore = datastore
 
+    def strip_ignore_text(self, content, list_ignore_text):
+        ignore = []
+        for k in list_ignore_text:
+            ignore.append(k.encode('utf8'))
+
+        output = []
+        for line in content.splitlines():
+            line = line.encode('utf8')
+
+            # Always ignore blank lines in this mode. (when this function gets called)
+            if len(line.strip()):
+                if not any(skip_text in line for skip_text in ignore):
+                    output.append(line)
+
+        return "\n".encode('utf8').join(output)
+
+
+
     def run(self, uuid):
         timestamp = int(time.time())  # used for storage etc too
         stripped_text_from_html = False
@@ -76,7 +95,15 @@ class perform_site_check():
             if not len(r.text):
                 update_obj["last_error"] = "Empty reply"
 
-            fetched_md5 = hashlib.md5(stripped_text_from_html.encode('utf-8')).hexdigest()
+            # If there's text to skip
+            # @todo we could abstract out the get_text() to handle this cleaner
+            if len(self.datastore.data['watching'][uuid]['ignore_text']):
+                content = self.strip_ignore_text(stripped_text_from_html,
+                                                 self.datastore.data['watching'][uuid]['ignore_text'])
+            else:
+                content = stripped_text_from_html.encode('utf8')
+
+            fetched_md5 = hashlib.md5(content).hexdigest()
 
             # could be None or False depending on JSON type
             if self.datastore.data['watching'][uuid]['previous_md5'] != fetched_md5:

backend/pytest.ini

@@ -1,2 +1,12 @@
 [pytest]
-addopts = --no-start-live-server --live-server-port=5005
+addopts = --no-start-live-server --live-server-port=5005
+#testpaths = tests pytest_invenio
+#live_server_scope = session
+
+filterwarnings =
+    ignore::DeprecationWarning:urllib3.*:
+
+; logging options
+log_cli = 1
+log_cli_level = DEBUG
+log_cli_format = %(asctime)s %(name)s: %(levelname)s %(message)s

backend/static/css/styles.css

@@ -88,11 +88,16 @@ section.content {
   margin: 0 3px 0 5px;
 }
 
-#check-all-button {
-  text-align:right;
+#post-list-buttons {
+  text-align: right;
+  padding: 0px;
+  margin: 0px;
+}
+#post-list-buttons li {
+  display: inline-block;
 }
 
-#check-all-button a {
+#post-list-buttons a {
    border-top-left-radius: initial;
    border-top-right-radius: initial;
    border-bottom-left-radius: 5px;
@@ -237,3 +242,32 @@ body:after, body:before {
     user-select:none;
     -o-user-select:none;
 }
+
+footer {
+    padding: 10px;
+    background: #fff;
+    color: #444;
+    text-align: center;
+}
+
+#feed-icon {
+    vertical-align: middle;
+}
+
+#version {
+    position: absolute;
+    top: 80px;
+    right: 0px;
+    font-size: 8px;
+    background: #fff;
+    padding: 10px;
+}
+
+#new-version-text a{
+    color: #e07171;
+}
+
+#diff-stream {
+    font-size: 10px;
+    white-space: pre-wrap;
+}

backend/static/images/Generic_Feed-icon.svg

@@ -0,0 +1,18 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<svg xmlns="http://www.w3.org/2000/svg"
+     id="RSSicon"
+     viewBox="0 0 8 8" width="256" height="256">
+
+  <title>RSS feed icon</title>
+
+  <style type="text/css">
+    .button {stroke: none; fill: orange;}
+    .symbol {stroke: none; fill: white;}
+  </style>
+
+  <rect   class="button" width="8" height="8" rx="1.5" />
+  <circle class="symbol" cx="2" cy="6" r="1" />
+  <path   class="symbol" d="m 1,4 a 3,3 0 0 1 3,3 h 1 a 4,4 0 0 0 -4,-4 z" />
+  <path   class="symbol" d="m 1,2 a 5,5 0 0 1 5,5 h 1 a 6,6 0 0 0 -6,-6 z" />
+
+</svg>

backend/store.py

@@ -22,10 +22,10 @@ class ChangeDetectionStore:
         self.datastore_path = datastore_path
         self.json_store_path = "{}/url-watches.json".format(self.datastore_path)
         self.stop_thread = False
+
         self.__data = {
             'note': "Hello! If you change this file manually, please be sure to restart your changedetection.io instance!",
             'watching': {},
-            'tag': "0.25",
             'settings': {
                 'headers': {
                     'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.66 Safari/537.36',
@@ -53,7 +53,8 @@ class ChangeDetectionStore:
             'previous_md5': "",
             'uuid': str(uuid_builder.uuid4()),
             'headers': {},  # Extra headers to send
-            'history': {}  # Dict of timestamp and output stripped filename
+            'history': {},  # Dict of timestamp and output stripped filename
+            'ignore_text': [] # List of text to ignore when calculating the comparison checksum
         }
 
         if path.isfile('/source.txt'):
@@ -63,6 +64,7 @@ class ChangeDetectionStore:
                 self.__data['build_sha'] = f.read()
 
         try:
+            # @todo retest with ", encoding='utf-8'"
             with open(self.json_store_path) as json_file:
                 from_disk = json.load(json_file)
 
@@ -71,6 +73,9 @@ class ChangeDetectionStore:
                 if 'watching' in from_disk:
                     self.__data['watching'].update(from_disk['watching'])
 
+                if 'app_guid' in from_disk:
+                    self.__data['app_guid'] = from_disk['app_guid']
+
                 if 'settings' in from_disk:
                     if 'headers' in from_disk['settings']:
                         self.__data['settings']['headers'].update(from_disk['settings']['headers'])
@@ -80,8 +85,7 @@ class ChangeDetectionStore:
 
                 # Reinitialise each `watching` with our generic_definition in the case that we add a new var in the future.
                 # @todo pretty sure theres a python we todo this with an abstracted(?) object!
-
-                for uuid, watch in self.data['watching'].items():
+                for uuid, watch in self.__data['watching'].items():
                     _blank = deepcopy(self.generic_definition)
                     _blank.update(watch)
                     self.__data['watching'].update({uuid: _blank})
@@ -98,6 +102,14 @@ class ChangeDetectionStore:
                 self.add_watch(url='https://www.gov.uk/coronavirus', tag='Covid')
                 self.add_watch(url='https://changedetection.io', tag='Tech news')
 
+
+        self.__data['version_tag'] = "0.27"
+
+        if not 'app_guid' in self.__data:
+            self.__data['app_guid'] = str(uuid_builder.uuid4())
+
+        self.needs_write = True
+
         # Finally start the thread that will manage periodic data saves to JSON
         save_data_thread = threading.Thread(target=self.save_datastore).start()
 
@@ -117,7 +129,7 @@ class ChangeDetectionStore:
         return 0
 
     def set_last_viewed(self, uuid, timestamp):
-        self.data['watching'][uuid].update({'last_viewed': str(timestamp)})
+        self.data['watching'][uuid].update({'last_viewed': int(timestamp)})
         self.needs_write = True
 
     def update_watch(self, uuid, update_obj):
@@ -139,6 +151,19 @@ class ChangeDetectionStore:
     @property
     def data(self):
 
+        has_unviewed = False
+
+        for uuid, v in self.__data['watching'].items():
+            self.__data['watching'][uuid]['newest_history_key'] = self.get_newest_history_key(uuid)
+            if int(v['newest_history_key']) <= int(v['last_viewed']):
+                self.__data['watching'][uuid]['viewed'] = True
+
+            else:
+                self.__data['watching'][uuid]['viewed'] = False
+                has_unviewed = True
+
+        self.__data['has_unviewed'] = has_unviewed
+
         return self.__data
 
     def get_all_tags(self):
@@ -156,7 +181,11 @@ class ChangeDetectionStore:
 
     def delete(self, uuid):
         with self.lock:
-            del (self.__data['watching'][uuid])
+            if uuid == 'all':
+                self.__data['watching'] = {}
+            else:
+                del (self.__data['watching'][uuid])
+
             self.needs_write = True
 
     def url_exists(self, url):
@@ -199,6 +228,12 @@ class ChangeDetectionStore:
     # result_obj from fetch_site_status.run()
     def save_history_text(self, uuid, result_obj, contents):
 
+        output_path = "{}/{}".format(self.datastore_path, uuid)
+        try:
+            os.mkdir(output_path)
+        except FileExistsError:
+            pass
+
         output_path = "{}/{}".format(self.datastore_path, uuid)
         fname = "{}/{}-{}.stripped.txt".format(output_path, result_obj['previous_md5'], str(time.time()))
         with open(fname, 'w') as f:

backend/templates/base.html

@@ -19,7 +19,11 @@
     <div class="home-menu pure-menu pure-menu-horizontal pure-menu-fixed">
         <a class="pure-menu-heading" href="/"><strong>Change</strong>Detection.io</a>
         {% if current_diff_url %}
-            <a class=current-diff-url href="{{ current_diff_url }}"><span style="max-width: 30%; overflow: hidden;">{{ current_diff_url }}</a>
+        <a class=current-diff-url href="{{ current_diff_url }}"><span style="max-width: 30%; overflow: hidden;">{{ current_diff_url }}</span></a>
+        {% else %}
+        {% if new_version_available %}
+        <span id="new-version-text" class="pure-menu-heading"><a href="https://github.com/dgtlmoon/changedetection.io">A new version is available</a></span>
+        {% endif %}
         {% endif %}
 
         <ul class="pure-menu-list">
@@ -34,7 +38,8 @@
                 <a href="/settings" class="pure-menu-link">SETTINGS</a>
             </li>
             <li class="pure-menu-item"><a class="github-link" href="https://github.com/dgtlmoon/changedetection.io">
-                <svg class="octicon octicon-mark-github v-align-middle" height="32" viewBox="0 0 16 16" version="1.1"
+                <svg class="octicon octicon-mark-github v-align-middle" height="32" viewBox="0 0 16 16"
+                     version="1.1"
                      width="32" aria-hidden="true">
                     <path fill-rule="evenodd"
                           d="M8 0C3.58 0 0 3.58 0 8c0 3.54 2.29 6.53 5.47 7.59.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2.01.37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.59.82-2.15-.08-.2-.36-1.02.08-2.12 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.92.08 2.12.51.56.82 1.27.82 2.15 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.46.55.38A8.013 8.013 0 0016 8c0-4.42-3.58-8-8-8z"></path>
@@ -47,7 +52,7 @@
         </ul>
     </div>
 </div>
-
+<div id="version">v{{ version }}</div>
 <section class="content">
     <header>
         {% block header %}{% endblock %}
@@ -66,5 +71,6 @@
 
     {% endblock %}
 </section>
+
 </body>
 </html>

backend/templates/edit.html

@@ -4,7 +4,7 @@
 <div class="edit-form">
 
 
-    <form class="pure-form pure-form-stacked" action="/edit?uuid={{uuid}}" method="POST">
+    <form class="pure-form pure-form-stacked" action="/edit/{{uuid}}" method="POST">
         <fieldset>
             <div class="pure-control-group">
                 <label for="url">URL</label>
@@ -18,10 +18,26 @@
                 <span class="pure-form-message-inline">Grouping tags, can be a comma separated list.</span>
             </div>
 
+            <!-- @todo: move to tabs --->
+            <fieldset class="pure-group">
+                <label for="ignore-text">Ignore text</label>
+
+                <textarea id="ignore-text" name="ignore-text" class="pure-input-1-2" placeholder=""
+                          style="width: 100%;
+                            font-family:monospace;
+                            white-space: pre;
+                            overflow-wrap: normal;
+                            overflow-x: scroll;" rows="5">{% for value in watch.ignore_text %}{{ value }}
+{% endfor %}</textarea>
+                <span class="pure-form-message-inline">Each line will be processed separately as an ignore rule.</span>
+
+            </fieldset>
+
+            <!-- @todo: move to tabs --->
             <fieldset class="pure-group">
                 <label for="headers">Extra request headers</label>
 
-                <textarea id=headers name="headers" class="pure-input-1-2" placeholder="Example
+                <textarea id="headers" name="headers" class="pure-input-1-2" placeholder="Example
 Cookie: foobar
 User-Agent: wonderbra 1.0"
                           style="width: 100%;
@@ -33,6 +49,8 @@ User-Agent: wonderbra 1.0"
                 <br/>
 
             </fieldset>
+
+
             <div class="pure-control-group">
                 <button type="submit" class="pure-button pure-button-primary">Save</button>
             </div>

backend/templates/watch-diff-stream.html

@@ -0,0 +1,12 @@
+{% extends 'watch-overview.html' %}
+{% block innercontent %}
+Entries: {{ streams|length }}
+
+<div id="diff-stream" class="edit-form">
+    {% for item in streams %}
+        {{ loop.index }}
+{% for diff in item %}{% if diff[0] =='+' %}<ins>{{ diff }}</ins>{% endif %}{% if diff[0] =='-' %}<del>{{ diff }}</del>{% endif %}
+        {% endfor %}
+    {% endfor %}
+</div>
+{% endblock %}

backend/templates/watch-overview.html

@@ -15,71 +15,18 @@
         <!-- user/pass r = requests.get('https://api.github.com/user', auth=('user', 'pass')) -->
     </form>
     <div>
-
+        <a href="/" class="pure-button button-tag {{'active' if not active_tag }}">All</a>
         {% for tag in tags %}
-        {% if tag == "" %}
-        <a href="/" class="pure-button button-tag {{'active' if active_tag == tag }}">All</a>
-        {% else %}
-        <a href="/?tag={{ tag}}" class="pure-button button-tag {{'active' if active_tag == tag }}">{{ tag }}</a>
-        {% endif %}
+            {% if tag != "" %}
+                <a href="/?tag={{ tag}}" class="pure-button button-tag {{'active' if active_tag == tag }}">{{ tag }}</a>
+            {% endif %}
         {% endfor %}
     </div>
 
     <div id="watch-table-wrapper">
-        <table class="pure-table pure-table-striped watch-table">
-            <thead>
-            <tr>
-                <th>#</th>
-                <th></th>
-                <th>Last Checked</th>
-                <th>Last Changed</th>
-                <th></th>
-            </tr>
-            </thead>
-            <tbody>
+    {% block innercontent %}
 
-
-            {% for watch in watches %}
-            <tr id="{{ watch.uuid }}"
-                class="{{ loop.cycle('pure-table-odd', 'pure-table-even') }}
-                {% if watch.last_error is defined and watch.last_error != False %}error{% endif %}
-                {% if watch.newest_history_key| int > watch.last_viewed| int %}unviewed{% endif %}">
-                <td>{{ loop.index }}</td>
-                <td class="title-col">{{watch.title if watch.title is not none else watch.url}}
-                    <a class="external" target=_blank href="{{ watch.url }}"></a>
-                    {% if watch.last_error is defined and watch.last_error != False %}
-                    <div class="fetch-error">{{ watch.last_error }}</div>
-                    {% endif %}
-                    {% if not active_tag %}
-                    <span class="watch-tag-list">{{ watch.tag}}</span>
-                    {% endif %}
-                </td>
-                <td>{{watch|format_last_checked_time}}</td>
-                <td>{% if watch.history|length >= 2 and watch.last_changed %}
-                    {{watch.last_changed|format_timestamp_timeago}}
-                    {% else %}
-                    Not yet
-                    {% endif %}
-                </td>
-                <td>
-                    <a href="/api/checknow?uuid={{ watch.uuid}}{% if request.args.get('tag') %}&tag={{request.args.get('tag')}}{% endif %}"
-                       class="pure-button button-small pure-button-primary">Recheck</a>
-                    <a href="/edit?uuid={{ watch.uuid}}" class="pure-button button-small pure-button-primary">Edit</a>
-                    {% if watch.history|length >= 2 %}
-                    <a href="/diff/{{ watch.uuid}}" class="pure-button button-small pure-button-primary">Diff</a>
-                    {% endif %}
-                </td>
-            </tr>
-            {% endfor %}
-
-
-            </tbody>
-        </table>
-        <div id="check-all-button">
-
-            <a href="/api/checknow{% if active_tag%}?tag={{active_tag}}{%endif%}" class="pure-button button-tag ">Recheck
-                all {% if active_tag%}in "{{active_tag}}"{%endif%}</a>
-        </div>
+    {% endblock %}
     </div>
 </div>
 {% endblock %}

backend/templates/watch-table.html

@@ -0,0 +1,64 @@
+{% extends 'watch-overview.html' %}
+{% block innercontent %}
+        <table class="pure-table pure-table-striped watch-table">
+            <thead>
+            <tr>
+                <th>#</th>
+                <th></th>
+                <th>Last Checked</th>
+                <th>Last Changed</th>
+                <th></th>
+            </tr>
+            </thead>
+            <tbody>
+
+
+            {% for watch in watches %}
+            <tr id="{{ watch.uuid }}"
+                class="{{ loop.cycle('pure-table-odd', 'pure-table-even') }}
+                {% if watch.last_error is defined and watch.last_error != False %}error{% endif %}
+                {% if watch.newest_history_key| int > watch.last_viewed| int %}unviewed{% endif %}">
+                <td>{{ loop.index }}</td>
+                <td class="title-col">{{watch.title if watch.title is not none else watch.url}}
+                    <a class="external" target=_blank href="{{ watch.url }}"></a>
+                    {% if watch.last_error is defined and watch.last_error != False %}
+                    <div class="fetch-error">{{ watch.last_error }}</div>
+                    {% endif %}
+                    {% if not active_tag %}
+                    <span class="watch-tag-list">{{ watch.tag}}</span>
+                    {% endif %}
+                </td>
+                <td>{{watch|format_last_checked_time}}</td>
+                <td>{% if watch.history|length >= 2 and watch.last_changed %}
+                    {{watch.last_changed|format_timestamp_timeago}}
+                    {% else %}
+                    Not yet
+                    {% endif %}
+                </td>
+                <td>
+                    <a href="/api/checknow?uuid={{ watch.uuid}}{% if request.args.get('tag') %}&tag={{request.args.get('tag')}}{% endif %}"
+                       class="pure-button button-small pure-button-primary">Recheck</a>
+                    <a href="/edit/{{ watch.uuid}}" class="pure-button button-small pure-button-primary">Edit</a>
+                    {% if watch.history|length >= 2 %}
+                    <a href="/diff/{{ watch.uuid}}" class="pure-button button-small pure-button-primary">Diff</a>
+                    {% endif %}
+                </td>
+            </tr>
+            {% endfor %}
+            </tbody>
+        </table>
+        <ul id="post-list-buttons">
+            {% if has_unviewed %}
+            <li>
+                <a href="/api/mark-all-viewed" class="pure-button button-tag ">Mark all viewed</a>
+            </li>
+            {% endif %}
+            <li>
+               <a href="/api/checknow{% if active_tag%}?tag={{active_tag}}{%endif%}" class="pure-button button-tag ">Recheck
+                all {% if active_tag%}in "{{active_tag}}"{%endif%}</a>
+            </li>
+            <li>
+                <a href="{{ url_for('index', tag=active_tag , rss=true)}}"><img id="feed-icon" src="/static/images/Generic_Feed-icon.svg" height="15px"></a>
+            </li>
+        </ul>
+{% endblock %}

backend/tests/conftest.py

@@ -7,17 +7,14 @@ import os
 
 
 # https://github.com/pallets/flask/blob/1.1.2/examples/tutorial/tests/test_auth.py
-
 # Much better boilerplate than the docs
 # https://www.python-boilerplate.com/py3+flask+pytest/
 
 global app
 
-
 @pytest.fixture(scope='session')
 def app(request):
     """Create application for the tests."""
-
     datastore_path = "./test-datastore"
 
     try:
@@ -33,11 +30,19 @@ def app(request):
     app_config = {'datastore_path': datastore_path}
     datastore = store.ChangeDetectionStore(datastore_path=app_config['datastore_path'], include_default_watches=False)
     app = changedetection_app(app_config, datastore)
+    app.config['STOP_THREADS'] = True
 
     def teardown():
         datastore.stop_thread = True
-        app.config['STOP_THREADS'] = True
+        app.config.exit.set()
+        try:
+            os.unlink("{}/url-watches.json".format(datastore_path))
+        except FileNotFoundError:
+            # This is fine in the case of a failure.
+            pass
+
+        assert 1 == 1
 
     request.addfinalizer(teardown)
+    yield app
 
-    return app

backend/tests/test_backend.py

@@ -3,6 +3,21 @@
 import time
 from flask import url_for
 from urllib.request import urlopen
+import pytest
+
+sleep_time_for_fetch_thread = 3
+
+
+def test_setup_liveserver(live_server):
+    @live_server.app.route('/test-endpoint')
+    def test_endpoint():
+        # Tried using a global var here but didn't seem to work, so reading from a file instead.
+        with open("test-datastore/output.txt", "r") as f:
+            return f.read()
+
+    live_server.start()
+
+    assert 1 == 1
 
 
 def set_original_response():
@@ -14,7 +29,6 @@ def set_original_response():
      So let's see what happens.  </br>
      </body>
      </html>
-
     """
 
     with open("test-datastore/output.txt", "w") as f:
@@ -30,7 +44,6 @@ def set_modified_response():
      So let's see what happens.  </br>
      </body>
      </html>
-
     """
 
     with open("test-datastore/output.txt", "w") as f:
@@ -38,18 +51,8 @@ def set_modified_response():
 
 
 def test_check_basic_change_detection_functionality(client, live_server):
-    sleep_time_for_fetch_thread = 5
-
-    @live_server.app.route('/test-endpoint')
-    def test_endpoint():
-        # Tried using a global var here but didn't seem to work, so reading from a file instead.
-        with open("test-datastore/output.txt", "r") as f:
-            return f.read()
-
     set_original_response()
 
-    live_server.start()
-
     # Add our URL to the import page
     res = client.post(
         url_for("import_page"),
@@ -91,13 +94,13 @@ def test_check_basic_change_detection_functionality(client, live_server):
     assert b'unviewed' in res.data
 
     # Following the 'diff' link, it should no longer display as 'unviewed' even after we recheck it a few times
-    res = client.get(url_for("diff_history_page", uuid="first") )
+    res = client.get(url_for("diff_history_page", uuid="first"))
     assert b'Compare newest' in res.data
 
     time.sleep(2)
 
     # Do this a few times.. ensures we dont accidently set the status
-    for n in range(3):
+    for n in range(2):
         client.get(url_for("api_watch_checknow"), follow_redirects=True)
 
         # Give the thread time to pick it up
@@ -108,10 +111,13 @@ def test_check_basic_change_detection_functionality(client, live_server):
         assert b'unviewed' not in res.data
         assert b'test-endpoint' in res.data
 
-
     set_original_response()
 
     client.get(url_for("api_watch_checknow"), follow_redirects=True)
     time.sleep(sleep_time_for_fetch_thread)
     res = client.get(url_for("index"))
-    assert b'unviewed' in res.data
+    assert b'unviewed' in res.data
+
+    # Cleanup everything
+    res = client.get(url_for("api_delete", uuid="all"), follow_redirects=True)
+    assert b'Deleted' in res.data

backend/tests/test_ignore_text.py

@@ -0,0 +1,152 @@
+#!/usr/bin/python3
+
+import time
+from flask import url_for
+from urllib.request import urlopen
+import pytest
+
+
+# Unit test of the stripper
+# Always we are dealing in utf-8
+def test_strip_text_func():
+    from backend import fetch_site_status
+
+    test_content = """
+    Some content
+    is listed here
+
+    but sometimes we want to remove the lines.
+
+    but not always."""
+
+    ignore_lines = ["sometimes"]
+
+    fetcher = fetch_site_status.perform_site_check(datastore=False)
+    stripped_content = fetcher.strip_ignore_text(test_content, ignore_lines)
+
+    assert b"sometimes" not in stripped_content
+    assert b"Some content" in stripped_content
+
+
+def set_original_ignore_response():
+    test_return_data = """<html>
+       <body>
+     Some initial text</br>
+     <p>Which is across multiple lines</p>
+     </br>
+     So let's see what happens.  </br>
+     </body>
+     </html>
+
+    """
+
+    with open("test-datastore/output.txt", "w") as f:
+        f.write(test_return_data)
+
+
+def set_modified_original_ignore_response():
+    test_return_data = """<html>
+       <body>
+     Some NEW nice initial text</br>
+     <p>Which is across multiple lines</p>
+     </br>
+     So let's see what happens.  </br>
+     </body>
+     </html>
+
+    """
+
+    with open("test-datastore/output.txt", "w") as f:
+        f.write(test_return_data)
+
+
+# Is the same but includes ZZZZZ, 'ZZZZZ' is the last line in ignore_text
+def set_modified_ignore_response():
+    test_return_data = """<html>
+       <body>
+     Some initial text</br>
+     <p>Which is across multiple lines</p>
+     <P>ZZZZZ</P>
+     </br>
+     So let's see what happens.  </br>
+     </body>
+     </html>
+
+    """
+
+    with open("test-datastore/output.txt", "w") as f:
+        f.write(test_return_data)
+
+
+def test_check_ignore_text_functionality(client, live_server):
+    sleep_time_for_fetch_thread = 3
+
+    ignore_text = "XXXXX\nYYYYY\nZZZZZ"
+    set_original_ignore_response()
+
+    # Give the endpoint time to spin up
+    time.sleep(1)
+
+    # Add our URL to the import page
+    test_url = url_for('test_endpoint', _external=True)
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
+
+    # Goto the edit page, add our ignore text
+    # Add our URL to the import page
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={"ignore-text": ignore_text, "url": test_url, "tag": "", "headers": ""},
+        follow_redirects=True
+    )
+    assert b"Updated watch." in res.data
+
+    # Check it saved
+    res = client.get(
+        url_for("edit_page", uuid="first"),
+    )
+    assert bytes(ignore_text.encode('utf-8')) in res.data
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
+
+    # It should report nothing found (no new 'unviewed' class)
+    res = client.get(url_for("index"))
+    assert b'unviewed' not in res.data
+    assert b'/test-endpoint' in res.data
+
+    #  Make a change
+    set_modified_ignore_response()
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
+
+    # It should report nothing found (no new 'unviewed' class)
+    res = client.get(url_for("index"))
+    assert b'unviewed' not in res.data
+    assert b'/test-endpoint' in res.data
+
+    # Just to be sure.. set a regular modified change..
+    set_modified_original_ignore_response()
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    time.sleep(sleep_time_for_fetch_thread)
+    res = client.get(url_for("index"))
+    assert b'unviewed' in res.data
+
+    res = client.get(url_for("api_delete", uuid="all"), follow_redirects=True)
+    assert b'Deleted' in res.data

changedetection.py

@@ -50,6 +50,14 @@ def main(argv):
     datastore = store.ChangeDetectionStore(datastore_path=app_config['datastore_path'])
     app = backend.changedetection_app(app_config, datastore)
 
+    @app.context_processor
+    def inject_version():
+        return dict(version=datastore.data['version_tag'])
+
+    @app.context_processor
+    def inject_new_version_available():
+        return dict(new_version_available=app.config['NEW_VERSION_AVAILABLE'])
+
     if ssl_mode:
         # @todo finalise SSL config, but this should get you in the right direction if you need it.
         eventlet.wsgi.server(eventlet.wrap_ssl(eventlet.listen(('', port)),

docker-compose-development.yml

@@ -9,7 +9,7 @@ services:
     image: dgtlmoon/changedetection.io:dev
     container_name: changedetection.io-dev
     volumes:
-      - ./backend:/app
+      - .:/app
       - ./requirements.txt:/requirements.txt # Normally COPY'ed in the Dockerfile
       - ./datastore:/datastore
 

requirements.txt

@@ -1,24 +1,13 @@
-aiohttp
-async-timeout
 chardet==2.3.0
-multidict
-python-engineio
-six==1.10.0
 yarl
-flask
-
-pytest
-pytest-flask # for live_server
-
-eventlet
+flask~= 1.0
+pytest ~=6.2
+pytest-flask ~=1.1
+eventlet ~= 0.30
 requests
 validators
-
-bleach==3.2.1
-html5lib==0.9999999       # via bleach
-timeago
-html2text
-inscriptis
-
-# @notes
-# - Dont install socketio, it interferes with flask_socketio
+timeago ~=1.0
+inscriptis ~= 1.1
+feedgen ~= 0.9
+pytz
+urllib3