logic fix

refactor handling

changedetection.py

@@ -6,34 +6,6 @@
 # Read more https://github.com/dgtlmoon/changedetection.io/wiki
 
 from changedetectionio import changedetection
-import multiprocessing
-import signal
-import os
-
-def sigterm_handler(_signo, _stack_frame):
-    import sys
-    print('Shutdown: Got SIGCHLD')
-    # https://stackoverflow.com/questions/40453496/python-multiprocessing-capturing-signals-to-restart-child-processes-or-shut-do
-    pid, status = os.waitpid(-1, os.WNOHANG | os.WUNTRACED | os.WCONTINUED)
-
-    print('Sub-process: pid %d status %d' % (pid, status))
-    if status != 0:
-        sys.exit(1)
-
-    raise SystemExit
 
 if __name__ == '__main__':
-    signal.signal(signal.SIGCHLD, sigterm_handler)
-    # The only way I could find to get Flask to shutdown, is to wrap it and then rely on the subsystem issuing SIGTERM/SIGKILL
-    parse_process = multiprocessing.Process(target=changedetection.main)
-    parse_process.daemon = True
-    parse_process.start()
-    import time
-
-    try:
-        while True:
-            time.sleep(1)
-
-    except KeyboardInterrupt:
-        #parse_process.terminate() not needed, because this process will issue it to the sub-process anyway
-        print ("Exited - CTRL+C")
+    changedetection.main()

changedetectionio/__init__.py

@@ -54,7 +54,7 @@ ticker_thread = None
 
 extra_stylesheets = []
 
-update_q = queue.PriorityQueue()
+update_q = queue.Queue()
 
 notification_q = queue.Queue()
 
@@ -406,6 +406,7 @@ def changedetection_app(config=None, datastore_o=None):
         existing_tags = datastore.get_all_tags()
 
         form = forms.quickWatchForm(request.form)
+
         output = render_template("watch-overview.html",
                                  form=form,
                                  watches=sorted_watches,
@@ -416,7 +417,7 @@ def changedetection_app(config=None, datastore_o=None):
                                  # Don't link to hosting when we're on the hosting environment
                                  hosted_sticky=os.getenv("SALTED_PASS", False) == False,
                                  guid=datastore.data['app_guid'],
-                                 queued_uuids=[uuid for p,uuid in update_q.queue])
+                                 queued_uuids=update_q.queue)
 
 
         if session.get('share-link'):
@@ -630,14 +631,18 @@ def changedetection_app(config=None, datastore_o=None):
             # But in the case something is added we should save straight away
             datastore.needs_write_urgent = True
 
-            # Queue the watch for immediate recheck, with a higher priority
-            update_q.put((1, uuid))
+            # Queue the watch for immediate recheck
+            update_q.put(uuid)
 
             # Diff page [edit] link should go back to diff page
-            if request.args.get("next") and request.args.get("next") == 'diff':
+            if request.args.get("next") and request.args.get("next") == 'diff' and not form.save_and_preview_button.data:
                 return redirect(url_for('diff_history_page', uuid=uuid))
-
-            return redirect(url_for('index'))
+            else:
+                if form.save_and_preview_button.data:
+                    flash('You may need to reload this page to see the new content.')
+                    return redirect(url_for('preview_page', uuid=uuid))
+                else:
+                    return redirect(url_for('index'))
 
         else:
             if request.method == 'POST' and not form.validate():
@@ -703,14 +708,7 @@ def changedetection_app(config=None, datastore_o=None):
                     return redirect(url_for('settings_page'))
 
             if form.validate():
-                # Don't set password to False when a password is set - should be only removed with the `removepassword` button
-                app_update = dict(deepcopy(form.data['application']))
-
-                # Never update password with '' or False (Added by wtforms when not in submission)
-                if 'password' in app_update and not app_update['password']:
-                    del (app_update['password'])
-
-                datastore.data['settings']['application'].update(app_update)
+                datastore.data['settings']['application'].update(form.data['application'])
                 datastore.data['settings']['requests'].update(form.data['requests'])
 
                 if not os.getenv("SALTED_PASS", False) and len(form.application.form.password.encrypted_password):
@@ -748,7 +746,7 @@ def changedetection_app(config=None, datastore_o=None):
                 importer = import_url_list()
                 importer.run(data=request.values.get('urls'), flash=flash, datastore=datastore)
                 for uuid in importer.new_uuids:
-                    update_q.put((1, uuid))
+                    update_q.put(uuid)
 
                 if len(importer.remaining_data) == 0:
                     return redirect(url_for('index'))
@@ -761,7 +759,7 @@ def changedetection_app(config=None, datastore_o=None):
                 d_importer = import_distill_io_json()
                 d_importer.run(data=request.values.get('distill-io'), flash=flash, datastore=datastore)
                 for uuid in d_importer.new_uuids:
-                    update_q.put((1, uuid))
+                    update_q.put(uuid)
 
 
 
@@ -879,40 +877,42 @@ def changedetection_app(config=None, datastore_o=None):
             flash("No history found for the specified link, bad link?", "error")
             return redirect(url_for('index'))
 
+        if watch.history_n >0:
+            timestamps = sorted(watch.history.keys(), key=lambda x: int(x))
+            filename = watch.history[timestamps[-1]]
+            try:
+                with open(filename, 'r') as f:
+                    tmp = f.readlines()
 
-        timestamp = list(watch.history.keys())[-1]
-        filename = watch.history[timestamp]
-        try:
-            with open(filename, 'r') as f:
-                tmp = f.readlines()
+                    # Get what needs to be highlighted
+                    ignore_rules = watch.get('ignore_text', []) + datastore.data['settings']['application']['global_ignore_text']
 
-                # Get what needs to be highlighted
-                ignore_rules = watch.get('ignore_text', []) + datastore.data['settings']['application']['global_ignore_text']
+                    # .readlines will keep the \n, but we will parse it here again, in the future tidy this up
+                    ignored_line_numbers = html_tools.strip_ignore_text(content="".join(tmp),
+                                                                        wordlist=ignore_rules,
+                                                                        mode='line numbers'
+                                                                        )
 
-                # .readlines will keep the \n, but we will parse it here again, in the future tidy this up
-                ignored_line_numbers = html_tools.strip_ignore_text(content="".join(tmp),
-                                                                    wordlist=ignore_rules,
-                                                                    mode='line numbers'
-                                                                    )
+                    trigger_line_numbers = html_tools.strip_ignore_text(content="".join(tmp),
+                                                                        wordlist=watch['trigger_text'],
+                                                                        mode='line numbers'
+                                                                        )
+                    # Prepare the classes and lines used in the template
+                    i=0
+                    for l in tmp:
+                        classes=[]
+                        i+=1
+                        if i in ignored_line_numbers:
+                            classes.append('ignored')
+                        if i in trigger_line_numbers:
+                            classes.append('triggered')
+                        content.append({'line': l, 'classes': ' '.join(classes)})
 
-                trigger_line_numbers = html_tools.strip_ignore_text(content="".join(tmp),
-                                                                    wordlist=watch['trigger_text'],
-                                                                    mode='line numbers'
-                                                                    )
-                # Prepare the classes and lines used in the template
-                i=0
-                for l in tmp:
-                    classes=[]
-                    i+=1
-                    if i in ignored_line_numbers:
-                        classes.append('ignored')
-                    if i in trigger_line_numbers:
-                        classes.append('triggered')
-                    content.append({'line': l, 'classes': ' '.join(classes)})
-
-        except Exception as e:
-            content.append({'line': "File doesnt exist or unable to read file {}".format(filename), 'classes': ''})
 
+            except Exception as e:
+                content.append({'line': "File doesnt exist or unable to read file {}".format(filename), 'classes': ''})
+        else:
+            content.append({'line': "No history found", 'classes': ''})
 
         screenshot_url = datastore.get_screenshot(uuid)
         system_uses_webdriver = datastore.data['settings']['application']['fetch_backend'] == 'html_webdriver'
@@ -1090,7 +1090,7 @@ def changedetection_app(config=None, datastore_o=None):
 
         if not add_paused and new_uuid:
             # Straight into the queue.
-            update_q.put((1, new_uuid))
+            update_q.put(new_uuid)
             flash("Watch added.")
 
         if add_paused:
@@ -1127,7 +1127,7 @@ def changedetection_app(config=None, datastore_o=None):
             uuid = list(datastore.data['watching'].keys()).pop()
 
         new_uuid = datastore.clone(uuid)
-        update_q.put((5, new_uuid))
+        update_q.put(new_uuid)
         flash('Cloned.')
 
         return redirect(url_for('index'))
@@ -1148,7 +1148,7 @@ def changedetection_app(config=None, datastore_o=None):
 
         if uuid:
             if uuid not in running_uuids:
-                update_q.put((1, uuid))
+                update_q.put(uuid)
             i = 1
 
         elif tag != None:
@@ -1156,7 +1156,7 @@ def changedetection_app(config=None, datastore_o=None):
             for watch_uuid, watch in datastore.data['watching'].items():
                 if (tag != None and tag in watch['tag']):
                     if watch_uuid not in running_uuids and not datastore.data['watching'][watch_uuid]['paused']:
-                        update_q.put((1, watch_uuid))
+                        update_q.put(watch_uuid)
                         i += 1
 
         else:
@@ -1164,7 +1164,7 @@ def changedetection_app(config=None, datastore_o=None):
             for watch_uuid, watch in datastore.data['watching'].items():
 
                 if watch_uuid not in running_uuids and not datastore.data['watching'][watch_uuid]['paused']:
-                    update_q.put((1, watch_uuid))
+                    update_q.put(watch_uuid)
                     i += 1
         flash("{} watches are queued for rechecking.".format(i))
         return redirect(url_for('index', tag=tag))
@@ -1266,6 +1266,7 @@ def notification_runner():
     global notification_debug_log
     from datetime import datetime
     import json
+
     while not app.config.exit.is_set():
         try:
             # At the moment only one thread runs (single runner)
@@ -1367,14 +1368,14 @@ def ticker_thread_check_time_launch_checks():
 
             seconds_since_last_recheck = now - watch['last_checked']
             if seconds_since_last_recheck >= (threshold + watch.jitter_seconds) and seconds_since_last_recheck >= recheck_time_minimum_seconds:
-                if not uuid in running_uuids and uuid not in [q_uuid for p,q_uuid in update_q.queue]:
-                    print("> Queued watch UUID {} last checked at {} queued at {:0.2f} jitter {:0.2f}s, {:0.2f}s since last checked".format(uuid,
+                if not uuid in running_uuids and uuid not in update_q.queue:
+                    print("Queued watch UUID {} last checked at {} queued at {:0.2f} jitter {:0.2f}s, {:0.2f}s since last checked".format(uuid,
                                                                                                          watch['last_checked'],
                                                                                                          now,
                                                                                                          watch.jitter_seconds,
                                                                                                          now - watch['last_checked']))
                     # Into the queue with you
-                    update_q.put((5, uuid))
+                    update_q.put(uuid)
 
                     # Reset for next time
                     watch.jitter_seconds = 0

changedetectionio/api/api_v1.py

@@ -24,7 +24,7 @@ class Watch(Resource):
             abort(404, message='No watch exists with the UUID of {}'.format(uuid))
 
         if request.args.get('recheck'):
-            self.update_q.put((1, uuid))
+            self.update_q.put(uuid)
             return "OK", 200
 
         # Return without history, get that via another API call
@@ -100,7 +100,7 @@ class CreateWatch(Resource):
         extras = {'title': json_data['title'].strip()} if json_data.get('title') else {}
 
         new_uuid = self.datastore.add_watch(url=json_data['url'].strip(), tag=tag, extras=extras)
-        self.update_q.put((1, new_uuid))
+        self.update_q.put(new_uuid)
         return {'uuid': new_uuid}, 201
 
     # Return concise list of available watches and some very basic info
@@ -118,7 +118,7 @@ class CreateWatch(Resource):
 
         if request.args.get('recheck_all'):
             for uuid in self.datastore.data['watching'].keys():
-                self.update_q.put((1, uuid))
+                self.update_q.put(uuid)
             return {'status': "OK"}, 200
 
         return list, 200

changedetectionio/changedetection.py

@@ -4,7 +4,6 @@
 
 import getopt
 import os
-import signal
 import sys
 
 import eventlet
@@ -12,22 +11,7 @@ import eventlet.wsgi
 from . import store, changedetection_app, content_fetcher
 from . import __version__
 
-# Only global so we can access it in the signal handler
-datastore = None
-app = None
-
-def sigterm_handler(_signo, _stack_frame):
-    global app
-    global datastore
-
-    app.config.exit.set()
-    datastore.sync_to_json()
-    print('Shutdown: Got SIGTERM, DB saved to disk')
-    raise SystemExit
-
 def main():
-    global datastore
-    global app
     ssl_mode = False
     host = ''
     port = os.environ.get('PORT') or 5000
@@ -88,10 +72,8 @@ def main():
                 "Or use the -C parameter to create the directory.".format(app_config['datastore_path']), file=sys.stderr)
             sys.exit(2)
 
-
     datastore = store.ChangeDetectionStore(datastore_path=app_config['datastore_path'], version_tag=__version__)
     app = changedetection_app(app_config, datastore)
-    signal.signal(signal.SIGTERM, sigterm_handler)
 
     # Go into cleanup mode
     if do_cleanup:
@@ -129,3 +111,4 @@ def main():
     else:
         eventlet.wsgi.server(eventlet.listen((host, int(port))), app)
 
+

changedetectionio/forms.py

@@ -350,7 +350,7 @@ class watchForm(commonSettingsForm):
     webdriver_js_execute_code = TextAreaField('Execute JavaScript before change detection', render_kw={"rows": "5"}, validators=[validators.Optional()])
 
     save_button = SubmitField('Save', render_kw={"class": "pure-button pure-button-primary"})
-
+    save_and_preview_button = SubmitField('Save & Preview', render_kw={"class": "pure-button pure-button-primary"})
     proxy = RadioField('Proxy')
     filter_failure_notification_send = BooleanField(
         'Send a notification when the filter can no longer be found on the page', default=False)

changedetectionio/templates/edit.html

@@ -307,7 +307,9 @@ Unavailable") }}
 
             <div id="actions">
                 <div class="pure-control-group">
-                    {{ render_button(form.save_button) }}
+
+                      {{ render_button(form.save_button) }} {{ render_button(form.save_and_preview_button) }}
+
                     <a href="{{url_for('form_delete', uuid=uuid)}}"
                        class="pure-button button-small button-error ">Delete</a>
                     <a href="{{url_for('clear_watch_history', uuid=uuid)}}"

changedetectionio/tests/test_access_control.py

@@ -19,6 +19,7 @@ def test_check_access_control(app, client):
         )
 
         assert b"Password protection enabled." in res.data
+        assert b"LOG OUT" not in res.data
 
         # Check we hit the login
         res = c.get(url_for("index"), follow_redirects=True)
@@ -37,42 +38,7 @@ def test_check_access_control(app, client):
             follow_redirects=True
         )
 
-        # Yes we are correctly logged in
         assert b"LOG OUT" in res.data
-
-        # 598 - Password should be set and not accidently removed
-        res = c.post(
-            url_for("settings_page"),
-            data={
-                  "requests-time_between_check-minutes": 180,
-                  'application-fetch_backend': "html_requests"},
-            follow_redirects=True
-        )
-
-        res = c.get(url_for("logout"),
-            follow_redirects=True)
-
-        res = c.get(url_for("settings_page"),
-            follow_redirects=True)
-
-
-        assert b"Login" in res.data
-
-        res = c.get(url_for("login"))
-        assert b"Login" in res.data
-
-
-        res = c.post(
-            url_for("login"),
-            data={"password": "foobar"},
-            follow_redirects=True
-        )
-
-        # Yes we are correctly logged in
-        assert b"LOG OUT" in res.data
-        return
-
-
         res = c.get(url_for("settings_page"))
 
         # Menu should be available now

changedetectionio/tests/test_backend.py

@@ -90,14 +90,6 @@ def test_check_basic_change_detection_functionality(client, live_server):
     res = client.get(url_for("diff_history_page", uuid="first"))
     assert b'Compare newest' in res.data
 
-    # Check the [preview] pulls the right one
-    res = client.get(
-        url_for("preview_page", uuid="first"),
-        follow_redirects=True
-    )
-    assert b'which has this one new line' in res.data
-    assert b'Which is across multiple lines' not in res.data
-
     time.sleep(2)
 
     # Do this a few times.. ensures we dont accidently set the status

changedetectionio/update_worker.py

@@ -115,7 +115,7 @@ class update_worker(threading.Thread):
         while not self.app.config.exit.is_set():
 
             try:
-                priority, uuid = self.q.get(block=False)
+                uuid = self.q.get(block=False)
             except queue.Empty:
                 pass
 
@@ -123,13 +123,14 @@ class update_worker(threading.Thread):
                 self.current_uuid = uuid
 
                 if uuid in list(self.datastore.data['watching'].keys()):
+
                     changed_detected = False
                     contents = b''
                     screenshot = False
                     update_obj= {}
                     xpath_data = False
                     process_changedetection_results = True
-                    print("> Processing UUID {} Priority {} URL {}".format(uuid, priority, self.datastore.data['watching'][uuid]['url']))
+
                     now = time.time()
 
                     try: