Move global vars to all live under the 'app' var

changedetectionio/__init__.py

@@ -2,7 +2,7 @@
 
 # Read more https://github.com/dgtlmoon/changedetection.io/wiki
 
-__version__ = '0.47.01'
+__version__ = '0.47.03'
 
 from changedetectionio.strtobool import strtobool
 from json.decoder import JSONDecodeError

changedetectionio/forms.py

@@ -496,7 +496,7 @@ class processor_text_json_diff_form(commonSettingsForm):
     text_should_not_be_present = StringListField('Block change-detection while text matches', [validators.Optional(), ValidateListRegex()])
     webdriver_js_execute_code = TextAreaField('Execute JavaScript before change detection', render_kw={"rows": "5"}, validators=[validators.Optional()])
 
-    save_button = SubmitField('Save', render_kw={"class": "pure-button pure-button-primary"})
+    save_button = SubmitField('Save', render_kw={"class": "pure-button button-small pure-button-primary"})
 
     proxy = RadioField('Proxy')
     filter_failure_notification_send = BooleanField(
@@ -616,7 +616,7 @@ class globalSettingsForm(Form):
 
     requests = FormField(globalSettingsRequestForm)
     application = FormField(globalSettingsApplicationForm)
-    save_button = SubmitField('Save', render_kw={"class": "pure-button pure-button-primary"})
+    save_button = SubmitField('Save', render_kw={"class": "pure-button button-small pure-button-primary"})
 
 
 class extractDataForm(Form):

changedetectionio/processors/text_json_diff/processor.py

@@ -331,13 +331,21 @@ class perform_site_check(difference_detection_processor):
             if result:
                 blocked = True
 
-        # The main thing that all this at the moment comes down to :)
-        if watch.get('previous_md5') != fetched_md5:
-            changed_detected = True
 
         # Looks like something changed, but did it match all the rules?
         if blocked:
             changed_detected = False
+        else:
+            # The main thing that all this at the moment comes down to :)
+            if watch.get('previous_md5') != fetched_md5:
+                changed_detected = True
+
+            # Always record the new checksum
+            update_obj["previous_md5"] = fetched_md5
+
+            # On the first run of a site, watch['previous_md5'] will be None, set it the current one.
+            if not watch.get('previous_md5'):
+                watch['previous_md5'] = fetched_md5
 
         logger.debug(f"Watch UUID {watch.get('uuid')} content check - Previous MD5: {watch.get('previous_md5')}, Fetched MD5 {fetched_md5}")
 
@@ -357,12 +365,6 @@ class perform_site_check(difference_detection_processor):
                 else:
                     logger.debug(f"check_unique_lines: UUID {watch.get('uuid')} had unique content")
 
-        # Always record the new checksum
-        update_obj["previous_md5"] = fetched_md5
-
-        # On the first run of a site, watch['previous_md5'] will be None, set it the current one.
-        if not watch.get('previous_md5'):
-            watch['previous_md5'] = fetched_md5
 
         # stripped_text_from_html - Everything after filters and NO 'ignored' content
         return changed_detected, update_obj, stripped_text_from_html

changedetectionio/static/js/tabs.js

@@ -26,8 +26,7 @@ function set_active_tab() {
     if (tab.length) {
         tab[0].parentElement.className = "active";
     }
-    // hash could move the page down
-    window.scrollTo(0, 0);
+
 }
 
 function focus_error_tab() {

changedetectionio/static/styles/diff.css

@@ -153,7 +153,8 @@ html[data-darkmode="true"] {
     border: 1px solid transparent;
     vertical-align: top;
     font: 1em monospace;
-    text-align: left; }
+    text-align: left;
+    overflow: clip; }
   #diff-ui pre {
     white-space: pre-wrap; }
 
@@ -172,7 +173,9 @@ ins {
   text-decoration: none; }
 
 #result {
-  white-space: pre-wrap; }
+  white-space: pre-wrap;
+  word-break: break-word;
+  overflow-wrap: break-word; }
 
 #settings {
   background: rgba(0, 0, 0, 0.05);
@@ -231,3 +234,12 @@ td#diff-col div {
   border-radius: 5px;
   background: var(--color-background);
   box-shadow: 1px 1px 4px var(--color-shadow-jump); }
+
+.pure-form button.reset-margin {
+  margin: 0px; }
+
+.diff-fieldset {
+  display: flex;
+  align-items: center;
+  gap: 4px;
+  flex-wrap: wrap; }

changedetectionio/static/styles/scss/diff.scss

@@ -24,6 +24,7 @@
     vertical-align: top;
     font: 1em monospace;
     text-align: left;
+    overflow: clip; // clip overflowing contents to cell boundariess
   }
 
   pre {
@@ -50,6 +51,8 @@ ins {
 
 #result {
   white-space: pre-wrap;
+  word-break: break-word;
+  overflow-wrap: break-word;
 
   .change {
     span {}
@@ -134,3 +137,15 @@ td#diff-col div {
   background: var(--color-background);
   box-shadow: 1px 1px 4px var(--color-shadow-jump);
 }
+
+// resets button margin to 0px
+.pure-form button.reset-margin {
+  margin: 0px;
+}
+
+.diff-fieldset {
+  display: flex;
+  align-items: center;
+  gap: 4px;
+  flex-wrap: wrap;
+}

changedetectionio/static/styles/scss/parts/_extra_browsers.scss

@@ -11,7 +11,22 @@ ul#requests-extra_browsers {
   /* each proxy entry is a `table` */
   table {
     tr {
-      display: inline;
+      display: table-row; // default display for small screens
+      input[type=text] {
+        width: 100%;
+      }
+    }
+  }
+  
+  // apply inline display for larger screens
+  @media only screen and (min-width: 1280px) {
+    table {
+      tr {
+        display: inline;
+        input[type=text] {
+          width: 100%;
+        }
+      }
     }
   }
 }

changedetectionio/static/styles/scss/parts/_extra_proxies.scss

@@ -11,7 +11,19 @@ ul#requests-extra_proxies {
   /* each proxy entry is a `table` */
   table {
     tr {
-      display: inline;
+      display: table-row; // default display for small screens
+      input[type=text] {
+        width: 100%;
+      }
+    }
+  }
+  
+  // apply inline display for large screens
+  @media only screen and (min-width: 1024px) {
+    table {
+      tr {
+        display: inline;
+      }
     }
   }
 }

changedetectionio/static/styles/scss/styles.scss

@@ -148,7 +148,7 @@ body.spinner-active {
 }
 
 
-.tabs ul li a {
+.tab-pane-inner {
   // .tab-pane-inner will have the #id that the tab button jumps/anchors to
   scroll-margin-top: 200px;
 }

changedetectionio/static/styles/styles.css

@@ -112,7 +112,12 @@ ul#requests-extra_proxies {
   ul#requests-extra_proxies li > label {
     display: none; }
   ul#requests-extra_proxies table tr {
-    display: inline; }
+    display: table-row; }
+    ul#requests-extra_proxies table tr input[type=text] {
+      width: 100%; }
+  @media only screen and (min-width: 1024px) {
+    ul#requests-extra_proxies table tr {
+      display: inline; } }
 
 #request {
   /* Auto proxy scan/checker */ }
@@ -161,7 +166,14 @@ ul#requests-extra_browsers {
   ul#requests-extra_browsers li > label {
     display: none; }
   ul#requests-extra_browsers table tr {
-    display: inline; }
+    display: table-row; }
+    ul#requests-extra_browsers table tr input[type=text] {
+      width: 100%; }
+  @media only screen and (min-width: 1280px) {
+    ul#requests-extra_browsers table tr {
+      display: inline; }
+      ul#requests-extra_browsers table tr input[type=text] {
+        width: 100%; } }
 
 #extra-browsers-setting {
   border: 1px solid var(--color-grey-800);
@@ -605,8 +617,7 @@ body.spinner-active #pure-menu-horizontal-spinner {
     background-color: var(--color-background-menu-link-hover);
     color: var(--color-text-menu-link-hover); }
 
-
-.tabs ul li a {
+.tab-pane-inner {
   scroll-margin-top: 200px; }
 
 section.content {

changedetectionio/templates/diff.html

@@ -14,7 +14,7 @@
 
 <div id="settings">
     <form class="pure-form " action="" method="GET" id="diff-form">
-        <fieldset>
+        <fieldset class="diff-fieldset">
             {% if versions|length >= 1 %}
                 <strong>Compare</strong>
                 <del class="change"><span>from</span></del>
@@ -33,7 +33,7 @@
                         </option>
                     {% endfor %}
                 </select>
-                <button type="submit" class="pure-button pure-button-primary">Go</button>
+                <button type="submit" class="pure-button pure-button-primary reset-margin">Go</button>
             {% endif %}
         </fieldset>
         <fieldset>

changedetectionio/templates/settings.html

@@ -276,7 +276,7 @@ nav
                 <div class="pure-control-group">
                     {{ render_button(form.save_button) }}
                     <a href="{{url_for('index')}}" class="pure-button button-small button-cancel">Back</a>
-                    <a href="{{url_for('clear_all_history')}}" class="pure-button button-small button-cancel">Clear Snapshot History</a>
+                    <a href="{{url_for('clear_all_history')}}" class="pure-button button-small button-error">Clear Snapshot History</a>
                 </div>
             </div>
         </form>

changedetectionio/tests/test_block_while_text_present.py

@@ -65,11 +65,8 @@ def test_check_block_changedetection_text_NOT_present(client, live_server, measu
     live_server_setup(live_server)
     # Use a mix of case in ZzZ to prove it works case-insensitive.
     ignore_text = "out of stoCk\r\nfoobar"
-
     set_original_ignore_response()
 
-    # Give the endpoint time to spin up
-    time.sleep(1)
 
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)
@@ -127,13 +124,24 @@ def test_check_block_changedetection_text_NOT_present(client, live_server, measu
     assert b'unviewed' not in res.data
     assert b'/test-endpoint' in res.data
 
+    # 2548
+    # Going back to the ORIGINAL should NOT trigger a change
+    set_original_ignore_response()
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    wait_for_all_checks(client)
+    res = client.get(url_for("index"))
+    assert b'unviewed' not in res.data
 
-    # Now we set a change where the text is gone, it should now trigger
+
+    # Now we set a change where the text is gone AND its different content, it should now trigger
     set_modified_response_minus_block_text()
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
     res = client.get(url_for("index"))
     assert b'unviewed' in res.data
 
+
+
+
     res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data

changedetectionio/tests/test_notification.py

@@ -429,3 +429,24 @@ def test_global_send_test_notification(client, live_server, measure_memory_usage
         follow_redirects=True
     )
 
+    #2727 - be sure a test notification when there are zero watches works ( should all be deleted now)
+
+    os.unlink("test-datastore/notification.txt")
+
+
+    ######### Test global/system settings
+    res = client.post(
+        url_for("ajax_callback_send_notification_test")+"?mode=global-settings",
+        data={"notification_urls": test_notification_url},
+        follow_redirects=True
+    )
+
+    assert res.status_code != 400
+    assert res.status_code != 500
+
+    # Give apprise time to fire
+    time.sleep(4)
+
+    with open("test-datastore/notification.txt", 'r') as f:
+        x = f.read()
+        assert 'change detection is cool 网站监测 内容更新了' in x

changedetectionio/update_worker.py

@@ -19,11 +19,9 @@ from loguru import logger
 class update_worker(threading.Thread):
     current_uuid = None
 
-    def __init__(self, q, notification_q, app, datastore, *args, **kwargs):
-        self.q = q
+    def __init__(self, app, *args, **kwargs):
+
         self.app = app
-        self.notification_q = notification_q
-        self.datastore = datastore
         super().__init__(*args, **kwargs)
 
     def queue_notification_for_watch(self, notification_q, n_object, watch):
@@ -81,7 +79,8 @@ class update_worker(threading.Thread):
             'watch_url': watch.get('url') if watch else None,
         })
 
-        n_object.update(watch.extra_notification_token_values())
+        if watch:
+            n_object.update(watch.extra_notification_token_values())
 
         logger.trace(f"Main rendered notification placeholders (diff_added etc) calculated in {time.time()-now:.3f}s")
         logger.debug("Queued notification for sending")
@@ -101,19 +100,19 @@ class update_worker(threading.Thread):
         v = watch.get(var_name)
         if v and not watch.get('notification_muted'):
             if var_name == 'notification_format' and v == default_notification_format_for_watch:
-                return self.datastore.data['settings']['application'].get('notification_format')
+                return self.app.datastore.data['settings']['application'].get('notification_format')
 
             return v
 
-        tags = self.datastore.get_all_tags_for_watch(uuid=watch.get('uuid'))
+        tags = self.app.datastore.get_all_tags_for_watch(uuid=watch.get('uuid'))
         if tags:
             for tag_uuid, tag in tags.items():
                 v = tag.get(var_name)
                 if v and not tag.get('notification_muted'):
                     return v
 
-        if self.datastore.data['settings']['application'].get(var_name):
-            return self.datastore.data['settings']['application'].get(var_name)
+        if self.app.datastore.data['settings']['application'].get(var_name):
+            return self.app.datastore.data['settings']['application'].get(var_name)
 
         # Otherwise could be defaults
         if var_name == 'notification_format':
@@ -128,7 +127,7 @@ class update_worker(threading.Thread):
     def send_content_changed_notification(self, watch_uuid):
 
         n_object = {}
-        watch = self.datastore.data['watching'].get(watch_uuid)
+        watch = self.app.datastore.data['watching'].get(watch_uuid)
         if not watch:
             return
 
@@ -155,17 +154,17 @@ class update_worker(threading.Thread):
             queued = True
 
             count = watch.get('notification_alert_count', 0) + 1
-            self.datastore.update_watch(uuid=watch_uuid, update_obj={'notification_alert_count': count})
+            self.app.datastore.update_watch(uuid=watch_uuid, update_obj={'notification_alert_count': count})
 
-            self.queue_notification_for_watch(notification_q=self.notification_q, n_object=n_object, watch=watch)
+            self.queue_notification_for_watch(notification_q=self.app.notification_q, n_object=n_object, watch=watch)
 
         return queued
 
 
     def send_filter_failure_notification(self, watch_uuid):
 
-        threshold = self.datastore.data['settings']['application'].get('filter_failure_notification_threshold_attempts')
-        watch = self.datastore.data['watching'].get(watch_uuid)
+        threshold = self.app.datastore.data['settings']['application'].get('filter_failure_notification_threshold_attempts')
+        watch = self.app.datastore.data['watching'].get(watch_uuid)
         if not watch:
             return
 
@@ -178,8 +177,8 @@ class update_worker(threading.Thread):
         if len(watch['notification_urls']):
             n_object['notification_urls'] = watch['notification_urls']
 
-        elif len(self.datastore.data['settings']['application']['notification_urls']):
-            n_object['notification_urls'] = self.datastore.data['settings']['application']['notification_urls']
+        elif len(self.app.datastore.data['settings']['application']['notification_urls']):
+            n_object['notification_urls'] = self.app.datastore.data['settings']['application']['notification_urls']
 
         # Only prepare to notify if the rules above matched
         if 'notification_urls' in n_object:
@@ -188,16 +187,16 @@ class update_worker(threading.Thread):
                 'uuid': watch_uuid,
                 'screenshot': None
             })
-            self.notification_q.put(n_object)
+            self.app.notification_q.put(n_object)
             logger.debug(f"Sent filter not found notification for {watch_uuid}")
         else:
             logger.debug(f"NOT sending filter not found notification for {watch_uuid} - no notification URLs")
 
     def send_step_failure_notification(self, watch_uuid, step_n):
-        watch = self.datastore.data['watching'].get(watch_uuid, False)
+        watch = self.app.datastore.data['watching'].get(watch_uuid, False)
         if not watch:
             return
-        threshold = self.datastore.data['settings']['application'].get('filter_failure_notification_threshold_attempts')
+        threshold = self.app.datastore.data['settings']['application'].get('filter_failure_notification_threshold_attempts')
         n_object = {'notification_title': "Changedetection.io - Alert - Browser step at position {} could not be run".format(step_n+1),
                     'notification_body': "Your configured browser step at position {} for {{{{watch_url}}}} "
                                          "did not appear on the page after {} attempts, did the page change layout? "
@@ -208,8 +207,8 @@ class update_worker(threading.Thread):
         if len(watch['notification_urls']):
             n_object['notification_urls'] = watch['notification_urls']
 
-        elif len(self.datastore.data['settings']['application']['notification_urls']):
-            n_object['notification_urls'] = self.datastore.data['settings']['application']['notification_urls']
+        elif len(self.app.datastore.data['settings']['application']['notification_urls']):
+            n_object['notification_urls'] = self.app.datastore.data['settings']['application']['notification_urls']
 
         # Only prepare to notify if the rules above matched
         if 'notification_urls' in n_object:
@@ -217,7 +216,7 @@ class update_worker(threading.Thread):
                 'watch_url': watch['url'],
                 'uuid': watch_uuid
             })
-            self.notification_q.put(n_object)
+            self.app.notification_q.put(n_object)
             logger.error(f"Sent step not found notification for {watch_uuid}")
 
 
@@ -225,7 +224,7 @@ class update_worker(threading.Thread):
         # All went fine, remove error artifacts
         cleanup_files = ["last-error-screenshot.png", "last-error.txt"]
         for f in cleanup_files:
-            full_path = os.path.join(self.datastore.datastore_path, uuid, f)
+            full_path = os.path.join(self.app.datastore.datastore_path, uuid, f)
             if os.path.isfile(full_path):
                 os.unlink(full_path)
 
@@ -236,23 +235,23 @@ class update_worker(threading.Thread):
             update_handler = None
 
             try:
-                queued_item_data = self.q.get(block=False)
+                queued_item_data = self.app.update_q.get(block=False)
             except queue.Empty:
                 pass
 
             else:
                 uuid = queued_item_data.item.get('uuid')
                 self.current_uuid = uuid
-                if uuid in list(self.datastore.data['watching'].keys()) and self.datastore.data['watching'][uuid].get('url'):
+                if uuid in list(self.app.datastore.data['watching'].keys()) and self.app.datastore.data['watching'][uuid].get('url'):
                     changed_detected = False
                     contents = b''
                     process_changedetection_results = True
                     update_obj = {}
 
                     # Clear last errors (move to preflight func?)
-                    self.datastore.data['watching'][uuid]['browser_steps_last_error_step'] = None
+                    self.app.datastore.data['watching'][uuid]['browser_steps_last_error_step'] = None
 
-                    watch = self.datastore.data['watching'].get(uuid)
+                    watch = self.app.datastore.data['watching'].get(uuid)
 
                     logger.info(f"Processing watch UUID {uuid} Priority {queued_item_data.priority} URL {watch['url']}")
                     now = time.time()
@@ -269,7 +268,7 @@ class update_worker(threading.Thread):
                             print(f"Processor module '{processor}' not found.")
                             raise e
 
-                        update_handler = processor_module.perform_site_check(datastore=self.datastore,
+                        update_handler = processor_module.perform_site_check(datastore=self.app.datastore,
                                                                              watch_uuid=uuid
                                                                              )
 
@@ -293,7 +292,7 @@ class update_worker(threading.Thread):
                             watch.save_screenshot(screenshot=e.screenshot)
                         if e.xpath_data:
                             watch.save_xpath_data(data=e.xpath_data)
-                        self.datastore.update_watch(uuid=uuid, update_obj={'last_error': e.message})
+                        self.app.datastore.update_watch(uuid=uuid, update_obj={'last_error': e.message})
                         process_changedetection_results = False
 
                     except content_fetchers_exceptions.ReplyWithContentButNoText as e:
@@ -310,7 +309,7 @@ class update_worker(threading.Thread):
                             else:
                                 extra_help = ", it's possible that the filters were found, but contained no usable text."
 
-                        self.datastore.update_watch(uuid=uuid, update_obj={
+                        self.app.datastore.update_watch(uuid=uuid, update_obj={
                             'last_error': f"Got HTML content but no text found (With {e.status_code} reply code){extra_help}"
                         })
 
@@ -342,15 +341,15 @@ class update_worker(threading.Thread):
                         if e.page_text:
                             watch.save_error_text(contents=e.page_text)
 
-                        self.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text})
+                        self.app.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text})
                         process_changedetection_results = False
 
                     except FilterNotFoundInResponse as e:
-                        if not self.datastore.data['watching'].get(uuid):
+                        if not self.app.datastore.data['watching'].get(uuid):
                             continue
 
                         err_text = "Warning, no filters were found, no change detection ran - Did the page change layout? update your Visual Filter if necessary."
-                        self.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text})
+                        self.app.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text})
 
                         # Filter wasnt found, but we should still update the visual selector so that they can have a chance to set it up again
                         if e.screenshot:
@@ -364,7 +363,7 @@ class update_worker(threading.Thread):
                             c = watch.get('consecutive_filter_failures', 0)
                             c += 1
                             # Send notification if we reached the threshold?
-                            threshold = self.datastore.data['settings']['application'].get('filter_failure_notification_threshold_attempts', 0)
+                            threshold = self.app.datastore.data['settings']['application'].get('filter_failure_notification_threshold_attempts', 0)
                             logger.debug(f"Filter for {uuid} not found, consecutive_filter_failures: {c} of threshold {threshold}")
                             if c >= threshold:
                                 if not watch.get('notification_muted'):
@@ -373,7 +372,7 @@ class update_worker(threading.Thread):
                                 c = 0
                                 logger.debug(f"Reset filter failure count back to zero")
 
-                            self.datastore.update_watch(uuid=uuid, update_obj={'consecutive_filter_failures': c})
+                            self.app.datastore.update_watch(uuid=uuid, update_obj={'consecutive_filter_failures': c})
                         else:
                             logger.trace(f"{uuid} - filter_failure_notification_send not enabled, skipping")
 
@@ -385,20 +384,20 @@ class update_worker(threading.Thread):
                         process_changedetection_results = False
                         changed_detected = False
                     except content_fetchers_exceptions.BrowserConnectError as e:
-                        self.datastore.update_watch(uuid=uuid,
+                        self.app.datastore.update_watch(uuid=uuid,
                                                     update_obj={'last_error': e.msg
                                                                 }
                                                     )
                         process_changedetection_results = False
                     except content_fetchers_exceptions.BrowserFetchTimedOut as e:
-                        self.datastore.update_watch(uuid=uuid,
+                        self.app.datastore.update_watch(uuid=uuid,
                                                     update_obj={'last_error': e.msg
                                                                 }
                                                     )
                         process_changedetection_results = False
                     except content_fetchers_exceptions.BrowserStepsStepException as e:
 
-                        if not self.datastore.data['watching'].get(uuid):
+                        if not self.app.datastore.data['watching'].get(uuid):
                             continue
 
                         error_step = e.step_n + 1
@@ -416,7 +415,7 @@ class update_worker(threading.Thread):
 
                         logger.debug(f"BrowserSteps exception at step {error_step} {str(e.original_e)}")
 
-                        self.datastore.update_watch(uuid=uuid,
+                        self.app.datastore.update_watch(uuid=uuid,
                                                     update_obj={'last_error': err_text,
                                                                 'browser_steps_last_error_step': error_step
                                                                 }
@@ -426,7 +425,7 @@ class update_worker(threading.Thread):
                             c = watch.get('consecutive_filter_failures', 0)
                             c += 1
                             # Send notification if we reached the threshold?
-                            threshold = self.datastore.data['settings']['application'].get('filter_failure_notification_threshold_attempts',
+                            threshold = self.app.datastore.data['settings']['application'].get('filter_failure_notification_threshold_attempts',
                                                                                            0)
                             logger.error(f"Step for {uuid} not found, consecutive_filter_failures: {c}")
                             if threshold > 0 and c >= threshold:
@@ -434,26 +433,26 @@ class update_worker(threading.Thread):
                                     self.send_step_failure_notification(watch_uuid=uuid, step_n=e.step_n)
                                 c = 0
 
-                            self.datastore.update_watch(uuid=uuid, update_obj={'consecutive_filter_failures': c})
+                            self.app.datastore.update_watch(uuid=uuid, update_obj={'consecutive_filter_failures': c})
 
                         process_changedetection_results = False
 
                     except content_fetchers_exceptions.EmptyReply as e:
                         # Some kind of custom to-str handler in the exception handler that does this?
                         err_text = "EmptyReply - try increasing 'Wait seconds before extracting text', Status Code {}".format(e.status_code)
-                        self.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text,
+                        self.app.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text,
                                                                            'last_check_status': e.status_code})
                         process_changedetection_results = False
                     except content_fetchers_exceptions.ScreenshotUnavailable as e:
                         err_text = "Screenshot unavailable, page did not render fully in the expected time or page was too long - try increasing 'Wait seconds before extracting text'"
-                        self.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text,
+                        self.app.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text,
                                                                            'last_check_status': e.status_code})
                         process_changedetection_results = False
                     except content_fetchers_exceptions.JSActionExceptions as e:
                         err_text = "Error running JS Actions - Page request - "+e.message
                         if e.screenshot:
                             watch.save_screenshot(screenshot=e.screenshot, as_error=True)
-                        self.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text,
+                        self.app.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text,
                                                                            'last_check_status': e.status_code})
                         process_changedetection_results = False
                     except content_fetchers_exceptions.PageUnloadable as e:
@@ -464,26 +463,26 @@ class update_worker(threading.Thread):
                         if e.screenshot:
                             watch.save_screenshot(screenshot=e.screenshot, as_error=True)
 
-                        self.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text,
+                        self.app.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text,
                                                                            'last_check_status': e.status_code,
                                                                            'has_ldjson_price_data': None})
                         process_changedetection_results = False
                     except content_fetchers_exceptions.BrowserStepsInUnsupportedFetcher as e:
                         err_text = "This watch has Browser Steps configured and so it cannot run with the 'Basic fast Plaintext/HTTP Client', either remove the Browser Steps or select a Chrome fetcher."
-                        self.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text})
+                        self.app.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text})
                         process_changedetection_results = False
                         logger.error(f"Exception (BrowserStepsInUnsupportedFetcher) reached processing watch UUID: {uuid}")
 
                     except Exception as e:
                         logger.error(f"Exception reached processing watch UUID: {uuid}")
                         logger.error(str(e))
-                        self.datastore.update_watch(uuid=uuid, update_obj={'last_error': "Exception: " + str(e)})
+                        self.app.datastore.update_watch(uuid=uuid, update_obj={'last_error': "Exception: " + str(e)})
                         # Other serious error
                         process_changedetection_results = False
 
                     else:
                         # Crash protection, the watch entry could have been removed by this point (during a slow chrome fetch etc)
-                        if not self.datastore.data['watching'].get(uuid):
+                        if not self.app.datastore.data['watching'].get(uuid):
                             continue
 
                         update_obj['content-type'] = update_handler.fetcher.get_all_headers().get('content-type', '').lower()
@@ -497,14 +496,14 @@ class update_worker(threading.Thread):
 
                         self.cleanup_error_artifacts(uuid)
 
-                    if not self.datastore.data['watching'].get(uuid):
+                    if not self.app.datastore.data['watching'].get(uuid):
                         continue
                     #
                     # Different exceptions mean that we may or may not want to bump the snapshot, trigger notifications etc
                     if process_changedetection_results:
 
                         # Extract <title> as title if possible/requested.
-                        if self.datastore.data['settings']['application'].get('extract_title_as_title') or watch['extract_title_as_title']:
+                        if self.app.datastore.data['settings']['application'].get('extract_title_as_title') or watch['extract_title_as_title']:
                             if not watch['title'] or not len(watch['title']):
                                 try:
                                     update_obj['title'] = html_tools.extract_element(find='title', html_content=update_handler.fetcher.content)
@@ -515,7 +514,7 @@ class update_worker(threading.Thread):
                         # Now update after running everything
                         timestamp = round(time.time())
                         try:
-                            self.datastore.update_watch(uuid=uuid, update_obj=update_obj)
+                            self.app.datastore.update_watch(uuid=uuid, update_obj=update_obj)
 
 
                             # Also save the snapshot on the first time checked, "last checked" will always be updated, so we just check history length.
@@ -553,7 +552,7 @@ class update_worker(threading.Thread):
                             # Catch everything possible here, so that if a worker crashes, we don't lose it until restart!
                             logger.critical("!!!! Exception in update_worker while processing process_changedetection_results !!!")
                             logger.critical(str(e))
-                            self.datastore.update_watch(uuid=uuid, update_obj={'last_error': str(e)})
+                            self.app.datastore.update_watch(uuid=uuid, update_obj={'last_error': str(e)})
 
 
                     # Always record that we atleast tried
@@ -562,13 +561,13 @@ class update_worker(threading.Thread):
                     # Record the 'server' header reply, can be used for actions in the future like cloudflare/akamai workarounds
                     try:
                         server_header = update_handler.fetcher.headers.get('server', '').strip().lower()[:255]
-                        self.datastore.update_watch(uuid=uuid,
+                        self.app.datastore.update_watch(uuid=uuid,
                                                     update_obj={'remote_server_reply': server_header}
                                                     )
                     except Exception as e:
                         pass
 
-                    self.datastore.update_watch(uuid=uuid, update_obj={'fetch_time': round(time.time() - now, 3),
+                    self.app.datastore.update_watch(uuid=uuid, update_obj={'fetch_time': round(time.time() - now, 3),
                                                                        'last_checked': round(time.time()),
                                                                        'check_count': count
                                                                        })

requirements.txt

@@ -82,7 +82,7 @@ pytest-flask ~=1.2
 # Anything 4.0 and up but not 5.0
 jsonschema ~= 4.0
 
-
+apscheduler ~= 3.9
 loguru
 
 # For scraping all possible metadata relating to products so we can do better restock detection