Attempt to save last data retrieved- WIP

README.md

@@ -1,21 +1,15 @@
-#  changedetection.io
+## Web Site Change Detection, Monitoring and Notification.
+
+[**Try our $6.99/month subscription - Unlimited checks and watches!**](https://lemonade.changedetection.io/start)
+ 
+
+[<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/docs/screenshot.png" style="max-width:100%;" alt="Self-hosted web page change monitoring"  title="Self-hosted web page change monitoring"  />](https://lemonade.changedetection.io/start)
+
 [![Release Version][release-shield]][release-link] [![Docker Pulls][docker-pulls]][docker-link] [![License][license-shield]](LICENSE.md)
 
 ![changedetection.io](https://github.com/dgtlmoon/changedetection.io/actions/workflows/test-only.yml/badge.svg?branch=master)
 
-## Web Site Change Detection, Monitoring and Notification - Self-Hosted or SaaS.
+Know when important content changes, we support notifications via Discord, Telegram, Home-Assistant, Slack, Email and 70+ more
-
-_Know when web pages change! Stay ontop of new information! get notifications when important website content changes_ 
-
-Live your data-life *pro-actively* instead of *re-actively*.
-
-Free, Open-source web page monitoring, notification and change detection. Don't have time? [**Try our $6.99/month subscription - unlimited checks and watches!**](https://lemonade.changedetection.io/start)
-
-
-[<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/docs/screenshot.png" style="max-width:100%;" alt="Self-hosted web page change monitoring"  title="Self-hosted web page change monitoring"  />](https://lemonade.changedetection.io/start)
-
-
-**Get your own private instance now! Let us host it for you!**
 
 [**Try our $6.99/month subscription - unlimited checks and watches!**](https://lemonade.changedetection.io/start) , _half the price of other website change monitoring services and comes with unlimited watches & checks!_
 

changedetectionio/__init__.py

@@ -44,7 +44,7 @@ from flask_wtf import CSRFProtect
 from changedetectionio import html_tools
 from changedetectionio.api import api_v1
 
-__version__ = '0.39.17.2'
+__version__ = '0.39.18'
 
 datastore = None
 
@@ -1186,6 +1186,36 @@ def changedetection_app(config=None, datastore_o=None):
         flash("{} watches are queued for rechecking.".format(i))
         return redirect(url_for('index', tag=tag))
 
+    @app.route("/form/checkbox-operations", methods=['POST'])
+    @login_required
+    def form_watch_list_checkbox_operations():
+        op = request.form['op']
+        uuids = request.form.getlist('uuids')
+
+        if (op == 'delete'):
+            for uuid in uuids:
+                uuid = uuid.strip()
+                if datastore.data['watching'].get(uuid):
+                    datastore.delete(uuid.strip())
+            flash("{} watches deleted".format(len(uuids)))
+
+        if (op == 'pause'):
+            for uuid in uuids:
+                uuid = uuid.strip()
+                if datastore.data['watching'].get(uuid):
+                    datastore.data['watching'][uuid.strip()]['paused'] = True
+
+            flash("{} watches paused".format(len(uuids)))
+
+        if (op == 'unpause'):
+            for uuid in uuids:
+                uuid = uuid.strip()
+                if datastore.data['watching'].get(uuid):
+                    datastore.data['watching'][uuid.strip()]['paused'] = False
+            flash("{} watches unpaused".format(len(uuids)))
+
+        return redirect(url_for('index'))
+
     @app.route("/api/share-url", methods=['GET'])
     @login_required
     def form_share_put_watch():
@@ -1385,13 +1415,18 @@ def ticker_thread_check_time_launch_checks():
             seconds_since_last_recheck = now - watch['last_checked']
             if seconds_since_last_recheck >= (threshold + watch.jitter_seconds) and seconds_since_last_recheck >= recheck_time_minimum_seconds:
                 if not uuid in running_uuids and uuid not in [q_uuid for p,q_uuid in update_q.queue]:
-                    print("> Queued watch UUID {} last checked at {} queued at {:0.2f} jitter {:0.2f}s, {:0.2f}s since last checked".format(uuid,
+                    # Use Epoch time as priority, so we get a "sorted" PriorityQueue, but we can still push a priority 1 into it.
-                                                                                                         watch['last_checked'],
+                    priority = int(time.time())
-                                                                                                         now,
+                    print(
-                                                                                                         watch.jitter_seconds,
+                        "> Queued watch UUID {} last checked at {} queued at {:0.2f} priority {} jitter {:0.2f}s, {:0.2f}s since last checked".format(
-                                                                                                         now - watch['last_checked']))
+                            uuid,
+                            watch['last_checked'],
+                            now,
+                            priority,
+                            watch.jitter_seconds,
+                            now - watch['last_checked']))
                     # Into the queue with you
-                    update_q.put((5, uuid))
+                    update_q.put((priority, uuid))
 
                     # Reset for next time
                     watch.jitter_seconds = 0

changedetectionio/content_fetcher.py

@@ -31,11 +31,12 @@ class JSActionExceptions(Exception):
         return
 
 class PageUnloadable(Exception):
-    def __init__(self, status_code, url, screenshot=False):
+    def __init__(self, status_code, url, screenshot=False, message=False):
         # Set this so we can use it in other parts of the app
         self.status_code = status_code
         self.url = url
         self.screenshot = screenshot
+        self.message = message
         return
 
 class EmptyReply(Exception):
@@ -292,7 +293,15 @@ class base_html_playwright(Fetcher):
 
         # allow per-watch proxy selection override
         if proxy_override:
-            self.proxy = {'server': proxy_override}
+            # https://playwright.dev/docs/network#http-proxy
+            from urllib.parse import urlparse
+            parsed = urlparse(proxy_override)
+            proxy_url = "{}://{}:{}".format(parsed.scheme, parsed.hostname, parsed.port)
+            self.proxy = {'server': proxy_url}
+            if parsed.username:
+                self.proxy['username'] = parsed.username
+            if parsed.password:
+                self.proxy['password'] = parsed.password
 
     def run(self,
             url,
@@ -356,7 +365,7 @@ class base_html_playwright(Fetcher):
                 print(str(e))
                 context.close()
                 browser.close()
-                raise PageUnloadable(url=url, status_code=None)
+                raise PageUnloadable(url=url, status_code=None, message=e.message)
 
             if response is None:
                 context.close()

changedetectionio/fetch_site_status.py

@@ -13,6 +13,9 @@ urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 # Some common stuff here that can be moved to a base class
 # (set_proxy_from_list)
 class perform_site_check():
+    screenshot = None
+    xpath_data = None
+    fetched_response = None
 
     def __init__(self, *args, datastore, **kwargs):
         super().__init__(*args, **kwargs)
@@ -127,6 +130,10 @@ class perform_site_check():
         fetcher.run(url, timeout, request_headers, request_body, request_method, ignore_status_codes, watch['css_filter'])
         fetcher.quit()
 
+        self.screenshot = fetcher.screenshot
+        self.xpath_data = fetcher.xpath_data
+        self.fetched_response = fetcher.content
+
         # Fetching complete, now filters
         # @todo move to class / maybe inside of fetcher abstract base?
 
@@ -312,4 +319,4 @@ class perform_site_check():
         if not watch.get('previous_md5'):
             watch['previous_md5'] = fetched_md5
 
-        return changed_detected, update_obj, text_content_before_ignored_filter, fetcher.screenshot, fetcher.xpath_data
+        return changed_detected, update_obj, text_content_before_ignored_filter

changedetectionio/run_all_tests.sh

@@ -38,13 +38,14 @@ docker kill $$-test_selenium
 
 echo "TESTING WEBDRIVER FETCH > PLAYWRIGHT/BROWSERLESS..."
 # Not all platforms support playwright (not ARM/rPI), so it's not packaged in requirements.txt
-pip3 install playwright~=1.22
+pip3 install playwright~=1.24
 docker run -d --name $$-test_browserless -e "DEFAULT_LAUNCH_ARGS=[\"--window-size=1920,1080\"]" --rm  -p 3000:3000  --shm-size="2g"  browserless/chrome:1.53-chrome-stable
 # takes a while to spin up
 sleep 5
 export PLAYWRIGHT_DRIVER_URL=ws://127.0.0.1:3000
 pytest tests/fetchers/test_content.py
 pytest tests/test_errorhandling.py
+pytest tests/visualselector/test_fetch_data.py
 
 unset PLAYWRIGHT_DRIVER_URL
 docker kill $$-test_browserless

changedetectionio/static/js/watch-overview.js

@@ -22,5 +22,18 @@ $(function () {
       });
   });
 
+    // checkboxes - check all
+    $("#check-all").click(function (e) {
+        $('input[type=checkbox]').not(this).prop('checked', this.checked);
+    });
+    // checkboxes - show/hide buttons
+    $("input[type=checkbox]").click(function (e) {
+        if ($('input[type=checkbox]:checked').length) {
+            $('#checkbox-operations').slideDown();
+        } else {
+            $('#checkbox-operations').slideUp();
+        }
+    });
+
 });
 

changedetectionio/static/styles/styles.css

@@ -555,3 +555,13 @@ ul {
   .snapshot-age.error {
     background-color: #ff0000;
     color: #fff; }
+
+#checkbox-operations {
+  background: rgba(0, 0, 0, 0.05);
+  padding: 1em;
+  border-radius: 10px;
+  margin-bottom: 1em;
+  display: none; }
+
+.checkbox-uuid > * {
+  vertical-align: middle; }

changedetectionio/static/styles/styles.scss

@@ -774,3 +774,15 @@ ul {
   }
 }
 
+#checkbox-operations {
+  background: rgba(0, 0, 0, 0.05);
+  padding: 1em;
+  border-radius: 10px;
+  margin-bottom: 1em;
+  display: none;
+}
+.checkbox-uuid {
+  > * {
+    vertical-align: middle;
+  }
+}

changedetectionio/store.py

@@ -341,6 +341,8 @@ class ChangeDetectionStore:
 
     # Save as PNG, PNG is larger but better for doing visual diff in the future
     def save_screenshot(self, watch_uuid, screenshot: bytes, as_error=False):
+        if not self.data['watching'].get(watch_uuid):
+            return
 
         if as_error:
             target_path = os.path.join(self.datastore_path, watch_uuid, "last-error-screenshot.png")
@@ -354,23 +356,36 @@ class ChangeDetectionStore:
             f.close()
 
     def save_error_text(self, watch_uuid, contents):
-
+        if not self.data['watching'].get(watch_uuid):
+            return
         target_path = os.path.join(self.datastore_path, watch_uuid, "last-error.txt")
 
         with open(target_path, 'w') as f:
             f.write(contents)
 
     def save_xpath_data(self, watch_uuid, data, as_error=False):
-
+        if not self.data['watching'].get(watch_uuid):
+            return
         if as_error:
-            target_path = os.path.join(self.datastore_path, watch_uuid, "elements.json")
-        else:
             target_path = os.path.join(self.datastore_path, watch_uuid, "elements-error.json")
+        else:
+            target_path = os.path.join(self.datastore_path, watch_uuid, "elements.json")
 
         with open(target_path, 'w') as f:
             f.write(json.dumps(data))
             f.close()
 
+    # Save whatever was returned from the fetcher
+    def save_last_response(self, watch_uuid, data):
+        if not self.data['watching'].get(watch_uuid):
+            return
+
+        target_path = os.path.join(self.datastore_path, watch_uuid, "last-response.bin")
+        # mimetype? binary? text? @todo
+        # gzip if its non-binary? auto get encoding?
+        with open(target_path, 'wb') as f:
+            f.write(data)
+            f.close()
 
     def sync_to_json(self):
         logging.info("Saving JSON..")

changedetectionio/templates/watch-overview.html

@@ -24,6 +24,14 @@
         </fieldset>
         <span style="color:#eee; font-size: 80%;"><img style="height: 1em;display:inline-block;" src="{{url_for('static_content', group='images', filename='spread-white.svg')}}" /> Tip: You can also add 'shared' watches. <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Sharing-a-Watch">More info</a></a></span>
     </form>
+
+    <form class="pure-form" action="{{ url_for('form_watch_list_checkbox_operations') }}" method="POST" id="watch-list-form">
+    <input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
+    <div id="checkbox-operations">
+        <button class="pure-button button-secondary button-xsmall" style="font-size: 70%"  name="op" value="pause">Pause</button>
+        <button class="pure-button button-secondary button-xsmall" style="font-size: 70%"  name="op" value="unpause">UnPause</button>
+        <button class="pure-button button-secondary button-xsmall" style="background: #dd4242; font-size: 70%" name="op" value="delete">Delete</button>
+    </div>
     <div>
         <a href="{{url_for('index')}}" class="pure-button button-tag {{'active' if not active_tag }}">All</a>
         {% for tag in tags %}
@@ -41,7 +49,7 @@
         <table class="pure-table pure-table-striped watch-table">
             <thead>
             <tr>
-                <th>#</th>
+                <th><input style="vertical-align: middle" type="checkbox" id="check-all"/> #</th>
                 <th></th>
                 {% set link_order = "desc" if sort_order else "asc" %}
                 {% set arrow_span = "" %}
@@ -66,7 +74,7 @@
                 {% if watch.paused is defined and watch.paused != False %}paused{% endif %}
                 {% if watch.newest_history_key| int > watch.last_viewed and watch.history_n>=2 %}unviewed{% endif %}
                 {% if watch.uuid in queued_uuids %}queued{% endif %}">
-                <td class="inline">{{ loop.index }}</td>
+                <td class="inline checkbox-uuid" ><input name="uuids"  type="checkbox" value="{{ watch.uuid}} "/> <span>{{ loop.index }}</span></td>
                 <td class="inline watch-controls">
                     <a class="state-{{'on' if watch.paused }}" href="{{url_for('index', op='pause', uuid=watch.uuid, tag=active_tag)}}"><img src="{{url_for('static_content', group='images', filename='pause.svg')}}" alt="Pause checks" title="Pause checks"/></a>
                     <a class="state-{{'on' if watch.notification_muted}}" href="{{url_for('index', op='mute', uuid=watch.uuid, tag=active_tag)}}"><img src="{{url_for('static_content', group='images', filename='bell-off.svg')}}" alt="Mute notifications" title="Mute notifications"/></a>
@@ -129,5 +137,6 @@
          #}
 
     </div>
+    </form>
 </div>
 {% endblock %}

changedetectionio/tests/fetchers/test_content.py

@@ -2,7 +2,7 @@
 
 import time
 from flask import url_for
-from ..util import live_server_setup
+from ..util import live_server_setup, wait_for_all_checks
 import logging
 
 
@@ -29,14 +29,8 @@ def test_fetch_webdriver_content(client, live_server):
 
     assert b"1 Imported" in res.data
     time.sleep(3)
-    attempt = 0
+
-    while attempt < 20:
+    wait_for_all_checks(client)
-        res = client.get(url_for("index"))
-        if not b'Checking now' in res.data:
-            break
-        logging.getLogger().info("Waiting for check to not say 'Checking now'..")
-        time.sleep(3)
-        attempt += 1
 
 
     res = client.get(

changedetectionio/tests/util.py

@@ -2,6 +2,8 @@
 
 from flask import make_response, request
 from flask import url_for
+import logging
+import time
 
 def set_original_response():
     test_return_data = """<html>
@@ -68,6 +70,31 @@ def extract_api_key_from_UI(client):
     api_key = m.group(1)
     return api_key.strip()
 
+
+# kinda funky, but works for now
+def extract_UUID_from_client(client):
+    import re
+    res = client.get(
+        url_for("index"),
+    )
+    # <span id="api-key">{{api_key}}</span>
+
+    m = re.search('edit/(.+?)"', str(res.data))
+    uuid = m.group(1)
+    return uuid.strip()
+
+def wait_for_all_checks(client):
+    # Loop waiting until done..
+    attempt=0
+    while attempt < 60:
+        time.sleep(1)
+        res = client.get(url_for("index"))
+        if not b'Checking now' in res.data:
+            break
+        logging.getLogger().info("Waiting for watch-list to not say 'Checking now'.. {}".format(attempt))
+
+        attempt += 1
+
 def live_server_setup(live_server):
 
     @live_server.app.route('/test-endpoint')
@@ -133,3 +160,4 @@ def live_server_setup(live_server):
         return ret
 
     live_server.start()
+

changedetectionio/tests/visualselector/__init__.py

@@ -0,0 +1,2 @@
+"""Tests for the app."""
+

changedetectionio/tests/visualselector/conftest.py

@@ -0,0 +1,3 @@
+#!/usr/bin/python3
+
+from .. import conftest

changedetectionio/tests/visualselector/test_fetch_data.py

@@ -0,0 +1,35 @@
+#!/usr/bin/python3
+
+import time
+from flask import url_for
+from ..util import live_server_setup, wait_for_all_checks, extract_UUID_from_client
+
+# Add a site in paused mode, add an invalid filter, we should still have visual selector data ready
+def test_visual_selector_content_ready(client, live_server):
+    import os
+
+    assert os.getenv('PLAYWRIGHT_DRIVER_URL'), "Needs PLAYWRIGHT_DRIVER_URL set for this test"
+    live_server_setup(live_server)
+    time.sleep(1)
+
+    # Add our URL to the import page, maybe better to use something we control?
+    # We use an external URL because the docker container is too difficult to setup to connect back to the pytest socket
+    test_url = 'https://news.ycombinator.com'
+    res = client.post(
+        url_for("form_quick_watch_add"),
+        data={"url": test_url, "tag": '', 'edit_and_watch_submit_button': 'Edit > Watch'},
+        follow_redirects=True
+    )
+    assert b"Watch added in Paused state, saving will unpause" in res.data
+
+    res = client.post(
+        url_for("edit_page", uuid="first", unpause_on_save=1),
+        data={"css_filter": ".does-not-exist", "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_webdriver"},
+        follow_redirects=True
+    )
+    assert b"unpaused" in res.data
+    time.sleep(1)
+    wait_for_all_checks(client)
+    uuid = extract_UUID_from_client(client)
+    assert os.path.isfile(os.path.join('test-datastore', uuid, 'last-screenshot.png')), "last-screenshot.png should exist"
+    assert os.path.isfile(os.path.join('test-datastore', uuid, 'elements.json')), "xpath elements.json data should exist"

changedetectionio/update_worker.py

@@ -142,7 +142,7 @@ class update_worker(threading.Thread):
                     now = time.time()
 
                     try:
-                        changed_detected, update_obj, contents, screenshot, xpath_data = update_handler.run(uuid)
+                        changed_detected, update_obj, contents = update_handler.run(uuid)
                         # Re #342
                         # In Python 3, all strings are sequences of Unicode characters. There is a bytes type that holds raw bytes.
                         # We then convert/.decode('utf-8') for the notification etc
@@ -223,6 +223,9 @@ class update_worker(threading.Thread):
                                                                            'last_check_status': e.status_code})
                     except content_fetcher.PageUnloadable as e:
                         err_text = "Page request from server didnt respond correctly"
+                        if e.message:
+                            err_text = "{} - {}".format(err_text, e.message)
+
                         if e.screenshot:
                             self.datastore.save_screenshot(watch_uuid=uuid, screenshot=e.screenshot, as_error=True)
 
@@ -234,6 +237,9 @@ class update_worker(threading.Thread):
                         # Other serious error
                         process_changedetection_results = False
                     else:
+                        # Crash protection, the watch entry could have been removed by this point (during a slow chrome fetch etc)
+                        if not self.datastore.data['watching'].get(uuid):
+                            continue
 
                         # Mark that we never had any failures
                         if not self.datastore.data['watching'][uuid].get('ignore_status_codes'):
@@ -241,10 +247,6 @@ class update_worker(threading.Thread):
 
                         self.cleanup_error_artifacts(uuid)
 
-                    # Crash protection, the watch entry could have been removed by this point (during a slow chrome fetch etc)
-                    if not self.datastore.data['watching'].get(uuid):
-                        continue
-
                     # Different exceptions mean that we may or may not want to bump the snapshot, trigger notifications etc
                     if process_changedetection_results:
                         try:
@@ -280,10 +282,13 @@ class update_worker(threading.Thread):
                                                                        'last_checked': round(time.time())})
 
                     # Always save the screenshot if it's available
-                    if screenshot:
+                    if update_handler.screenshot:
-                        self.datastore.save_screenshot(watch_uuid=uuid, screenshot=screenshot)
+                        self.datastore.save_screenshot(watch_uuid=uuid, screenshot=update_handler.screenshot)
-                    if xpath_data:
+                    if update_handler.xpath_data:
-                        self.datastore.save_xpath_data(watch_uuid=uuid, data=xpath_data)
+                        self.datastore.save_xpath_data(watch_uuid=uuid, data=update_handler.xpath_data)
+                    if update_handler.fetched_response:
+                        # @todo mimetype?
+                        self.datastore.save_last_response(watch_uuid=uuid, data=update_handler.fetched_response)
 
 
                 self.current_uuid = None  # Done