bump comments

- :latest should be :latest stable release according to a new release, and also the :0.xx.xx tag for the release
- :dev can be what ever is going on in `master`

.github/workflows/containers.yml

@@ -85,8 +85,8 @@ jobs:
           version: latest
           driver-opts: image=moby/buildkit:master
 
-      # master always builds :latest
-      - name: Build and push :latest
+      # master branch -> :dev container tag
+      - name: Build and push :dev
         id: docker_build
         if: ${{ github.ref }} == "refs/heads/master"
         uses: docker/build-push-action@v2
@@ -95,12 +95,12 @@ jobs:
           file: ./Dockerfile
           push: true
           tags: |
-            ${{ secrets.DOCKER_HUB_USERNAME }}/changedetection.io:latest,ghcr.io/${{ github.repository }}:latest
+            ${{ secrets.DOCKER_HUB_USERNAME }}/changedetection.io:dev,ghcr.io/${{ github.repository }}:dev
           platforms: linux/amd64,linux/arm64,linux/arm/v6,linux/arm/v7
           cache-from: type=local,src=/tmp/.buildx-cache
           cache-to: type=local,dest=/tmp/.buildx-cache
 
-      # A new tagged release is required, which builds :tag
+      # A new tagged release is required, which builds :tag and :latest
       - name: Build and push :tag
         id: docker_build_tag_release
         if: github.event_name == 'release' && startsWith(github.event.release.tag_name, '0.')
@@ -110,7 +110,10 @@ jobs:
           file: ./Dockerfile
           push: true
           tags: |
-            ${{ secrets.DOCKER_HUB_USERNAME }}/changedetection.io:${{ github.event.release.tag_name }},ghcr.io/dgtlmoon/changedetection.io:${{ github.event.release.tag_name }}
+            ${{ secrets.DOCKER_HUB_USERNAME }}/changedetection.io:${{ github.event.release.tag_name }}
+            ghcr.io/dgtlmoon/changedetection.io:${{ github.event.release.tag_name }}
+            ${{ secrets.DOCKER_HUB_USERNAME }}/changedetection.io:latest
+            ghcr.io/dgtlmoon/changedetection.io:latest
           platforms: linux/amd64,linux/arm64,linux/arm/v6,linux/arm/v7
           cache-from: type=local,src=/tmp/.buildx-cache
           cache-to: type=local,dest=/tmp/.buildx-cache
@@ -125,5 +128,3 @@ jobs:
           key: ${{ runner.os }}-buildx-${{ github.sha }}
           restore-keys: |
             ${{ runner.os }}-buildx-
-
-

README.md

@@ -67,14 +67,18 @@ Available when connected to a <a href="https://github.com/dgtlmoon/changedetecti
 ### Docker
 
 With Docker composer, just clone this repository and..
+
 ```bash
 $ docker-compose up -d
 ```
+
 Docker standalone
 ```bash
 $ docker run -d --restart always -p "127.0.0.1:5000:5000" -v datastore-volume:/datastore --name changedetection.io dgtlmoon/changedetection.io
 ```
 
+`:latest` tag is our latest stable release, `:dev` tag is our bleeding edge `master` branch.
+
 ### Windows
 
 See the install instructions at the wiki https://github.com/dgtlmoon/changedetection.io/wiki/Microsoft-Windows

changedetectionio/__init__.py

@@ -1267,7 +1267,7 @@ def notification_runner():
             try:
                 from changedetectionio import notification
 
-                notification.process_notification(n_object, datastore)
+                sent_obj = notification.process_notification(n_object, datastore)
 
             except Exception as e:
                 logging.error("Watch URL: {}  Error {}".format(n_object['watch_url'], str(e)))
@@ -1281,14 +1281,17 @@ def notification_runner():
                 notification_debug_log += log_lines
 
             # Process notifications
-            notification_debug_log+= ["{} - SENDING {}".format(now.strftime("%Y/%m/%d %H:%M:%S,000"), json.dumps(n_object))]
+            notification_debug_log+= ["{} - SENDING - {}".format(now.strftime("%Y/%m/%d %H:%M:%S,000"), json.dumps(sent_obj))]
             # Trim the log length
             notification_debug_log = notification_debug_log[-100:]
 
 # Thread runner to check every minute, look for new watches to feed into the Queue.
 def ticker_thread_check_time_launch_checks():
+    import random
     from changedetectionio import update_worker
-    import logging
+
+    recheck_time_minimum_seconds = int(os.getenv('MINIMUM_SECONDS_RECHECK_TIME', 20))
+    print("System env MINIMUM_SECONDS_RECHECK_TIME", recheck_time_minimum_seconds)
 
     # Spin up Workers that do the fetching
     # Can be overriden by ENV or use the default settings
@@ -1321,14 +1324,12 @@ def ticker_thread_check_time_launch_checks():
         while update_q.qsize() >= 2000:
             time.sleep(1)
 
+
+        recheck_time_system_seconds = int(datastore.threshold_seconds)
+
         # Check for watches outside of the time threshold to put in the thread queue.
-        now = time.time()
-
-        recheck_time_minimum_seconds = int(os.getenv('MINIMUM_SECONDS_RECHECK_TIME', 60))
-        recheck_time_system_seconds = datastore.threshold_seconds
-
         for uuid in watch_uuid_list:
-
+            now = time.time()
             watch = datastore.data['watching'].get(uuid)
             if not watch:
                 logging.error("Watch: {} no longer present.".format(uuid))
@@ -1339,20 +1340,33 @@ def ticker_thread_check_time_launch_checks():
                 continue
 
             # If they supplied an individual entry minutes to threshold.
-            threshold = now
-            watch_threshold_seconds = watch.threshold_seconds()
-            if watch_threshold_seconds:
-                threshold -= watch_threshold_seconds
-            else:
-                threshold -= recheck_time_system_seconds
 
-            # Yeah, put it in the queue, it's more than time
-            if watch['last_checked'] <= max(threshold, recheck_time_minimum_seconds):
+            watch_threshold_seconds = watch.threshold_seconds()
+            threshold = watch_threshold_seconds if watch_threshold_seconds > 0 else recheck_time_system_seconds
+
+            # #580 - Jitter plus/minus amount of time to make the check seem more random to the server
+            jitter = datastore.data['settings']['requests'].get('jitter_seconds', 0)
+            if jitter > 0:
+                if watch.jitter_seconds == 0:
+                    watch.jitter_seconds = random.uniform(-abs(jitter), jitter)
+
+
+            seconds_since_last_recheck = now - watch['last_checked']
+            if seconds_since_last_recheck >= (threshold + watch.jitter_seconds) and seconds_since_last_recheck >= recheck_time_minimum_seconds:
                 if not uuid in running_uuids and uuid not in update_q.queue:
+                    print("Queued watch UUID {} last checked at {} queued at {:0.2f} jitter {:0.2f}s, {:0.2f}s since last checked".format(uuid,
+                                                                                                         watch['last_checked'],
+                                                                                                         now,
+                                                                                                         watch.jitter_seconds,
+                                                                                                         now - watch['last_checked']))
+                    # Into the queue with you
                     update_q.put(uuid)
 
-        # Wait a few seconds before checking the list again
-        time.sleep(3)
+                    # Reset for next time
+                    watch.jitter_seconds = 0
+
+        # Wait before checking the list again - saves CPU
+        time.sleep(1)
 
         # Should be low so we can break this out in testing
         app.config.exit.wait(1)

changedetectionio/content_fetcher.py

@@ -281,13 +281,14 @@ class base_html_playwright(Fetcher):
         from playwright.sync_api import sync_playwright
         import playwright._impl._api_types
         from playwright._impl._api_types import Error, TimeoutError
-
+        response = None
         with sync_playwright() as p:
             browser_type = getattr(p, self.browser_type)
 
             # Seemed to cause a connection Exception even tho I can see it connect
             # self.browser = browser_type.connect(self.command_executor, timeout=timeout*1000)
-            browser = browser_type.connect_over_cdp(self.command_executor, timeout=timeout * 1000)
+            # 60,000 connection timeout only
+            browser = browser_type.connect_over_cdp(self.command_executor, timeout=60000)
 
             # Set user agent to prevent Cloudflare from blocking the browser
             # Use the default one configured in the App.py model that's passed from fetch_site_status.py
@@ -302,19 +303,24 @@ class base_html_playwright(Fetcher):
 
             page = context.new_page()
             try:
+                page.set_default_navigation_timeout(90000)
+                page.set_default_timeout(90000)
+
                # Bug - never set viewport size BEFORE page.goto
-                response = page.goto(url, timeout=timeout * 1000, wait_until='commit')
-                # Wait_until = commit
-                # - `'commit'` - consider operation to be finished when network response is received and the document started loading.
-                # Better to not use any smarts from Playwright and just wait an arbitrary number of seconds
-                # This seemed to solve nearly all 'TimeoutErrors'
-                extra_wait = int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)) + self.render_extract_delay
-                page.wait_for_timeout(extra_wait * 1000)
+
+                # Waits for the next navigation. Using Python context manager
+                # prevents a race condition between clicking and waiting for a navigation.
+                with page.expect_navigation():
+                    response = page.goto(url, wait_until='load')
+
             except playwright._impl._api_types.TimeoutError as e:
                 context.close()
                 browser.close()
-                raise EmptyReply(url=url, status_code=None)
+                # This can be ok, we will try to grab what we could retrieve
+                pass
             except Exception as e:
+                print ("other exception when page.goto")
+                print (str(e))
                 context.close()
                 browser.close()
                 raise PageUnloadable(url=url, status_code=None)
@@ -322,18 +328,22 @@ class base_html_playwright(Fetcher):
             if response is None:
                 context.close()
                 browser.close()
-                raise EmptyReply(url=url, status_code=None)
-
-            if len(page.content().strip()) == 0:
-                context.close()
-                browser.close()
+                print ("response object was none")
                 raise EmptyReply(url=url, status_code=None)
 
             # Bug 2(?) Set the viewport size AFTER loading the page
-            page.set_viewport_size({"width": 1280, "height": 1024})
-
-            self.status_code = response.status
+            page.set_viewport_size({"width": 1280, "height": 1024})            
+            extra_wait = int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)) + self.render_extract_delay
+            time.sleep(extra_wait)
             self.content = page.content()
+            self.status_code = response.status
+
+            if len(self.content.strip()) == 0:
+                context.close()
+                browser.close()
+                print ("Content was empty")
+                raise EmptyReply(url=url, status_code=None)
+            
             self.headers = response.all_headers()
 
             if current_css_filter is not None:
@@ -346,9 +356,15 @@ class base_html_playwright(Fetcher):
             # Bug 3 in Playwright screenshot handling
             # Some bug where it gives the wrong screenshot size, but making a request with the clip set first seems to solve it
             # JPEG is better here because the screenshots can be very very large
+
+            # Screenshots also travel via the ws:// (websocket) meaning that the binary data is base64 encoded
+            # which will significantly increase the IO size between the server and client, it's recommended to use the lowest
+            # acceptable screenshot quality here
             try:
-                page.screenshot(type='jpeg', clip={'x': 1.0, 'y': 1.0, 'width': 1280, 'height': 1024})
-                self.screenshot = page.screenshot(type='jpeg', full_page=True, quality=92)
+                # Quality set to 1 because it's not used, just used as a work-around for a bug, no need to change this.
+                page.screenshot(type='jpeg', clip={'x': 1.0, 'y': 1.0, 'width': 1280, 'height': 1024}, quality=1)
+                # The actual screenshot
+                self.screenshot = page.screenshot(type='jpeg', full_page=True, quality=int(os.getenv("PLAYWRIGHT_SCREENSHOT_QUALITY", 72)))
             except Exception as e:
                 context.close()
                 browser.close()

changedetectionio/forms.py

@@ -363,7 +363,9 @@ class watchForm(commonSettingsForm):
 class globalSettingsRequestForm(Form):
     time_between_check = FormField(TimeBetweenCheckForm)
     proxy = RadioField('Proxy')
-
+    jitter_seconds = IntegerField('Random jitter seconds ± check',
+                                  render_kw={"style": "width: 5em;"},
+                                  validators=[validators.NumberRange(min=0, message="Should contain zero or more seconds")])
 
 # datastore.data['settings']['application']..
 class globalSettingsApplicationForm(commonSettingsForm):

changedetectionio/model/App.py

@@ -23,6 +23,7 @@ class model(dict):
                 'requests': {
                     'timeout': 15,  # Default 15 seconds
                     'time_between_check': {'weeks': None, 'days': None, 'hours': 3, 'minutes': None, 'seconds': None},
+                    'jitter_seconds': 0,
                     'workers': 10,  # Number of threads, lower is better for slow connections
                     'proxy': None # Preferred proxy connection
                 },

changedetectionio/model/Watch.py

@@ -13,7 +13,6 @@ from changedetectionio.notification import (
 class model(dict):
     __newest_history_key = None
     __history_n=0
-
     __base_config = {
             'url': None,
             'tag': None,
@@ -48,7 +47,8 @@ class model(dict):
             'time_between_check': {'weeks': None, 'days': None, 'hours': None, 'minutes': None, 'seconds': None},
             'webdriver_delay': None
         }
-
+    jitter_seconds = 0
+    mtable = {'seconds': 1, 'minutes': 60, 'hours': 3600, 'days': 86400, 'weeks': 86400 * 7}
     def __init__(self, *arg, **kw):
         import uuid
         self.update(self.__base_config)
@@ -157,8 +157,7 @@ class model(dict):
 
     def threshold_seconds(self):
         seconds = 0
-        mtable = {'seconds': 1, 'minutes': 60, 'hours': 3600, 'days': 86400, 'weeks': 86400 * 7}
-        for m, n in mtable.items():
+        for m, n in self.mtable.items():
             x = self.get('time_between_check', {}).get(m, None)
             if x:
                 seconds += x * n

changedetectionio/notification.py

@@ -67,6 +67,11 @@ def process_notification(n_object, datastore):
                     url += k + 'avatar_url=https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/changedetectionio/static/images/avatar-256x256.png'
 
                 if url.startswith('tgram://'):
+                    # Telegram only supports a limit subset of HTML, remove the '<br/>' we place in.
+                    # re https://github.com/dgtlmoon/changedetection.io/issues/555
+                    # @todo re-use an existing library we have already imported to strip all non-allowed tags
+                    n_body = n_body.replace('<br/>', '\n')
+                    n_body = n_body.replace('</br>', '\n')
                     # real limit is 4096, but minus some for extra metadata
                     payload_max_size = 3600
                     body_limit = max(0, payload_max_size - len(n_title))
@@ -97,6 +102,10 @@ def process_notification(n_object, datastore):
                 if log_value and 'WARNING' in log_value or 'ERROR' in log_value:
                     raise Exception(log_value)
 
+                # Return what was sent for better logging
+                return {'title': n_title,
+                        'body': n_body,
+                        'body_format': n_format}
 
 
 # Notification title + body content parameters get created here.

changedetectionio/run_all_tests.sh

@@ -9,6 +9,8 @@
 # exit when any command fails
 set -e
 
+export MINIMUM_SECONDS_RECHECK_TIME=0
+
 find tests/test_*py -type f|while read test_name
 do
   echo "TEST RUNNING $test_name"

changedetectionio/store.py

@@ -159,12 +159,11 @@ class ChangeDetectionStore:
     def threshold_seconds(self):
         seconds = 0
         mtable = {'seconds': 1, 'minutes': 60, 'hours': 3600, 'days': 86400, 'weeks': 86400 * 7}
-        minimum_seconds_recheck_time = int(os.getenv('MINIMUM_SECONDS_RECHECK_TIME', 60))
         for m, n in mtable.items():
             x = self.__data['settings']['requests']['time_between_check'].get(m)
             if x:
                 seconds += x * n
-        return max(seconds, minimum_seconds_recheck_time)
+        return seconds
 
     @property
     def has_unviewed(self):

changedetectionio/templates/_common_fields.jinja

@@ -14,7 +14,7 @@
                                 <li>Use <a target=_new href="https://github.com/caronc/apprise">AppRise URLs</a> for notification to just about any service! <i><a target=_new href="https://github.com/dgtlmoon/changedetection.io/wiki/Notification-configuration-notes">Please read the notification services wiki here for important configuration notes</a></i>.</li>
                                 <li><code>discord://</code> only supports a maximum <strong>2,000 characters</strong> of notification text, including the title.</li>
                                 <li><code>tgram://</code> bots cant send messages to other bots, so you should specify chat ID of non-bot user.</li>
-
+                                <li><code>tgram://</code> only supports very limited HTML and can fail when extra tags are sent, <a href="https://core.telegram.org/bots/api#html-style">read more here</a> (or use plaintext/markdown format)</li>
                               </ul>
                             </div>
                             <br/>

changedetectionio/templates/settings.html

@@ -32,6 +32,11 @@
                         {{ render_field(form.requests.form.time_between_check, class="time-check-widget") }}
                         <span class="pure-form-message-inline">Default time for all watches, when the watch does not have a specific time setting.</span>
                     </div>
+                    <div class="pure-control-group">
+                        {{ render_field(form.requests.form.jitter_seconds, class="jitter_seconds") }}
+                        <span class="pure-form-message-inline">Example - 3 seconds random jitter could trigger up to 3 seconds earlier or up to 3 seconds later</span>
+                    </div>
+
                     <div class="pure-control-group">
                         {% if not hide_remove_pass %}
                             {% if current_user.is_authenticated %}

changedetectionio/tests/test_notification.py

@@ -154,6 +154,10 @@ def test_check_notification(client, live_server):
     time.sleep(1)
     assert os.path.exists("test-datastore/notification.txt") == False
 
+    res = client.get(url_for("notification_logs"))
+    # be sure we see it in the output log
+    assert b'New ChangeDetection.io Notification - ' + test_url.encode('utf-8') in res.data
+
     # cleanup for the next
     client.get(
         url_for("form_delete", uuid="all"),

changedetectionio/update_worker.py

@@ -45,7 +45,6 @@ class update_worker(threading.Thread):
 
                     try:
                         changed_detected, update_obj, contents, screenshot, xpath_data = update_handler.run(uuid)
-
                         # Re #342
                         # In Python 3, all strings are sequences of Unicode characters. There is a bytes type that holds raw bytes.
                         # We then convert/.decode('utf-8') for the notification etc
@@ -56,18 +55,18 @@ class update_worker(threading.Thread):
                     except content_fetcher.ReplyWithContentButNoText as e:
                         # Totally fine, it's by choice - just continue on, nothing more to care about
                         # Page had elements/content but no renderable text
-                        if self.datastore.data['watching'][uuid].get('css_filter'):
+                        if self.datastore.data['watching'].get(uuid, False) and self.datastore.data['watching'][uuid].get('css_filter'):
                             self.datastore.update_watch(uuid=uuid, update_obj={'last_error': "Got HTML content but no text found (CSS / xPath Filter not found in page?)"})
                         else:
                             self.datastore.update_watch(uuid=uuid, update_obj={'last_error': "Got HTML content but no text found."})
                         pass
                     except content_fetcher.EmptyReply as e:
                         # Some kind of custom to-str handler in the exception handler that does this?
-                        err_text = "EmptyReply: Status Code {}".format(e.status_code)
+                        err_text = "EmptyReply - try increasing 'Wait seconds before extracting text', Status Code {}".format(e.status_code)
                         self.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text,
                                                                            'last_check_status': e.status_code})
                     except content_fetcher.ScreenshotUnavailable as e:
-                        err_text = "Screenshot unavailable, page did not render fully in the expected time"
+                        err_text = "Screenshot unavailable, page did not render fully in the expected time - try increasing 'Wait seconds before extracting text'"
                         self.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text,
                                                                            'last_check_status': e.status_code})
                     except content_fetcher.PageUnloadable as e: