Fix test

Dockerfile

@@ -9,6 +9,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     gcc \
     libc-dev \
     libffi-dev \
+    libjpeg-dev \
     libssl-dev \
     libxslt-dev \
     make \
@@ -36,13 +37,14 @@ ARG CRYPTOGRAPHY_DONT_BUILD_RUST=1
 
 # Re #93, #73, excluding rustc (adds another 430Mb~)
 RUN apt-get update && apt-get install -y --no-install-recommends \
-    libssl-dev \
-    libffi-dev \
+    g++ \
     gcc \
     libc-dev \
+    libffi-dev \
+    libjpeg-dev \
+    libssl-dev \
     libxslt-dev \
-    zlib1g-dev \
-    g++
+    zlib1g-dev
 
 # https://stackoverflow.com/questions/58701233/docker-logs-erroneously-appears-empty-until-container-stops
 ENV PYTHONUNBUFFERED=1

README.md

@@ -40,6 +40,8 @@ Easily see what changed, examine by word, line, or individual character.
 - Create RSS feeds based on changes in web content
 - Monitor HTML source code for unexpected changes, strengthen your PCI compliance
 - You have a very sensitive list of URLs to watch and you do _not_ want to use the paid alternatives. (Remember, _you_ are the product)
+- Get notified when certain keywords appear in Twitter search results
+- Proactively search for jobs, get notified when companies update their careers page, search job portals for keywords.
 
 _Need an actual Chrome runner with Javascript support? We support fetching via WebDriver and Playwright!</a>_
 
@@ -53,6 +55,7 @@ _Need an actual Chrome runner with Javascript support? We support fetching via W
 - Override Request Headers, Specify `POST` or `GET` and other methods
 - Use the "Visual Selector" to help target specific elements
 - Configurable [proxy per watch](https://github.com/dgtlmoon/changedetection.io/wiki/Proxy-configuration)
+- Send a screenshot with the notification when a change is detected in the web page
 
 We [recommend and use Bright Data](https://brightdata.grsm.io/n0r16zf7eivq) global proxy services, Bright Data will match any first deposit up to $100 using our signup link.
 

changedetectionio/__init__.py

@@ -33,7 +33,7 @@ from flask_wtf import CSRFProtect
 from changedetectionio import html_tools
 from changedetectionio.api import api_v1
 
-__version__ = '0.39.21.1'
+__version__ = '0.39.22'
 
 datastore = None
 
@@ -644,12 +644,18 @@ def changedetection_app(config=None, datastore_o=None):
             except ModuleNotFoundError:
                 jq_support = False
 
+            watch = datastore.data['watching'].get(uuid)
+            system_uses_webdriver = datastore.data['settings']['application']['fetch_backend'] == 'html_webdriver'
+            is_html_webdriver = True if watch.get('fetch_backend') == 'html_webdriver' or (
+                    watch.get('fetch_backend', None) is None and system_uses_webdriver) else False
+
             output = render_template("edit.html",
                                      current_base_url=datastore.data['settings']['application']['base_url'],
                                      emailprefix=os.getenv('NOTIFICATION_MAIL_BUTTON_PREFIX', False),
                                      form=form,
                                      has_default_notification_urls=True if len(datastore.data['settings']['application']['notification_urls']) else False,
                                      has_empty_checktime=using_default_check_time,
+                                     is_html_webdriver=is_html_webdriver,
                                      jq_support=jq_support,
                                      playwright_enabled=os.getenv('PLAYWRIGHT_DRIVER_URL', False),
                                      settings_application=datastore.data['settings']['application'],
@@ -657,7 +663,7 @@ def changedetection_app(config=None, datastore_o=None):
                                      uuid=uuid,
                                      visualselector_data_is_ready=visualselector_data_is_ready,
                                      visualselector_enabled=visualselector_enabled,
-                                     watch=datastore.data['watching'][uuid],
+                                     watch=watch
                                      )
 
         return output

changedetectionio/forms.py

@@ -375,6 +375,7 @@ class watchForm(commonSettingsForm):
         'Send a notification when the filter can no longer be found on the page', default=False)
 
     notification_muted = BooleanField('Notifications Muted / Off', default=False)
+    notification_screenshot = BooleanField('Attach screenshot to notification (where possible)', default=False)
 
     def validate(self, **kwargs):
         if not super().validate():

changedetectionio/model/Watch.py

@@ -38,6 +38,7 @@ class model(dict):
             'notification_format': default_notification_format_for_watch,
             'notification_muted': False,
             'notification_title': None,
+            'notification_screenshot': False, # Include the latest screenshot if available and supported by the apprise URL
             'notification_urls': [],  # List of URLs to add to the notification Queue (Usually AppRise)
             'paused': False,
             'previous_md5': False,
@@ -247,7 +248,19 @@ class model(dict):
         if os.path.isfile(fname):
             return fname
 
-        return False
+        # False is not an option for AppRise, must be type None
+        return None
+
+    def get_screenshot_as_jpeg(self):
+
+        # Created by save_screenshot()
+        fname = os.path.join(self.watch_data_dir, "last-screenshot.jpg")
+        if os.path.isfile(fname):
+            return fname
+
+        # False is not an option for AppRise, must be type None
+        return None
+
 
     def __get_file_ctime(self, filename):
         fname = os.path.join(self.watch_data_dir, filename)

changedetectionio/notification.py

@@ -101,7 +101,10 @@ def process_notification(n_object, datastore):
                 apobj.notify(
                     title=n_title,
                     body=n_body,
-                    body_format=n_format)
+                    body_format=n_format,
+                    # False is not an option for AppRise, must be type None
+                    attach=n_object.get('screenshot', None)
+                )
 
                 apobj.clear()
 

changedetectionio/static/js/visual-selector.js

@@ -13,7 +13,7 @@ $(document).ready(function() {
     // redline highlight context
     var ctx;
 
-    var current_default_xpath;
+    var current_default_xpath=[];
     var x_scale=1;
     var y_scale=1;
     var selector_image;
@@ -57,21 +57,24 @@ $(document).ready(function() {
     bootstrap_visualselector();
 
 
-
     function bootstrap_visualselector() {
-        if ( 1 ) {
+        if (1) {
             // bootstrap it, this will trigger everything else
             $("img#selector-background").bind('load', function () {
                 console.log("Loaded background...");
-               c = document.getElementById("selector-canvas");
+                c = document.getElementById("selector-canvas");
                 // greyed out fill context
-               xctx = c.getContext("2d");
+                xctx = c.getContext("2d");
                 // redline highlight context
-               ctx = c.getContext("2d");
-               current_default_xpath =$("#include_filters").val().split(/\r?\n/g);
-               fetch_data();
-               $('#selector-canvas').off("mousemove mousedown");
-               // screenshot_url defined in the edit.html template
+                ctx = c.getContext("2d");
+                if ($("#include_filters").val().trim().length) {
+                    current_default_xpath = $("#include_filters").val().split(/\r?\n/g);
+                } else {
+                    current_default_xpath = [];
+                }
+                fetch_data();
+                $('#selector-canvas').off("mousemove mousedown");
+                // screenshot_url defined in the edit.html template
             }).attr("src", screenshot_url);
         }
     }

changedetectionio/store.py

@@ -368,6 +368,12 @@ class ChangeDetectionStore:
             f.write(screenshot)
             f.close()
 
+        # Make a JPEG that's used in notifications (due to being a smaller size) available
+        from PIL import Image
+        im1 = Image.open(target_path)
+        im1.convert('RGB').save(target_path.replace('.png','.jpg'), quality=int(os.getenv("NOTIFICATION_SCREENSHOT_JPG_QUALITY", 75)))
+
+
     def save_error_text(self, watch_uuid, contents):
         if not self.data['watching'].get(watch_uuid):
             return

changedetectionio/templates/edit.html

@@ -141,6 +141,14 @@ User-Agent: wonderbra 1.0") }}
                     <div  class="pure-control-group inline-radio">
                       {{ render_checkbox_field(form.notification_muted) }}
                     </div>
+                    {% if is_html_webdriver %}
+                    <div class="pure-control-group inline-radio">
+                      {{ render_checkbox_field(form.notification_screenshot) }}
+                        <span class="pure-form-message-inline">
+                            <strong>Use with caution!</strong> This will easily fill up your email storage quota or flood other storages.
+                        </span>
+                    </div>
+                    {% endif %}
                     <div class="field-group" id="notification-field-group">
                         {% if has_default_notification_urls %}
                         <div class="inline-warning">

changedetectionio/tests/test_auth.py

@@ -19,6 +19,7 @@ def test_basic_auth(client, live_server):
         follow_redirects=True
     )
     assert b"1 Imported" in res.data
+    time.sleep(1)
 
     # Check form validation
     res = client.post(
@@ -28,8 +29,6 @@ def test_basic_auth(client, live_server):
     )
     assert b"Updated watch." in res.data
 
-    # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
     time.sleep(1)
     res = client.get(
         url_for("preview_page", uuid="first"),

changedetectionio/tests/test_css_selector.py

@@ -89,9 +89,6 @@ def test_check_markup_include_filters_restriction(client, live_server):
     )
     assert b"1 Imported" in res.data
 
-    # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
-
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
 
@@ -103,16 +100,13 @@ def test_check_markup_include_filters_restriction(client, live_server):
         follow_redirects=True
     )
     assert b"Updated watch." in res.data
-
+    time.sleep(1)
     # Check it saved
     res = client.get(
         url_for("edit_page", uuid="first"),
     )
     assert bytes(include_filters.encode('utf-8')) in res.data
 
-    # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
-
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
     #  Make a change

changedetectionio/tests/test_encoding.py

@@ -70,9 +70,6 @@ def test_check_encoding_detection_missing_content_type_header(client, live_serve
         follow_redirects=True
     )
 
-    # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
-
     # Give the thread time to pick it up
     time.sleep(2)
 

changedetectionio/tests/test_notification.py

@@ -1,9 +1,12 @@
+import json
 import os
 import time
 import re
 from flask import url_for
 from . util import set_original_response, set_modified_response, set_more_modified_response, live_server_setup
+from . util import  extract_UUID_from_client
 import logging
+import base64
 
 from changedetectionio.notification import (
     default_notification_body,
@@ -18,7 +21,6 @@ def test_setup(live_server):
 # Hard to just add more live server URLs when one test is already running (I think)
 # So we add our test here (was in a different file)
 def test_check_notification(client, live_server):
-
     set_original_response()
 
     # Give the endpoint time to spin up
@@ -68,6 +70,20 @@ def test_check_notification(client, live_server):
     # Give the thread time to pick up the first version
     time.sleep(3)
 
+    # We write the PNG to disk, but a JPEG should appear in the notification
+    # Write the last screenshot png
+    testimage_png = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII='
+    # This one is created when we save the screenshot from the webdriver/playwright session (converted from PNG)
+    testimage_jpg = '/9j/4AAQSkZJRgABAQEASABIAAD/2wBDAAMCAgMCAgMDAwMEAwMEBQgFBQQEBQoHBwYIDAoMDAsKCwsNDhIQDQ4RDgsLEBYQERMUFRUVDA8XGBYUGBIUFRT/wAALCAABAAEBAREA/8QAFAABAAAAAAAAAAAAAAAAAAAACf/EABQQAQAAAAAAAAAAAAAAAAAAAAD/2gAIAQEAAD8AKp//2Q=='
+
+
+    uuid = extract_UUID_from_client(client)
+    datastore = 'test-datastore'
+    with open(os.path.join(datastore, str(uuid), 'last-screenshot.png'), 'wb') as f:
+        f.write(base64.b64decode(testimage_png))
+    with open(os.path.join(datastore, str(uuid), 'last-screenshot.jpg'), 'wb') as f:
+        f.write(base64.b64decode(testimage_jpg))
+
     # Goto the edit page, add our ignore text
     # Add our URL to the import page
 
@@ -86,6 +102,7 @@ def test_check_notification(client, live_server):
                                                    "Diff: {diff}\n"
                                                    "Diff Full: {diff_full}\n"
                                                    ":-)",
+                              "notification_screenshot": True,
                               "notification_format": "Text"}
 
     notification_form_data.update({
@@ -116,8 +133,6 @@ def test_check_notification(client, live_server):
     time.sleep(3)
     set_modified_response()
 
-    notification_submission = None
-
     # Trigger a check
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
     time.sleep(3)
@@ -143,6 +158,19 @@ def test_check_notification(client, live_server):
     assert ":-)" in notification_submission
     assert "New ChangeDetection.io Notification - {}".format(test_url) in notification_submission
 
+    # Check the attachment was added, and that it is a JPEG from the original PNG
+    notification_submission_object = json.loads(notification_submission)
+    assert notification_submission_object['attachments'][0]['filename'] == 'last-screenshot.jpg'
+    assert len(notification_submission_object['attachments'][0]['base64'])
+    assert notification_submission_object['attachments'][0]['mimetype'] == 'image/jpeg'
+    jpeg_in_attachment = base64.b64decode(notification_submission_object['attachments'][0]['base64'])
+    assert b'JFIF' in jpeg_in_attachment
+    assert testimage_png not in notification_submission
+    # Assert that the JPEG is readable (didn't get chewed up somewhere)
+    from PIL import Image
+    import io
+    assert Image.open(io.BytesIO(jpeg_in_attachment))
+
     if env_base_url:
         # Re #65 - did we see our BASE_URl ?
         logging.debug (">>> BASE_URL checking in notification: %s", env_base_url)

changedetectionio/update_worker.py

@@ -74,6 +74,7 @@ class update_worker(threading.Thread):
             n_object.update({
                 'watch_url': watch['url'],
                 'uuid': watch_uuid,
+                'screenshot': watch.get_screenshot_as_jpeg() if watch.get('notification_screenshot') else None,
                 'current_snapshot': snapshot_contents.decode('utf-8'),
                 'diff': diff.render_diff(watch_history[dates[-2]], watch_history[dates[-1]], line_feed_sep=line_feed_sep),
                 'diff_full': diff.render_diff(watch_history[dates[-2]], watch_history[dates[-1]], True, line_feed_sep=line_feed_sep)
@@ -106,7 +107,8 @@ class update_worker(threading.Thread):
         if 'notification_urls' in n_object:
             n_object.update({
                 'watch_url': watch['url'],
-                'uuid': watch_uuid
+                'uuid': watch_uuid,
+                'screenshot': None
             })
             self.notification_q.put(n_object)
             print("Sent filter not found notification for {}".format(watch_uuid))

requirements.txt

@@ -54,5 +54,7 @@ jinja2-time
 # https://github.com/dgtlmoon/changedetection.io/pull/1009
 jq~=1.3 ;python_version >= "3.8" and sys_platform == "linux"
 
+# Any current modern version, required so far for screenshot PNG->JPEG conversion but will be used more in the future
+pillow
 # playwright is installed at Dockerfile build time because it's not available on all platforms