Add test

.github/test/Dockerfile-alpine

@@ -2,7 +2,7 @@
 # Test that we can still build on Alpine (musl modified libc https://musl.libc.org/)
 # Some packages wont install via pypi because they dont have a wheel available under this architecture.
 
-FROM ghcr.io/linuxserver/baseimage-alpine:3.21
+FROM ghcr.io/linuxserver/baseimage-alpine:3.22
 ENV PYTHONUNBUFFERED=1
 
 COPY requirements.txt /requirements.txt
@@ -24,12 +24,13 @@ RUN \
   apk add --update --no-cache \
     libjpeg \
     libxslt \
+    file \
     nodejs \
     poppler-utils \
     python3 && \
   echo "**** pip3 install test of changedetection.io ****" && \
   python3 -m venv /lsiopy  && \
   pip install -U pip wheel setuptools && \
-  pip install -U --no-cache-dir --find-links https://wheel-index.linuxserver.io/alpine-3.21/ -r /requirements.txt && \
+  pip install -U --no-cache-dir --find-links https://wheel-index.linuxserver.io/alpine-3.22/ -r /requirements.txt && \
   apk del --purge \
     build-dependencies

Dockerfile

@@ -84,6 +84,9 @@ COPY changedetection.py /app/changedetection.py
 ARG LOGGER_LEVEL=''
 ENV LOGGER_LEVEL="$LOGGER_LEVEL"
 
+# Default
+ENV LC_ALL=en_US.UTF-8
+
 WORKDIR /app
 CMD ["python", "./changedetection.py", "-d", "/datastore"]
 

changedetectionio/__init__.py

@@ -2,7 +2,7 @@
 
 # Read more https://github.com/dgtlmoon/changedetection.io/wiki
 
-__version__ = '0.50.6'
+__version__ = '0.50.8'
 
 from changedetectionio.strtobool import strtobool
 from json.decoder import JSONDecodeError

changedetectionio/blueprint/ui/views.py

@@ -93,12 +93,15 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMe
             return redirect(url_for('watchlist.index'))
 
         # For submission of requesting an extract
-        extract_form = forms.extractDataForm(request.form)
+        extract_form = forms.extractDataForm(formdata=request.form,
+                                             data={'extract_regex': request.form.get('extract_regex', '')}
+                                             )
         if not extract_form.validate():
             flash("An error occurred, please see below.", "error")
+            return _render_diff_template(uuid, extract_form)
 
         else:
-            extract_regex = request.form.get('extract_regex').strip()
+            extract_regex = request.form.get('extract_regex', '').strip()
             output = watch.extract_regex_from_all_history(extract_regex)
             if output:
                 watch_dir = os.path.join(datastore.datastore_path, uuid)
@@ -109,12 +112,11 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMe
                 response.headers['Expires'] = "0"
                 return response
 
-            flash('Nothing matches that RegEx', 'error')
-        redirect(url_for('ui_views.diff_history_page', uuid=uuid) + '#extract')
+            flash('No matches found while scanning all of the watch history for that RegEx.', 'error')
+        return redirect(url_for('ui.ui_views.diff_history_page', uuid=uuid) + '#extract')
 
-    @views_blueprint.route("/diff/<string:uuid>", methods=['GET'])
-    @login_optionally_required
-    def diff_history_page(uuid):
+    def _render_diff_template(uuid, extract_form=None):
+        """Helper function to render the diff template with all required data"""
         from changedetectionio import forms
 
         # More for testing, possible to return the first/only
@@ -128,8 +130,11 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMe
             flash("No history found for the specified link, bad link?", "error")
             return redirect(url_for('watchlist.index'))
 
-        # For submission of requesting an extract
-        extract_form = forms.extractDataForm(request.form)
+        # Use provided form or create a new one
+        if extract_form is None:
+            extract_form = forms.extractDataForm(formdata=request.form,
+                                                 data={'extract_regex': request.form.get('extract_regex', '')}
+                                                 )
 
         history = watch.history
         dates = list(history.keys())
@@ -170,7 +175,7 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMe
 
         datastore.set_last_viewed(uuid, time.time())
 
-        output = render_template("diff.html",
+        return render_template("diff.html",
                                  current_diff_url=watch['url'],
                                  from_version=str(from_version),
                                  to_version=str(to_version),
@@ -193,7 +198,10 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMe
                                  watch_a=watch
                                  )
 
-        return output
+    @views_blueprint.route("/diff/<string:uuid>", methods=['GET'])
+    @login_optionally_required
+    def diff_history_page(uuid):
+        return _render_diff_template(uuid)
 
     @views_blueprint.route("/form/add/quickwatch", methods=['POST'])
     @login_optionally_required

changedetectionio/blueprint/watchlist/templates/watch-overview.html

@@ -81,10 +81,11 @@ document.addEventListener('DOMContentLoaded', function() {
     {%- if any_has_restock_price_processor -%}
         {%- set cols_required = cols_required + 1 -%}
     {%- endif -%}
+    {%- set ui_settings = datastore.data['settings']['application']['ui'] -%}
 
     <div id="watch-table-wrapper">
         {%- set table_classes = [
-            'favicon-enabled' if  datastore.data['settings']['application']['ui'].get('favicons_enabled') else 'favicon-not-enabled',
+            'favicon-enabled' if 'favicons_enabled' not in ui_settings or ui_settings['favicons_enabled'] else 'favicon-not-enabled',
         ] -%}
         <table class="pure-table pure-table-striped watch-table {{ table_classes | reject('equalto', '') | join(' ') }}">
             <thead>
@@ -147,7 +148,7 @@ document.addEventListener('DOMContentLoaded', function() {
 
                 <td class="title-col inline">
                     <div class="flex-wrapper">
-                    {%  if datastore.data['settings']['application']['ui'].get('favicons_enabled') %}
+                    {% if 'favicons_enabled' not in ui_settings or ui_settings['favicons_enabled'] %}
                         <div>{# A page might have hundreds of these images, set IMG options for lazy loading, don't set SRC if we dont have it so it doesnt fetch the placeholder'  #}
                             <img alt="Favicon thumbnail" class="favicon" loading="lazy" decoding="async" fetchpriority="low" {% if favicon %} src="{{url_for('static_content', group='favicon', filename=watch.uuid)}}" {% else %} src='data:image/svg+xml;utf8,%3Csvg xmlns="http://www.w3.org/2000/svg" width="7.087" height="7.087" viewBox="0 0 7.087 7.087"%3E%3Ccircle cx="3.543" cy="3.543" r="3.279" stroke="%23e1e1e1" stroke-width="0.45" fill="none" opacity="0.74"/%3E%3C/svg%3E' {%  endif %} />
                         </div>

changedetectionio/forms.py

@@ -396,6 +396,19 @@ def validate_url(test_url):
         # This should be wtforms.validators.
         raise ValidationError('Watch protocol is not permitted by SAFE_PROTOCOL_REGEX or incorrect URL format')
 
+
+class ValidateSinglePythonRegexString(object):
+    def __init__(self, message=None):
+        self.message = message
+
+    def __call__(self, form, field):
+        try:
+            re.compile(field.data)
+        except re.error:
+            message = field.gettext('RegEx \'%s\' is not a valid regular expression.')
+            raise ValidationError(message % (field.data))
+
+
 class ValidateListRegex(object):
     """
     Validates that anything that looks like a regex passes as a regex
@@ -414,6 +427,7 @@ class ValidateListRegex(object):
                     message = field.gettext('RegEx \'%s\' is not a valid regular expression.')
                     raise ValidationError(message % (line))
 
+
 class ValidateCSSJSONXPATHInput(object):
     """
     Filter validation
@@ -791,5 +805,5 @@ class globalSettingsForm(Form):
 
 
 class extractDataForm(Form):
-    extract_regex = StringField('RegEx to extract', validators=[validators.Length(min=1, message="Needs a RegEx")])
+    extract_regex = StringField('RegEx to extract', validators=[validators.DataRequired(), ValidateSinglePythonRegexString()])
     extract_submit_button = SubmitField('Extract as CSV', render_kw={"class": "pure-button pure-button-primary"})

changedetectionio/model/Watch.py

@@ -639,7 +639,7 @@ class model(watch_base):
                     if res:
                         if not csv_writer:
                             # A file on the disk can be transferred much faster via flask than a string reply
-                            csv_output_filename = 'report.csv'
+                            csv_output_filename = f"report-{self.get('uuid')}.csv"
                             f = open(os.path.join(self.watch_data_dir, csv_output_filename), 'w')
                             # @todo some headers in the future
                             #fieldnames = ['Epoch seconds', 'Date']

changedetectionio/model/__init__.py

@@ -3,6 +3,7 @@ import uuid
 
 from changedetectionio import strtobool
 default_notification_format_for_watch = 'System default'
+CONDITIONS_MATCH_LOGIC_DEFAULT = 'ALL'
 
 class watch_base(dict):
 
@@ -15,6 +16,8 @@ class watch_base(dict):
             'body': None,
             'browser_steps': [],
             'browser_steps_last_error_step': None,
+            'conditions' : {},
+            'conditions_match_logic': CONDITIONS_MATCH_LOGIC_DEFAULT,
             'check_count': 0,
             'check_unique_lines': False,  # On change-detected, compare against all history if its something new
             'consecutive_filter_failures': 0,  # Every time the CSS/xPath filter cannot be located, reset when all is fine.

changedetectionio/static/styles/scss/parts/_lister_extra.scss

@@ -6,19 +6,19 @@
       }
     }
   }
-  &.favicon-enabled {
-    tr {
-      /* make the icons and the text inline-ish */
-      td.inline.title-col {
-        .flex-wrapper {
-          display: flex;
-          align-items: center;
-          gap: 4px;
-        }
+
+  tr {
+    /* make the icons and the text inline-ish */
+    td.inline.title-col {
+      .flex-wrapper {
+        display: flex;
+        align-items: center;
+        gap: 4px;
       }
     }
   }
 
+
   td,
   th {
     vertical-align: middle;

changedetectionio/store.py

@@ -44,8 +44,6 @@ class ChangeDetectionStore:
     def __init__(self, datastore_path="/datastore", include_default_watches=True, version_tag="0.0.0"):
         # Should only be active for docker
         # logging.basicConfig(filename='/dev/stdout', level=logging.INFO)
-        from deepmerge import always_merger
-
         self.__data = App.model()
         self.datastore_path = datastore_path
         self.json_store_path = os.path.join(self.datastore_path, "url-watches.json")
@@ -77,8 +75,14 @@ class ChangeDetectionStore:
                     self.__data['app_guid'] = from_disk['app_guid']
 
                 if 'settings' in from_disk:
-                    # update the modal with whats on disk
-                    self.__data['settings'] = always_merger.merge(from_disk['settings'], self.__data['settings'])
+                    if 'headers' in from_disk['settings']:
+                        self.__data['settings']['headers'].update(from_disk['settings']['headers'])
+
+                    if 'requests' in from_disk['settings']:
+                        self.__data['settings']['requests'].update(from_disk['settings']['requests'])
+
+                    if 'application' in from_disk['settings']:
+                        self.__data['settings']['application'].update(from_disk['settings']['application'])
 
                 # Convert each existing watch back to the Watch.model object
                 for uuid, watch in self.__data['watching'].items():

changedetectionio/tests/test_api.py

@@ -292,9 +292,7 @@ def test_access_denied(client, live_server, measure_memory_usage):
 
 def test_api_watch_PUT_update(client, live_server, measure_memory_usage):
 
-    
     api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')
-
     # Create a watch
     set_original_response()
     test_url = url_for('test_endpoint', _external=True)
@@ -302,14 +300,27 @@ def test_api_watch_PUT_update(client, live_server, measure_memory_usage):
     # Create new
     res = client.post(
         url_for("createwatch"),
-        data=json.dumps({"url": test_url, 'tag': "One, Two", "title": "My test URL", 'headers': {'cookie': 'yum'} }),
+        data=json.dumps({"url": test_url,
+                         'tag': "One, Two",
+                         "title": "My test URL",
+                         'headers': {'cookie': 'yum'},
+                         "conditions": [
+                             {
+                                 "field": "page_filtered_text",
+                                 "operator": "contains_regex",
+                                 "value": "."  # contains anything
+                             }
+                         ],
+                         "conditions_match_logic": "ALL"
+                         }
+                        ),
         headers={'content-type': 'application/json', 'x-api-key': api_key},
         follow_redirects=True
     )
 
     assert res.status_code == 201
 
-
+    wait_for_all_checks(client)
     # Get a listing, it will be the first one
     res = client.get(
         url_for("createwatch"),

changedetectionio/tests/test_conditions.py

@@ -4,6 +4,8 @@ import time
 
 from flask import url_for
 from .util import live_server_setup, wait_for_all_checks
+from ..model import CONDITIONS_MATCH_LOGIC_DEFAULT
+
 
 def set_original_response(number="50"):
     test_return_data = f"""<html>
@@ -76,7 +78,7 @@ def test_conditions_with_text_and_number(client, live_server):
             "fetch_backend": "html_requests",
             "include_filters": ".number-container",
             "title": "Number AND Text Condition Test",
-            "conditions_match_logic": "ALL",  # ALL = AND logic
+            "conditions_match_logic": CONDITIONS_MATCH_LOGIC_DEFAULT,  # ALL = AND logic
             "conditions-0-operator": "in",
             "conditions-0-field": "page_filtered_text",
             "conditions-0-value": "5",
@@ -283,7 +285,7 @@ def test_lev_conditions_plugin(client, live_server, measure_memory_usage):
         data={
             "url": test_url,
             "fetch_backend": "html_requests",
-            "conditions_match_logic": "ALL",  # ALL = AND logic
+            "conditions_match_logic": CONDITIONS_MATCH_LOGIC_DEFAULT,  # ALL = AND logic
             "conditions-0-field": "levenshtein_ratio",
             "conditions-0-operator": "<",
             "conditions-0-value": "0.8" # needs to be more of a diff to trigger a change

changedetectionio/tests/test_extract_csv.py

@@ -46,7 +46,7 @@ def test_check_extract_text_from_diff(client, live_server, measure_memory_usage)
         follow_redirects=False
     )
 
-    assert b'Nothing matches that RegEx' not in res.data
+    assert b'No matches found while scanning all of the watch history for that RegEx.' not in res.data
     assert res.content_type == 'text/csv'
 
     # Read the csv reply as stringio

changedetectionio/tests/unit/test_conditions.py

@@ -1,4 +1,5 @@
 from changedetectionio.conditions import execute_ruleset_against_all_plugins
+from changedetectionio.model import CONDITIONS_MATCH_LOGIC_DEFAULT
 from changedetectionio.store import ChangeDetectionStore
 import shutil
 import tempfile
@@ -59,7 +60,7 @@ class TestTriggerConditions(unittest.TestCase):
 
         self.store.data['watching'][self.watch_uuid].update(
             {
-                "conditions_match_logic": "ALL",
+                "conditions_match_logic": CONDITIONS_MATCH_LOGIC_DEFAULT,
                 "conditions": [
                     {"operator": ">=", "field": "extracted_number", "value": "10"},
                     {"operator": "<=", "field": "extracted_number", "value": "5000"},

docker-compose.yml

@@ -66,6 +66,9 @@ services:
   #        A valid timezone name to run as (for scheduling watch checking) see https://en.wikipedia.org/wiki/List_of_tz_database_time_zones
   #      - TZ=America/Los_Angeles
   #
+  #        Text processing locale, en_US.UTF-8 used by default unless defined as something else here, UTF-8 should cover 99.99% of cases.
+  #      - LC_ALL=en_US.UTF-8
+  #
   #        Maximum height of screenshots, default is 16000 px, screenshots will be clipped to this if exceeded.
   #        RAM usage will be higher if you increase this.
   #      - SCREENSHOT_MAX_HEIGHT=16000

requirements.txt

@@ -78,7 +78,6 @@ jq~=1.3; python_version >= "3.8" and sys_platform == "linux"
 # playwright is installed at Dockerfile build time because it's not available on all platforms
 
 pyppeteer-ng==2.0.0rc10
-deepmerge
 
 pyppeteerstealth>=0.0.4