Re #462 - check encoding

changedetectionio/__init__.py

@@ -35,9 +35,10 @@ from flask import (
     url_for,
 )
 from flask_login import login_required
+
 from changedetectionio import html_tools
 
-__version__ = '0.39.9'
+__version__ = '0.39.10'
 
 datastore = None
 
@@ -127,7 +128,7 @@ def _jinja2_filter_datetimestamp(timestamp, format="%Y-%m-%d %H:%M:%S"):
     # return timeago.format(timestamp, time.time())
     # return datetime.datetime.utcfromtimestamp(timestamp).strftime(format)
 
-
+# When nobody is logged in Flask-Login's current_user is set to an AnonymousUser object.
 class User(flask_login.UserMixin):
     id=None
 
@@ -136,7 +137,6 @@ class User(flask_login.UserMixin):
     def get_user(self, email="defaultuser@changedetection.io"):
         return self
     def is_authenticated(self):
-
         return True
     def is_active(self):
         return True
@@ -215,6 +215,10 @@ def changedetection_app(config=None, datastore_o=None):
             return redirect(url_for('index'))
 
         if request.method == 'GET':
+            if flask_login.current_user.is_authenticated:
+                flash("Already logged in")
+                return redirect(url_for("index"))
+
             output = render_template("login.html")
             return output
 
@@ -250,6 +254,11 @@ def changedetection_app(config=None, datastore_o=None):
         # (No password in settings or env var)
         app.config['LOGIN_DISABLED'] = datastore.data['settings']['application']['password'] == False and os.getenv("SALTED_PASS", False) == False
 
+        # Set the auth cookie path if we're running as X-settings/X-Forwarded-Prefix
+        if os.getenv('USE_X_SETTINGS') and 'X-Forwarded-Prefix' in request.headers:
+            app.config['REMEMBER_COOKIE_PATH'] = request.headers['X-Forwarded-Prefix']
+            app.config['SESSION_COOKIE_PATH'] = request.headers['X-Forwarded-Prefix']
+
         # For the RSS path, allow access via a token
         if request.path == '/rss' and request.args.get('token'):
             app_rss_token = datastore.data['settings']['application']['rss_access_token']
@@ -518,6 +527,7 @@ def changedetection_app(config=None, datastore_o=None):
 
 
             datastore.data['watching'][uuid]['css_filter'] = form.css_filter.data.strip()
+            datastore.data['watching'][uuid]['subtractive_selectors'] = form.subtractive_selectors.data
 
             # Reset the previous_md5 so we process a new snapshot including stripping ignore text.
             if form.css_filter.data.strip() != datastore.data['watching'][uuid]['css_filter']:
@@ -590,6 +600,7 @@ def changedetection_app(config=None, datastore_o=None):
         if request.method == 'GET':
             form.minutes_between_check.data = int(datastore.data['settings']['requests']['minutes_between_check'])
             form.notification_urls.data = datastore.data['settings']['application']['notification_urls']
+            form.global_subtractive_selectors.data = datastore.data['settings']['application']['global_subtractive_selectors']
             form.global_ignore_text.data = datastore.data['settings']['application']['global_ignore_text']
             form.ignore_whitespace.data = datastore.data['settings']['application']['ignore_whitespace']
             form.extract_title_as_title.data = datastore.data['settings']['application']['extract_title_as_title']
@@ -618,6 +629,7 @@ def changedetection_app(config=None, datastore_o=None):
             datastore.data['settings']['application']['notification_format'] = form.notification_format.data
             datastore.data['settings']['application']['notification_urls'] = form.notification_urls.data
             datastore.data['settings']['application']['base_url'] = form.base_url.data
+            datastore.data['settings']['application']['global_subtractive_selectors'] = form.global_subtractive_selectors.data
             datastore.data['settings']['application']['global_ignore_text'] =  form.global_ignore_text.data
             datastore.data['settings']['application']['ignore_whitespace'] = form.ignore_whitespace.data
 

changedetectionio/fetch_site_status.py

@@ -1,11 +1,11 @@
-import time
-from changedetectionio import content_fetcher
-from changedetectionio import html_tools
 import hashlib
-from inscriptis import get_text
-import urllib3
-from . import html_tools
 import re
+import time
+
+import urllib3
+from inscriptis import get_text
+
+from changedetectionio import content_fetcher, html_tools
 
 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 
@@ -72,8 +72,15 @@ class perform_site_check():
             is_json = 'application/json' in fetcher.headers.get('Content-Type', '')
             is_html = not is_json
             css_filter_rule = watch['css_filter']
+            subtractive_selectors = watch.get(
+                "subtractive_selectors", []
+            ) + self.datastore.data["settings"]["application"].get(
+                "global_subtractive_selectors", []
+            )
 
             has_filter_rule = css_filter_rule and len(css_filter_rule.strip())
+            has_subtractive_selectors = subtractive_selectors and len(subtractive_selectors[0].strip())
+            
             if is_json and not has_filter_rule:
                 css_filter_rule = "json:$"
                 has_filter_rule = True
@@ -100,11 +107,11 @@ class perform_site_check():
                         else:
                             # CSS Filter, extract the HTML that matches and feed that into the existing inscriptis::get_text
                             html_content = html_tools.css_filter(css_filter=css_filter_rule, html_content=fetcher.content)
-
+                    if has_subtractive_selectors:
+                        html_content = html_tools.element_removal(subtractive_selectors, html_content)
                     # get_text() via inscriptis
                     stripped_text_from_html = get_text(html_content)
 
-
             # Re #340 - return the content before the 'ignore text' was applied
             text_content_before_ignored_filter = stripped_text_from_html.encode('utf-8')
 

changedetectionio/forms.py

@@ -1,13 +1,30 @@
-from wtforms import Form, SelectField, RadioField, BooleanField, StringField, PasswordField, validators, IntegerField, fields, TextAreaField, \
-    Field
-
-from wtforms import widgets, SubmitField
-from wtforms.validators import ValidationError
-from wtforms.fields import html5
-from changedetectionio import content_fetcher
 import re
 
-from changedetectionio.notification import default_notification_format, valid_notification_formats, default_notification_body, default_notification_title
+from wtforms import (
+    BooleanField,
+    Field,
+    Form,
+    IntegerField,
+    PasswordField,
+    RadioField,
+    SelectField,
+    StringField,
+    SubmitField,
+    TextAreaField,
+    fields,
+    validators,
+    widgets,
+)
+from wtforms.fields import html5
+from wtforms.validators import ValidationError
+
+from changedetectionio import content_fetcher
+from changedetectionio.notification import (
+    default_notification_body,
+    default_notification_format,
+    default_notification_title,
+    valid_notification_formats,
+)
 
 valid_method = {
     'GET',
@@ -45,8 +62,8 @@ class SaltyPasswordField(StringField):
     encrypted_password = ""
 
     def build_password(self, password):
-        import hashlib
         import base64
+        import hashlib
         import secrets
 
         # Make a new salt on every new password and store it with the password
@@ -104,9 +121,10 @@ class ValidateContentFetcherIsReady(object):
         self.message = message
 
     def __call__(self, form, field):
-        from changedetectionio import content_fetcher
         import urllib3.exceptions
 
+        from changedetectionio import content_fetcher
+
         # Better would be a radiohandler that keeps a reference to each class
         if field.data is not None:
             klass = getattr(content_fetcher, field.data)
@@ -213,52 +231,69 @@ class ValidateListRegex(object):
                 except re.error:
                     message = field.gettext('RegEx \'%s\' is not a valid regular expression.')
                     raise ValidationError(message % (line))
-
+              
 class ValidateCSSJSONXPATHInput(object):
     """
     Filter validation
     @todo CSS validator ;)
     """
 
-    def __init__(self, message=None):
+    def __init__(self, message=None, allow_xpath=True, allow_json=True):
         self.message = message
+        self.allow_xpath = allow_xpath
+        self.allow_json = allow_json
 
     def __call__(self, form, field):
 
+        if isinstance(field.data, str):
+            data = [field.data]
+        else:
+            data = field.data
+
+        for line in data:
         # Nothing to see here
-        if not len(field.data.strip()):
-            return
+            if not len(line.strip()):
+                return
 
-        # Does it look like XPath?
-        if field.data.strip()[0] == '/':
-            from lxml import html, etree
-            tree = html.fromstring("<html></html>")
+            # Does it look like XPath?
+            if line.strip()[0] == '/':
+                if not self.allow_xpath:
+                    raise ValidationError("XPath not permitted in this field!")
+                from lxml import etree, html
+                tree = html.fromstring("<html></html>")
 
-            try:
-                tree.xpath(field.data.strip())
-            except etree.XPathEvalError as e:
-                message = field.gettext('\'%s\' is not a valid XPath expression. (%s)')
-                raise ValidationError(message % (field.data, str(e)))
-            except:
-                raise ValidationError("A system-error occurred when validating your XPath expression")
+                try:
+                    tree.xpath(line.strip())
+                except etree.XPathEvalError as e:
+                    message = field.gettext('\'%s\' is not a valid XPath expression. (%s)')
+                    raise ValidationError(message % (line, str(e)))
+                except:
+                    raise ValidationError("A system-error occurred when validating your XPath expression")
 
-        if 'json:' in field.data:
-            from jsonpath_ng.exceptions import JsonPathParserError, JsonPathLexerError
-            from jsonpath_ng.ext import parse
+            if 'json:' in line:
+                if not self.allow_json:
+                    raise ValidationError("JSONPath not permitted in this field!")
 
-            input = field.data.replace('json:', '')
+                from jsonpath_ng.exceptions import (
+                    JsonPathLexerError,
+                    JsonPathParserError,
+                )
+                from jsonpath_ng.ext import parse
 
-            try:
-                parse(input)
-            except (JsonPathParserError, JsonPathLexerError) as e:
-                message = field.gettext('\'%s\' is not a valid JSONPath expression. (%s)')
-                raise ValidationError(message % (input, str(e)))
-            except:
-                raise ValidationError("A system-error occurred when validating your JSONPath expression")
+                input = line.replace('json:', '')
 
-            # Re #265 - maybe in the future fetch the page and offer a
-            # warning/notice that its possible the rule doesnt yet match anything?
+                try:
+                    parse(input)
+                except (JsonPathParserError, JsonPathLexerError) as e:
+                    message = field.gettext('\'%s\' is not a valid JSONPath expression. (%s)')
+                    raise ValidationError(message % (input, str(e)))
+                except:
+                    raise ValidationError("A system-error occurred when validating your JSONPath expression")
 
+                # Re #265 - maybe in the future fetch the page and offer a
+                # warning/notice that its possible the rule doesnt yet match anything?
+
+            
 class quickWatchForm(Form):
     # https://wtforms.readthedocs.io/en/2.3.x/fields/#module-wtforms.fields.html5
     # `require_tld` = False is needed even for the test harness "http://localhost:5005.." to run
@@ -283,6 +318,7 @@ class watchForm(commonSettingsForm):
     minutes_between_check = html5.IntegerField('Maximum time in minutes until recheck',
                                                [validators.Optional(), validators.NumberRange(min=1)])
     css_filter = StringField('CSS/JSON/XPATH Filter', [ValidateCSSJSONXPATHInput()])
+    subtractive_selectors = StringListField('Remove elements', [ValidateCSSJSONXPATHInput(allow_xpath=False, allow_json=False)])
     title = StringField('Title')
 
     ignore_text = StringListField('Ignore Text', [ValidateListRegex()])
@@ -314,5 +350,6 @@ class globalSettingsForm(commonSettingsForm):
                                                [validators.NumberRange(min=1)])
     extract_title_as_title = BooleanField('Extract <title> from document and use as watch title')
     base_url = StringField('Base URL', validators=[validators.Optional()])
+    global_subtractive_selectors = StringListField('Ignore elements', [ValidateCSSJSONXPATHInput(allow_xpath=False, allow_json=False)])
     global_ignore_text = StringListField('Ignore Text', [ValidateListRegex()])
-    ignore_whitespace = BooleanField('Ignore whitespace')
+    ignore_whitespace = BooleanField('Ignore whitespace')

changedetectionio/html_tools.py

@@ -1,7 +1,10 @@
 import json
+import re
+from typing import List
+
 from bs4 import BeautifulSoup
 from jsonpath_ng.ext import parse
-import re
+
 
 class JSONNotFound(ValueError):
     def __init__(self, msg):
@@ -16,11 +19,22 @@ def css_filter(css_filter, html_content):
 
     return html_block + "\n"
 
+def subtractive_css_selector(css_selector, html_content):
+    soup = BeautifulSoup(html_content, "html.parser")
+    for item in soup.select(css_selector):
+        item.decompose()
+    return str(soup)
+
+    
+def element_removal(selectors: List[str], html_content):
+    """Joins individual filters into one css filter."""
+    selector = ",".join(selectors)
+    return subtractive_css_selector(selector, html_content)
+    
 
 # Return str Utf-8 of matched rules
 def xpath_filter(xpath_filter, html_content):
-    from lxml import html
-    from lxml import etree
+    from lxml import etree, html
 
     tree = html.fromstring(html_content)
     html_block = ""
@@ -64,7 +78,8 @@ def _parse_json(json_data, jsonpath_filter):
         # Re 265 - Just return an empty string when filter not found
         return ''
 
-    stripped_text_from_html = json.dumps(s, indent=4)
+    # Ticket #462 - allow the original encoding through, usually it's UTF-8 or similar
+    stripped_text_from_html = json.dumps(s, indent=4, ensure_ascii=False)
 
     return stripped_text_from_html
 
@@ -151,4 +166,4 @@ def strip_ignore_text(content, wordlist, mode="content"):
     if mode == "line numbers":
         return ignored_line_numbers
 
-    return "\n".encode('utf8').join(output)
+    return "\n".encode('utf8').join(output)

changedetectionio/static/styles/styles.css

@@ -39,13 +39,13 @@ section.content {
 
 /* table related */
 .watch-table {
-  width: 100%; }
+  width: 100%;
+  font-size: 80%; }
   .watch-table tr.unviewed {
     font-weight: bold; }
   .watch-table .error {
     color: #a00; }
   .watch-table td {
-    font-size: 80%;
     white-space: nowrap; }
   .watch-table td.title-col {
     word-break: break-all;
@@ -80,11 +80,11 @@ section.content {
 
 body:after {
   content: "";
-  background: linear-gradient(130deg, #ff7a18, #af002d 41.07%, #319197 76.05%); }
+  background: linear-gradient(130deg, #5ad8f7, #2f50af 41.07%, #9150bf 84.05%); }
 
 body:after, body:before {
   display: block;
-  height: 600px;
+  height: 650px;
   position: absolute;
   top: 0;
   left: 0;
@@ -96,9 +96,6 @@ body::after {
 
 body::before {
   content: "";
-  background-image: url(/static/images/gradient-border.png); }
-
-body:before {
   background-size: cover; }
 
 body:after, body:before {
@@ -199,7 +196,8 @@ body:after, body:before {
   #new-watch-form .label {
     display: none; }
   #new-watch-form legend {
-    color: #fff; }
+    color: #fff;
+    font-weight: bold; }
 
 #diff-col {
   padding-left: 40px; }
@@ -388,6 +386,11 @@ and also iPads specifically.
 .pure-form-stacked > div:first-child {
   display: block; }
 
+.login-form .inner {
+  background: #fff;
+  padding: 20px;
+  border-radius: 5px; }
+
 .edit-form {
   min-width: 70%; }
   .edit-form .tab-pane-inner {
@@ -404,6 +407,8 @@ and also iPads specifically.
   .edit-form #actions {
     display: block;
     background: #fff; }
+  .edit-form .pure-form-message-inline {
+    padding-left: 0; }
 
 ul {
   padding-left: 1em;

changedetectionio/static/styles/styles.scss

@@ -45,6 +45,7 @@ section.content {
 /* table related */
 .watch-table {
   width: 100%;
+  font-size: 80%;
 
   tr.unviewed {
     font-weight: bold;
@@ -55,7 +56,6 @@ section.content {
   }
 
   td {
-    font-size: 80%;
     white-space: nowrap;
   }
 
@@ -107,12 +107,12 @@ section.content {
 
 body:after {
   content: "";
-  background: linear-gradient(130deg, #ff7a18, #af002d 41.07%, #319197 76.05%)
+  background: linear-gradient(130deg, #5ad8f7, #2f50af 41.07%, #9150bf 84.05%);
 }
 
 body:after, body:before {
   display: block;
-  height: 600px;
+  height: 650px;
   position: absolute;
   top: 0;
   left: 0;
@@ -125,11 +125,8 @@ body::after {
 }
 
 body::before {
+  // background-image set in base.html so it works with reverse proxies etc
   content: "";
-  background-image: url(/static/images/gradient-border.png);
-}
-
-body:before {
   background-size: cover
 }
 
@@ -265,6 +262,7 @@ body:after, body:before {
   }
   legend {
     color: #fff;
+    font-weight: bold;
   }
 }
 
@@ -545,6 +543,16 @@ $form-edge-padding: 20px;
     display: block;
   }
 }
+
+.login-form {
+  .inner {
+    background: #fff;;
+    padding: $form-edge-padding;
+    border-radius: 5px;
+  }
+}
+
+
 .edit-form {
   min-width: 70%;
   .tab-pane-inner {
@@ -568,6 +576,10 @@ $form-edge-padding: 20px;
     display: block;
     background: #fff;
   }
+
+  .pure-form-message-inline {
+    padding-left: 0;
+  }
 }
 
 ul {

changedetectionio/store.py

@@ -1,15 +1,19 @@
-from os import unlink, path, mkdir
 import json
-import uuid as uuid_builder
-from threading import Lock
-from copy import deepcopy
-
 import logging
-import time
-import threading
 import os
+import threading
+import time
+import uuid as uuid_builder
+from copy import deepcopy
+from os import mkdir, path, unlink
+from threading import Lock
+
+from changedetectionio.notification import (
+    default_notification_body,
+    default_notification_format,
+    default_notification_title,
+)
 
-from changedetectionio.notification import default_notification_format, default_notification_body, default_notification_title
 
 # Is there an existing library to ensure some data store (JSON etc) is in sync with CRUD methods?
 # Open a github issue if you know something :)
@@ -46,6 +50,7 @@ class ChangeDetectionStore:
                     'extract_title_as_title': False,
                     'fetch_backend': 'html_requests',
                     'global_ignore_text': [], # List of text to ignore when calculating the comparison checksum
+                    'global_subtractive_selectors': [],
                     'ignore_whitespace': False,
                     'notification_urls': [], # Apprise URL list
                     # Custom notification content
@@ -82,6 +87,7 @@ class ChangeDetectionStore:
             'notification_body': default_notification_body,
             'notification_format': default_notification_format,
             'css_filter': "",
+            'subtractive_selectors': [],
             'trigger_text': [],  # List of text or regex to wait for until a change is detected
             'fetch_backend': None,
             'extract_title_as_title': False
@@ -144,8 +150,8 @@ class ChangeDetectionStore:
             unlink(password_reset_lockfile)
 
         if not 'app_guid' in self.__data:
-            import sys
             import os
+            import sys
             if "pytest" in sys.modules or "PYTEST_CURRENT_TEST" in os.environ:
                 self.__data['app_guid'] = "test-" + str(uuid_builder.uuid4())
             else:
@@ -430,6 +436,7 @@ class ChangeDetectionStore:
                 index.append(self.data['watching'][uuid]['history'][str(id)])
 
         import pathlib
+
         # Only in the sub-directories
         for item in pathlib.Path(self.datastore_path).rglob("*/*txt"):
             if not str(item) in index:

changedetectionio/templates/_common_fields.jinja

@@ -34,9 +34,8 @@
                             </div>
                             <div class="pure-controls">
                             <span class="pure-form-message-inline">
-                                These tokens can be used in the notification body and title to
-                                customise the notification text.
-                            </span>
+                                These tokens can be used in the notification body and title to customise the notification text.
+
                                 <table class="pure-table" id="token-table">
                                     <thead>
                                     <tr>
@@ -88,7 +87,7 @@
                                     </tr>
                                     </tbody>
                                 </table>
-                                <span class="pure-form-message-inline">
+                                <br/>
                                 URLs generated by changedetection.io (such as <code>{diff_url}</code>) require the <code>BASE_URL</code> environment variable set.<br/>
                                 Your <code>BASE_URL</code> var is currently "{{current_base_url}}"
                             </span>

changedetectionio/templates/base.html

@@ -12,7 +12,13 @@
         <link rel="stylesheet" href="{{ m }}?ver=1000">
         {% endfor %}
     {% endif %}
+    <style>
+    body::before {
+        background-image: url({{url_for('static_content', group='images', filename='gradient-border.png')}});
+    }
+    </style>
 </head>
+
 <body>
 
 <div class="header">
@@ -35,13 +41,13 @@
         {% if current_user.is_authenticated or not has_password %}
             {% if not current_diff_url %}
             <li class="pure-menu-item">
-                <a href="{{ url_for('get_backup')}}" class="pure-menu-link">BACKUP</a>
+                <a href="{{ url_for('settings_page')}}" class="pure-menu-link">SETTINGS</a>
             </li>
             <li class="pure-menu-item">
                 <a href="{{ url_for('import_page')}}" class="pure-menu-link">IMPORT</a>
             </li>
             <li class="pure-menu-item">
-                <a href="{{ url_for('settings_page')}}" class="pure-menu-link">SETTINGS</a>
+                <a href="{{ url_for('get_backup')}}" class="pure-menu-link">BACKUP</a>
             </li>
             {% else %}
             <li class="pure-menu-item">

changedetectionio/templates/edit.html

@@ -58,24 +58,30 @@
                         </span>
                     </div>
 
+                <hr/>
                 <fieldset class="pure-group">
-                                    <div class="pure-control-group">
-                    {{ render_field(form.method) }}
-                </div>
-                    <strong>Note: <i>Request Headers and Body settings are ONLY used by Basic fast Plaintext/HTTP Client fetch method.</i></strong>
-                    {{ render_field(form.headers, rows=5, placeholder="Example
+
+                    <span class="pure-form-message-inline">
+                        <strong>Request override is currently only used by the <i>Basic fast Plaintext/HTTP Client</i> method.</strong>
+                    </span>
+                    <div class="pure-control-group">
+                        {{ render_field(form.method) }}
+                    </div>
+                    <div class="pure-control-group">
+{{ render_field(form.headers, rows=5, placeholder="Example
 Cookie: foobar
 User-Agent: wonderbra 1.0") }}
-                </fieldset>
-                <div class="pure-control-group">
-                    {{ render_field(form.body, rows=5, placeholder="Example
+                    </div>
+                    <div class="pure-control-group">
+                                        {{ render_field(form.body, rows=5, placeholder="Example
 {
    \"name\":\"John\",
    \"age\":30,
    \"car\":null
 }") }}
-                </div>
-
+                    </div>
+                </fieldset>
+                <br/>
             </div>
 
             <div class="tab-pane-inner" id="notifications">
@@ -109,14 +115,25 @@ User-Agent: wonderbra 1.0") }}
                         <li>CSS - Limit text to this CSS rule, only text matching this CSS rule is included.</li>
                         <li>JSON - Limit text to this JSON rule, using <a href="https://pypi.org/project/jsonpath-ng/">JSONPath</a>, prefix with <b>"json:"</b>, <a
                                 href="https://jsonpath.com/" target="new">test your JSONPath here</a></li>
-                        <li>XPATH - Limit text to this XPath rule, simply start with a forward-slash, example  <b>//*[contains(@class, 'sametext')]</b>, <a
+                        <li>XPath - Limit text to this XPath rule, simply start with a forward-slash, example  <b>//*[contains(@class, 'sametext')]</b>, <a
                                 href="http://xpather.com/" target="new">test your XPath here</a></li>
                     </ul>
                     Please be sure that you thoroughly understand how to write CSS or JSONPath, XPath selector rules before filing an issue on GitHub! <a
                                 href="https://github.com/dgtlmoon/changedetection.io/wiki/CSS-Selector-help">here for more CSS selector help</a>.<br/>
                 </span>
                     </div>
-
+                    <fieldset class="pure-group">
+                      {{ render_field(form.subtractive_selectors, rows=5, placeholder="header
+footer
+nav
+.stockticker") }}
+                      <span class="pure-form-message-inline">
+                        <ul>
+                          <li> Remove HTML element(s) by CSS selector before text conversion. </li>
+                          <li> Add multiple elements or CSS selectors per line to ignore multiple parts of the HTML. </li>
+                        </ul>
+                      </span>
+                    </fieldset>
                 </fieldset>
                 <fieldset class="pure-group">
                     {{ render_field(form.ignore_text, rows=5, placeholder="Some text to ignore in a line

changedetectionio/templates/login.html

@@ -1,8 +1,7 @@
 {% extends 'base.html' %}
 
 {% block content %}
-<div class="edit-form">
-
+<div class="login-form">
  <div class="inner">
     <form class="pure-form pure-form-stacked" action="{{url_for('login')}}" method="POST">
         <fieldset>

changedetectionio/templates/settings.html

@@ -83,7 +83,18 @@
                     </span>
                     </fieldset>
 
-
+                    <fieldset class="pure-group">
+                      {{ render_field(form.global_subtractive_selectors, rows=5, placeholder="header
+footer
+nav
+.stockticker") }}
+                      <span class="pure-form-message-inline">
+                        <ul>
+                          <li> Remove HTML element(s) by CSS selector before text conversion. </li>
+                          <li> Add multiple elements or CSS selectors per line to ignore multiple parts of the HTML. </li>
+                        </ul>
+                      </span>
+                    </fieldset>
                     <fieldset class="pure-group">
                     {{ render_field(form.global_ignore_text, rows=5, placeholder="Some text to ignore in a line
 /some.regex\d{2}/ for case-INsensitive regex

changedetectionio/tests/test_element_removal.py

@@ -0,0 +1,168 @@
+#!/usr/bin/python3
+
+import time
+
+from flask import url_for
+
+from ..html_tools import *
+from .util import live_server_setup
+
+
+def test_setup(live_server):
+    live_server_setup(live_server)
+
+
+def set_original_response():
+    test_return_data = """<html>
+    <header>
+    <h2>Header</h2>
+    </header>
+    <nav>
+    <ul>
+      <li><a href="#">A</a></li>
+      <li><a href="#">B</a></li>
+      <li><a href="#">C</a></li>
+    </ul>
+    </nav>
+       <body>
+     Some initial text</br>
+     <p>Which is across multiple lines</p>
+     </br>
+     So let's see what happens.  </br>
+    <div id="changetext">Some text that will change</div>
+     </body>
+    <footer>
+    <p>Footer</p>
+    </footer>
+     </html>
+    """
+
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write(test_return_data)
+
+
+def set_modified_response():
+    test_return_data = """<html>
+    <header>
+    <h2>Header changed</h2>
+    </header>
+    <nav>
+    <ul>
+      <li><a href="#">A changed</a></li>
+      <li><a href="#">B</a></li>
+      <li><a href="#">C</a></li>
+    </ul>
+    </nav>
+       <body>
+     Some initial text</br>
+     <p>Which is across multiple lines</p>
+     </br>
+     So let's see what happens.  </br>
+    <div id="changetext">Some text that changes</div>
+     </body>
+    <footer>
+    <p>Footer changed</p>
+    </footer>
+     </html>
+    """
+
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write(test_return_data)
+
+
+def test_element_removal_output():
+    from changedetectionio import fetch_site_status
+    from inscriptis import get_text
+
+    # Check text with sub-parts renders correctly
+    content = """<html>
+    <header>
+    <h2>Header</h2>
+    </header>
+    <nav>
+    <ul>
+      <li><a href="#">A</a></li>
+    </ul>
+    </nav>
+       <body>
+     Some initial text</br>
+     <p>across multiple lines</p>
+     <div id="changetext">Some text that changes</div>
+     </body>
+    <footer>
+    <p>Footer</p>
+    </footer>
+     </html>
+    """
+    html_blob = element_removal(
+        ["header", "footer", "nav", "#changetext"], html_content=content
+    )
+    text = get_text(html_blob)
+    assert (
+        text
+        == """Some initial text
+
+across multiple lines
+"""
+    )
+
+
+def test_element_removal_full(client, live_server):
+    sleep_time_for_fetch_thread = 3
+
+    set_original_response()
+
+    # Give the endpoint time to spin up
+    time.sleep(1)
+
+    # Add our URL to the import page
+    test_url = url_for("test_endpoint", _external=True)
+    res = client.post(
+        url_for("import_page"), data={"urls": test_url}, follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+    # Goto the edit page, add the filter data
+    # Not sure why \r needs to be added - absent of the #changetext this is not necessary
+    subtractive_selectors_data = "header\r\nfooter\r\nnav\r\n#changetext"
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={
+            "subtractive_selectors": subtractive_selectors_data,
+            "url": test_url,
+            "tag": "",
+            "headers": "",
+            "fetch_backend": "html_requests",
+        },
+        follow_redirects=True,
+    )
+    assert b"Updated watch." in res.data
+
+    # Check it saved
+    res = client.get(
+        url_for("edit_page", uuid="first"),
+    )
+    assert bytes(subtractive_selectors_data.encode("utf-8")) in res.data
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
+
+    # No change yet - first check
+    res = client.get(url_for("index"))
+    assert b"unviewed" not in res.data
+
+    #  Make a change to header/footer/nav
+    set_modified_response()
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
+
+    # There should not be an unviewed change, as changes should be removed
+    res = client.get(url_for("index"))
+    assert b"unviewed" not in res.data

changedetectionio/tests/test_jsonpath_selector.py

@@ -1,4 +1,5 @@
 #!/usr/bin/python3
+# coding=utf-8
 
 import time
 from flask import url_for
@@ -142,7 +143,7 @@ def set_modified_response():
         }
       ],
       "boss": {
-        "name": "Foobar"
+        "name": "Örnsköldsvik"
       },
       "available": false
     }
@@ -246,8 +247,10 @@ def test_check_json_filter(client, live_server):
 
     # Should not see this, because its not in the JSONPath we entered
     res = client.get(url_for("diff_history_page", uuid="first"))
+
     # But the change should be there, tho its hard to test the change was detected because it will show old and new versions
-    assert b'Foobar' in res.data
+    # And #462 - check we see the proper utf-8 string there
+    assert "Örnsköldsvik".encode('utf-8') in res.data
 
 
 def test_check_json_filter_bool_val(client, live_server):

changedetectionio/tests/test_request.py

@@ -77,14 +77,6 @@ def test_body_in_request(client, live_server):
     # Add our URL to the import page
     test_url = url_for('test_body', _external=True)
 
-    # Add the test URL twice, we will check
-    res = client.post(
-        url_for("import_page"),
-        data={"urls": test_url},
-        follow_redirects=True
-    )
-    assert b"1 Imported" in res.data
-
     res = client.post(
         url_for("import_page"),
         data={"urls": test_url},
@@ -94,19 +86,6 @@ def test_body_in_request(client, live_server):
 
     body_value = 'Test Body Value'
 
-    # Attempt to add a body with a GET method
-    res = client.post(
-        url_for("edit_page", uuid="first"),
-        data={
-              "url": test_url,
-              "tag": "",
-              "method": "GET",
-              "fetch_backend": "html_requests",
-              "body": "invalid"},
-        follow_redirects=True
-    )
-    assert b"Body must be empty when Request Method is set to GET" in res.data
-
     # Add a properly formatted body with a proper method
     res = client.post(
         url_for("edit_page", uuid="first"),
@@ -120,8 +99,7 @@ def test_body_in_request(client, live_server):
     )
     assert b"Updated watch." in res.data
 
-    # Give the thread time to pick up the first version
-    time.sleep(5)
+    time.sleep(3)
 
     # The service should echo back the body
     res = client.get(
@@ -129,9 +107,20 @@ def test_body_in_request(client, live_server):
         follow_redirects=True
     )
 
-    # Check if body returned contains the specified data
+    # If this gets stuck something is wrong, something should always be there
+    assert b"No history found" not in res.data
+    # We should see what we sent in the reply
     assert str.encode(body_value) in res.data
 
+    ####### data sanity checks
+    # Add the test URL twice, we will check
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
     watches_with_body = 0
     with open('test-datastore/url-watches.json') as f:
         app_struct = json.load(f)
@@ -142,6 +131,20 @@ def test_body_in_request(client, live_server):
     # Should be only one with body set
     assert watches_with_body==1
 
+    # Attempt to add a body with a GET method
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={
+              "url": test_url,
+              "tag": "",
+              "method": "GET",
+              "fetch_backend": "html_requests",
+              "body": "invalid"},
+        follow_redirects=True
+    )
+    assert b"Body must be empty when Request Method is set to GET" in res.data
+
+
 def test_method_in_request(client, live_server):
     # Add our URL to the import page
     test_url = url_for('test_method', _external=True)

docker-compose.yml

@@ -1,9 +1,9 @@
 version: '2'
 services:
-    changedetection.io:
+    changedetection:
       image: ghcr.io/dgtlmoon/changedetection.io
       container_name: changedetection.io
-      hostname: changedetection.io
+      hostname: changedetection
       volumes:
         - changedetection-data:/datastore
 

requirements.txt

@@ -3,7 +3,7 @@ flask~= 2.0
 eventlet>=0.31.0
 validators
 timeago ~=1.0
-inscriptis ~= 1.2
+inscriptis ~= 2.2
 feedgen ~= 0.9
 flask-login ~= 0.5
 pytz
@@ -17,7 +17,7 @@ wtforms ~= 2.3.3
 jsonpath-ng ~= 1.5.3
 
 # Notification library
-apprise ~= 0.9.6
+apprise ~= 0.9.7
 
 # apprise mqtt https://github.com/dgtlmoon/changedetection.io/issues/315
 paho-mqtt
@@ -34,5 +34,4 @@ lxml
 
 # 3.141 was missing socksVersion, 3.150 was not in pypi, so we try 4.1.0
 selenium ~= 4.1.0
-pytest ~=6.2
-pytest-flask ~=1.2
+