Stock Alert (UK): RPi CM4' in res.data
+ assert b'
Stock Alert (UK): Big monitor' in res.data
+
+ res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+ assert b'Deleted' in res.data
def test_check_markup_xpath_filter_restriction(client, live_server):
sleep_time_for_fetch_thread = 3
diff --git a/changedetectionio/update_worker.py b/changedetectionio/update_worker.py
index 7202142dc..02ea036ce 100644
--- a/changedetectionio/update_worker.py
+++ b/changedetectionio/update_worker.py
@@ -3,6 +3,8 @@ import queue
import time
from changedetectionio import content_fetcher
+from changedetectionio.html_tools import FilterNotFoundInResponse
+
# A single update worker
#
# Requests for checking on a single site(watch) from a queue of watches
@@ -19,6 +21,32 @@ class update_worker(threading.Thread):
self.datastore = datastore
super().__init__(*args, **kwargs)
+ def send_filter_failure_notification(self, uuid):
+
+ threshold = self.datastore.data['settings']['application'].get('filter_failure_notification_threshold_attempts')
+ watch = self.datastore.data['watching'].get(uuid, False)
+
+ n_object = {'notification_title': 'Changedetection.io - Alert - CSS/xPath filter was not present in the page',
+ 'notification_body': "Your configured CSS/xPath filter of '{}' for {{watch_url}} did not appear on the page after {} attempts, did the page change layout?\n\nLink: {{base_url}}/edit/{{watch_uuid}}\n\nThanks - Your omniscient changedetection.io installation :)\n".format(
+ watch['css_filter'],
+ threshold),
+ 'notification_format': 'text'}
+
+ if len(watch['notification_urls']):
+ n_object['notification_urls'] = watch['notification_urls']
+
+ elif len(self.datastore.data['settings']['application']['notification_urls']):
+ n_object['notification_urls'] = self.datastore.data['settings']['application']['notification_urls']
+
+ # Only prepare to notify if the rules above matched
+ if 'notification_urls' in n_object:
+ n_object.update({
+ 'watch_url': watch['url'],
+ 'uuid': uuid
+ })
+ self.notification_q.put(n_object)
+ print("Sent filter not found notification for {}".format(uuid))
+
def run(self):
from changedetectionio import fetch_site_status
@@ -37,10 +65,12 @@ class update_worker(threading.Thread):
if uuid in list(self.datastore.data['watching'].keys()):
changed_detected = False
- contents = ""
+ contents = b''
screenshot = False
update_obj= {}
xpath_data = False
+ process_changedetection_results = True
+
now = time.time()
try:
@@ -52,14 +82,35 @@ class update_worker(threading.Thread):
raise Exception("Error - returned data from the fetch handler SHOULD be bytes")
except PermissionError as e:
self.app.logger.error("File permission error updating", uuid, str(e))
+ process_changedetection_results = False
except content_fetcher.ReplyWithContentButNoText as e:
# Totally fine, it's by choice - just continue on, nothing more to care about
# Page had elements/content but no renderable text
- if self.datastore.data['watching'].get(uuid, False) and self.datastore.data['watching'][uuid].get('css_filter'):
- self.datastore.update_watch(uuid=uuid, update_obj={'last_error': "Got HTML content but no text found (CSS / xPath Filter not found in page?)"})
- else:
- self.datastore.update_watch(uuid=uuid, update_obj={'last_error': "Got HTML content but no text found."})
- pass
+ # Backend (not filters) gave zero output
+ self.datastore.update_watch(uuid=uuid, update_obj={'last_error': "Got HTML content but no text found."})
+ process_changedetection_results = False
+
+ except FilterNotFoundInResponse as e:
+ err_text = "Warning, filter '{}' not found".format(str(e))
+ self.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text,
+ # So that we get a trigger when the content is added again
+ 'previous_md5': ''})
+
+ # Only when enabled, send the notification
+ if self.datastore.data['watching'][uuid].get('filter_failure_notification_send', False):
+ c = self.datastore.data['watching'][uuid].get('consecutive_filter_failures', 5)
+ c += 1
+ # Send notification if we reached the threshold?
+ threshold = self.datastore.data['settings']['application'].get('filter_failure_notification_threshold_attempts',
+ 0)
+ print("Filter for {} not found, consecutive_filter_failures: {}".format(uuid, c))
+ if threshold > 0 and c >= threshold:
+ self.send_filter_failure_notification(uuid)
+ c = 0
+ self.datastore.update_watch(uuid=uuid, update_obj={'consecutive_filter_failures': c})
+
+ process_changedetection_results = True
+
except content_fetcher.EmptyReply as e:
# Some kind of custom to-str handler in the exception handler that does this?
err_text = "EmptyReply - try increasing 'Wait seconds before extracting text', Status Code {}".format(e.status_code)
@@ -69,16 +120,22 @@ class update_worker(threading.Thread):
err_text = "Screenshot unavailable, page did not render fully in the expected time - try increasing 'Wait seconds before extracting text'"
self.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text,
'last_check_status': e.status_code})
+ process_changedetection_results = False
except content_fetcher.PageUnloadable as e:
err_text = "Page request from server didnt respond correctly"
self.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text,
'last_check_status': e.status_code})
-
except Exception as e:
self.app.logger.error("Exception reached processing watch UUID: %s - %s", uuid, str(e))
self.datastore.update_watch(uuid=uuid, update_obj={'last_error': str(e)})
-
+ # Other serious error
+ process_changedetection_results = False
else:
+ # Mark that we never had any failures
+ update_obj['consecutive_filter_failures'] = 0
+
+ # Different exceptions mean that we may or may not want to bump the snapshot, trigger notifications etc
+ if process_changedetection_results:
try:
watch = self.datastore.data['watching'][uuid]
fname = "" # Saved history text filename
@@ -88,7 +145,6 @@ class update_worker(threading.Thread):
# A change was detected
fname = watch.save_history_text(contents=contents, timestamp=str(round(time.time())))
- # Generally update anything interesting returned
self.datastore.update_watch(uuid=uuid, update_obj=update_obj)
# A change was detected
@@ -154,7 +210,7 @@ class update_worker(threading.Thread):
self.app.logger.error("Exception reached processing watch UUID: %s - %s", uuid, str(e))
self.datastore.update_watch(uuid=uuid, update_obj={'last_error': str(e)})
- finally:
+
# Always record that we atleast tried
self.datastore.update_watch(uuid=uuid, update_obj={'fetch_time': round(time.time() - now, 3),
'last_checked': round(time.time())})
diff --git a/docker-compose.yml b/docker-compose.yml
index 437d3e64e..04f37d138 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -24,7 +24,7 @@ services:
# https://selenium-python.readthedocs.io/api.html#module-selenium.webdriver.common.proxy
#
# Alternative Playwright URL, do not use "'s or 's!
- # - PLAYWRIGHT_DRIVER_URL=ws://playwright-chrome:3000/
+ # - PLAYWRIGHT_DRIVER_URL=ws://playwright-chrome:3000/?stealth=1&--disable-web-security=true
#
# Playwright proxy settings playwright_proxy_server, playwright_proxy_bypass, playwright_proxy_username, playwright_proxy_password
#
diff --git a/x b/x
deleted file mode 100644
index 0219e7895..000000000
--- a/x
+++ /dev/null
@@ -1,16 +0,0 @@
-diff --git a/changedetectionio/content_fetcher.py b/changedetectionio/content_fetcher.py
-index 331ef959..ca43edc8 100644
---- a/changedetectionio/content_fetcher.py
-+++ b/changedetectionio/content_fetcher.py
-@@ -309,7 +309,10 @@ class base_html_playwright(Fetcher):
- page.set_default_navigation_timeout(90000)
- page.set_default_timeout(90000)
-
-- # Bug - never set viewport size BEFORE page.goto
-+ # Listen for all console events and handle errors
-+ page.on("console", lambda msg: print(f"Playwright console: Watch URL: {url} {msg.type}: {msg.text} {msg.args}"))
-+
-+ # Bug - never set viewport size BEFORE page.goto
-
- # Waits for the next navigation. Using Python context manager
- # prevents a race condition between clicking and waiting for a navigation.
](https://lemonade.changedetection.io/start)
diff --git a/changedetectionio/.gitignore b/changedetectionio/.gitignore
index 1d4637848..0d3c1d4eb 100644
--- a/changedetectionio/.gitignore
+++ b/changedetectionio/.gitignore
@@ -1 +1,2 @@
test-datastore
+package-lock.json
diff --git a/changedetectionio/__init__.py b/changedetectionio/__init__.py
index 2b10ace39..556503122 100644
--- a/changedetectionio/__init__.py
+++ b/changedetectionio/__init__.py
@@ -44,7 +44,7 @@ from flask_wtf import CSRFProtect
from changedetectionio import html_tools
from changedetectionio.api import api_v1
-__version__ = '0.39.16'
+__version__ = '0.39.17.1'
datastore = None
@@ -580,6 +580,9 @@ def changedetection_app(config=None, datastore_o=None):
if request.method == 'POST' and form.validate():
extra_update_obj = {}
+ if request.args.get('unpause_on_save'):
+ extra_update_obj['paused'] = False
+
# Re #110, if they submit the same as the default value, set it to None, so we continue to follow the default
# Assume we use the default value, unless something relevant is different, then use the form value
# values could be None, 0 etc.
@@ -619,7 +622,10 @@ def changedetection_app(config=None, datastore_o=None):
datastore.data['watching'][uuid].update(form.data)
datastore.data['watching'][uuid].update(extra_update_obj)
- flash("Updated watch.")
+ if request.args.get('unpause_on_save'):
+ flash("Updated watch - unpaused!.")
+ else:
+ flash("Updated watch.")
# Re #286 - We wait for syncing new data to disk in another thread every 60 seconds
# But in the case something is added we should save straight away
@@ -1063,9 +1069,9 @@ def changedetection_app(config=None, datastore_o=None):
except FileNotFoundError:
abort(404)
- @app.route("/api/add", methods=['POST'])
+ @app.route("/form/add/quickwatch", methods=['POST'])
@login_required
- def form_watch_add():
+ def form_quick_watch_add():
from changedetectionio import forms
form = forms.quickWatchForm(request.form)
@@ -1078,13 +1084,19 @@ def changedetection_app(config=None, datastore_o=None):
flash('The URL {} already exists'.format(url), "error")
return redirect(url_for('index'))
- # @todo add_watch should throw a custom Exception for validation etc
- new_uuid = datastore.add_watch(url=url, tag=request.form.get('tag').strip())
- if new_uuid:
+ add_paused = request.form.get('edit_and_watch_submit_button') != None
+ new_uuid = datastore.add_watch(url=url, tag=request.form.get('tag').strip(), extras={'paused': add_paused})
+
+
+ if not add_paused and new_uuid:
# Straight into the queue.
update_q.put(new_uuid)
flash("Watch added.")
+ if add_paused:
+ flash('Watch added in Paused state, saving will unpause.')
+ return redirect(url_for('edit_page', uuid=new_uuid, unpause_on_save=1))
+
return redirect(url_for('index'))
diff --git a/changedetectionio/content_fetcher.py b/changedetectionio/content_fetcher.py
index ca43edc87..909881794 100644
--- a/changedetectionio/content_fetcher.py
+++ b/changedetectionio/content_fetcher.py
@@ -63,12 +63,12 @@ class Fetcher():
break;
}
if('' !==r.id) {
- chained_css.unshift("#"+r.id);
- final_selector= chained_css.join('>');
+ chained_css.unshift("#"+CSS.escape(r.id));
+ final_selector= chained_css.join(' > ');
// Be sure theres only one, some sites have multiples of the same ID tag :-(
if (window.document.querySelectorAll(final_selector).length ==1 ) {
return final_selector;
- }
+ }
return null;
} else {
chained_css.unshift(r.tagName.toLowerCase());
diff --git a/changedetectionio/fetch_site_status.py b/changedetectionio/fetch_site_status.py
index 551cb5a5a..ab12afb83 100644
--- a/changedetectionio/fetch_site_status.py
+++ b/changedetectionio/fetch_site_status.py
@@ -12,6 +12,7 @@ urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
# Some common stuff here that can be moved to a base class
+# (set_proxy_from_list)
class perform_site_check():
def __init__(self, *args, datastore, **kwargs):
@@ -46,6 +47,20 @@ class perform_site_check():
return proxy_args
+ # Doesn't look like python supports forward slash auto enclosure in re.findall
+ # So convert it to inline flag "foobar(?i)" type configuration
+ def forward_slash_enclosed_regex_to_options(self, regex):
+ res = re.search(r'^/(.*?)/(\w+)$', regex, re.IGNORECASE)
+
+ if res:
+ regex = res.group(1)
+ regex += '(?{})'.format(res.group(2))
+ else:
+ regex += '(?{})'.format('i')
+
+ return regex
+
+
def run(self, uuid):
timestamp = int(time.time()) # used for storage etc too
@@ -151,7 +166,9 @@ class perform_site_check():
is_html = False
if is_html or is_source:
+
# CSS Filter, extract the HTML that matches and feed that into the existing inscriptis::get_text
+ fetcher.content = html_tools.workarounds_for_obfuscations(fetcher.content)
html_content = fetcher.content
# If not JSON, and if it's not text/plain..
@@ -214,15 +231,27 @@ class perform_site_check():
if len(extract_text) > 0:
regex_matched_output = []
for s_re in extract_text:
- result = re.findall(s_re.encode('utf8'), stripped_text_from_html,
- flags=re.MULTILINE | re.DOTALL | re.LOCALE)
- if result:
- regex_matched_output = regex_matched_output + result
+ # incase they specified something in '/.../x'
+ regex = self.forward_slash_enclosed_regex_to_options(s_re)
+ result = re.findall(regex.encode('utf-8'), stripped_text_from_html)
+ for l in result:
+ if type(l) is tuple:
+ #@todo - some formatter option default (between groups)
+ regex_matched_output += list(l) + [b'\n']
+ else:
+ # @todo - some formatter option default (between each ungrouped result)
+ regex_matched_output += [l] + [b'\n']
+
+ # Now we will only show what the regex matched
+ stripped_text_from_html = b''
+ text_content_before_ignored_filter = b''
if regex_matched_output:
- stripped_text_from_html = b'\n'.join(regex_matched_output)
+ # @todo some formatter for presentation?
+ stripped_text_from_html = b''.join(regex_matched_output)
text_content_before_ignored_filter = stripped_text_from_html
+
# Re #133 - if we should strip whitespaces from triggering the change detected comparison
if self.datastore.data['settings']['application'].get('ignore_whitespace', False):
fetched_md5 = hashlib.md5(stripped_text_from_html.translate(None, b'\r\n\t ')).hexdigest()
diff --git a/changedetectionio/forms.py b/changedetectionio/forms.py
index 00d2e4fef..75d57ef34 100644
--- a/changedetectionio/forms.py
+++ b/changedetectionio/forms.py
@@ -308,6 +308,9 @@ class ValidateCSSJSONXPATHInput(object):
class quickWatchForm(Form):
url = fields.URLField('URL', validators=[validateURL()])
tag = StringField('Group tag', [validators.Optional()])
+ watch_submit_button = SubmitField('Watch', render_kw={"class": "pure-button pure-button-primary"})
+ edit_and_watch_submit_button = SubmitField('Edit > Watch', render_kw={"class": "pure-button pure-button-primary"})
+
# Common to a single watch and the global settings
class commonSettingsForm(Form):
@@ -351,6 +354,8 @@ class watchForm(commonSettingsForm):
save_button = SubmitField('Save', render_kw={"class": "pure-button pure-button-primary"})
save_and_preview_button = SubmitField('Save & Preview', render_kw={"class": "pure-button pure-button-primary"})
proxy = RadioField('Proxy')
+ filter_failure_notification_send = BooleanField(
+ 'Send a notification when the filter can no longer be found on the page', default=False)
def validate(self, **kwargs):
if not super().validate():
@@ -389,6 +394,11 @@ class globalSettingsApplicationForm(commonSettingsForm):
api_access_token_enabled = BooleanField('API access token security check enabled', default=True, validators=[validators.Optional()])
password = SaltyPasswordField()
+ filter_failure_notification_threshold_attempts = IntegerField('Number of times the filter can be missing before sending a notification',
+ render_kw={"style": "width: 5em;"},
+ validators=[validators.NumberRange(min=0,
+ message="Should contain zero or more attempts")])
+
class globalSettingsForm(Form):
# Define these as FormFields/"sub forms", this way it matches the JSON storage
diff --git a/changedetectionio/html_tools.py b/changedetectionio/html_tools.py
index 27496e1a2..a851a4d65 100644
--- a/changedetectionio/html_tools.py
+++ b/changedetectionio/html_tools.py
@@ -1,5 +1,4 @@
import json
-import re
from typing import List
from bs4 import BeautifulSoup
@@ -8,16 +7,23 @@ import re
from inscriptis import get_text
from inscriptis.model.config import ParserConfig
+class FilterNotFoundInResponse(ValueError):
+ def __init__(self, msg):
+ ValueError.__init__(self, msg)
class JSONNotFound(ValueError):
def __init__(self, msg):
ValueError.__init__(self, msg)
+
# Given a CSS Rule, and a blob of HTML, return the blob of HTML that matches
def css_filter(css_filter, html_content):
soup = BeautifulSoup(html_content, "html.parser")
html_block = ""
- for item in soup.select(css_filter, separator=""):
+ r = soup.select(css_filter, separator="")
+ if len(html_content) > 0 and len(r) == 0:
+ raise FilterNotFoundInResponse(css_filter)
+ for item in r:
html_block += str(item)
return html_block + "\n"
@@ -42,8 +48,19 @@ def xpath_filter(xpath_filter, html_content):
tree = html.fromstring(bytes(html_content, encoding='utf-8'))
html_block = ""
- for item in tree.xpath(xpath_filter.strip(), namespaces={'re':'http://exslt.org/regular-expressions'}):
- html_block+= etree.tostring(item, pretty_print=True).decode('utf-8')+"