Compare commits

..

29 Commits

Author SHA1 Message Date
dgtlmoon
5454b0fc75 Closes #1224 2022-12-14 00:02:51 +01:00
dgtlmoon
6562d6e0d4 Improve ARM/rust build comment 2022-12-13 12:28:20 +01:00
dgtlmoon
6c217cc3b6 README.md - Improving JSONPath example for LD+JSON product data 2022-12-11 11:14:52 +01:00
dgtlmoon
f30cdf0674 0.40.0.2 2022-12-08 22:36:59 +01:00
dgtlmoon
14da0646a7 Price follower - Dont scan for ldjson data when 'no' was clicked on the suggestion (#1207) 2022-12-08 22:35:37 +01:00
dgtlmoon
b413cdecc7 Adding missing parts for pip build Re #1206 2022-12-08 21:54:55 +01:00
dgtlmoon
7bf52d9275 0.40.0 2022-12-08 20:09:42 +01:00
dgtlmoon
09e6624afd VisualSelector - Exclude items that are not interactable or visible 2022-12-08 20:08:41 +01:00
dgtlmoon
b58fd995b5 Automatically offer to track LD+JSON product price data (#1204) 2022-12-08 19:28:20 +01:00
dgtlmoon
f7bb8a0afa UI - favicon callback no longer needed 2022-12-07 12:14:36 +01:00
dgtlmoon
3e333496c1 Test cleanups (#1196) 2022-12-07 12:03:28 +01:00
Amro Hendawi
ee776a9627 Update runtime.txt (#1198) 2022-12-07 00:17:58 +01:00
dgtlmoon
65db4d68e3 Dark mode - HTML template tidy up (#1197) 2022-12-06 23:50:49 +01:00
dgtlmoon
74d93d10c3 UI - watch tags also known as watch tag / label 2022-12-06 23:16:22 +01:00
dgtlmoon
37aef0530a Notification templates - bug in update, was updating the main system instead of the watch notification_title incorrectly 2022-12-06 18:29:09 +01:00
dgtlmoon
f86763dc7a Extract data - minor improvement to example 2022-12-06 10:53:23 +01:00
dgtlmoon
13c25f9b92 Darkmode - Pause/Mute notification colour fix, re #1195 2022-12-06 10:49:24 +01:00
dgtlmoon
265f622e75 Notification - Support for standard API calls post:// posts:// get:// gets:// delete:// deletes:// put:// puts:// (#1194) 2022-12-05 20:49:08 +01:00
dgtlmoon
c12db2b725 Notifications - tokens/jinja2 templating (#1184) 2022-12-05 19:58:43 +01:00
dgtlmoon
a048e4a02d Dark mode - more colour fixes 2022-12-05 19:10:36 +01:00
dgtlmoon
69662ff91c Test improvement - improving notification error network test 2022-12-05 17:45:30 +01:00
dgtlmoon
fc94c57d7f Extract text as CSV - Extra validation (#1192) 2022-12-05 16:36:00 +01:00
dgtlmoon
7b94ba6f23 Dark mode - make watch list easier to read when theres 'unviewed' entries 2022-12-05 15:13:47 +01:00
dgtlmoon
2345b6b558 New feature - Simple extract data by regex from all historical watch text into CSV (#1191) 2022-12-05 14:48:03 +01:00
dgtlmoon
b8d5a12ad0 UI - Cursor over labels should be pointer 2022-12-05 10:42:48 +01:00
dgtlmoon
9e67a572c5 Dark mode - Make watches with errors easier to read 2022-12-05 09:53:53 +01:00
dgtlmoon
378d7b7362 Dark mode - cookie path should be all site 2022-12-04 20:54:15 +01:00
dgtlmoon
d1d4045c49 Tweaks - adding hover/title to dark mode button 2022-12-04 18:53:56 +01:00
dgtlmoon
77409eeb3a UI - Dark Mode (#1187) 2022-12-04 16:39:25 +01:00
44 changed files with 1735 additions and 1326 deletions

View File

@@ -1,7 +1,7 @@
# pip dependencies install stage
FROM python:3.8-slim as builder
# rustc compiler would be needed on ARM type devices but theres an issue with some deps not building..
# See `cryptography` pin comment in requirements.txt
ARG CRYPTOGRAPHY_DONT_BUILD_RUST=1
RUN apt-get update && apt-get install -y --no-install-recommends \
@@ -31,8 +31,7 @@ RUN pip install --target=/dependencies playwright~=1.27.1 \
# Final image stage
FROM python:3.8-slim
# Actual packages needed at runtime, usually due to the notification (apprise) backend
# rustc compiler would be needed on ARM type devices but theres an issue with some deps not building..
# See `cryptography` pin comment in requirements.txt
ARG CRYPTOGRAPHY_DONT_BUILD_RUST=1
# Re #93, #73, excluding rustc (adds another 430Mb~)

View File

@@ -1,9 +1,10 @@
recursive-include changedetectionio/api *
recursive-include changedetectionio/templates *
recursive-include changedetectionio/static *
recursive-include changedetectionio/blueprint *
recursive-include changedetectionio/model *
recursive-include changedetectionio/tests *
recursive-include changedetectionio/res *
recursive-include changedetectionio/static *
recursive-include changedetectionio/templates *
recursive-include changedetectionio/tests *
prune changedetectionio/static/package-lock.json
prune changedetectionio/static/styles/node_modules
prune changedetectionio/static/styles/package-lock.json

View File

@@ -159,7 +159,7 @@ Just some examples
<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/docs/screenshot-notifications.png" style="max-width:100%;" alt="Self-hosted web page change monitoring notifications" title="Self-hosted web page change monitoring notifications" />
Now you can also customise your notification content!
Now you can also customise your notification content and use <a target="_new" href="https://jinja.palletsprojects.com/en/3.0.x/templates/">Jinja2 templating</a> for their title and body!
## JSON API Monitoring
@@ -187,11 +187,29 @@ When you enable a `json:` or `jq:` filter, you can even automatically extract an
<html>
...
<script type="application/ld+json">
{"@context":"http://schema.org","@type":"Product","name":"Nan Optipro Stage 1 Baby Formula 800g","price": 23.50 }
{
"@context":"http://schema.org/",
"@type":"Product",
"offers":{
"@type":"Offer",
"availability":"http://schema.org/InStock",
"price":"3949.99",
"priceCurrency":"USD",
"url":"https://www.newegg.com/p/3D5-000D-001T1"
},
"description":"Cobratype King Cobra Hero Desktop Gaming PC",
"name":"Cobratype King Cobra Hero Desktop Gaming PC",
"sku":"3D5-000D-001T1",
"itemCondition":"NewCondition"
}
</script>
```
`json:$.price` or `jq:.price` would give `23.50`, or you can extract the whole structure
`json:$..price` or `jq:..price` would give `3949.99`, or you can extract the whole structure (use a JSONpath test website to validate with)
The application also supports notifying you that it can follow this information automatically
## Proxy Configuration

View File

@@ -35,7 +35,7 @@ from flask_wtf import CSRFProtect
from changedetectionio import html_tools
from changedetectionio.api import api_v1
__version__ = '0.39.22.1'
__version__ = '0.40.0.2'
datastore = None
@@ -95,6 +95,12 @@ def init_app_secret(datastore_path):
return secret
@app.template_global()
def get_darkmode_state():
css_dark_mode = request.cookies.get('css_dark_mode', 'false')
return 'true' if css_dark_mode and strtobool(css_dark_mode) else 'false'
# We use the whole watch object from the store/JSON so we can see if there's some related status in terms of a thread
# running or something similar.
@app.template_filter('format_last_checked_time')
@@ -202,10 +208,6 @@ def changedetection_app(config=None, datastore_o=None):
watch_api.add_resource(api_v1.SystemInfo, '/api/v1/systeminfo',
resource_class_kwargs={'datastore': datastore, 'update_q': update_q})
def getDarkModeSetting():
css_dark_mode = request.cookies.get('css_dark_mode')
return True if (css_dark_mode == 'true' or css_dark_mode == True) else False
# Setup cors headers to allow all domains
# https://flask-cors.readthedocs.io/en/latest/
# CORS(app)
@@ -405,7 +407,6 @@ def changedetection_app(config=None, datastore_o=None):
form = forms.quickWatchForm(request.form)
output = render_template("watch-overview.html",
dark_mode=getDarkModeSetting(),
form=form,
watches=sorted_watches,
tags=existing_tags,
@@ -664,7 +665,6 @@ def changedetection_app(config=None, datastore_o=None):
browser_steps_config=browser_step_ui_config,
current_base_url=datastore.data['settings']['application']['base_url'],
emailprefix=os.getenv('NOTIFICATION_MAIL_BUTTON_PREFIX', False),
dark_mode=getDarkModeSetting(),
form=form,
has_default_notification_urls=True if len(datastore.data['settings']['application']['notification_urls']) else False,
has_empty_checktime=using_default_check_time,
@@ -752,7 +752,6 @@ def changedetection_app(config=None, datastore_o=None):
output = render_template("settings.html",
form=form,
dark_mode=getDarkModeSetting(),
current_base_url = datastore.data['settings']['application']['base_url'],
hide_remove_pass=os.getenv("SALTED_PASS", False),
api_key=datastore.data['settings']['application'].get('api_access_token'),
@@ -793,7 +792,6 @@ def changedetection_app(config=None, datastore_o=None):
# Could be some remaining, or we could be on GET
output = render_template("import.html",
dark_mode=getDarkModeSetting(),
import_url_list_remaining="\n".join(remaining_urls),
original_distill_json=''
)
@@ -810,10 +808,12 @@ def changedetection_app(config=None, datastore_o=None):
return redirect(url_for('index'))
@app.route("/diff/<string:uuid>", methods=['GET'])
@app.route("/diff/<string:uuid>", methods=['GET', 'POST'])
@login_required
def diff_history_page(uuid):
from changedetectionio import forms
# More for testing, possible to return the first/only
if uuid == 'first':
uuid = list(datastore.data['watching'].keys()).pop()
@@ -825,6 +825,28 @@ def changedetection_app(config=None, datastore_o=None):
flash("No history found for the specified link, bad link?", "error")
return redirect(url_for('index'))
# For submission of requesting an extract
extract_form = forms.extractDataForm(request.form)
if request.method == 'POST':
if not extract_form.validate():
flash("An error occurred, please see below.", "error")
else:
extract_regex = request.form.get('extract_regex').strip()
output = watch.extract_regex_from_all_history(extract_regex)
if output:
watch_dir = os.path.join(datastore_o.datastore_path, uuid)
response = make_response(send_from_directory(directory=watch_dir, path=output, as_attachment=True))
response.headers['Content-type'] = 'text/csv'
response.headers['Cache-Control'] = 'no-cache, no-store, must-revalidate'
response.headers['Pragma'] = 'no-cache'
response.headers['Expires'] = 0
return response
flash('Nothing matches that RegEx', 'error')
redirect(url_for('diff_history_page', uuid=uuid)+'#extract')
history = watch.history
dates = list(history.keys())
@@ -867,23 +889,23 @@ def changedetection_app(config=None, datastore_o=None):
watch.get('fetch_backend', None) is None and system_uses_webdriver) else False
output = render_template("diff.html",
watch_a=watch,
newest=newest_version_file_contents,
previous=previous_version_file_contents,
extra_stylesheets=extra_stylesheets,
dark_mode=getDarkModeSetting(),
versions=dates[:-1], # All except current/last
uuid=uuid,
newest_version_timestamp=dates[-1],
current_previous_version=str(previous_version),
current_diff_url=watch['url'],
current_previous_version=str(previous_version),
extra_stylesheets=extra_stylesheets,
extra_title=" - Diff - {}".format(watch['title'] if watch['title'] else watch['url']),
left_sticky=True,
screenshot=screenshot_url,
extract_form=extract_form,
is_html_webdriver=is_html_webdriver,
last_error=watch['last_error'],
last_error_screenshot=watch.get_error_snapshot(),
last_error_text=watch.get_error_text(),
last_error_screenshot=watch.get_error_snapshot()
left_sticky=True,
newest=newest_version_file_contents,
newest_version_timestamp=dates[-1],
previous=previous_version_file_contents,
screenshot=screenshot_url,
uuid=uuid,
versions=dates[:-1], # All except current/last
watch_a=watch
)
return output
@@ -919,7 +941,6 @@ def changedetection_app(config=None, datastore_o=None):
content=content,
history_n=watch.history_n,
extra_stylesheets=extra_stylesheets,
dark_mode=getDarkModeSetting(),
# current_diff_url=watch['url'],
watch=watch,
uuid=uuid,
@@ -966,7 +987,6 @@ def changedetection_app(config=None, datastore_o=None):
content=content,
history_n=watch.history_n,
extra_stylesheets=extra_stylesheets,
dark_mode=getDarkModeSetting(),
ignored_line_numbers=ignored_line_numbers,
triggered_line_numbers=trigger_line_numbers,
current_diff_url=watch['url'],
@@ -985,15 +1005,10 @@ def changedetection_app(config=None, datastore_o=None):
def notification_logs():
global notification_debug_log
output = render_template("notification-log.html",
dark_mode=getDarkModeSetting(),
logs=notification_debug_log if len(notification_debug_log) else ["Notification logs are empty - no notifications sent yet."])
return output
@app.route("/favicon.ico", methods=['GET'])
def favicon():
return send_from_directory("static/images", path="favicon.ico")
# We're good but backups are even better!
@app.route("/backup", methods=['GET'])
@login_required
@@ -1328,6 +1343,10 @@ def changedetection_app(config=None, datastore_o=None):
import changedetectionio.blueprint.browser_steps as browser_steps
app.register_blueprint(browser_steps.construct_blueprint(datastore), url_prefix='/browser-steps')
import changedetectionio.blueprint.price_data_follower as price_data_follower
app.register_blueprint(price_data_follower.construct_blueprint(datastore), url_prefix='/price_data_follower')
# @todo handle ctrl break
ticker_thread = threading.Thread(target=ticker_thread_check_time_launch_checks).start()
threading.Thread(target=notification_runner).start()

View File

@@ -26,7 +26,7 @@ browser_step_ui_config = {'Choose one': '0 0',
# 'Extract text and use as filter': '1 0',
'Goto site': '0 0',
'Press Enter': '0 0',
'Select by label': '1 1',
'Select option': '1 1',
'Scroll down': '0 0',
'Uncheck checkbox': '1 0',
'Wait for seconds': '0 1',
@@ -236,7 +236,7 @@ class browsersteps_live_ui(steppable_browser_interface):
self.page.evaluate("var include_filters=''")
# Go find the interactive elements
# @todo in the future, something smarter that can scan for elements with .click/focus etc event handlers?
elements = 'a,button,input,select,textarea,i,th,td,p,li,h1,h2,h3,h4,div,span'
elements = 'a,button,input,select,textarea,i,th,td,p,li,h1,h2,h3,h4,div,span,select'
xpath_element_js = xpath_element_js.replace('%ELEMENTS%', elements)
xpath_data = self.page.evaluate("async () => {" + xpath_element_js + "}")
# So the JS will find the smallest one first

View File

@@ -0,0 +1,30 @@
from distutils.util import strtobool
from flask import Blueprint, flash, redirect, url_for
from flask_login import login_required
from changedetectionio.store import ChangeDetectionStore
PRICE_DATA_TRACK_ACCEPT = 'accepted'
PRICE_DATA_TRACK_REJECT = 'rejected'
def construct_blueprint(datastore: ChangeDetectionStore):
price_data_follower_blueprint = Blueprint('price_data_follower', __name__)
@login_required
@price_data_follower_blueprint.route("/<string:uuid>/accept", methods=['GET'])
def accept(uuid):
datastore.data['watching'][uuid]['track_ldjson_price_data'] = PRICE_DATA_TRACK_ACCEPT
return redirect(url_for("form_watch_checknow", uuid=uuid))
@login_required
@price_data_follower_blueprint.route("/<string:uuid>/reject", methods=['GET'])
def reject(uuid):
datastore.data['watching'][uuid]['track_ldjson_price_data'] = PRICE_DATA_TRACK_REJECT
return redirect(url_for("index"))
return price_data_follower_blueprint

View File

@@ -2,10 +2,10 @@ import hashlib
import logging
import os
import re
import time
import urllib3
from changedetectionio import content_fetcher, html_tools
from changedetectionio.blueprint.price_data_follower import PRICE_DATA_TRACK_ACCEPT, PRICE_DATA_TRACK_REJECT
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
@@ -140,7 +140,7 @@ class perform_site_check():
is_html = False
is_json = False
include_filters_rule = watch.get('include_filters', [])
include_filters_rule = deepcopy(watch.get('include_filters', []))
# include_filters_rule = watch['include_filters']
subtractive_selectors = watch.get(
"subtractive_selectors", []
@@ -148,6 +148,10 @@ class perform_site_check():
"global_subtractive_selectors", []
)
# Inject a virtual LD+JSON price tracker rule
if watch.get('track_ldjson_price_data', '') == PRICE_DATA_TRACK_ACCEPT:
include_filters_rule.append(html_tools.LD_JSON_PRODUCT_OFFER_SELECTOR)
has_filter_rule = include_filters_rule and len("".join(include_filters_rule).strip())
has_subtractive_selectors = subtractive_selectors and len(subtractive_selectors[0].strip())
@@ -173,9 +177,13 @@ class perform_site_check():
# Don't run get_text or xpath/css filters on plaintext
stripped_text_from_html = html_content
else:
# Does it have some ld+json price data? used for easier monitoring
update_obj['has_ldjson_price_data'] = html_tools.has_ldjson_product_info(fetcher.content)
# Then we assume HTML
if has_filter_rule:
html_content = ""
for filter_rule in include_filters_rule:
# For HTML/XML we offer xpath as an option, just start a regular xPath "/.."
if filter_rule[0] == '/' or filter_rule.startswith('xpath:'):

View File

@@ -193,7 +193,7 @@ class ValidateAppRiseServers(object):
message = field.gettext('\'%s\' is not a valid AppRise URL.' % (server_url))
raise ValidationError(message)
class ValidateTokensList(object):
class ValidateJinja2Template(object):
"""
Validates that a {token} is from a valid set
"""
@@ -202,11 +202,24 @@ class ValidateTokensList(object):
def __call__(self, form, field):
from changedetectionio import notification
regex = re.compile('{.*?}')
for p in re.findall(regex, field.data):
if not p.strip('{}') in notification.valid_tokens:
message = field.gettext('Token \'%s\' is not a valid token.')
raise ValidationError(message % (p))
from jinja2 import Environment, BaseLoader, TemplateSyntaxError
from jinja2.meta import find_undeclared_variables
try:
jinja2_env = Environment(loader=BaseLoader)
jinja2_env.globals.update(notification.valid_tokens)
rendered = jinja2_env.from_string(field.data).render()
except TemplateSyntaxError as e:
raise ValidationError(f"This is not a valid Jinja2 template: {e}") from e
ast = jinja2_env.parse(field.data)
undefined = ", ".join(find_undeclared_variables(ast))
if undefined:
raise ValidationError(
f"The following tokens used in the notification are not valid: {undefined}"
)
class validateURL(object):
@@ -225,6 +238,7 @@ class validateURL(object):
message = field.gettext('\'%s\' is not a valid URL.' % (field.data.strip()))
raise ValidationError(message)
class ValidateListRegex(object):
"""
Validates that anything that looks like a regex passes as a regex
@@ -333,11 +347,11 @@ class quickWatchForm(Form):
# Common to a single watch and the global settings
class commonSettingsForm(Form):
notification_urls = StringListField('Notification URL list', validators=[validators.Optional(), ValidateAppRiseServers()])
notification_title = StringField('Notification title', validators=[validators.Optional(), ValidateTokensList()])
notification_body = TextAreaField('Notification body', validators=[validators.Optional(), ValidateTokensList()])
notification_urls = StringListField('Notification URL List', validators=[validators.Optional(), ValidateAppRiseServers()])
notification_title = StringField('Notification Title', default='ChangeDetection.io Notification - {{ watch_url }}', validators=[validators.Optional(), ValidateJinja2Template()])
notification_body = TextAreaField('Notification Body', default='{{ watch_url }} had a change.', validators=[validators.Optional(), ValidateJinja2Template()])
notification_format = SelectField('Notification format', choices=valid_notification_formats.keys())
fetch_backend = RadioField(u'Fetch method', choices=content_fetcher.available_fetchers(), validators=[ValidateContentFetcherIsReady()])
fetch_backend = RadioField(u'Fetch Method', choices=content_fetcher.available_fetchers(), validators=[ValidateContentFetcherIsReady()])
extract_title_as_title = BooleanField('Extract <title> from document and use as watch title', default=False)
webdriver_delay = IntegerField('Wait seconds before extracting text', validators=[validators.Optional(), validators.NumberRange(min=1,
message="Should contain one or more seconds")])
@@ -448,3 +462,8 @@ class globalSettingsForm(Form):
requests = FormField(globalSettingsRequestForm)
application = FormField(globalSettingsApplicationForm)
save_button = SubmitField('Save', render_kw={"class": "pure-button pure-button-primary"})
class extractDataForm(Form):
extract_regex = StringField('RegEx to extract', validators=[validators.Length(min=1, message="Needs a RegEx")])
extract_submit_button = SubmitField('Extract as CSV', render_kw={"class": "pure-button pure-button-primary"})

View File

@@ -10,6 +10,10 @@ import re
# HTML added to be sure each result matching a filter (.example) gets converted to a new line by Inscriptis
TEXT_FILTER_LIST_LINE_SUFFIX = "<br/>"
# 'price' , 'lowPrice', 'highPrice' are usually under here
# all of those may or may not appear on different websites
LD_JSON_PRODUCT_OFFER_SELECTOR = "json:$..offers"
class JSONNotFound(ValueError):
def __init__(self, msg):
ValueError.__init__(self, msg)
@@ -127,8 +131,10 @@ def _get_stripped_text_from_json_match(match):
return stripped_text_from_html
def extract_json_as_string(content, json_filter):
# content - json
# json_filter - ie json:$..price
# ensure_is_ldjson_info_type - str "product", optional, "@type == product" (I dont know how to do that as a json selector)
def extract_json_as_string(content, json_filter, ensure_is_ldjson_info_type=None):
stripped_text_from_html = False
# Try to parse/filter out the JSON, if we get some parser error, then maybe it's embedded <script type=ldjson>
@@ -139,7 +145,12 @@ def extract_json_as_string(content, json_filter):
# Foreach <script json></script> blob.. just return the first that matches json_filter
s = []
soup = BeautifulSoup(content, 'html.parser')
bs_result = soup.findAll('script')
if ensure_is_ldjson_info_type:
bs_result = soup.findAll('script', {"type": "application/ld+json"})
else:
bs_result = soup.findAll('script')
if not bs_result:
raise JSONNotFound("No parsable JSON found in this document")
@@ -156,7 +167,14 @@ def extract_json_as_string(content, json_filter):
continue
else:
stripped_text_from_html = _parse_json(json_data, json_filter)
if stripped_text_from_html:
if ensure_is_ldjson_info_type:
# Could sometimes be list, string or something else random
if isinstance(json_data, dict):
# If it has LD JSON 'key' @type, and @type is 'product', and something was found for the search
# (Some sites have multiple of the same ld+json @type='product', but some have the review part, some have the 'price' part)
if json_data.get('@type', False) and json_data.get('@type','').lower() == ensure_is_ldjson_info_type.lower() and stripped_text_from_html:
break
elif stripped_text_from_html:
break
if not stripped_text_from_html:
@@ -243,6 +261,18 @@ def html_to_text(html_content: str, render_anchor_tag_content=False) -> str:
return text_content
# Does LD+JSON exist with a @type=='product' and a .price set anywhere?
def has_ldjson_product_info(content):
try:
pricing_data = extract_json_as_string(content=content, json_filter=LD_JSON_PRODUCT_OFFER_SELECTOR, ensure_is_ldjson_info_type="product")
except JSONNotFound as e:
# Totally fine
return False
x=bool(pricing_data)
return x
def workarounds_for_obfuscations(content):
"""
Some sites are using sneaky tactics to make prices and other information un-renderable by Inscriptis

View File

@@ -26,6 +26,8 @@ class model(dict):
'extract_title_as_title': False,
'fetch_backend': None,
'filter_failure_notification_send': strtobool(os.getenv('FILTER_FAILURE_NOTIFICATION_SEND_DEFAULT', 'True')),
'has_ldjson_price_data': None,
'track_ldjson_price_data': None,
'headers': {}, # Extra headers to send
'ignore_text': [], # List of text to ignore when calculating the comparison checksum
'include_filters': [],
@@ -318,3 +320,47 @@ class model(dict):
if os.path.isfile(fname):
return fname
return False
def extract_regex_from_all_history(self, regex):
import csv
import re
import datetime
csv_output_filename = False
csv_writer = False
f = None
# self.history will be keyed with the full path
for k, fname in self.history.items():
if os.path.isfile(fname):
with open(fname, "r") as f:
contents = f.read()
res = re.findall(regex, contents, re.MULTILINE)
if res:
if not csv_writer:
# A file on the disk can be transferred much faster via flask than a string reply
csv_output_filename = 'report.csv'
f = open(os.path.join(self.watch_data_dir, csv_output_filename), 'w')
# @todo some headers in the future
#fieldnames = ['Epoch seconds', 'Date']
csv_writer = csv.writer(f,
delimiter=',',
quotechar='"',
quoting=csv.QUOTE_MINIMAL,
#fieldnames=fieldnames
)
csv_writer.writerow(['Epoch seconds', 'Date'])
# csv_writer.writeheader()
date_str = datetime.datetime.fromtimestamp(int(k)).strftime('%Y-%m-%d %H:%M:%S')
for r in res:
row = [k, date_str]
if isinstance(r, str):
row.append(r)
else:
row+=r
csv_writer.writerow(row)
if f:
f.close()
return csv_output_filename

View File

@@ -1,5 +1,7 @@
import apprise
from jinja2 import Environment, BaseLoader
from apprise import NotifyFormat
import json
valid_tokens = {
'base_url': '',
@@ -16,8 +18,8 @@ valid_tokens = {
default_notification_format_for_watch = 'System default'
default_notification_format = 'Text'
default_notification_body = '{watch_url} had a change.\n---\n{diff}\n---\n'
default_notification_title = 'ChangeDetection.io Notification - {watch_url}'
default_notification_body = '{{watch_url}} had a change.\n---\n{{diff}}\n---\n'
default_notification_title = 'ChangeDetection.io Notification - {{watch_url}}'
valid_notification_formats = {
'Text': NotifyFormat.TEXT,
@@ -27,25 +29,67 @@ valid_notification_formats = {
default_notification_format_for_watch: default_notification_format_for_watch
}
def process_notification(n_object, datastore):
# include the decorator
from apprise.decorators import notify
# Get the notification body from datastore
n_body = n_object.get('notification_body', default_notification_body)
n_title = n_object.get('notification_title', default_notification_title)
n_format = valid_notification_formats.get(
n_object['notification_format'],
valid_notification_formats[default_notification_format],
)
@notify(on="delete")
@notify(on="deletes")
@notify(on="get")
@notify(on="gets")
@notify(on="post")
@notify(on="posts")
@notify(on="put")
@notify(on="puts")
def apprise_custom_api_call_wrapper(body, title, notify_type, *args, **kwargs):
import requests
url = kwargs['meta'].get('url')
if url.startswith('post'):
r = requests.post
elif url.startswith('get'):
r = requests.get
elif url.startswith('put'):
r = requests.put
elif url.startswith('delete'):
r = requests.delete
url = url.replace('post://', 'http://')
url = url.replace('posts://', 'https://')
url = url.replace('put://', 'http://')
url = url.replace('puts://', 'https://')
url = url.replace('get://', 'http://')
url = url.replace('gets://', 'https://')
url = url.replace('put://', 'http://')
url = url.replace('puts://', 'https://')
url = url.replace('delete://', 'http://')
url = url.replace('deletes://', 'https://')
# Try to auto-guess if it's JSON
headers = {}
try:
json.loads(body)
headers = {'Content-Type': 'application/json; charset=utf-8'}
except ValueError as e:
pass
r(url, headers=headers, data=body)
def process_notification(n_object, datastore):
# Insert variables into the notification content
notification_parameters = create_notification_parameters(n_object, datastore)
for n_k in notification_parameters:
token = '{' + n_k + '}'
val = notification_parameters[n_k]
n_title = n_title.replace(token, val)
n_body = n_body.replace(token, val)
# Get the notification body from datastore
jinja2_env = Environment(loader=BaseLoader)
n_body = jinja2_env.from_string(n_object.get('notification_body', default_notification_body)).render(**notification_parameters)
n_title = jinja2_env.from_string(n_object.get('notification_title', default_notification_title)).render(**notification_parameters)
n_format = valid_notification_formats.get(
n_object['notification_format'],
valid_notification_formats[default_notification_format],
)
# https://github.com/caronc/apprise/wiki/Development_LogCapture
# Anything higher than or equal to WARNING (which covers things like Connection errors)
# raise it as an exception
@@ -53,6 +97,7 @@ def process_notification(n_object, datastore):
sent_objs=[]
from .apprise_asset import asset
for url in n_object['notification_urls']:
url = jinja2_env.from_string(url).render(**notification_parameters)
apobj = apprise.Apprise(debug=True, asset=asset)
url = url.strip()
if len(url):
@@ -66,7 +111,12 @@ def process_notification(n_object, datastore):
# So if no avatar_url is specified, add one so it can be correctly calculated into the total payload
k = '?' if not '?' in url else '&'
if not 'avatar_url' in url and not url.startswith('mail'):
if not 'avatar_url' in url \
and not url.startswith('mail') \
and not url.startswith('post') \
and not url.startswith('get') \
and not url.startswith('delete') \
and not url.startswith('put'):
url += k + 'avatar_url=https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/changedetectionio/static/images/avatar-256x256.png'
if url.startswith('tgram://'):
@@ -144,7 +194,7 @@ def create_notification_parameters(n_object, datastore):
watch_url = n_object['watch_url']
# Re #148 - Some people have just {base_url} in the body or title, but this may break some notification services
# Re #148 - Some people have just {{ base_url }} in the body or title, but this may break some notification services
# like 'Join', so it's always best to atleast set something obvious so that they are not broken.
if base_url == '':
base_url = "<base-url-env-var-not-set>"

View File

@@ -6,6 +6,10 @@
// will automatically force a scroll somewhere, so include the position offset
// Lets hope the position doesnt change while we iterate the bbox's, but this is better than nothing
function setModalChoiceFromList(items, element) {
}
var scroll_y=+document.documentElement.scrollTop || document.body.scrollTop
// Include the getXpath script directly, easier than fetching
@@ -81,6 +85,14 @@ var bbox;
for (var i = 0; i < elements.length; i++) {
bbox = elements[i].getBoundingClientRect();
// Exclude items that are not interactable or visible
if(elements[i].style.opacity === "0") {
continue
}
if(elements[i].style.display === "none" || elements[i].style.pointerEvents === "none" ) {
continue
}
// Forget really small ones
if (bbox['width'] < 10 && bbox['height'] < 10) {
continue;
@@ -125,7 +137,7 @@ for (var i = 0; i < elements.length; i++) {
}
// @todo Possible to ONLY list where it's clickable to save JSON xfer size
size_pos.push({
var n = {
xpath: xpath_result,
width: Math.round(bbox['width']),
height: Math.round(bbox['height']),
@@ -134,7 +146,16 @@ for (var i = 0; i < elements.length; i++) {
tagName: (elements[i].tagName) ? elements[i].tagName.toLowerCase() : '',
tagtype: (elements[i].tagName == 'INPUT' && elements[i].type) ? elements[i].type.toLowerCase() : '',
isClickable: (elements[i].onclick) || window.getComputedStyle(elements[i]).cursor == "pointer"
});
}
if (n['tagName'] == 'select') {
n['options']=[]
for (const v of elements[i].options) {
n['options'].push(v.text)
}
}
size_pos.push(n);
}

View File

@@ -0,0 +1,2 @@
<?xml version="1.0" encoding="UTF-8"?>
<svg width="83.39" height="89.648" enable-background="new 0 0 122.406 122.881" version="1.1" viewBox="0 0 83.39 89.648" xml:space="preserve" xmlns="http://www.w3.org/2000/svg"><g transform="translate(5e-4 -33.234)"><path d="m44.239 42.946-39.111 39.896 34.908 34.91 39.09-39.876-1.149-34.931zm-0.91791 42.273c0.979-0.979 1.507-1.99 1.577-3.027 0.077-1.043-0.248-2.424-0.967-4.135-0.725-1.717-1.348-3.346-1.87-4.885s-0.814-3.014-0.897-4.432c-0.07-1.42 0.134-2.768 0.624-4.045 0.477-1.279 1.348-2.545 2.607-3.804 2.099-2.099 4.535-3.123 7.314-3.065 2.773 0.063 5.457 1.158 8.04 3.294l2.881 3.034c1.946 2.607 2.799 5.33 2.557 8.166-0.235 2.83-1.532 5.426-3.893 7.785l-6.296-6.297c1.291-1.291 2.035-2.531 2.238-3.727 0.191-1.197-0.165-2.252-1.081-3.168-0.821-0.82-1.717-1.195-2.69-1.139-0.967 0.064-1.908 0.547-2.817 1.457-0.922 0.922-1.393 1.914-1.412 2.977s0.306 2.416 0.973 4.064c0.661 1.652 1.24 3.25 1.736 4.801 0.496 1.553 0.782 3.035 0.858 4.445 0.076 1.426-0.127 2.787-0.591 4.104-0.477 1.316-1.336 2.596-2.588 3.848-2.125 2.125-4.522 3.186-7.212 3.18s-5.311-1.063-7.855-3.16l-3.747 3.746-2.964-2.965 3.766-3.764c-2.423-2.996-3.568-5.998-3.447-9.02 0.127-3.014 1.476-5.813 4.045-8.383l6.278 6.277c-1.412 1.412-2.175 2.799-2.277 4.16-0.108 1.367 0.414 2.627 1.571 3.783 0.839 0.84 1.755 1.26 2.741 1.242 0.985-0.017 1.92-0.47 2.798-1.347zm21.127-46.435h17.457c-0.0269 2.2368 0.69936 16.025 0.69936 16.025l0.785 23.858c0.019 0.609-0.221 1.164-0.619 1.564l5e-3 4e-3 -41.236 42.022c-0.82213 0.8378-2.175 0.83-3.004 0l-37.913-37.91c-0.83-0.83-0.83-2.176 0-3.006l41.236-42.021c0.39287-0.42671 1.502-0.53568 1.502-0.53568zm18.011 11.59c-59.392-29.687-29.696-14.843 0 0z"/></g></svg>

After

Width:  |  Height:  |  Size: 1.7 KiB

View File

@@ -214,7 +214,12 @@ $(document).ready(function () {
$('input[placeholder="Value"]', first_available).addClass('ok').click().focus();
found_something = true;
} else {
if (x['isClickable'] || x['tagName'].startsWith('h') || x['tagName'] === 'a' || x['tagName'] === 'button' || x['tagtype'] === 'submit' || x['tagtype'] === 'checkbox' || x['tagtype'] === 'radio' || x['tagtype'] === 'li') {
if (x['tagName'] === 'select') {
$('select', first_available).val('Select option').change();
$('input[type=text]', first_available).first().val(x['xpath']);
$('#myModal').show();
found_something = true;
} else if (x['isClickable'] || x['tagName'].startsWith('h') || x['tagName'] === 'a' || x['tagName'] === 'button' || x['tagtype'] === 'submit' || x['tagtype'] === 'checkbox' || x['tagtype'] === 'radio' || x['tagtype'] === 'li') {
$('select', first_available).val('Click element').change();
$('input[type=text]', first_available).first().val(x['xpath']);
found_something = true;

View File

@@ -13,6 +13,8 @@ $(document).ready(function () {
} else if (hash_name === '#error-screenshot') {
$("img#error-screenshot-img").attr('src', error_screenshot_url);
$("#settings").hide();
} else if (hash_name === '#extract') {
$("#settings").hide();
}

View File

@@ -19,6 +19,6 @@ $(document).ready(function () {
};
const setCookieValue = (value) => {
document.cookie = `css_dark_mode=${value};max-age=31536000`
document.cookie = `css_dark_mode=${value};max-age=31536000;path=/`
}
});

View File

@@ -18,6 +18,8 @@
--color-grey-850: #eee;
--color-grey-900: #f2f2f2;
--color-black: #000;
--color-dark-red: #a00;
--color-light-red: #dd0000;
--color-background-page: var(--color-grey-100);
--color-background-gradient-first: #5ad8f7;
--color-background-gradient-second: #2f50af;
@@ -27,9 +29,9 @@
--color-link: #1b98f8;
--color-menu-accent: #ed5900;
--color-background-code: var(--color-grey-850);
--color-error: #a00;
--color-error: var(--color-dark-red);
--color-error-input: #ffebeb;
--color-error-list: #dd0000;
--color-error-list: var(--color-light-red);
--color-table-background: var(--color-background);
--color-table-stripe: var(--color-grey-900);
--color-text-tab: var(--color-white);
@@ -84,7 +86,9 @@
--color-text-menu-link-hover: var(--color-grey-300);
--color-shadow-jump: var(--color-grey-500);
--color-icon-github: var(--color-black);
--color-icon-github-hover: var(--color-grey-300); }
--color-icon-github-hover: var(--color-grey-300);
--color-watch-table-error: var(--color-dark-red);
--color-watch-table-row-text: var(--color-grey-100); }
html[data-darkmode="true"] {
--color-link: #59bdfb;
@@ -114,22 +118,31 @@ html[data-darkmode="true"] {
--color-background-snapshot-age: var(--color-grey-200);
--color-shadow-jump: var(--color-grey-200);
--color-icon-github: var(--color-white);
--color-icon-github-hover: var(--color-grey-700); }
html[data-darkmode="true"] .watch-controls img {
opacity: 0.4; }
--color-icon-github-hover: var(--color-grey-700);
--color-watch-table-error: var(--color-light-red);
--color-watch-table-row-text: var(--color-grey-800); }
html[data-darkmode="true"] .icon-spread {
filter: hue-rotate(-10deg) brightness(1.5); }
html[data-darkmode="true"] .watch-table .title-col a[target="_blank"]::after,
html[data-darkmode="true"] .watch-table .current-diff-url::after {
filter: invert(0.5) hue-rotate(10deg) brightness(2); }
html[data-darkmode="true"] .watch-table .watch-controls .state-off img {
opacity: 0.3; }
html[data-darkmode="true"] .watch-table .watch-controls .state-on img {
opacity: 1.0; }
html[data-darkmode="true"] .watch-table .unviewed {
color: #fff; }
html[data-darkmode="true"] .watch-table .unviewed.error {
color: var(--color-watch-table-error); }
#diff-ui {
background: var(--color-background);
padding: 2em;
margin-left: 1em;
margin-right: 1em;
border-radius: 5px;
font-size: 11px; }
border-radius: 5px; }
#diff-ui #text {
font-size: 11px; }
#diff-ui table {
table-layout: fixed;
width: 100%; }

View File

@@ -7,7 +7,11 @@
margin-left: 1em;
margin-right: 1em;
border-radius: 5px;
font-size: 11px;
// The first tab 'text' diff
#text {
font-size: 11px;
}
table {
table-layout: fixed;

View File

@@ -0,0 +1,37 @@
/* The Modal (background) */
.modal {
display: none; /* Hidden by default */
position: fixed; /* Stay in place */
z-index: 1; /* Sit on top */
left: 0;
top: 0;
width: 100%; /* Full width */
height: 100%; /* Full height */
overflow: auto; /* Enable scroll if needed */
background-color: rgb(0,0,0); /* Fallback color */
background-color: rgba(0,0,0,0.4); /* Black w/ opacity */
}
/* Modal Content/Box */
.modal-content {
background-color: #fefefe;
margin: 15% auto; /* 15% from the top and centered */
padding: 20px;
border: 1px solid #888;
max-width: 80%; /* Could be more or less, depending on screen size */
}
/* The Close Button */
.close {
color: #aaa;
float: right;
font-size: 28px;
font-weight: bold;
}
.close:hover,
.close:focus {
color: black;
text-decoration: none;
cursor: pointer;
}

View File

@@ -19,6 +19,8 @@
--color-grey-850: #eee;
--color-grey-900: #f2f2f2;
--color-black: #000;
--color-dark-red: #a00;
--color-light-red: #dd0000;
--color-background-page: var(--color-grey-100);
--color-background-gradient-first: #5ad8f7;
@@ -29,9 +31,9 @@
--color-link: #1b98f8;
--color-menu-accent: #ed5900;
--color-background-code: var(--color-grey-850);
--color-error: #a00;
--color-error: var(--color-dark-red);
--color-error-input: #ffebeb;
--color-error-list: #dd0000;
--color-error-list: var(--color-light-red);
--color-table-background: var(--color-background);
--color-table-stripe: var(--color-grey-900);
--color-text-tab: var(--color-white);
@@ -96,6 +98,9 @@
--color-shadow-jump: var(--color-grey-500);
--color-icon-github: var(--color-black);
--color-icon-github-hover: var(--color-grey-300);
--color-watch-table-error: var(--color-dark-red);
--color-watch-table-row-text: var(--color-grey-100);
}
html[data-darkmode="true"] {
@@ -123,7 +128,6 @@ html[data-darkmode="true"] {
--color-text-input-description: var(--color-grey-600);
--color-text-input-placeholder: var(--color-grey-600);
--color-text-watch-tag-list: #fa3e92;
--color-background-code: var(--color-grey-200);
--color-background-tab: rgba(0, 0, 0, 0.2);
@@ -133,13 +137,9 @@ html[data-darkmode="true"] {
--color-shadow-jump: var(--color-grey-200);
--color-icon-github: var(--color-white);
--color-icon-github-hover: var(--color-grey-700);
--color-watch-table-error: var(--color-light-red);
--color-watch-table-row-text: var(--color-grey-800);
// Anything that can't be manipulated through variables follows.
.watch-controls {
img {
opacity: 0.4;
}
}
.icon-spread {
filter: hue-rotate(-10deg) brightness(1.5);
@@ -151,5 +151,25 @@ html[data-darkmode="true"] {
.current-diff-url::after {
filter: invert(.5) hue-rotate(10deg) brightness(2);
}
.watch-controls {
.state-off {
img {
opacity: 0.3;
}
}
.state-on {
img {
opacity: 1.0;
}
}
}
.unviewed {
color: #fff;
&.error {
color: var(--color-watch-table-error);
}
}
}
}

View File

@@ -2,10 +2,11 @@
* -- BASE STYLES --
*/
@import "parts/_variables";
@import "parts/_spinners";
@import "parts/_browser-steps";
@import "parts/_arrows";
@import "parts/_browser-steps";
@import "parts/_modal";
@import "parts/_spinners";
@import "parts/_variables";
body {
color: var(--color-text);
@@ -121,22 +122,25 @@ code {
width: 100%;
font-size: 80%;
tr.unviewed {
font-weight: bold;
tr {
&.unviewed {
font-weight: bold;
}
&.error {
color: var(--color-watch-table-error);
}
color: var(--color-watch-table-row-text);
}
.error {
color: var(--color-error);
}
td {
white-space: nowrap;
&.title-col {
word-break: break-all;
white-space: normal;
}
}
td.title-col {
word-break: break-all;
white-space: normal;
}
th {
white-space: nowrap;
@@ -322,6 +326,12 @@ a.pure-button-selected {
padding: 0.5rem 0 1rem 0;
}
label {
&:hover {
cursor: pointer;
}
}
#notification-customisation {
border: 1px solid var(--color-border-notification);
padding: 0.5rem;
@@ -868,6 +878,9 @@ body.full-width {
.pure-form-message-inline {
padding-left: 0;
color: var(--color-text-input-description);
code {
font-size: .875em;
}
}
}
@@ -997,3 +1010,30 @@ ul {
border-radius: 5px;
color: var(--color-warning);
}
/* automatic price following helpers */
.tracking-ldjson-price-data {
background-color: var(--color-background-button-green);
color: #000;
padding: 3px;
border-radius: 3px;
white-space: nowrap;
}
.ldjson-price-track-offer {
a.pure-button {
border-radius: 3px;
padding: 3px;
background-color: var(--color-background-button-green);
}
font-weight: bold;
font-style: italic;
}
.price-follow-tag-icon {
display: inline-block;
height: 0.8rem;
vertical-align: middle;
}

View File

@@ -1,169 +1,23 @@
/*
* -- BASE STYLES --
*/
/**
* CSS custom properties (aka variables).
*/
:root {
--color-white: #fff;
--color-grey-50: #111;
--color-grey-100: #262626;
--color-grey-200: #333;
--color-grey-300: #444;
--color-grey-325: #555;
--color-grey-350: #565d64;
--color-grey-400: #666;
--color-grey-500: #777;
--color-grey-600: #999;
--color-grey-700: #cbcbcb;
--color-grey-750: #ddd;
--color-grey-800: #e0e0e0;
--color-grey-850: #eee;
--color-grey-900: #f2f2f2;
--color-black: #000;
--color-background-page: var(--color-grey-100);
--color-background-gradient-first: #5ad8f7;
--color-background-gradient-second: #2f50af;
--color-background-gradient-third: #9150bf;
--color-background: var(--color-white);
--color-text: var(--color-grey-200);
--color-link: #1b98f8;
--color-menu-accent: #ed5900;
--color-background-code: var(--color-grey-850);
--color-error: #a00;
--color-error-input: #ffebeb;
--color-error-list: #dd0000;
--color-table-background: var(--color-background);
--color-table-stripe: var(--color-grey-900);
--color-text-tab: var(--color-white);
--color-background-tab: rgba(255, 255, 255, 0.2);
--color-background-tab-hover: rgba(255, 255, 255, 0.5);
--color-text-tab-active: #222;
--color-api-key: #0078e7;
--color-background-button-primary: #0078e7;
--color-background-button-green: #42dd53;
--color-background-button-red: #dd4242;
--color-background-button-success: rgb(28, 184, 65);
--color-background-button-error: rgb(202, 60, 60);
--color-text-button-error: var(--color-white);
--color-background-button-warning: rgb(202, 60, 60);
--color-text-button-warning: var(--color-white);
--color-background-button-secondary: rgb(66, 184, 221);
--color-background-button-cancel: rgb(200, 200, 200);
--color-text-button: var(--color-white);
--color-background-button-tag: rgb(99, 99, 99);
--color-background-snapshot-age: #dfdfdf;
--color-error-text-snapshot-age: var(--color-white);
--color-error-background-snapshot-age: #ff0000;
--color-background-button-tag-active: #9c9c9c;
--color-text-messages: var(--color-white);
--color-background-messages-message: rgba(255, 255, 255, .2);
--color-background-messages-error: rgba(255, 1, 1, .5);
--color-background-messages-notice: rgba(255, 255, 255, .5);
--color-border-notification: #ccc;
--color-background-checkbox-operations: rgba(0, 0, 0, 0.05);
--color-warning: #ff3300;
--color-border-warning: var(--color-warning);
--color-text-legend: var(--color-white);
--color-link-new-version: #e07171;
--color-last-checked: #bbb;
--color-text-footer: #444;
--color-border-watch-table-cell: #eee;
--color-text-watch-tag-list: #e70069;
--color-background-new-watch-form: rgba(0, 0, 0, 0.05);
--color-background-new-watch-input: var(--color-white);
--color-text-new-watch-input: var(--color-text);
--color-border-input: var(--color-grey-500);
--color-shadow-input: var(--color-grey-400);
--color-background-input: var(--color-white);
--color-text-input: var(--color-text);
--color-text-input-description: var(--color-grey-500);
--color-text-input-placeholder: var(--color-grey-600);
--color-background-table-thead: var(--color-grey-800);
--color-border-table-cell: var(--color-grey-700);
--color-text-menu-heading: var(--color-grey-350);
--color-text-menu-link: var(--color-grey-500);
--color-background-menu-link-hover: var(--color-grey-850);
--color-text-menu-link-hover: var(--color-grey-300);
--color-shadow-jump: var(--color-grey-500);
--color-icon-github: var(--color-black);
--color-icon-github-hover: var(--color-grey-300); }
html[data-darkmode="true"] {
--color-link: #59bdfb;
--color-text: var(--color-white);
--color-background-gradient-first: #3f90a5;
--color-background-gradient-second: #1e316c;
--color-background-gradient-third: #4d2c64;
--color-background-new-watch-input: var(--color-grey-100);
--color-text-new-watch-input: var(--color-text);
--color-background-table-thead: var(--color-grey-200);
--color-table-background: var(--color-grey-300);
--color-table-stripe: var(--color-grey-325);
--color-background: var(--color-grey-300);
--color-text-menu-heading: var(--color-grey-850);
--color-text-menu-link: var(--color-grey-800);
--color-border-table-cell: var(--color-grey-400);
--color-text-tab-active: var(--color-text);
--color-border-input: var(--color-grey-400);
--color-shadow-input: var(--color-grey-50);
--color-background-input: var(--color-grey-350);
--color-text-input-description: var(--color-grey-600);
--color-text-input-placeholder: var(--color-grey-600);
--color-text-watch-tag-list: #fa3e92;
--color-background-code: var(--color-grey-200);
--color-background-tab: rgba(0, 0, 0, 0.2);
--color-background-tab-hover: rgba(0, 0, 0, 0.5);
--color-background-snapshot-age: var(--color-grey-200);
--color-shadow-jump: var(--color-grey-200);
--color-icon-github: var(--color-white);
--color-icon-github-hover: var(--color-grey-700); }
html[data-darkmode="true"] .watch-controls img {
opacity: 0.4; }
html[data-darkmode="true"] .icon-spread {
filter: hue-rotate(-10deg) brightness(1.5); }
html[data-darkmode="true"] .watch-table .title-col a[target="_blank"]::after,
html[data-darkmode="true"] .watch-table .current-diff-url::after {
filter: invert(0.5) hue-rotate(10deg) brightness(2); }
/* spinner */
.spinner,
.spinner:after {
border-radius: 50%;
width: 10px;
height: 10px; }
.spinner {
margin: 0px auto;
font-size: 3px;
vertical-align: middle;
.arrow {
border: solid #1b98f8;
border-width: 0 2px 2px 0;
display: inline-block;
text-indent: -9999em;
border-top: 1.1em solid rgba(38, 104, 237, 0.2);
border-right: 1.1em solid rgba(38, 104, 237, 0.2);
border-bottom: 1.1em solid rgba(38, 104, 237, 0.2);
border-left: 1.1em solid #2668ed;
-webkit-transform: translateZ(0);
-ms-transform: translateZ(0);
transform: translateZ(0);
-webkit-animation: load8 1.1s infinite linear;
animation: load8 1.1s infinite linear; }
@-webkit-keyframes load8 {
0% {
-webkit-transform: rotate(0deg);
transform: rotate(0deg); }
100% {
-webkit-transform: rotate(360deg);
transform: rotate(360deg); } }
@keyframes load8 {
0% {
-webkit-transform: rotate(0deg);
transform: rotate(0deg); }
100% {
-webkit-transform: rotate(360deg);
transform: rotate(360deg); } }
padding: 3px; }
.arrow.right {
transform: rotate(-45deg);
-webkit-transform: rotate(-45deg); }
.arrow.left {
transform: rotate(135deg);
-webkit-transform: rotate(135deg); }
.arrow.up, .arrow.asc {
transform: rotate(-135deg);
-webkit-transform: rotate(-135deg); }
.arrow.down, .arrow.desc {
transform: rotate(45deg);
-webkit-transform: rotate(45deg); }
#browser_steps {
/* convert rows to horizontal cells */ }
@@ -232,23 +86,225 @@ html[data-darkmode="true"] {
#browsersteps-selector-wrapper #browsersteps-click-start:hover {
cursor: pointer; }
.arrow {
border: solid #1b98f8;
border-width: 0 2px 2px 0;
/* The Modal (background) */
.modal {
display: none;
/* Hidden by default */
position: fixed;
/* Stay in place */
z-index: 1;
/* Sit on top */
left: 0;
top: 0;
width: 100%;
/* Full width */
height: 100%;
/* Full height */
overflow: auto;
/* Enable scroll if needed */
background-color: black;
/* Fallback color */
background-color: rgba(0, 0, 0, 0.4);
/* Black w/ opacity */ }
/* Modal Content/Box */
.modal-content {
background-color: #fefefe;
margin: 15% auto;
/* 15% from the top and centered */
padding: 20px;
border: 1px solid #888;
max-width: 80%;
/* Could be more or less, depending on screen size */ }
/* The Close Button */
.close {
color: #aaa;
float: right;
font-size: 28px;
font-weight: bold; }
.close:hover,
.close:focus {
color: black;
text-decoration: none;
cursor: pointer; }
/* spinner */
.spinner,
.spinner:after {
border-radius: 50%;
width: 10px;
height: 10px; }
.spinner {
margin: 0px auto;
font-size: 3px;
vertical-align: middle;
display: inline-block;
padding: 3px; }
.arrow.right {
transform: rotate(-45deg);
-webkit-transform: rotate(-45deg); }
.arrow.left {
transform: rotate(135deg);
-webkit-transform: rotate(135deg); }
.arrow.up, .arrow.asc {
transform: rotate(-135deg);
-webkit-transform: rotate(-135deg); }
.arrow.down, .arrow.desc {
transform: rotate(45deg);
-webkit-transform: rotate(45deg); }
text-indent: -9999em;
border-top: 1.1em solid rgba(38, 104, 237, 0.2);
border-right: 1.1em solid rgba(38, 104, 237, 0.2);
border-bottom: 1.1em solid rgba(38, 104, 237, 0.2);
border-left: 1.1em solid #2668ed;
-webkit-transform: translateZ(0);
-ms-transform: translateZ(0);
transform: translateZ(0);
-webkit-animation: load8 1.1s infinite linear;
animation: load8 1.1s infinite linear; }
@-webkit-keyframes load8 {
0% {
-webkit-transform: rotate(0deg);
transform: rotate(0deg); }
100% {
-webkit-transform: rotate(360deg);
transform: rotate(360deg); } }
@keyframes load8 {
0% {
-webkit-transform: rotate(0deg);
transform: rotate(0deg); }
100% {
-webkit-transform: rotate(360deg);
transform: rotate(360deg); } }
/**
* CSS custom properties (aka variables).
*/
:root {
--color-white: #fff;
--color-grey-50: #111;
--color-grey-100: #262626;
--color-grey-200: #333;
--color-grey-300: #444;
--color-grey-325: #555;
--color-grey-350: #565d64;
--color-grey-400: #666;
--color-grey-500: #777;
--color-grey-600: #999;
--color-grey-700: #cbcbcb;
--color-grey-750: #ddd;
--color-grey-800: #e0e0e0;
--color-grey-850: #eee;
--color-grey-900: #f2f2f2;
--color-black: #000;
--color-dark-red: #a00;
--color-light-red: #dd0000;
--color-background-page: var(--color-grey-100);
--color-background-gradient-first: #5ad8f7;
--color-background-gradient-second: #2f50af;
--color-background-gradient-third: #9150bf;
--color-background: var(--color-white);
--color-text: var(--color-grey-200);
--color-link: #1b98f8;
--color-menu-accent: #ed5900;
--color-background-code: var(--color-grey-850);
--color-error: var(--color-dark-red);
--color-error-input: #ffebeb;
--color-error-list: var(--color-light-red);
--color-table-background: var(--color-background);
--color-table-stripe: var(--color-grey-900);
--color-text-tab: var(--color-white);
--color-background-tab: rgba(255, 255, 255, 0.2);
--color-background-tab-hover: rgba(255, 255, 255, 0.5);
--color-text-tab-active: #222;
--color-api-key: #0078e7;
--color-background-button-primary: #0078e7;
--color-background-button-green: #42dd53;
--color-background-button-red: #dd4242;
--color-background-button-success: rgb(28, 184, 65);
--color-background-button-error: rgb(202, 60, 60);
--color-text-button-error: var(--color-white);
--color-background-button-warning: rgb(202, 60, 60);
--color-text-button-warning: var(--color-white);
--color-background-button-secondary: rgb(66, 184, 221);
--color-background-button-cancel: rgb(200, 200, 200);
--color-text-button: var(--color-white);
--color-background-button-tag: rgb(99, 99, 99);
--color-background-snapshot-age: #dfdfdf;
--color-error-text-snapshot-age: var(--color-white);
--color-error-background-snapshot-age: #ff0000;
--color-background-button-tag-active: #9c9c9c;
--color-text-messages: var(--color-white);
--color-background-messages-message: rgba(255, 255, 255, .2);
--color-background-messages-error: rgba(255, 1, 1, .5);
--color-background-messages-notice: rgba(255, 255, 255, .5);
--color-border-notification: #ccc;
--color-background-checkbox-operations: rgba(0, 0, 0, 0.05);
--color-warning: #ff3300;
--color-border-warning: var(--color-warning);
--color-text-legend: var(--color-white);
--color-link-new-version: #e07171;
--color-last-checked: #bbb;
--color-text-footer: #444;
--color-border-watch-table-cell: #eee;
--color-text-watch-tag-list: #e70069;
--color-background-new-watch-form: rgba(0, 0, 0, 0.05);
--color-background-new-watch-input: var(--color-white);
--color-text-new-watch-input: var(--color-text);
--color-border-input: var(--color-grey-500);
--color-shadow-input: var(--color-grey-400);
--color-background-input: var(--color-white);
--color-text-input: var(--color-text);
--color-text-input-description: var(--color-grey-500);
--color-text-input-placeholder: var(--color-grey-600);
--color-background-table-thead: var(--color-grey-800);
--color-border-table-cell: var(--color-grey-700);
--color-text-menu-heading: var(--color-grey-350);
--color-text-menu-link: var(--color-grey-500);
--color-background-menu-link-hover: var(--color-grey-850);
--color-text-menu-link-hover: var(--color-grey-300);
--color-shadow-jump: var(--color-grey-500);
--color-icon-github: var(--color-black);
--color-icon-github-hover: var(--color-grey-300);
--color-watch-table-error: var(--color-dark-red);
--color-watch-table-row-text: var(--color-grey-100); }
html[data-darkmode="true"] {
--color-link: #59bdfb;
--color-text: var(--color-white);
--color-background-gradient-first: #3f90a5;
--color-background-gradient-second: #1e316c;
--color-background-gradient-third: #4d2c64;
--color-background-new-watch-input: var(--color-grey-100);
--color-text-new-watch-input: var(--color-text);
--color-background-table-thead: var(--color-grey-200);
--color-table-background: var(--color-grey-300);
--color-table-stripe: var(--color-grey-325);
--color-background: var(--color-grey-300);
--color-text-menu-heading: var(--color-grey-850);
--color-text-menu-link: var(--color-grey-800);
--color-border-table-cell: var(--color-grey-400);
--color-text-tab-active: var(--color-text);
--color-border-input: var(--color-grey-400);
--color-shadow-input: var(--color-grey-50);
--color-background-input: var(--color-grey-350);
--color-text-input-description: var(--color-grey-600);
--color-text-input-placeholder: var(--color-grey-600);
--color-text-watch-tag-list: #fa3e92;
--color-background-code: var(--color-grey-200);
--color-background-tab: rgba(0, 0, 0, 0.2);
--color-background-tab-hover: rgba(0, 0, 0, 0.5);
--color-background-snapshot-age: var(--color-grey-200);
--color-shadow-jump: var(--color-grey-200);
--color-icon-github: var(--color-white);
--color-icon-github-hover: var(--color-grey-700);
--color-watch-table-error: var(--color-light-red);
--color-watch-table-row-text: var(--color-grey-800); }
html[data-darkmode="true"] .icon-spread {
filter: hue-rotate(-10deg) brightness(1.5); }
html[data-darkmode="true"] .watch-table .title-col a[target="_blank"]::after,
html[data-darkmode="true"] .watch-table .current-diff-url::after {
filter: invert(0.5) hue-rotate(10deg) brightness(2); }
html[data-darkmode="true"] .watch-table .watch-controls .state-off img {
opacity: 0.3; }
html[data-darkmode="true"] .watch-table .watch-controls .state-on img {
opacity: 1.0; }
html[data-darkmode="true"] .watch-table .unviewed {
color: #fff; }
html[data-darkmode="true"] .watch-table .unviewed.error {
color: var(--color-watch-table-error); }
body {
color: var(--color-text);
@@ -331,15 +387,17 @@ code {
.watch-table {
width: 100%;
font-size: 80%; }
.watch-table tr.unviewed {
font-weight: bold; }
.watch-table .error {
color: var(--color-error); }
.watch-table tr {
color: var(--color-watch-table-row-text); }
.watch-table tr.unviewed {
font-weight: bold; }
.watch-table tr.error {
color: var(--color-watch-table-error); }
.watch-table td {
white-space: nowrap; }
.watch-table td.title-col {
word-break: break-all;
white-space: normal; }
.watch-table td.title-col {
word-break: break-all;
white-space: normal; }
.watch-table th {
white-space: nowrap; }
.watch-table th a {
@@ -470,6 +528,9 @@ a.pure-button-selected {
.notifications-wrapper {
padding: 0.5rem 0 1rem 0; }
label:hover {
cursor: pointer; }
#notification-customisation {
border: 1px solid var(--color-border-notification);
padding: 0.5rem;
@@ -836,6 +897,8 @@ body.full-width .edit-form {
.edit-form .pure-form-message-inline {
padding-left: 0;
color: var(--color-text-input-description); }
.edit-form .pure-form-message-inline code {
font-size: .875em; }
ul {
padding-left: 1em;
@@ -926,3 +989,24 @@ ul {
display: inline;
height: 26px;
vertical-align: middle; }
/* automatic price following helpers */
.tracking-ldjson-price-data {
background-color: var(--color-background-button-green);
color: #000;
padding: 3px;
border-radius: 3px;
white-space: nowrap; }
.ldjson-price-track-offer {
font-weight: bold;
font-style: italic; }
.ldjson-price-track-offer a.pure-button {
border-radius: 3px;
padding: 3px;
background-color: var(--color-background-button-green); }
.price-follow-tag-icon {
display: inline-block;
height: 0.8rem;
vertical-align: middle; }

View File

@@ -250,12 +250,15 @@ class ChangeDetectionStore:
def clear_watch_history(self, uuid):
import pathlib
self.__data['watching'][uuid].update(
{'last_checked': 0,
'last_viewed': 0,
'previous_md5': False,
'last_notification_error': False,
'last_error': False})
self.__data['watching'][uuid].update({
'last_checked': 0,
'has_ldjson_price_data': None,
'last_error': False,
'last_notification_error': False,
'last_viewed': 0,
'previous_md5': False,
'track_ldjson_price_data': None,
})
# JSON Data, Screenshots, Textfiles (history index and snapshots), HTML in the future etc
for item in pathlib.Path(os.path.join(self.datastore_path, uuid)).rglob("*.*"):
@@ -621,4 +624,44 @@ class ChangeDetectionStore:
watch['include_filters'] = [existing_filter]
except:
continue
return
return
# Convert old static notification tokens to jinja2 tokens
def update_9(self):
# Each watch
import re
# only { } not {{ or }}
r = r'(?<!{){(?!{)(\w+)(?<!})}(?!})'
for uuid, watch in self.data['watching'].items():
try:
n_body = watch.get('notification_body', '')
if n_body:
watch['notification_body'] = re.sub(r, r'{{\1}}', n_body)
n_title = watch.get('notification_title')
if n_title:
watch['notification_title'] = re.sub(r, r'{{\1}}', n_title)
n_urls = watch.get('notification_urls')
if n_urls:
for i, url in enumerate(n_urls):
watch['notification_urls'][i] = re.sub(r, r'{{\1}}', url)
except:
continue
# System wide
n_body = self.data['settings']['application'].get('notification_body')
if n_body:
self.data['settings']['application']['notification_body'] = re.sub(r, r'{{\1}}', n_body)
n_title = self.data['settings']['application'].get('notification_title')
if n_body:
self.data['settings']['application']['notification_title'] = re.sub(r, r'{{\1}}', n_title)
n_urls = self.data['settings']['application'].get('notification_urls')
if n_urls:
for i, url in enumerate(n_urls):
self.data['settings']['application']['notification_urls'][i] = re.sub(r, r'{{\1}}', url)
return

View File

@@ -16,6 +16,7 @@
<li><code>discord://</code> only supports a maximum <strong>2,000 characters</strong> of notification text, including the title.</li>
<li><code>tgram://</code> bots cant send messages to other bots, so you should specify chat ID of non-bot user.</li>
<li><code>tgram://</code> only supports very limited HTML and can fail when extra tags are sent, <a href="https://core.telegram.org/bots/api#html-style">read more here</a> (or use plaintext/markdown format)</li>
<li><code>gets://</code>, <code>posts://</code>, <code>puts://</code>, <code>deletes://</code> for direct API calls (or omit the "<code>s</code>" for non-SSL ie <code>get://</code>)</li>
</ul>
</div>
<div class="notifications-wrapper">
@@ -41,8 +42,9 @@
<span class="pure-form-message-inline">Format for all notifications</span>
</div>
<div class="pure-controls">
<span class="pure-form-message-inline">
These tokens can be used in the notification body and title to customise the notification text.
<p class="pure-form-message-inline">
You can use <a target="_new" href="https://jinja.palletsprojects.com/en/3.0.x/templates/">Jinja2</a> templating in the notification title, body and URL.
</p>
<table class="pure-table" id="token-table">
<thead>
@@ -53,52 +55,49 @@
</thead>
<tbody>
<tr>
<td><code>{base_url}</code></td>
<td><code>{{ '{{ base_url }}' }}</code></td>
<td>The URL of the changedetection.io instance you are running.</td>
</tr>
<tr>
<td><code>{watch_url}</code></td>
<td><code>{{ '{{ watch_url }}' }}</code></td>
<td>The URL being watched.</td>
</tr>
<tr>
<td><code>{watch_uuid}</code></td>
<td><code>{{ '{{ watch_uuid }}' }}</code></td>
<td>The UUID of the watch.</td>
</tr>
<tr>
<td><code>{watch_title}</code></td>
<td><code>{{ '{{ watch_title }}' }}</code></td>
<td>The title of the watch.</td>
</tr>
<tr>
<td><code>{watch_tag}</code></td>
<td>The tag of the watch.</td>
<td><code>{{ '{{ watch_tag }}' }}</code></td>
<td>The watch label / tag</td>
</tr>
<tr>
<td><code>{preview_url}</code></td>
<td><code>{{ '{{ preview_url }}' }}</code></td>
<td>The URL of the preview page generated by changedetection.io.</td>
</tr>
<tr>
<td><code>{diff}</code></td>
<td><code>{{ '{{ diff_url }}' }}</code></td>
<td>The diff output - differences only</td>
</tr>
<tr>
<td><code>{diff_full}</code></td>
<td><code>{{ '{{ diff_full }}' }}</code></td>
<td>The diff output - full difference output</td>
</tr>
<tr>
<td><code>{diff_url}</code></td>
<td>The URL of the diff page generated by changedetection.io.</td>
</tr>
<tr>
<td><code>{current_snapshot}</code></td>
<td><code>{{ '{{ current_snapshot }}' }}</code></td>
<td>The current snapshot value, useful when combined with JSON or CSS filters
</td>
</tr>
</tbody>
</table>
<br/>
URLs generated by changedetection.io (such as <code>{diff_url}</code>) require the <code>BASE_URL</code> environment variable set.<br/>
Your <code>BASE_URL</code> var is currently "{{settings_application['current_base_url']}}"
</span>
<div class="pure-form-message-inline">
<br>
URLs generated by changedetection.io (such as <code>{{ '{{ diff_url }}' }}</code>) require the <code>BASE_URL</code> environment variable set.<br/>
Your <code>BASE_URL</code> var is currently "{{settings_application['current_base_url']}}"
</div>
</div>
</div>
{% endmacro %}

View File

@@ -1,5 +1,5 @@
<!DOCTYPE html>
<html lang="en" data-darkmode="{{ dark_mode|lower }}">
<html lang="en" data-darkmode="{{ get_darkmode_state() }}">
<head>
<meta charset="utf-8"/>
@@ -34,6 +34,17 @@
</head>
<body>
<div id="myModal" class="modal" style="display: none;">
<!-- Modal content -->
<div class="modal-content">
<span class="close">&times;</span>
<div class="inner">
xxx
</div>
</div>
</div>
<div class="header">
<div class="home-menu pure-menu pure-menu-horizontal pure-menu-fixed" id="nav-menu">
{% if has_password and not current_user.is_authenticated %}
@@ -86,7 +97,7 @@
{% if dark_mode %}
{% set darkClass = 'dark' %}
{% endif %}
<button class="toggle-theme {{darkClass}}" type="button">
<button class="toggle-theme {{darkClass}}" type="button" title="Toggle Light/Dark Mode">
<span class="visually-hidden">Toggle light/dark mode</span>
<span class="icon-light">
{% include "svgs/light-mode-toggle-icon.svg" %}

View File

@@ -1,5 +1,5 @@
{% extends 'base.html' %}
{% from '_helpers.jinja' import render_field, render_checkbox_field, render_button %}
{% block content %}
<script>
const screenshot_url="{{url_for('static_content', group='screenshot', filename=uuid)}}";
@@ -58,6 +58,7 @@
{% if last_error_screenshot %}<li class="tab" id="error-screenshot-tab"><a href="#error-screenshot">Error Screenshot</a></li> {% endif %}
<li class="tab" id=""><a href="#text">Text</a></li>
<li class="tab" id="screenshot-tab"><a href="#screenshot">Screenshot</a></li>
<li class="tab" id="extract-tab"><a href="#extract">Extract Data</a></li>
</ul>
</div>
@@ -108,6 +109,37 @@
<strong>Screenshot requires Playwright/WebDriver enabled</strong>
{% endif %}
</div>
<div class="tab-pane-inner" id="extract">
<form id="extract-data-form" class="pure-form pure-form-stacked edit-form"
action="{{ url_for('diff_history_page', uuid=uuid) }}#extract"
method="POST">
<input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
<p>This tool will extract text data from all of the watch history.</p>
<div class="pure-control-group">
{{ render_field(extract_form.extract_regex) }}
<span class="pure-form-message-inline">
A <strong>RegEx</strong> is a pattern that identifies exactly which part inside of the text that you want to extract.<br/>
<p>
For example, to extract only the numbers from text &dash;</br>
<strong>Raw text</strong>: <code>Temperature <span style="color: red">5.5</span>°C in Sydney</code></br>
<strong>RegEx to extract:</strong> <code>Temperature <span style="color: red">([0-9\.]+)</span></code><br/>
</p>
<p>
<a href="https://RegExr.com/">Be sure to test your RegEx here.</a>
</p>
<p>
Each RegEx group bracket <code>()</code> will be in its own column, the first column value is always the date.
</p>
</span>
</div>
<div class="pure-control-group">
{{ render_button(extract_form.extract_submit_button) }}
</div>
</form>
</div>
</div>
<script>

File diff suppressed because it is too large Load Diff

View File

@@ -1,290 +1,186 @@
{% extends 'base.html' %} {% block content %} {% from '_helpers.jinja' import
render_field, render_checkbox_field, render_button %} {% from
'_common_fields.jinja' import render_common_settings_form %}
{% extends 'base.html' %}
{% block content %}
{% from '_helpers.jinja' import render_field, render_checkbox_field, render_button %}
{% from '_common_fields.jinja' import render_common_settings_form %}
<script>
const notification_base_url="{{url_for('ajax_callback_send_notification_test')}}";
{% if emailprefix %}
const email_notification_prefix=JSON.parse('{{emailprefix|tojson}}');
{% endif %}
const notification_base_url="{{url_for('ajax_callback_send_notification_test')}}";
{% if emailprefix %}
const email_notification_prefix=JSON.parse('{{emailprefix|tojson}}');
{% endif %}
</script>
<script
type="text/javascript"
src="{{url_for('static_content', group='js', filename='tabs.js')}}"
defer
></script>
<script
type="text/javascript"
src="{{url_for('static_content', group='js', filename='notifications.js')}}"
defer
></script>
<script type="text/javascript" src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
<script type="text/javascript" src="{{url_for('static_content', group='js', filename='notifications.js')}}" defer></script>
<script
type="text/javascript"
src="{{url_for('static_content', group='js', filename='global-settings.js')}}"
defer
></script>
<script type="text/javascript" src="{{url_for('static_content', group='js', filename='global-settings.js')}}" defer></script>
<div class="edit-form">
<div class="tabs collapsable">
<ul>
<li class="tab" id=""><a href="#general">General</a></li>
<li class="tab"><a href="#notifications">Notifications</a></li>
<li class="tab"><a href="#fetching">Fetching</a></li>
<li class="tab"><a href="#filters">Global Filters</a></li>
<li class="tab"><a href="#api">API</a></li>
</ul>
</div>
<div class="box-wrap inner">
<form
class="pure-form pure-form-stacked settings"
action="{{url_for('settings_page')}}"
method="POST"
>
<input type="hidden" name="csrf_token" value="{{ csrf_token() }}" />
<div class="tab-pane-inner" id="general">
<fieldset>
<div class="pure-control-group">
{{ render_field(form.requests.form.time_between_check,
class="time-check-widget") }}
<span class="pure-form-message-inline"
>Default time for all watches, when the watch does not have a
specific time setting.</span
>
</div>
<div class="pure-control-group">
{{ render_field(form.requests.form.jitter_seconds,
class="jitter_seconds") }}
<span class="pure-form-message-inline"
>Example - 3 seconds random jitter could trigger up to 3 seconds
earlier or up to 3 seconds later</span
>
</div>
<div class="pure-control-group">
{{
render_field(form.application.form.filter_failure_notification_threshold_attempts,
class="filter_failure_notification_threshold_attempts") }}
<span class="pure-form-message-inline"
>After this many consecutive times that the CSS/xPath filter is
missing, send a notification
<br />
Set to <strong>0</strong> to disable
</span>
</div>
<div class="pure-control-group">
{% if not hide_remove_pass %} {% if current_user.is_authenticated %}
{{ render_button(form.application.form.removepassword_button) }} {%
else %} {{ render_field(form.application.form.password) }}
<span class="pure-form-message-inline"
>Password protection for your changedetection.io
application.</span
>
{% endif %} {% else %}
<span class="pure-form-message-inline">Password is locked.</span>
{% endif %}
</div>
<div class="tabs collapsable">
<ul>
<li class="tab" id=""><a href="#general">General</a></li>
<li class="tab"><a href="#notifications">Notifications</a></li>
<li class="tab"><a href="#fetching">Fetching</a></li>
<li class="tab"><a href="#filters">Global Filters</a></li>
<li class="tab"><a href="#api">API</a></li>
</ul>
</div>
<div class="box-wrap inner">
<form class="pure-form pure-form-stacked settings" action="{{url_for('settings_page')}}" method="POST">
<input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
<div class="tab-pane-inner" id="general">
<fieldset>
<div class="pure-control-group">
{{ render_field(form.requests.form.time_between_check, class="time-check-widget") }}
<span class="pure-form-message-inline">Default time for all watches, when the watch does not have a specific time setting.</span>
</div>
<div class="pure-control-group">
{{ render_field(form.requests.form.jitter_seconds, class="jitter_seconds") }}
<span class="pure-form-message-inline">Example - 3 seconds random jitter could trigger up to 3 seconds earlier or up to 3 seconds later</span>
</div>
<div class="pure-control-group">
{{ render_field(form.application.form.filter_failure_notification_threshold_attempts, class="filter_failure_notification_threshold_attempts") }}
<span class="pure-form-message-inline">After this many consecutive times that the CSS/xPath filter is missing, send a notification
<br/>
Set to <strong>0</strong> to disable
</span>
</div>
<div class="pure-control-group">
{% if not hide_remove_pass %}
{% if current_user.is_authenticated %}
{{ render_button(form.application.form.removepassword_button) }}
{% else %}
{{ render_field(form.application.form.password) }}
<span class="pure-form-message-inline">Password protection for your changedetection.io application.</span>
{% endif %}
{% else %}
<span class="pure-form-message-inline">Password is locked.</span>
{% endif %}
</div>
<div class="pure-control-group">
{{ render_field(form.application.form.base_url,
placeholder="http://yoursite.com:5000/", class="m-d") }}
<span class="pure-form-message-inline">
Base URL used for the <code>{base_url}</code> token in
notifications and RSS links.<br />Default value is the ENV var
'BASE_URL' (Currently
"{{settings_application['current_base_url']}}"),
<a
href="https://github.com/dgtlmoon/changedetection.io/wiki/Configurable-BASE_URL-setting"
>read more here</a
>.
</span>
</div>
<div class="pure-control-group">
{{ render_field(form.application.form.base_url, placeholder="http://yoursite.com:5000/",
class="m-d") }}
<span class="pure-form-message-inline">
Base URL used for the <code>{{ '{{ base_url }}' }}</code> token in notifications and RSS links.<br/>Default value is the ENV var 'BASE_URL' (Currently "{{settings_application['current_base_url']}}"),
<a href="https://github.com/dgtlmoon/changedetection.io/wiki/Configurable-BASE_URL-setting">read more here</a>.
</span>
</div>
<div class="pure-control-group">
{{
render_checkbox_field(form.application.form.extract_title_as_title)
}}
<span class="pure-form-message-inline"
>Note: This will automatically apply to all existing
watches.</span
>
</div>
<div class="pure-control-group">
{{
render_checkbox_field(form.application.form.empty_pages_are_a_change)
}}
<span class="pure-form-message-inline"
>When a page contains HTML, but no renderable text appears (empty
page), is this considered a change?</span
>
</div>
{% if form.requests.proxy %}
<div class="pure-control-group inline-radio">
{{ render_field(form.requests.form.proxy,
class="fetch-backend-proxy") }}
<span class="pure-form-message-inline">
Choose a default proxy for all watches
</span>
</div>
{% endif %}
</fieldset>
</div>
<div class="pure-control-group">
{{ render_checkbox_field(form.application.form.extract_title_as_title) }}
<span class="pure-form-message-inline">Note: This will automatically apply to all existing watches.</span>
</div>
<div class="pure-control-group">
{{ render_checkbox_field(form.application.form.empty_pages_are_a_change) }}
<span class="pure-form-message-inline">When a page contains HTML, but no renderable text appears (empty page), is this considered a change?</span>
</div>
{% if form.requests.proxy %}
<div class="pure-control-group inline-radio">
{{ render_field(form.requests.form.proxy, class="fetch-backend-proxy") }}
<span class="pure-form-message-inline">
Choose a default proxy for all watches
</span>
</div>
{% endif %}
</fieldset>
</div>
<div class="tab-pane-inner" id="notifications">
<fieldset>
<div class="field-group">
{{ render_common_settings_form(form.application.form, emailprefix,
settings_application) }}
</div>
</fieldset>
</div>
<div class="tab-pane-inner" id="notifications">
<fieldset>
<div class="field-group">
{{ render_common_settings_form(form.application.form, emailprefix, settings_application) }}
</div>
</fieldset>
</div>
<div class="tab-pane-inner" id="fetching">
<div class="pure-control-group inline-radio">
{{ render_field(form.application.form.fetch_backend,
class="fetch-backend") }}
<span class="pure-form-message-inline">
<p>
Use the <strong>Basic</strong> method (default) where your watched
sites don't need Javascript to render.
</p>
<p>
The <strong>Chrome/Javascript</strong> method requires a network
connection to a running WebDriver+Chrome server, set by the ENV
var 'WEBDRIVER_URL'.
</p>
</span>
<br />
Tip:
<a
href="https://github.com/dgtlmoon/changedetection.io/wiki/Proxy-configuration#brightdata-proxy-support"
>Connect using BrightData Proxies, find out more here.</a
>
</div>
<fieldset class="pure-group" id="webdriver-override-options">
<div class="pure-form-message-inline">
<strong
>If you're having trouble waiting for the page to be fully
rendered (text missing etc), try increasing the 'wait' time
here.</strong
>
<br />
This will wait <i>n</i> seconds before extracting the text.
</div>
<div class="pure-control-group">
{{ render_field(form.application.form.webdriver_delay) }}
</div>
</fieldset>
</div>
<div class="tab-pane-inner" id="fetching">
<div class="pure-control-group inline-radio">
{{ render_field(form.application.form.fetch_backend, class="fetch-backend") }}
<span class="pure-form-message-inline">
<p>Use the <strong>Basic</strong> method (default) where your watched sites don't need Javascript to render.</p>
<p>The <strong>Chrome/Javascript</strong> method requires a network connection to a running WebDriver+Chrome server, set by the ENV var 'WEBDRIVER_URL'. </p>
</span>
<br/>
Tip: <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Proxy-configuration#brightdata-proxy-support">Connect using BrightData Proxies, find out more here.</a>
</div>
<fieldset class="pure-group" id="webdriver-override-options">
<div class="pure-form-message-inline">
<strong>If you're having trouble waiting for the page to be fully rendered (text missing etc), try increasing the 'wait' time here.</strong>
<br/>
This will wait <i>n</i> seconds before extracting the text.
</div>
<div class="pure-control-group">
{{ render_field(form.application.form.webdriver_delay) }}
</div>
</fieldset>
</div>
<div class="tab-pane-inner" id="filters">
<fieldset class="pure-group">
{{ render_checkbox_field(form.application.form.ignore_whitespace) }}
<span class="pure-form-message-inline"
>Ignore whitespace, tabs and new-lines/line-feeds when considering
if a change was detected.<br />
<i>Note:</i> Changing this will change the status of your existing
watches, possibly trigger alerts etc.
</span>
</fieldset>
<fieldset class="pure-group">
{{
render_checkbox_field(form.application.form.render_anchor_tag_content)
}}
<span class="pure-form-message-inline"
>Render anchor tag content, default disabled, when enabled renders
links as <code>(link text)[https://somesite.com]</code>
<br />
<i>Note:</i> Changing this could affect the content of your existing
watches, possibly trigger alerts etc.
</span>
</fieldset>
<fieldset class="pure-group">
{{ render_field(form.application.form.global_subtractive_selectors,
rows=5, placeholder="header
<div class="tab-pane-inner" id="filters">
<fieldset class="pure-group">
{{ render_checkbox_field(form.application.form.ignore_whitespace) }}
<span class="pure-form-message-inline">Ignore whitespace, tabs and new-lines/line-feeds when considering if a change was detected.<br/>
<i>Note:</i> Changing this will change the status of your existing watches, possibly trigger alerts etc.
</span>
</fieldset>
<fieldset class="pure-group">
{{ render_checkbox_field(form.application.form.render_anchor_tag_content) }}
<span class="pure-form-message-inline">Render anchor tag content, default disabled, when enabled renders links as <code>(link text)[https://somesite.com]</code>
<br/>
<i>Note:</i> Changing this could affect the content of your existing watches, possibly trigger alerts etc.
</span>
</fieldset>
<fieldset class="pure-group">
{{ render_field(form.application.form.global_subtractive_selectors, rows=5, placeholder="header
footer
nav
.stockticker") }}
<span class="pure-form-message-inline">
<ul>
<li>
Remove HTML element(s) by CSS selector before text conversion.
</li>
<li>
Add multiple elements or CSS selectors per line to ignore
multiple parts of the HTML.
</li>
</ul>
</span>
</fieldset>
<fieldset class="pure-group">
{{ render_field(form.application.form.global_ignore_text, rows=5,
placeholder="Some text to ignore in a line /some.regex\d{2}/ for
case-INsensitive regex ") }}
<span class="pure-form-message-inline"
>Note: This is applied globally in addition to the per-watch
rules.</span
><br />
<span class="pure-form-message-inline">
<ul>
<li>
Note: This is applied globally in addition to the per-watch
rules.
</li>
<li>
Each line processed separately, any line matching will be
ignored (removed before creating the checksum)
</li>
<li>
Regular Expression support, wrap the entire line in forward
slash <code>/regex/</code>
</li>
<li>
Changing this will affect the comparison checksum which may
trigger an alert
</li>
<li>Use the preview/show current tab to see ignores</li>
</ul>
</span>
</fieldset>
</div>
<span class="pure-form-message-inline">
<ul>
<li> Remove HTML element(s) by CSS selector before text conversion. </li>
<li> Add multiple elements or CSS selectors per line to ignore multiple parts of the HTML. </li>
</ul>
</span>
</fieldset>
<fieldset class="pure-group">
{{ render_field(form.application.form.global_ignore_text, rows=5, placeholder="Some text to ignore in a line
/some.regex\d{2}/ for case-INsensitive regex
") }}
<span class="pure-form-message-inline">Note: This is applied globally in addition to the per-watch rules.</span><br/>
<span class="pure-form-message-inline">
<ul>
<li>Note: This is applied globally in addition to the per-watch rules.</li>
<li>Each line processed separately, any line matching will be ignored (removed before creating the checksum)</li>
<li>Regular Expression support, wrap the entire line in forward slash <code>/regex/</code></li>
<li>Changing this will affect the comparison checksum which may trigger an alert</li>
<li>Use the preview/show current tab to see ignores</li>
</ul>
</span>
</fieldset>
</div>
<div class="tab-pane-inner" id="api">
<p>
Drive your changedetection.io via API, More about
<a
href="https://github.com/dgtlmoon/changedetection.io/wiki/API-Reference"
>API access here</a
>
</p>
<div class="tab-pane-inner" id="api">
<div class="pure-control-group">
{{
render_checkbox_field(form.application.form.api_access_token_enabled)
}}
<div class="pure-form-message-inline">
Restrict API access limit by using <code>x-api-key</code> header
</div>
<br />
<div class="pure-form-message-inline">
<br />API Key <span id="api-key">{{api_key}}</span>
<span style="display: none" id="api-key-copy">copy</span>
</div>
</div>
</div>
<p>Drive your changedetection.io via API, More about <a href="https://github.com/dgtlmoon/changedetection.io/wiki/API-Reference">API access here</a></p>
<div id="actions">
<div class="pure-control-group">
{{ render_button(form.save_button) }}
<a href="{{url_for('index')}}" class="pure-button button-cancel"
>Back</a
>
<a
href="{{url_for('clear_all_history')}}"
class="pure-button button-cancel"
>Clear Snapshot History</a
>
</div>
</div>
</form>
</div>
<div class="pure-control-group">
{{ render_checkbox_field(form.application.form.api_access_token_enabled) }}
<div class="pure-form-message-inline">Restrict API access limit by using <code>x-api-key</code> header</div><br/>
<div class="pure-form-message-inline"><br/>API Key <span id="api-key">{{api_key}}</span>
<span style="display:none;" id="api-key-copy" >copy</span>
</div>
</div>
</div>
<div id="actions">
<div class="pure-control-group">
{{ render_button(form.save_button) }}
<a href="{{url_for('index')}}" class="pure-button button-small button-cancel">Back</a>
<a href="{{url_for('clear_all_history')}}" class="pure-button button-small button-cancel">Clear Snapshot History</a>
</div>
</div>
</form>
</div>
</div>
{% endblock %}

View File

@@ -14,7 +14,7 @@
<div id="watch-add-wrapper-zone">
<div>
{{ render_simple_field(form.url, placeholder="https://...", required=true) }}
{{ render_simple_field(form.tag, value=active_tag if active_tag else '', placeholder="watch group") }}
{{ render_simple_field(form.tag, value=active_tag if active_tag else '', placeholder="watch label / tag") }}
</div>
<div>
{{ render_simple_field(form.watch_submit_button, title="Watch this URL!" ) }}
@@ -88,9 +88,9 @@
</td>
<td class="title-col inline">{{watch.title if watch.title is not none and watch.title|length > 0 else watch.url}}
<a class="external" target="_blank" rel="noopener" href="{{ watch.link.replace('source:','') }}"></a>
<a class="link-spread" href="{{url_for('form_share_put_watch', uuid=watch.uuid)}}"><img style="height: 1em;display:inline-block;" src="{{url_for('static_content', group='images', filename='spread.svg')}}" class="icon icon-spread" /></a>
<a class="link-spread" href="{{url_for('form_share_put_watch', uuid=watch.uuid)}}"><img style="height: 1em;display:inline-block;" src="{{url_for('static_content', group='images', filename='spread.svg')}}" class="icon icon-spread" title="Create a link to share watch config with others" /></a>
{%if watch.fetch_backend == "html_webdriver" %}<img style="height: 1em; display:inline-block;" src="{{url_for('static_content', group='images', filename='Google-Chrome-icon.png')}}" />{% endif %}
{%if watch.fetch_backend == "html_webdriver" %}<img style="height: 1em; display:inline-block;" src="{{url_for('static_content', group='images', filename='Google-Chrome-icon.png')}}" title="Using a chrome browser" />{% endif %}
{% if watch.last_error is defined and watch.last_error != False %}
<div class="fetch-error">{{ watch.last_error }}</div>
@@ -98,6 +98,12 @@
{% if watch.last_notification_error is defined and watch.last_notification_error != False %}
<div class="fetch-error notification-error"><a href="{{url_for('notification_logs')}}">{{ watch.last_notification_error }}</a></div>
{% endif %}
{% if watch['has_ldjson_price_data'] and not watch['track_ldjson_price_data'] %}
<div class="ldjson-price-track-offer">Embedded price data detected, follow only price data? <a href="{{url_for('price_data_follower.accept', uuid=watch.uuid)}}" class="pure-button button-xsmall">Yes</a> <a href="{{url_for('price_data_follower.reject', uuid=watch.uuid)}}" class="">No</a></div>
{% endif %}
{% if watch['track_ldjson_price_data'] == 'accepted' %}
<span class="tracking-ldjson-price-data" title="Automatically following embedded price information"><img src="{{url_for('static_content', group='images', filename='price-tag-icon.svg')}}" class="price-follow-tag-icon"/> Price</span>
{% endif %}
{% if not active_tag %}
<span class="watch-tag-list">{{ watch.tag}}</span>
{% endif %}

View File

@@ -0,0 +1,146 @@
#!/usr/bin/python3
import time
from flask import url_for
from .util import live_server_setup, extract_UUID_from_client, extract_api_key_from_UI
def set_response_with_ldjson():
test_return_data = """<html>
<body>
Some initial text</br>
<p>Which is across multiple lines</p>
</br>
So let's see what happens. </br>
<div class="sametext">Some text thats the same</div>
<div class="changetext">Some text that will change</div>
<script type="application/ld+json">
{
"@context":"https://schema.org/",
"@type":"Product",
"@id":"https://www.some-virtual-phone-shop.com/celular-iphone-14/p",
"name":"Celular Iphone 14 Pro Max 256Gb E Sim A16 Bionic",
"brand":{
"@type":"Brand",
"name":"APPLE"
},
"image":"https://www.some-virtual-phone-shop.com/15509426/image.jpg",
"description":"You dont need it",
"mpn":"111111",
"sku":"22222",
"offers":{
"@type":"AggregateOffer",
"lowPrice":8097000,
"highPrice":8099900,
"priceCurrency":"COP",
"offers":[
{
"@type":"Offer",
"price":8097000,
"priceCurrency":"COP",
"availability":"http://schema.org/InStock",
"sku":"102375961",
"itemCondition":"http://schema.org/NewCondition",
"seller":{
"@type":"Organization",
"name":"ajax"
}
}
],
"offerCount":1
}
}
</script>
</body>
</html>
"""
with open("test-datastore/endpoint-content.txt", "w") as f:
f.write(test_return_data)
return None
def set_response_without_ldjson():
test_return_data = """<html>
<body>
Some initial text</br>
<p>Which is across multiple lines</p>
</br>
So let's see what happens. </br>
<div class="sametext">Some text thats the same</div>
<div class="changetext">Some text that will change</div>
</body>
</html>
"""
with open("test-datastore/endpoint-content.txt", "w") as f:
f.write(test_return_data)
return None
# actually only really used by the distll.io importer, but could be handy too
def test_check_ldjson_price_autodetect(client, live_server):
live_server_setup(live_server)
# Give the endpoint time to spin up
time.sleep(1)
set_response_with_ldjson()
# Add our URL to the import page
test_url = url_for('test_endpoint', _external=True)
res = client.post(
url_for("import_page"),
data={"urls": test_url},
follow_redirects=True
)
assert b"1 Imported" in res.data
time.sleep(3)
# Should get a notice that it's available
res = client.get(url_for("index"))
assert b'ldjson-price-track-offer' in res.data
# Accept it
uuid = extract_UUID_from_client(client)
client.get(url_for('price_data_follower.accept', uuid=uuid, follow_redirects=True))
time.sleep(2)
# Trigger a check
client.get(url_for("form_watch_checknow"), follow_redirects=True)
time.sleep(2)
# Offer should be gone
res = client.get(url_for("index"))
assert b'Embedded price data' not in res.data
assert b'tracking-ldjson-price-data' in res.data
# and last snapshop (via API) should be just the price
api_key = extract_api_key_from_UI(client)
res = client.get(
url_for("watchsinglehistory", uuid=uuid, timestamp='latest'),
headers={'x-api-key': api_key},
)
# Should see this (dont know where the whitespace came from)
assert b'"highPrice": 8099900' in res.data
# And not this cause its not the ld-json
assert b"So let's see what happens" not in res.data
client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
##########################################################################################
# And we shouldnt see the offer
set_response_without_ldjson()
# Add our URL to the import page
test_url = url_for('test_endpoint', _external=True)
res = client.post(
url_for("import_page"),
data={"urls": test_url},
follow_redirects=True
)
assert b"1 Imported" in res.data
time.sleep(3)
res = client.get(url_for("index"))
assert b'ldjson-price-track-offer' not in res.data
##########################################################################################
client.get(url_for("form_delete", uuid="all"), follow_redirects=True)

View File

@@ -121,7 +121,7 @@ def test_element_removal_full(client, live_server):
url_for("import_page"), data={"urls": test_url}, follow_redirects=True
)
assert b"1 Imported" in res.data
time.sleep(1)
# Goto the edit page, add the filter data
# Not sure why \r needs to be added - absent of the #changetext this is not necessary
subtractive_selectors_data = "header\r\nfooter\r\nnav\r\n#changetext"

View File

@@ -38,9 +38,6 @@ def test_check_encoding_detection(client, live_server):
follow_redirects=True
)
# Trigger a check
client.get(url_for("form_watch_checknow"), follow_redirects=True)
# Give the thread time to pick it up
time.sleep(2)

View File

@@ -77,7 +77,8 @@ def test_DNS_errors(client, live_server):
time.sleep(3)
res = client.get(url_for("index"))
assert b'Name or service not known' in res.data
found_name_resolution_error = b"Temporary failure in name resolution" in res.data or b"Name or service not known" in res.data
assert found_name_resolution_error
# Should always record that we tried
assert bytes("just now".encode('utf-8')) in res.data

View File

@@ -0,0 +1,70 @@
#!/usr/bin/python3
import time
from flask import url_for
from urllib.request import urlopen
from .util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks
sleep_time_for_fetch_thread = 3
def test_check_extract_text_from_diff(client, live_server):
import time
with open("test-datastore/endpoint-content.txt", "w") as f:
f.write("Now it's {} seconds since epoch, time flies!".format(str(time.time())))
live_server_setup(live_server)
# Add our URL to the import page
res = client.post(
url_for("import_page"),
data={"urls": url_for('test_endpoint', _external=True)},
follow_redirects=True
)
assert b"1 Imported" in res.data
time.sleep(1)
# Load in 5 different numbers/changes
last_date=""
for n in range(5):
# Give the thread time to pick it up
print("Bumping snapshot and checking.. ", n)
last_date = str(time.time())
with open("test-datastore/endpoint-content.txt", "w") as f:
f.write("Now it's {} seconds since epoch, time flies!".format(last_date))
client.get(url_for("form_watch_checknow"), follow_redirects=True)
wait_for_all_checks(client)
res = client.post(
url_for("diff_history_page", uuid="first"),
data={"extract_regex": "Now it's ([0-9\.]+)",
"extract_submit_button": "Extract as CSV"},
follow_redirects=False
)
assert b'Nothing matches that RegEx' not in res.data
assert res.content_type == 'text/csv'
# Read the csv reply as stringio
from io import StringIO
import csv
f = StringIO(res.data.decode('utf-8'))
reader = csv.reader(f, delimiter=',')
output=[]
for row in reader:
output.append(row)
assert output[0][0] == 'Epoch seconds'
# Header line + 1 origin/first + 5 changes
assert(len(output) == 7)
# We expect to find the last bumped date in the changes in the last field of the spreadsheet
assert(output[6][2] == last_date)
# And nothing else, only that group () of the decimal and .
assert "time flies" not in output[6][2]

View File

@@ -73,17 +73,17 @@ def test_filter_doesnt_exist_then_exists_should_get_notification(client, live_se
# Just a regular notification setting, this will be used by the special 'filter not found' notification
notification_form_data = {"notification_urls": notification_url,
"notification_title": "New ChangeDetection.io Notification - {watch_url}",
"notification_body": "BASE URL: {base_url}\n"
"Watch URL: {watch_url}\n"
"Watch UUID: {watch_uuid}\n"
"Watch title: {watch_title}\n"
"Watch tag: {watch_tag}\n"
"Preview: {preview_url}\n"
"Diff URL: {diff_url}\n"
"Snapshot: {current_snapshot}\n"
"Diff: {diff}\n"
"Diff Full: {diff_full}\n"
"notification_title": "New ChangeDetection.io Notification - {{watch_url}}",
"notification_body": "BASE URL: {{base_url}}\n"
"Watch URL: {{watch_url}}\n"
"Watch UUID: {{watch_uuid}}\n"
"Watch title: {{watch_title}}\n"
"Watch tag: {{watch_tag}}\n"
"Preview: {{preview_url}}\n"
"Diff URL: {{diff_url}}\n"
"Snapshot: {{current_snapshot}}\n"
"Diff: {{diff}}\n"
"Diff Full: {{diff_full}}\n"
":-)",
"notification_format": "Text"}

View File

@@ -56,17 +56,17 @@ def run_filter_test(client, content_filter):
# Just a regular notification setting, this will be used by the special 'filter not found' notification
notification_form_data = {"notification_urls": notification_url,
"notification_title": "New ChangeDetection.io Notification - {watch_url}",
"notification_body": "BASE URL: {base_url}\n"
"Watch URL: {watch_url}\n"
"Watch UUID: {watch_uuid}\n"
"Watch title: {watch_title}\n"
"Watch tag: {watch_tag}\n"
"Preview: {preview_url}\n"
"Diff URL: {diff_url}\n"
"Snapshot: {current_snapshot}\n"
"Diff: {diff}\n"
"Diff Full: {diff_full}\n"
"notification_title": "New ChangeDetection.io Notification - {{watch_url}}",
"notification_body": "BASE URL: {{base_url}}\n"
"Watch URL: {{watch_url}}\n"
"Watch UUID: {{watch_uuid}}\n"
"Watch title: {{watch_title}}\n"
"Watch tag: {{watch_tag}}\n"
"Preview: {{preview_url}}\n"
"Diff URL: {{diff_url}}\n"
"Snapshot: {{current_snapshot}}\n"
"Diff: {{diff}}\n"
"Diff Full: {{diff_full}}\n"
":-)",
"notification_format": "Text"}
@@ -84,6 +84,7 @@ def run_filter_test(client, content_filter):
data=notification_form_data,
follow_redirects=True
)
assert b"Updated watch." in res.data
time.sleep(3)

View File

@@ -101,9 +101,6 @@ def test_check_ignore_text_functionality(client, live_server):
)
assert b"1 Imported" in res.data
# Trigger a check
client.get(url_for("form_watch_checknow"), follow_redirects=True)
# Give the thread time to pick it up
time.sleep(sleep_time_for_fetch_thread)
@@ -199,9 +196,6 @@ def test_check_global_ignore_text_functionality(client, live_server):
)
assert b"1 Imported" in res.data
# Trigger a check
client.get(url_for("form_watch_checknow"), follow_redirects=True)
# Give the thread time to pick it up
time.sleep(sleep_time_for_fetch_thread)

View File

@@ -69,8 +69,6 @@ def test_normal_page_check_works_with_ignore_status_code(client, live_server):
assert b"1 Imported" in res.data
time.sleep(sleep_time_for_fetch_thread)
# Trigger a check
client.get(url_for("form_watch_checknow"), follow_redirects=True)
set_some_changed_response()
time.sleep(sleep_time_for_fetch_thread)
@@ -104,9 +102,6 @@ def test_403_page_check_works_with_ignore_status_code(client, live_server):
)
assert b"1 Imported" in res.data
# Trigger a check
client.get(url_for("form_watch_checknow"), follow_redirects=True)
# Give the thread time to pick it up
time.sleep(sleep_time_for_fetch_thread)
@@ -119,11 +114,9 @@ def test_403_page_check_works_with_ignore_status_code(client, live_server):
)
assert b"Updated watch." in res.data
# Trigger a check
client.get(url_for("form_watch_checknow"), follow_redirects=True)
# Give the thread time to pick it up
time.sleep(sleep_time_for_fetch_thread)
# Make a change
set_some_changed_response()

View File

@@ -90,17 +90,17 @@ def test_check_notification(client, live_server):
print (">>>> Notification URL: "+notification_url)
notification_form_data = {"notification_urls": notification_url,
"notification_title": "New ChangeDetection.io Notification - {watch_url}",
"notification_body": "BASE URL: {base_url}\n"
"Watch URL: {watch_url}\n"
"Watch UUID: {watch_uuid}\n"
"Watch title: {watch_title}\n"
"Watch tag: {watch_tag}\n"
"Preview: {preview_url}\n"
"Diff URL: {diff_url}\n"
"Snapshot: {current_snapshot}\n"
"Diff: {diff}\n"
"Diff Full: {diff_full}\n"
"notification_title": "New ChangeDetection.io Notification - {{watch_url}}",
"notification_body": "BASE URL: {{base_url}}\n"
"Watch URL: {{watch_url}}\n"
"Watch UUID: {{watch_uuid}}\n"
"Watch title: {{watch_title}}\n"
"Watch tag: {{watch_tag}}\n"
"Preview: {{preview_url}}\n"
"Diff URL: {{diff_url}}\n"
"Snapshot: {{current_snapshot}}\n"
"Diff: {{diff}}\n"
"Diff Full: {{diff_full}}\n"
":-)",
"notification_screenshot": True,
"notification_format": "Text"}
@@ -179,7 +179,6 @@ def test_check_notification(client, live_server):
logging.debug(">>> Skipping BASE_URL check")
# This should insert the {current_snapshot}
set_more_modified_response()
client.get(url_for("form_watch_checknow"), follow_redirects=True)
@@ -237,10 +236,10 @@ def test_check_notification(client, live_server):
follow_redirects=True
)
def test_notification_validation(client, live_server):
#live_server_setup(live_server)
time.sleep(3)
time.sleep(1)
# re #242 - when you edited an existing new entry, it would not correctly show the notification settings
# Add our URL to the import page
test_url = url_for('test_endpoint', _external=True)
@@ -268,21 +267,6 @@ def test_notification_validation(client, live_server):
# )
# assert b"Notification Body and Title is required when a Notification URL is used" in res.data
# Now adding a wrong token should give us an error
res = client.post(
url_for("settings_page"),
data={"application-notification_title": "New ChangeDetection.io Notification - {watch_url}",
"application-notification_body": "Rubbish: {rubbish}\n",
"application-notification_format": "Text",
"application-notification_urls": "json://localhost/foobar",
"requests-time_between_check-minutes": 180,
"fetch_backend": "html_requests"
},
follow_redirects=True
)
assert bytes("is not a valid token".encode('utf-8')) in res.data
# cleanup for the next
client.get(
url_for("form_delete", uuid="all"),
@@ -290,3 +274,58 @@ def test_notification_validation(client, live_server):
)
def test_notification_custom_endpoint_and_jinja2(client, live_server):
time.sleep(1)
# test_endpoint - that sends the contents of a file
# test_notification_endpoint - that takes a POST and writes it to file (test-datastore/notification.txt)
# CUSTOM JSON BODY CHECK for POST://
set_original_response()
test_notification_url = url_for('test_notification_endpoint', _external=True).replace('http://', 'post://')+"?xxx={{ watch_url }}"
res = client.post(
url_for("settings_page"),
data={"application-notification_title": "New ChangeDetection.io Notification - {{ watch_url }}",
"application-notification_body": '{ "url" : "{{ watch_url }}", "secret": 444 }',
# https://github.com/caronc/apprise/wiki/Notify_Custom_JSON#get-parameter-manipulation
"application-notification_urls": test_notification_url,
"application-minutes_between_check": 180,
"application-fetch_backend": "html_requests"
},
follow_redirects=True
)
assert b'Settings updated' in res.data
# Add a watch and trigger a HTTP POST
test_url = url_for('test_endpoint', _external=True)
res = client.post(
url_for("form_quick_watch_add"),
data={"url": test_url, "tag": 'nice one'},
follow_redirects=True
)
assert b"Watch added" in res.data
time.sleep(2)
set_modified_response()
client.get(url_for("form_watch_checknow"), follow_redirects=True)
time.sleep(2)
with open("test-datastore/notification.txt", 'r') as f:
x=f.read()
j = json.loads(x)
assert j['url'].startswith('http://localhost')
assert j['secret'] == 444
# URL check, this will always be converted to lowercase
assert os.path.isfile("test-datastore/notification-url.txt")
with open("test-datastore/notification-url.txt", 'r') as f:
notification_url = f.read()
assert 'xxx=http' in notification_url
os.unlink("test-datastore/notification-url.txt")

View File

@@ -11,23 +11,23 @@ def test_check_notification_error_handling(client, live_server):
set_original_response()
# Give the endpoint time to spin up
time.sleep(3)
time.sleep(2)
# use a different URL so that it doesnt interfere with the actual check until we are ready
# Set a URL and fetch it, then set a notification URL which is going to give errors
test_url = url_for('test_endpoint', _external=True)
res = client.post(
url_for("form_quick_watch_add"),
data={"url": "https://changedetection.io/CHANGELOG.txt", "tag": ''},
data={"url": test_url, "tag": ''},
follow_redirects=True
)
assert b"Watch added" in res.data
time.sleep(10)
time.sleep(2)
set_modified_response()
# Check we capture the failure, we can just use trigger_check = y here
res = client.post(
url_for("edit_page", uuid="first"),
data={"notification_urls": "jsons://broken-url.changedetection.io/test",
data={"notification_urls": "jsons://broken-url-xxxxxxxx123/test",
"notification_title": "xxx",
"notification_body": "xxxxx",
"notification_format": "Text",
@@ -36,15 +36,14 @@ def test_check_notification_error_handling(client, live_server):
"title": "",
"headers": "",
"time_between_check-minutes": "180",
"fetch_backend": "html_requests",
"trigger_check": "y"},
"fetch_backend": "html_requests"},
follow_redirects=True
)
assert b"Updated watch." in res.data
found=False
for i in range(1, 10):
time.sleep(1)
logging.debug("Fetching watch overview....")
res = client.get(
url_for("index"))
@@ -53,6 +52,7 @@ def test_check_notification_error_handling(client, live_server):
found=True
break
time.sleep(1)
assert found
@@ -60,7 +60,7 @@ def test_check_notification_error_handling(client, live_server):
# The error should show in the notification logs
res = client.get(
url_for("notification_logs"))
assert bytes("Name or service not known".encode('utf-8')) in res.data
found_name_resolution_error = b"Temporary failure in name resolution" in res.data or b"Name or service not known" in res.data
assert found_name_resolution_error
# And it should be listed on the watch overview
client.get(url_for("form_delete", uuid="all"), follow_redirects=True)

View File

@@ -20,6 +20,8 @@ def test_headers_in_request(client, live_server):
)
assert b"1 Imported" in res.data
time.sleep(1)
res = client.post(
url_for("import_page"),
data={"urls": test_url},
@@ -174,6 +176,7 @@ def test_method_in_request(client, live_server):
)
assert b"1 Imported" in res.data
time.sleep(2)
res = client.post(
url_for("import_page"),
data={"urls": test_url},
@@ -181,6 +184,8 @@ def test_method_in_request(client, live_server):
)
assert b"1 Imported" in res.data
time.sleep(2)
# Attempt to add a method which is not valid
res = client.post(
url_for("edit_page", uuid="first"),
@@ -206,7 +211,7 @@ def test_method_in_request(client, live_server):
assert b"Updated watch." in res.data
# Give the thread time to pick up the first version
time.sleep(5)
time.sleep(2)
# The service should echo back the request verb
res = client.get(
@@ -217,7 +222,7 @@ def test_method_in_request(client, live_server):
# The test call service will return the verb as the body
assert b"PATCH" in res.data
time.sleep(5)
time.sleep(2)
watches_with_method = 0
with open('test-datastore/url-watches.json') as f:

View File

@@ -149,6 +149,9 @@ def live_server_setup(live_server):
if data != None:
f.write(data)
with open("test-datastore/notification-url.txt", "w") as f:
f.write(request.url)
print("\n>> Test notification endpoint was hit.\n", data)
return "Text was set"

View File

@@ -29,8 +29,9 @@ apprise~=1.2.0
# apprise mqtt https://github.com/dgtlmoon/changedetection.io/issues/315
paho-mqtt
# Pinned version of cryptography otherwise
# ERROR: Could not build wheels for cryptography which use PEP 517 and cannot be installed directly
# This mainly affects some ARM builds, which unlike the other builds ignores "ARG CRYPTOGRAPHY_DONT_BUILD_RUST=1"
# so without this pinning, the newer versions on ARM will forcefully try to build rust, which results in "rust compiler not found"
# (introduced once apprise became a dep)
cryptography~=3.4
# Used for CSS filtering

View File

@@ -1 +1 @@
python-3.8.12
python-3.9.15