From 73189672c38b85d2f134e697bb9c0d8614f812f9 Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Tue, 18 Mar 2025 10:40:22 +0100 Subject: [PATCH] Refactor code layout, add extra tests --- changedetectionio/auth_decorator.py | 36 + .../blueprint/imports/__init__.py | 74 + .../{ => blueprint/imports}/importer.py | 7 +- .../blueprint/imports/templates/import.html | 125 ++ changedetectionio/blueprint/rss/__init__.py | 103 ++ .../blueprint/settings/__init__.py | 120 ++ .../settings/templates/notification-log.html | 19 + .../settings/templates/settings.html | 310 ++++ .../blueprint/tags/templates/edit-tag.html | 4 +- changedetectionio/blueprint/ui/__init__.py | 301 ++++ changedetectionio/blueprint/ui/edit.py | 333 +++++ .../blueprint/ui/notification.py | 107 ++ .../ui/templates/clear_all_history.html | 49 + changedetectionio/blueprint/ui/views.py | 220 +++ changedetectionio/conditions/blueprint.py | 2 + changedetectionio/flask_app.py | 1242 +---------------- changedetectionio/model/Watch.py | 2 +- .../templates/_common_fields.html | 2 +- changedetectionio/templates/_helpers.html | 2 +- changedetectionio/templates/base.html | 10 +- .../templates/clear_all_history.html | 2 +- changedetectionio/templates/diff.html | 4 +- changedetectionio/templates/edit.html | 16 +- changedetectionio/templates/import.html | 2 +- changedetectionio/templates/preview.html | 2 +- changedetectionio/templates/settings.html | 8 +- .../templates/watch-overview.html | 30 +- .../test_custom_browser_url.py | 14 +- .../tests/fetchers/test_content.py | 6 +- .../fetchers/test_custom_js_before_content.py | 8 +- .../tests/proxy_list/test_multiple_proxy.py | 4 +- .../tests/proxy_list/test_noproxy.py | 14 +- .../tests/proxy_list/test_proxy.py | 2 +- .../proxy_list/test_select_custom_proxy.py | 6 +- .../tests/proxy_socks5/test_socks5_proxy.py | 12 +- .../proxy_socks5/test_socks5_proxy_sources.py | 10 +- .../tests/restock/test_restock.py | 8 +- .../tests/smtp/test_notification_smtp.py | 18 +- .../tests/test_access_control.py | 29 +- .../tests/test_add_replace_remove_filter.py | 32 +- changedetectionio/tests/test_api.py | 15 +- changedetectionio/tests/test_auth.py | 6 +- .../test_automatic_follow_ldjson_price.py | 14 +- changedetectionio/tests/test_backend.py | 32 +- changedetectionio/tests/test_backup.py | 2 +- .../tests/test_block_while_text_present.py | 16 +- changedetectionio/tests/test_clone.py | 4 +- changedetectionio/tests/test_conditions.py | 77 +- changedetectionio/tests/test_css_selector.py | 20 +- .../tests/test_element_removal.py | 24 +- changedetectionio/tests/test_encoding.py | 8 +- changedetectionio/tests/test_errorhandling.py | 18 +- changedetectionio/tests/test_extract_csv.py | 6 +- changedetectionio/tests/test_extract_regex.py | 20 +- .../tests/test_filter_exist_changes.py | 6 +- .../tests/test_filter_failure_notification.py | 16 +- changedetectionio/tests/test_group.py | 50 +- .../tests/test_history_consistency.py | 4 +- changedetectionio/tests/test_ignore.py | 8 +- changedetectionio/tests/test_ignore_text.py | 32 +- .../tests/test_ignorehyperlinks.py | 18 +- .../tests/test_ignorestatuscode.py | 12 +- .../tests/test_ignorewhitespace.py | 8 +- changedetectionio/tests/test_import.py | 24 +- changedetectionio/tests/test_jinja2.py | 6 +- .../tests/test_jsonpath_jq_selector.py | 54 +- changedetectionio/tests/test_live_preview.py | 10 +- .../tests/test_nonrenderable_pages.py | 16 +- changedetectionio/tests/test_notification.py | 64 +- .../tests/test_notification_errors.py | 8 +- changedetectionio/tests/test_obfuscations.py | 4 +- changedetectionio/tests/test_pdf.py | 12 +- .../tests/test_preview_endpoints.py | 12 +- changedetectionio/tests/test_request.py | 64 +- .../tests/test_restock_itemprop.py | 88 +- changedetectionio/tests/test_rss.py | 24 +- changedetectionio/tests/test_scheduler.py | 24 +- changedetectionio/tests/test_search.py | 8 +- changedetectionio/tests/test_security.py | 18 +- changedetectionio/tests/test_share_watch.py | 14 +- changedetectionio/tests/test_source.py | 16 +- changedetectionio/tests/test_trigger.py | 20 +- changedetectionio/tests/test_trigger_regex.py | 12 +- .../tests/test_trigger_regex_with_filter.py | 12 +- changedetectionio/tests/test_unique_lines.py | 30 +- .../tests/test_watch_fields_storage.py | 6 +- .../tests/test_xpath_selector.py | 96 +- changedetectionio/tests/util.py | 2 +- .../tests/visualselector/test_fetch_data.py | 22 +- 89 files changed, 2541 insertions(+), 1836 deletions(-) create mode 100644 changedetectionio/auth_decorator.py create mode 100644 changedetectionio/blueprint/imports/__init__.py rename changedetectionio/{ => blueprint/imports}/importer.py (98%) create mode 100644 changedetectionio/blueprint/imports/templates/import.html create mode 100644 changedetectionio/blueprint/rss/__init__.py create mode 100644 changedetectionio/blueprint/settings/__init__.py create mode 100644 changedetectionio/blueprint/settings/templates/notification-log.html create mode 100644 changedetectionio/blueprint/settings/templates/settings.html create mode 100644 changedetectionio/blueprint/ui/__init__.py create mode 100644 changedetectionio/blueprint/ui/edit.py create mode 100644 changedetectionio/blueprint/ui/notification.py create mode 100644 changedetectionio/blueprint/ui/templates/clear_all_history.html create mode 100644 changedetectionio/blueprint/ui/views.py diff --git a/changedetectionio/auth_decorator.py b/changedetectionio/auth_decorator.py new file mode 100644 index 00000000..3358e76a --- /dev/null +++ b/changedetectionio/auth_decorator.py @@ -0,0 +1,36 @@ +import os +from functools import wraps +from flask import current_app, redirect, request +from loguru import logger + +def login_optionally_required(func): + """ + If password authentication is enabled, verify the user is logged in. + To be used as a decorator for routes that should optionally require login. + This version is blueprint-friendly as it uses current_app instead of directly accessing app. + """ + @wraps(func) + def decorated_view(*args, **kwargs): + from flask import current_app + import flask_login + from flask_login import current_user + + # Access datastore through the app config + datastore = current_app.config['DATASTORE'] + has_password_enabled = datastore.data['settings']['application'].get('password') or os.getenv("SALTED_PASS", False) + + # Permitted + if request.endpoint and 'static_content' in request.endpoint and request.view_args and request.view_args.get('group') == 'styles': + return func(*args, **kwargs) + # Permitted + elif request.endpoint and 'diff_history_page' in request.endpoint and datastore.data['settings']['application'].get('shared_diff_access'): + return func(*args, **kwargs) + elif request.method in flask_login.config.EXEMPT_METHODS: + return func(*args, **kwargs) + elif current_app.config.get('LOGIN_DISABLED'): + return func(*args, **kwargs) + elif has_password_enabled and not current_user.is_authenticated: + return current_app.login_manager.unauthorized() + + return func(*args, **kwargs) + return decorated_view \ No newline at end of file diff --git a/changedetectionio/blueprint/imports/__init__.py b/changedetectionio/blueprint/imports/__init__.py new file mode 100644 index 00000000..e0dd12bd --- /dev/null +++ b/changedetectionio/blueprint/imports/__init__.py @@ -0,0 +1,74 @@ +from flask import Blueprint, request, redirect, url_for, flash, render_template +from changedetectionio.store import ChangeDetectionStore +from changedetectionio.auth_decorator import login_optionally_required +from changedetectionio.blueprint.imports.importer import ( + import_url_list, + import_distill_io_json, + import_xlsx_wachete, + import_xlsx_custom +) + +def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMetaData): + import_blueprint = Blueprint('imports', __name__, template_folder="templates") + + @import_blueprint.route("/import", methods=['GET', 'POST']) + @login_optionally_required + def import_page(): + remaining_urls = [] + from changedetectionio import forms + + if request.method == 'POST': + # URL List import + if request.values.get('urls') and len(request.values.get('urls').strip()): + # Import and push into the queue for immediate update check + importer_handler = import_url_list() + importer_handler.run(data=request.values.get('urls'), flash=flash, datastore=datastore, processor=request.values.get('processor', 'text_json_diff')) + for uuid in importer_handler.new_uuids: + update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid})) + + if len(importer_handler.remaining_data) == 0: + return redirect(url_for('index')) + else: + remaining_urls = importer_handler.remaining_data + + # Distill.io import + if request.values.get('distill-io') and len(request.values.get('distill-io').strip()): + # Import and push into the queue for immediate update check + d_importer = import_distill_io_json() + d_importer.run(data=request.values.get('distill-io'), flash=flash, datastore=datastore) + for uuid in d_importer.new_uuids: + update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid})) + + # XLSX importer + if request.files and request.files.get('xlsx_file'): + file = request.files['xlsx_file'] + + if request.values.get('file_mapping') == 'wachete': + w_importer = import_xlsx_wachete() + w_importer.run(data=file, flash=flash, datastore=datastore) + else: + w_importer = import_xlsx_custom() + # Building mapping of col # to col # type + map = {} + for i in range(10): + c = request.values.get(f"custom_xlsx[col_{i}]") + v = request.values.get(f"custom_xlsx[col_type_{i}]") + if c and v: + map[int(c)] = v + + w_importer.import_profile = map + w_importer.run(data=file, flash=flash, datastore=datastore) + + for uuid in w_importer.new_uuids: + update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid})) + + # Could be some remaining, or we could be on GET + form = forms.importForm(formdata=request.form if request.method == 'POST' else None) + output = render_template("import.html", + form=form, + import_url_list_remaining="\n".join(remaining_urls), + original_distill_json='' + ) + return output + + return import_blueprint \ No newline at end of file diff --git a/changedetectionio/importer.py b/changedetectionio/blueprint/imports/importer.py similarity index 98% rename from changedetectionio/importer.py rename to changedetectionio/blueprint/imports/importer.py index 42a062be..4824d138 100644 --- a/changedetectionio/importer.py +++ b/changedetectionio/blueprint/imports/importer.py @@ -1,6 +1,5 @@ -from abc import ABC, abstractmethod +from abc import abstractmethod import time -import validators from wtforms import ValidationError from loguru import logger @@ -241,7 +240,7 @@ class import_xlsx_custom(Importer): return # @todo cehck atleast 2 rows, same in other method - from .forms import validate_url + from changedetectionio.forms import validate_url row_i = 1 try: @@ -300,4 +299,4 @@ class import_xlsx_custom(Importer): row_i += 1 flash( - "{} imported from custom .xlsx in {:.2f}s".format(len(self.new_uuids), time.time() - now)) + "{} imported from custom .xlsx in {:.2f}s".format(len(self.new_uuids), time.time() - now)) \ No newline at end of file diff --git a/changedetectionio/blueprint/imports/templates/import.html b/changedetectionio/blueprint/imports/templates/import.html new file mode 100644 index 00000000..42080885 --- /dev/null +++ b/changedetectionio/blueprint/imports/templates/import.html @@ -0,0 +1,125 @@ +{% extends 'base.html' %} +{% block content %} +{% from '_helpers.html' import render_field %} + +
+ +
+ +
+ +
+
+ +
+ + Enter one URL per line, and optionally add tags for each URL after a space, delineated by comma + (,): +
+ https://example.com tag1, tag2, last tag +
+ URLs which do not pass validation will stay in the textarea. + + {{ render_field(form.processor, class="processor") }} + + + +
+ +
+ +
+ +
+ + + + + Copy and Paste your Distill.io watch 'export' file, this should be a JSON file.
+ This is experimental, supported fields are name, uri, tags, config:selections, the rest (including schedule) are ignored. +
+

+ How to export? https://distill.io/docs/web-monitor/how-export-and-import-monitors/
+ Be sure to set your default fetcher to Chrome if required.
+

+ + + + + +
+
+
+
+ {{ render_field(form.xlsx_file, class="processor") }} +
+
+ {{ render_field(form.file_mapping, class="processor") }} +
+
+
+ + Table of custom column and data types mapping for the Custom mapping File mapping type. + + + + + {% for n in range(4) %} + + {% endfor %} + + + + {% for n in range(4) %} + + {% endfor %} + +
Column #
Type
+
+
+ +
+ +
+
+ +{% endblock %} \ No newline at end of file diff --git a/changedetectionio/blueprint/rss/__init__.py b/changedetectionio/blueprint/rss/__init__.py new file mode 100644 index 00000000..d9a87695 --- /dev/null +++ b/changedetectionio/blueprint/rss/__init__.py @@ -0,0 +1,103 @@ +import time +import datetime +import pytz +from flask import Blueprint, make_response, request, url_for +from loguru import logger +from feedgen.feed import FeedGenerator + +from changedetectionio.store import ChangeDetectionStore +from changedetectionio.safe_jinja import render as jinja_render + +def construct_blueprint(datastore: ChangeDetectionStore): + rss_blueprint = Blueprint('rss', __name__) + + # Import the login decorator if needed + # from changedetectionio.auth_decorator import login_optionally_required + + @rss_blueprint.route("/", methods=['GET']) + def feed(): + now = time.time() + # Always requires token set + app_rss_token = datastore.data['settings']['application'].get('rss_access_token') + rss_url_token = request.args.get('token') + if rss_url_token != app_rss_token: + return "Access denied, bad token", 403 + + from changedetectionio import diff + limit_tag = request.args.get('tag', '').lower().strip() + # Be sure limit_tag is a uuid + for uuid, tag in datastore.data['settings']['application'].get('tags', {}).items(): + if limit_tag == tag.get('title', '').lower().strip(): + limit_tag = uuid + + # Sort by last_changed and add the uuid which is usually the key.. + sorted_watches = [] + + # @todo needs a .itemsWithTag() or something - then we can use that in Jinaj2 and throw this away + for uuid, watch in datastore.data['watching'].items(): + # @todo tag notification_muted skip also (improve Watch model) + if datastore.data['settings']['application'].get('rss_hide_muted_watches') and watch.get('notification_muted'): + continue + if limit_tag and not limit_tag in watch['tags']: + continue + watch['uuid'] = uuid + sorted_watches.append(watch) + + sorted_watches.sort(key=lambda x: x.last_changed, reverse=False) + + fg = FeedGenerator() + fg.title('changedetection.io') + fg.description('Feed description') + fg.link(href='https://changedetection.io') + + for watch in sorted_watches: + + dates = list(watch.history.keys()) + # Re #521 - Don't bother processing this one if theres less than 2 snapshots, means we never had a change detected. + if len(dates) < 2: + continue + + if not watch.viewed: + # Re #239 - GUID needs to be individual for each event + # @todo In the future make this a configurable link back (see work on BASE_URL https://github.com/dgtlmoon/changedetection.io/pull/228) + guid = "{}/{}".format(watch['uuid'], watch.last_changed) + fe = fg.add_entry() + + # Include a link to the diff page, they will have to login here to see if password protection is enabled. + # Description is the page you watch, link takes you to the diff JS UI page + # Dict val base_url will get overriden with the env var if it is set. + ext_base_url = datastore.data['settings']['application'].get('active_base_url') + + # Because we are called via whatever web server, flask should figure out the right path ( + diff_link = {'href': url_for('ui.ui_views.diff_history_page', uuid=watch['uuid'], _external=True)} + + fe.link(link=diff_link) + + # @todo watch should be a getter - watch.get('title') (internally if URL else..) + + watch_title = watch.get('title') if watch.get('title') else watch.get('url') + fe.title(title=watch_title) + + html_diff = diff.render_diff(previous_version_file_contents=watch.get_history_snapshot(dates[-2]), + newest_version_file_contents=watch.get_history_snapshot(dates[-1]), + include_equal=False, + line_feed_sep="
") + + # @todo Make this configurable and also consider html-colored markup + # @todo User could decide if goes to the diff page, or to the watch link + rss_template = "\n

{{watch_title}}

\n

{{html_diff}}

\n\n" + content = jinja_render(template_str=rss_template, watch_title=watch_title, html_diff=html_diff, watch_url=watch.link) + + fe.content(content=content, type='CDATA') + + fe.guid(guid, permalink=False) + dt = datetime.datetime.fromtimestamp(int(watch.newest_history_key)) + dt = dt.replace(tzinfo=pytz.UTC) + fe.pubDate(dt) + + response = make_response(fg.rss_str()) + response.headers.set('Content-Type', 'application/rss+xml;charset=utf-8') + logger.trace(f"RSS generated in {time.time() - now:.3f}s") + return response + + return rss_blueprint \ No newline at end of file diff --git a/changedetectionio/blueprint/settings/__init__.py b/changedetectionio/blueprint/settings/__init__.py new file mode 100644 index 00000000..5375b565 --- /dev/null +++ b/changedetectionio/blueprint/settings/__init__.py @@ -0,0 +1,120 @@ +import os +from copy import deepcopy +from datetime import datetime +from zoneinfo import ZoneInfo, available_timezones +import secrets +import flask_login +from flask import Blueprint, render_template, request, redirect, url_for, flash + +from changedetectionio.store import ChangeDetectionStore +from changedetectionio.auth_decorator import login_optionally_required + + +def construct_blueprint(datastore: ChangeDetectionStore): + settings_blueprint = Blueprint('settings', __name__, template_folder="templates") + + @settings_blueprint.route("/", methods=['GET', "POST"]) + @login_optionally_required + def settings_page(): + from changedetectionio import forms + + default = deepcopy(datastore.data['settings']) + if datastore.proxy_list is not None: + available_proxies = list(datastore.proxy_list.keys()) + # When enabled + system_proxy = datastore.data['settings']['requests']['proxy'] + # In the case it doesnt exist anymore + if not system_proxy in available_proxies: + system_proxy = None + + default['requests']['proxy'] = system_proxy if system_proxy is not None else available_proxies[0] + # Used by the form handler to keep or remove the proxy settings + default['proxy_list'] = available_proxies[0] + + # Don't use form.data on POST so that it doesnt overrid the checkbox status from the POST status + form = forms.globalSettingsForm(formdata=request.form if request.method == 'POST' else None, + data=default, + extra_notification_tokens=datastore.get_unique_notification_tokens_available() + ) + + # Remove the last option 'System default' + form.application.form.notification_format.choices.pop() + + if datastore.proxy_list is None: + # @todo - Couldn't get setattr() etc dynamic addition working, so remove it instead + del form.requests.form.proxy + else: + form.requests.form.proxy.choices = [] + for p in datastore.proxy_list: + form.requests.form.proxy.choices.append(tuple((p, datastore.proxy_list[p]['label']))) + + if request.method == 'POST': + # Password unset is a GET, but we can lock the session to a salted env password to always need the password + if form.application.form.data.get('removepassword_button', False): + # SALTED_PASS means the password is "locked" to what we set in the Env var + if not os.getenv("SALTED_PASS", False): + datastore.remove_password() + flash("Password protection removed.", 'notice') + flask_login.logout_user() + return redirect(url_for('settings.settings_page')) + + if form.validate(): + # Don't set password to False when a password is set - should be only removed with the `removepassword` button + app_update = dict(deepcopy(form.data['application'])) + + # Never update password with '' or False (Added by wtforms when not in submission) + if 'password' in app_update and not app_update['password']: + del (app_update['password']) + + datastore.data['settings']['application'].update(app_update) + datastore.data['settings']['requests'].update(form.data['requests']) + + if not os.getenv("SALTED_PASS", False) and len(form.application.form.password.encrypted_password): + datastore.data['settings']['application']['password'] = form.application.form.password.encrypted_password + datastore.needs_write_urgent = True + flash("Password protection enabled.", 'notice') + flask_login.logout_user() + return redirect(url_for('index')) + + datastore.needs_write_urgent = True + flash("Settings updated.") + + else: + flash("An error occurred, please see below.", "error") + + # Convert to ISO 8601 format, all date/time relative events stored as UTC time + utc_time = datetime.now(ZoneInfo("UTC")).isoformat() + + output = render_template("settings.html", + api_key=datastore.data['settings']['application'].get('api_access_token'), + available_timezones=sorted(available_timezones()), + emailprefix=os.getenv('NOTIFICATION_MAIL_BUTTON_PREFIX', False), + extra_notification_token_placeholder_info=datastore.get_unique_notification_token_placeholders_available(), + form=form, + hide_remove_pass=os.getenv("SALTED_PASS", False), + min_system_recheck_seconds=int(os.getenv('MINIMUM_SECONDS_RECHECK_TIME', 3)), + settings_application=datastore.data['settings']['application'], + timezone_default_config=datastore.data['settings']['application'].get('timezone'), + utc_time=utc_time, + ) + + return output + + @settings_blueprint.route("/reset-api-key", methods=['GET']) + @login_optionally_required + def settings_reset_api_key(): + secret = secrets.token_hex(16) + datastore.data['settings']['application']['api_access_token'] = secret + datastore.needs_write_urgent = True + flash("API Key was regenerated.") + return redirect(url_for('settings.settings_page')+'#api') + + @settings_blueprint.route("/notification-logs", methods=['GET']) + @login_optionally_required + def notification_logs(): + from changedetectionio.flask_app import notification_debug_log + output = render_template("notification-log.html", + logs=notification_debug_log if len(notification_debug_log) else ["Notification logs are empty - no notifications sent yet."]) + return output + + return settings_blueprint \ No newline at end of file diff --git a/changedetectionio/blueprint/settings/templates/notification-log.html b/changedetectionio/blueprint/settings/templates/notification-log.html new file mode 100644 index 00000000..ee76e259 --- /dev/null +++ b/changedetectionio/blueprint/settings/templates/notification-log.html @@ -0,0 +1,19 @@ +{% extends 'base.html' %} + +{% block content %} +
+
+ +

Notification debug log

+
+
    + {% for log in logs|reverse %} +
  • {{log}}
    • + {% endfor %} +
+
+ +
+
+ +{% endblock %} diff --git a/changedetectionio/blueprint/settings/templates/settings.html b/changedetectionio/blueprint/settings/templates/settings.html new file mode 100644 index 00000000..1dfeba0d --- /dev/null +++ b/changedetectionio/blueprint/settings/templates/settings.html @@ -0,0 +1,310 @@ +{% extends 'base.html' %} + +{% block content %} +{% from '_helpers.html' import render_field, render_checkbox_field, render_button, render_time_schedule_form %} +{% from '_common_fields.html' import render_common_settings_form %} + + + + + + + +
+
+ +
+
+
+ +
+
+
+ {{ render_field(form.requests.form.time_between_check, class="time-check-widget") }} + Default recheck time for all watches, current system minimum is {{min_system_recheck_seconds}} seconds (more info). +
+ +
+ {{ render_time_schedule_form(form.requests, available_timezones, timezone_default_config) }} +
+
+
+
+ {{ render_field(form.requests.form.jitter_seconds, class="jitter_seconds") }} + Example - 3 seconds random jitter could trigger up to 3 seconds earlier or up to 3 seconds later +
+
+ {{ render_field(form.application.form.filter_failure_notification_threshold_attempts, class="filter_failure_notification_threshold_attempts") }} + After this many consecutive times that the CSS/xPath filter is missing, send a notification +
+ Set to 0 to disable + +
+
+ {% if not hide_remove_pass %} + {% if current_user.is_authenticated %} + {{ render_button(form.application.form.removepassword_button) }} + {% else %} + {{ render_field(form.application.form.password) }} + Password protection for your changedetection.io application. + {% endif %} + {% else %} + Password is locked. + {% endif %} +
+ +
+ {{ render_checkbox_field(form.application.form.shared_diff_access, class="shared_diff_access") }} + Allow access to view watch diff page when password is enabled (Good for sharing the diff page) + +
+
+ {{ render_checkbox_field(form.application.form.rss_hide_muted_watches) }} +
+
+ {{ render_field(form.application.form.pager_size) }} + Number of items per page in the watch overview list, 0 to disable. +
+ +
+ {{ render_checkbox_field(form.application.form.extract_title_as_title) }} + Note: This will automatically apply to all existing watches. +
+
+ {{ render_checkbox_field(form.application.form.empty_pages_are_a_change) }} + When a request returns no content, or the HTML does not contain any text, is this considered a change? +
+ {% if form.requests.proxy %} +
+ {{ render_field(form.requests.form.proxy, class="fetch-backend-proxy") }} + + Choose a default proxy for all watches + +
+ {% endif %} +
+
+ +
+
+
+ {{ render_common_settings_form(form.application.form, emailprefix, settings_application, extra_notification_token_placeholder_info) }} +
+
+
+ {{ render_field(form.application.form.base_url, class="m-d") }} + + Base URL used for the {{ '{{ base_url }}' }} token in notification links.
+ Default value is the system environment variable 'BASE_URL' - read more here. + +
+
+ +
+
+ {{ render_field(form.application.form.fetch_backend, class="fetch-backend") }} + +

Use the Basic method (default) where your watched sites don't need Javascript to render.

+

The Chrome/Javascript method requires a network connection to a running WebDriver+Chrome server, set by the ENV var 'WEBDRIVER_URL'.

+ +
+
+
+ If you're having trouble waiting for the page to be fully rendered (text missing etc), try increasing the 'wait' time here. +
+ This will wait n seconds before extracting the text. +
+
+ {{ render_field(form.application.form.webdriver_delay) }} +
+
+
+ {{ render_field(form.requests.form.default_ua) }} + + Applied to all requests.

+ Note: Simply changing the User-Agent often does not defeat anti-robot technologies, it's important to consider all of the ways that the browser is detected. + +
+
+
+ Tip: Connect using Bright Data and Oxylabs Proxies, find out more here. + +
+
+ +
+ +
+ {{ render_checkbox_field(form.application.form.ignore_whitespace) }} + Ignore whitespace, tabs and new-lines/line-feeds when considering if a change was detected.
+ Note: Changing this will change the status of your existing watches, possibly trigger alerts etc. + +
+
+ {{ render_checkbox_field(form.application.form.render_anchor_tag_content) }} + Render anchor tag content, default disabled, when enabled renders links as (link text)[https://somesite.com] +
+ Note: Changing this could affect the content of your existing watches, possibly trigger alerts etc. + +
+
+ {{ render_field(form.application.form.global_subtractive_selectors, rows=5, placeholder="header +footer +nav +.stockticker +//*[contains(text(), 'Advertisement')]") }} + +
    +
  • Remove HTML element(s) by CSS and XPath selectors before text conversion.
    • +
  • Don't paste HTML here, use only CSS and XPath selectors
    • +
  • Add multiple elements, CSS or XPath selectors per line to ignore multiple parts of the HTML.
    • +
+ +
+
+ {{ render_field(form.application.form.global_ignore_text, rows=5, placeholder="Some text to ignore in a line +/some.regex\d{2}/ for case-INsensitive regex + ") }} + Note: This is applied globally in addition to the per-watch rules.
+ +
    +
  • Matching text will be ignored in the text snapshot (you can still see it but it wont trigger a change)
    • +
  • Note: This is applied globally in addition to the per-watch rules.
    • +
  • Each line processed separately, any line matching will be ignored (removed before creating the checksum)
    • +
  • Regular Expression support, wrap the entire line in forward slash /regex/
    • +
  • Changing this will affect the comparison checksum which may trigger an alert
    • +
+ +
+
+ +
+

API Access

+

Drive your changedetection.io via API, More about API access here

+ +
+ {{ render_checkbox_field(form.application.form.api_access_token_enabled) }} +
Restrict API access limit by using x-api-key header - required for the Chrome Extension to work

+

API Key {{api_key}} + +
+
+
+ Regenerate API key +
+
+

Chrome Extension

+

Easily add any web-page to your changedetection.io installation from within Chrome.

+ Step 1 Install the extension, Step 2 Navigate to this page, + Step 3 Open the extension from the toolbar and click "Sync API Access" +

+ + Chrome store icon + Chrome Webstore + +

+
+
+
+
+ Ensure the settings below are correct, they are used to manage the time schedule for checking your web page watches. +
+
+

UTC Time & Date from Server: {{ utc_time }}

+

Local Time & Date in Browser:

+

+ {{ render_field(form.application.form.timezone) }} + +

+
+
+
+ + +

Tip: "Residential" and "Mobile" proxy type can be more successfull than "Data Center" for blocked websites. + +

+ {{ render_field(form.requests.form.extra_proxies) }} + "Name" will be used for selecting the proxy in the Watch Edit settings
+ SOCKS5 proxies with authentication are only supported with 'plain requests' fetcher, for other fetchers you should whitelist the IP access instead +
+
+

+ Extra Browsers can be attached to further defeat CAPTCHA's on websites that are particularly hard to scrape.
+ Simply paste the connection address into the box, More instructions and examples here +

+ {{ render_field(form.requests.form.extra_browsers) }} +
+
+
+
+ {{ render_button(form.save_button) }} + Back + Clear Snapshot History +
+
+
+
+
+ +{% endblock %} diff --git a/changedetectionio/blueprint/tags/templates/edit-tag.html b/changedetectionio/blueprint/tags/templates/edit-tag.html index e527ea52..08c42a2c 100644 --- a/changedetectionio/blueprint/tags/templates/edit-tag.html +++ b/changedetectionio/blueprint/tags/templates/edit-tag.html @@ -3,7 +3,7 @@ {% from '_helpers.html' import render_field, render_checkbox_field, render_button %} {% from '_common_fields.html' import render_common_settings_form %} @@ -124,7 +124,7 @@ nav {% if has_default_notification_urls %}
Look out! - There are system-wide notification URLs enabled, this form will override notification settings for this watch only ‐ an empty Notification URL list here will still send notifications. + There are system-wide notification URLs enabled, this form will override notification settings for this watch only ‐ an empty Notification URL list here will still send notifications.
{% endif %} Use system defaults diff --git a/changedetectionio/blueprint/ui/__init__.py b/changedetectionio/blueprint/ui/__init__.py new file mode 100644 index 00000000..16f07f37 --- /dev/null +++ b/changedetectionio/blueprint/ui/__init__.py @@ -0,0 +1,301 @@ +import time +from flask import Blueprint, request, redirect, url_for, flash, render_template, session +from loguru import logger +from functools import wraps + +from changedetectionio.store import ChangeDetectionStore +from changedetectionio.blueprint.ui.edit import construct_blueprint as construct_edit_blueprint +from changedetectionio.blueprint.ui.notification import construct_blueprint as construct_notification_blueprint +from changedetectionio.blueprint.ui.views import construct_blueprint as construct_views_blueprint + +def construct_blueprint(datastore: ChangeDetectionStore, update_q, running_update_threads, queuedWatchMetaData): + ui_blueprint = Blueprint('ui', __name__, template_folder="templates") + + # Register the edit blueprint + edit_blueprint = construct_edit_blueprint(datastore, update_q, queuedWatchMetaData) + ui_blueprint.register_blueprint(edit_blueprint) + + # Register the notification blueprint + notification_blueprint = construct_notification_blueprint(datastore) + ui_blueprint.register_blueprint(notification_blueprint) + + # Register the views blueprint + views_blueprint = construct_views_blueprint(datastore, update_q, queuedWatchMetaData) + ui_blueprint.register_blueprint(views_blueprint) + + # Import the login decorator + from changedetectionio.auth_decorator import login_optionally_required + + @ui_blueprint.route("/clear_history/", methods=['GET']) + @login_optionally_required + def clear_watch_history(uuid): + try: + datastore.clear_watch_history(uuid) + except KeyError: + flash('Watch not found', 'error') + else: + flash("Cleared snapshot history for watch {}".format(uuid)) + + return redirect(url_for('index')) + + @ui_blueprint.route("/clear_history", methods=['GET', 'POST']) + @login_optionally_required + def clear_all_history(): + if request.method == 'POST': + confirmtext = request.form.get('confirmtext') + + if confirmtext == 'clear': + for uuid in datastore.data['watching'].keys(): + datastore.clear_watch_history(uuid) + + flash("Cleared snapshot history for all watches") + else: + flash('Incorrect confirmation text.', 'error') + + return redirect(url_for('index')) + + output = render_template("clear_all_history.html") + return output + + # Clear all statuses, so we do not see the 'unviewed' class + @ui_blueprint.route("/form/mark-all-viewed", methods=['GET']) + @login_optionally_required + def mark_all_viewed(): + # Save the current newest history as the most recently viewed + with_errors = request.args.get('with_errors') == "1" + for watch_uuid, watch in datastore.data['watching'].items(): + if with_errors and not watch.get('last_error'): + continue + datastore.set_last_viewed(watch_uuid, int(time.time())) + + return redirect(url_for('index')) + + @ui_blueprint.route("/delete", methods=['GET']) + @login_optionally_required + def form_delete(): + uuid = request.args.get('uuid') + + if uuid != 'all' and not uuid in datastore.data['watching'].keys(): + flash('The watch by UUID {} does not exist.'.format(uuid), 'error') + return redirect(url_for('index')) + + # More for testing, possible to return the first/only + if uuid == 'first': + uuid = list(datastore.data['watching'].keys()).pop() + datastore.delete(uuid) + flash('Deleted.') + + return redirect(url_for('index')) + + @ui_blueprint.route("/clone", methods=['GET']) + @login_optionally_required + def form_clone(): + uuid = request.args.get('uuid') + # More for testing, possible to return the first/only + if uuid == 'first': + uuid = list(datastore.data['watching'].keys()).pop() + + new_uuid = datastore.clone(uuid) + if new_uuid: + if not datastore.data['watching'].get(uuid).get('paused'): + update_q.put(queuedWatchMetaData.PrioritizedItem(priority=5, item={'uuid': new_uuid})) + flash('Cloned.') + + return redirect(url_for('index')) + + @ui_blueprint.route("/checknow", methods=['GET']) + @login_optionally_required + def form_watch_checknow(): + # Forced recheck will skip the 'skip if content is the same' rule (, 'reprocess_existing_data': True}))) + tag = request.args.get('tag') + uuid = request.args.get('uuid') + with_errors = request.args.get('with_errors') == "1" + + i = 0 + + running_uuids = [] + for t in running_update_threads: + running_uuids.append(t.current_uuid) + + if uuid: + if uuid not in running_uuids: + update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid})) + i += 1 + + else: + # Recheck all, including muted + for watch_uuid, watch in datastore.data['watching'].items(): + if not watch['paused']: + if watch_uuid not in running_uuids: + if with_errors and not watch.get('last_error'): + continue + + if tag != None and tag not in watch['tags']: + continue + + update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': watch_uuid})) + i += 1 + + if i == 1: + flash("Queued 1 watch for rechecking.") + if i > 1: + flash("Queued {} watches for rechecking.".format(i)) + if i == 0: + flash("No watches available to recheck.") + + return redirect(url_for('index')) + + @ui_blueprint.route("/form/checkbox-operations", methods=['POST']) + @login_optionally_required + def form_watch_list_checkbox_operations(): + op = request.form['op'] + uuids = request.form.getlist('uuids') + + if (op == 'delete'): + for uuid in uuids: + uuid = uuid.strip() + if datastore.data['watching'].get(uuid): + datastore.delete(uuid.strip()) + flash("{} watches deleted".format(len(uuids))) + + elif (op == 'pause'): + for uuid in uuids: + uuid = uuid.strip() + if datastore.data['watching'].get(uuid): + datastore.data['watching'][uuid.strip()]['paused'] = True + flash("{} watches paused".format(len(uuids))) + + elif (op == 'unpause'): + for uuid in uuids: + uuid = uuid.strip() + if datastore.data['watching'].get(uuid): + datastore.data['watching'][uuid.strip()]['paused'] = False + flash("{} watches unpaused".format(len(uuids))) + + elif (op == 'mark-viewed'): + for uuid in uuids: + uuid = uuid.strip() + if datastore.data['watching'].get(uuid): + datastore.set_last_viewed(uuid, int(time.time())) + flash("{} watches updated".format(len(uuids))) + + elif (op == 'mute'): + for uuid in uuids: + uuid = uuid.strip() + if datastore.data['watching'].get(uuid): + datastore.data['watching'][uuid.strip()]['notification_muted'] = True + flash("{} watches muted".format(len(uuids))) + + elif (op == 'unmute'): + for uuid in uuids: + uuid = uuid.strip() + if datastore.data['watching'].get(uuid): + datastore.data['watching'][uuid.strip()]['notification_muted'] = False + flash("{} watches un-muted".format(len(uuids))) + + elif (op == 'recheck'): + for uuid in uuids: + uuid = uuid.strip() + if datastore.data['watching'].get(uuid): + # Recheck and require a full reprocessing + update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid})) + flash("{} watches queued for rechecking".format(len(uuids))) + + elif (op == 'clear-errors'): + for uuid in uuids: + uuid = uuid.strip() + if datastore.data['watching'].get(uuid): + datastore.data['watching'][uuid]["last_error"] = False + flash(f"{len(uuids)} watches errors cleared") + + elif (op == 'clear-history'): + for uuid in uuids: + uuid = uuid.strip() + if datastore.data['watching'].get(uuid): + datastore.clear_watch_history(uuid) + flash("{} watches cleared/reset.".format(len(uuids))) + + elif (op == 'notification-default'): + from changedetectionio.notification import ( + default_notification_format_for_watch + ) + for uuid in uuids: + uuid = uuid.strip() + if datastore.data['watching'].get(uuid): + datastore.data['watching'][uuid.strip()]['notification_title'] = None + datastore.data['watching'][uuid.strip()]['notification_body'] = None + datastore.data['watching'][uuid.strip()]['notification_urls'] = [] + datastore.data['watching'][uuid.strip()]['notification_format'] = default_notification_format_for_watch + flash("{} watches set to use default notification settings".format(len(uuids))) + + elif (op == 'assign-tag'): + op_extradata = request.form.get('op_extradata', '').strip() + if op_extradata: + tag_uuid = datastore.add_tag(name=op_extradata) + if op_extradata and tag_uuid: + for uuid in uuids: + uuid = uuid.strip() + if datastore.data['watching'].get(uuid): + # Bug in old versions caused by bad edit page/tag handler + if isinstance(datastore.data['watching'][uuid]['tags'], str): + datastore.data['watching'][uuid]['tags'] = [] + + datastore.data['watching'][uuid]['tags'].append(tag_uuid) + + flash(f"{len(uuids)} watches were tagged") + + return redirect(url_for('index')) + + + @ui_blueprint.route("/share-url/", methods=['GET']) + @login_optionally_required + def form_share_put_watch(uuid): + """Given a watch UUID, upload the info and return a share-link + the share-link can be imported/added""" + import requests + import json + from copy import deepcopy + + # more for testing + if uuid == 'first': + uuid = list(datastore.data['watching'].keys()).pop() + + # copy it to memory as trim off what we dont need (history) + watch = deepcopy(datastore.data['watching'].get(uuid)) + # For older versions that are not a @property + if (watch.get('history')): + del (watch['history']) + + # for safety/privacy + for k in list(watch.keys()): + if k.startswith('notification_'): + del watch[k] + + for r in['uuid', 'last_checked', 'last_changed']: + if watch.get(r): + del (watch[r]) + + # Add the global stuff which may have an impact + watch['ignore_text'] += datastore.data['settings']['application']['global_ignore_text'] + watch['subtractive_selectors'] += datastore.data['settings']['application']['global_subtractive_selectors'] + + watch_json = json.dumps(watch) + + try: + r = requests.request(method="POST", + data={'watch': watch_json}, + url="https://changedetection.io/share/share", + headers={'App-Guid': datastore.data['app_guid']}) + res = r.json() + + # Add to the flask session + session['share-link'] = f"https://changedetection.io/share/{res['share_key']}" + + + except Exception as e: + logger.error(f"Error sharing -{str(e)}") + flash(f"Could not share, something went wrong while communicating with the share server - {str(e)}", 'error') + + return redirect(url_for('index')) + + return ui_blueprint \ No newline at end of file diff --git a/changedetectionio/blueprint/ui/edit.py b/changedetectionio/blueprint/ui/edit.py new file mode 100644 index 00000000..73cd7853 --- /dev/null +++ b/changedetectionio/blueprint/ui/edit.py @@ -0,0 +1,333 @@ +import time +from copy import deepcopy +import os +import importlib.resources +from flask import Blueprint, request, redirect, url_for, flash, render_template, make_response, send_from_directory, abort +from loguru import logger +from jinja2 import Environment, FileSystemLoader + +from changedetectionio.store import ChangeDetectionStore +from changedetectionio.auth_decorator import login_optionally_required +from changedetectionio.time_handler import is_within_schedule + +def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMetaData): + edit_blueprint = Blueprint('ui_edit', __name__, template_folder="../ui/templates") + + def _watch_has_tag_options_set(watch): + """This should be fixed better so that Tag is some proper Model, a tag is just a Watch also""" + for tag_uuid, tag in datastore.data['settings']['application'].get('tags', {}).items(): + if tag_uuid in watch.get('tags', []) and (tag.get('include_filters') or tag.get('subtractive_selectors')): + return True + + @edit_blueprint.route("/edit/", methods=['GET', 'POST']) + @login_optionally_required + # https://stackoverflow.com/questions/42984453/wtforms-populate-form-with-data-if-data-exists + # https://wtforms.readthedocs.io/en/3.0.x/forms/#wtforms.form.Form.populate_obj ? + def edit_page(uuid): + from changedetectionio import forms + from changedetectionio.blueprint.browser_steps.browser_steps import browser_step_ui_config + from changedetectionio import processors + import importlib + + # More for testing, possible to return the first/only + if not datastore.data['watching'].keys(): + flash("No watches to edit", "error") + return redirect(url_for('index')) + + if uuid == 'first': + uuid = list(datastore.data['watching'].keys()).pop() + + if not uuid in datastore.data['watching']: + flash("No watch with the UUID %s found." % (uuid), "error") + return redirect(url_for('index')) + + switch_processor = request.args.get('switch_processor') + if switch_processor: + for p in processors.available_processors(): + if p[0] == switch_processor: + datastore.data['watching'][uuid]['processor'] = switch_processor + flash(f"Switched to mode - {p[1]}.") + datastore.clear_watch_history(uuid) + redirect(url_for('ui_edit.edit_page', uuid=uuid)) + + # be sure we update with a copy instead of accidently editing the live object by reference + default = deepcopy(datastore.data['watching'][uuid]) + + # Defaults for proxy choice + if datastore.proxy_list is not None: # When enabled + # @todo + # Radio needs '' not None, or incase that the chosen one no longer exists + if default['proxy'] is None or not any(default['proxy'] in tup for tup in datastore.proxy_list): + default['proxy'] = '' + # proxy_override set to the json/text list of the items + + # Does it use some custom form? does one exist? + processor_name = datastore.data['watching'][uuid].get('processor', '') + processor_classes = next((tpl for tpl in processors.find_processors() if tpl[1] == processor_name), None) + if not processor_classes: + flash(f"Cannot load the edit form for processor/plugin '{processor_classes[1]}', plugin missing?", 'error') + return redirect(url_for('index')) + + parent_module = processors.get_parent_module(processor_classes[0]) + + try: + # Get the parent of the "processor.py" go up one, get the form (kinda spaghetti but its reusing existing code) + forms_module = importlib.import_module(f"{parent_module.__name__}.forms") + # Access the 'processor_settings_form' class from the 'forms' module + form_class = getattr(forms_module, 'processor_settings_form') + except ModuleNotFoundError as e: + # .forms didnt exist + form_class = forms.processor_text_json_diff_form + except AttributeError as e: + # .forms exists but no useful form + form_class = forms.processor_text_json_diff_form + + form = form_class(formdata=request.form if request.method == 'POST' else None, + data=default, + extra_notification_tokens=default.extra_notification_token_values(), + default_system_settings=datastore.data['settings'] + ) + + # For the form widget tag UUID back to "string name" for the field + form.tags.datastore = datastore + + # Used by some forms that need to dig deeper + form.datastore = datastore + form.watch = default + + for p in datastore.extra_browsers: + form.fetch_backend.choices.append(p) + + form.fetch_backend.choices.append(("system", 'System settings default')) + + # form.browser_steps[0] can be assumed that we 'goto url' first + + if datastore.proxy_list is None: + # @todo - Couldn't get setattr() etc dynamic addition working, so remove it instead + del form.proxy + else: + form.proxy.choices = [('', 'Default')] + for p in datastore.proxy_list: + form.proxy.choices.append(tuple((p, datastore.proxy_list[p]['label']))) + + + if request.method == 'POST' and form.validate(): + + # If they changed processor, it makes sense to reset it. + if datastore.data['watching'][uuid].get('processor') != form.data.get('processor'): + datastore.data['watching'][uuid].clear_watch() + flash("Reset watch history due to change of processor") + + extra_update_obj = { + 'consecutive_filter_failures': 0, + 'last_error' : False + } + + if request.args.get('unpause_on_save'): + extra_update_obj['paused'] = False + + extra_update_obj['time_between_check'] = form.time_between_check.data + + # Ignore text + form_ignore_text = form.ignore_text.data + datastore.data['watching'][uuid]['ignore_text'] = form_ignore_text + + # Be sure proxy value is None + if datastore.proxy_list is not None and form.data['proxy'] == '': + extra_update_obj['proxy'] = None + + # Unsetting all filter_text methods should make it go back to default + # This particularly affects tests running + if 'filter_text_added' in form.data and not form.data.get('filter_text_added') \ + and 'filter_text_replaced' in form.data and not form.data.get('filter_text_replaced') \ + and 'filter_text_removed' in form.data and not form.data.get('filter_text_removed'): + extra_update_obj['filter_text_added'] = True + extra_update_obj['filter_text_replaced'] = True + extra_update_obj['filter_text_removed'] = True + + # Because wtforms doesn't support accessing other data in process_ , but we convert the CSV list of tags back to a list of UUIDs + tag_uuids = [] + if form.data.get('tags'): + # Sometimes in testing this can be list, dont know why + if type(form.data.get('tags')) == list: + extra_update_obj['tags'] = form.data.get('tags') + else: + for t in form.data.get('tags').split(','): + tag_uuids.append(datastore.add_tag(name=t)) + extra_update_obj['tags'] = tag_uuids + + datastore.data['watching'][uuid].update(form.data) + datastore.data['watching'][uuid].update(extra_update_obj) + + if not datastore.data['watching'][uuid].get('tags'): + # Force it to be a list, because form.data['tags'] will be string if nothing found + # And del(form.data['tags'] ) wont work either for some reason + datastore.data['watching'][uuid]['tags'] = [] + + # Recast it if need be to right data Watch handler + watch_class = processors.get_custom_watch_obj_for_processor(form.data.get('processor')) + datastore.data['watching'][uuid] = watch_class(datastore_path=datastore.datastore_path, default=datastore.data['watching'][uuid]) + flash("Updated watch - unpaused!" if request.args.get('unpause_on_save') else "Updated watch.") + + # Re #286 - We wait for syncing new data to disk in another thread every 60 seconds + # But in the case something is added we should save straight away + datastore.needs_write_urgent = True + + # Do not queue on edit if its not within the time range + + # @todo maybe it should never queue anyway on edit... + is_in_schedule = True + watch = datastore.data['watching'].get(uuid) + + if watch.get('time_between_check_use_default'): + time_schedule_limit = datastore.data['settings']['requests'].get('time_schedule_limit', {}) + else: + time_schedule_limit = watch.get('time_schedule_limit') + + tz_name = time_schedule_limit.get('timezone') + if not tz_name: + tz_name = datastore.data['settings']['application'].get('timezone', 'UTC') + + if time_schedule_limit and time_schedule_limit.get('enabled'): + try: + is_in_schedule = is_within_schedule(time_schedule_limit=time_schedule_limit, + default_tz=tz_name + ) + except Exception as e: + logger.error( + f"{uuid} - Recheck scheduler, error handling timezone, check skipped - TZ name '{tz_name}' - {str(e)}") + return False + + ############################# + if not datastore.data['watching'][uuid].get('paused') and is_in_schedule: + # Queue the watch for immediate recheck, with a higher priority + update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid})) + + # Diff page [edit] link should go back to diff page + if request.args.get("next") and request.args.get("next") == 'diff': + return redirect(url_for('ui.ui_views.diff_history_page', uuid=uuid)) + + return redirect(url_for('index', tag=request.args.get("tag",''))) + + else: + if request.method == 'POST' and not form.validate(): + flash("An error occurred, please see below.", "error") + + visualselector_data_is_ready = datastore.visualselector_data_is_ready(uuid) + + + # JQ is difficult to install on windows and must be manually added (outside requirements.txt) + jq_support = True + try: + import jq + except ModuleNotFoundError: + jq_support = False + + watch = datastore.data['watching'].get(uuid) + + system_uses_webdriver = datastore.data['settings']['application']['fetch_backend'] == 'html_webdriver' + + watch_uses_webdriver = False + if (watch.get('fetch_backend') == 'system' and system_uses_webdriver) or watch.get('fetch_backend') == 'html_webdriver' or watch.get('fetch_backend', '').startswith('extra_browser_'): + watch_uses_webdriver = True + + from zoneinfo import available_timezones + + # Only works reliably with Playwright + + template_args = { + 'available_processors': processors.available_processors(), + 'available_timezones': sorted(available_timezones()), + 'browser_steps_config': browser_step_ui_config, + 'emailprefix': os.getenv('NOTIFICATION_MAIL_BUTTON_PREFIX', False), + 'extra_notification_token_placeholder_info': datastore.get_unique_notification_token_placeholders_available(), + 'extra_processor_config': form.extra_tab_content(), + 'extra_title': f" - Edit - {watch.label}", + 'form': form, + 'has_default_notification_urls': True if len(datastore.data['settings']['application']['notification_urls']) else False, + 'has_extra_headers_file': len(datastore.get_all_headers_in_textfile_for_watch(uuid=uuid)) > 0, + 'has_special_tag_options': _watch_has_tag_options_set(watch=watch), + 'watch_uses_webdriver': watch_uses_webdriver, + 'jq_support': jq_support, + 'playwright_enabled': os.getenv('PLAYWRIGHT_DRIVER_URL', False), + 'settings_application': datastore.data['settings']['application'], + 'timezone_default_config': datastore.data['settings']['application'].get('timezone'), + 'using_global_webdriver_wait': not default['webdriver_delay'], + 'uuid': uuid, + 'watch': watch + } + + included_content = None + if form.extra_form_content(): + # So that the extra panels can access _helpers.html etc, we set the environment to load from templates/ + # And then render the code from the module + templates_dir = str(importlib.resources.files("changedetectionio").joinpath('templates')) + env = Environment(loader=FileSystemLoader(templates_dir)) + template = env.from_string(form.extra_form_content()) + included_content = template.render(**template_args) + + output = render_template("edit.html", + extra_tab_content=form.extra_tab_content() if form.extra_tab_content() else None, + extra_form_content=included_content, + **template_args + ) + + return output + + @edit_blueprint.route("/edit//get-html", methods=['GET']) + @login_optionally_required + def watch_get_latest_html(uuid): + from io import BytesIO + from flask import send_file + import brotli + + watch = datastore.data['watching'].get(uuid) + if watch and watch.history.keys() and os.path.isdir(watch.watch_data_dir): + latest_filename = list(watch.history.keys())[-1] + html_fname = os.path.join(watch.watch_data_dir, f"{latest_filename}.html.br") + with open(html_fname, 'rb') as f: + if html_fname.endswith('.br'): + # Read and decompress the Brotli file + decompressed_data = brotli.decompress(f.read()) + else: + decompressed_data = f.read() + + buffer = BytesIO(decompressed_data) + + return send_file(buffer, as_attachment=True, download_name=f"{latest_filename}.html", mimetype='text/html') + + # Return a 500 error + abort(500) + + # Ajax callback + @edit_blueprint.route("/edit//preview-rendered", methods=['POST']) + @login_optionally_required + def watch_get_preview_rendered(uuid): + '''For when viewing the "preview" of the rendered text from inside of Edit''' + from flask import jsonify + from changedetectionio.processors.text_json_diff import prepare_filter_prevew + result = prepare_filter_prevew(watch_uuid=uuid, form_data=request.form, datastore=datastore) + return jsonify(result) + + @edit_blueprint.route("/highlight_submit_ignore_url", methods=['POST']) + @login_optionally_required + def highlight_submit_ignore_url(): + import re + mode = request.form.get('mode') + selection = request.form.get('selection') + + uuid = request.args.get('uuid','') + if datastore.data["watching"].get(uuid): + if mode == 'exact': + for l in selection.splitlines(): + datastore.data["watching"][uuid]['ignore_text'].append(l.strip()) + elif mode == 'digit-regex': + for l in selection.splitlines(): + # Replace any series of numbers with a regex + s = re.escape(l.strip()) + s = re.sub(r'[0-9]+', r'\\d+', s) + datastore.data["watching"][uuid]['ignore_text'].append('/' + s + '/') + + return f"Click to preview" + + return edit_blueprint \ No newline at end of file diff --git a/changedetectionio/blueprint/ui/notification.py b/changedetectionio/blueprint/ui/notification.py new file mode 100644 index 00000000..a946917b --- /dev/null +++ b/changedetectionio/blueprint/ui/notification.py @@ -0,0 +1,107 @@ +from flask import Blueprint, request, make_response +import random +from loguru import logger + +from changedetectionio.store import ChangeDetectionStore +from changedetectionio.auth_decorator import login_optionally_required + +def construct_blueprint(datastore: ChangeDetectionStore): + notification_blueprint = Blueprint('ui_notification', __name__, template_folder="../ui/templates") + + # AJAX endpoint for sending a test + @notification_blueprint.route("/notification/send-test/", methods=['POST']) + @notification_blueprint.route("/notification/send-test", methods=['POST']) + @notification_blueprint.route("/notification/send-test/", methods=['POST']) + @login_optionally_required + def ajax_callback_send_notification_test(watch_uuid=None): + + # Watch_uuid could be unset in the case it`s used in tag editor, global settings + import apprise + from changedetectionio.apprise_asset import asset + apobj = apprise.Apprise(asset=asset) + + # so that the custom endpoints are registered + from changedetectionio.apprise_plugin import apprise_custom_api_call_wrapper + is_global_settings_form = request.args.get('mode', '') == 'global-settings' + is_group_settings_form = request.args.get('mode', '') == 'group-settings' + + # Use an existing random one on the global/main settings form + if not watch_uuid and (is_global_settings_form or is_group_settings_form) \ + and datastore.data.get('watching'): + logger.debug(f"Send test notification - Choosing random Watch {watch_uuid}") + watch_uuid = random.choice(list(datastore.data['watching'].keys())) + + if not watch_uuid: + return make_response("Error: You must have atleast one watch configured for 'test notification' to work", 400) + + watch = datastore.data['watching'].get(watch_uuid) + + notification_urls = None + + if request.form.get('notification_urls'): + notification_urls = request.form['notification_urls'].strip().splitlines() + + if not notification_urls: + logger.debug("Test notification - Trying by group/tag in the edit form if available") + # On an edit page, we should also fire off to the tags if they have notifications + if request.form.get('tags') and request.form['tags'].strip(): + for k in request.form['tags'].split(','): + tag = datastore.tag_exists_by_name(k.strip()) + notification_urls = tag.get('notifications_urls') if tag and tag.get('notifications_urls') else None + + if not notification_urls and not is_global_settings_form and not is_group_settings_form: + # In the global settings, use only what is typed currently in the text box + logger.debug("Test notification - Trying by global system settings notifications") + if datastore.data['settings']['application'].get('notification_urls'): + notification_urls = datastore.data['settings']['application']['notification_urls'] + + if not notification_urls: + return 'Error: No Notification URLs set/found' + + for n_url in notification_urls: + if len(n_url.strip()): + if not apobj.add(n_url): + return f'Error: {n_url} is not a valid AppRise URL.' + + try: + # use the same as when it is triggered, but then override it with the form test values + n_object = { + 'watch_url': request.form.get('window_url', "https://changedetection.io"), + 'notification_urls': notification_urls + } + + # Only use if present, if not set in n_object it should use the default system value + if 'notification_format' in request.form and request.form['notification_format'].strip(): + n_object['notification_format'] = request.form.get('notification_format', '').strip() + + if 'notification_title' in request.form and request.form['notification_title'].strip(): + n_object['notification_title'] = request.form.get('notification_title', '').strip() + elif datastore.data['settings']['application'].get('notification_title'): + n_object['notification_title'] = datastore.data['settings']['application'].get('notification_title') + else: + n_object['notification_title'] = "Test title" + + if 'notification_body' in request.form and request.form['notification_body'].strip(): + n_object['notification_body'] = request.form.get('notification_body', '').strip() + elif datastore.data['settings']['application'].get('notification_body'): + n_object['notification_body'] = datastore.data['settings']['application'].get('notification_body') + else: + n_object['notification_body'] = "Test body" + + n_object['as_async'] = False + n_object.update(watch.extra_notification_token_values()) + from changedetectionio.notification import process_notification + sent_obj = process_notification(n_object, datastore) + + except Exception as e: + e_str = str(e) + # Remove this text which is not important and floods the container + e_str = e_str.replace( + "DEBUG - .CustomNotifyPluginWrapper'>", + '') + + return make_response(e_str, 400) + + return 'OK - Sent test notifications' + + return notification_blueprint \ No newline at end of file diff --git a/changedetectionio/blueprint/ui/templates/clear_all_history.html b/changedetectionio/blueprint/ui/templates/clear_all_history.html new file mode 100644 index 00000000..fbbaa34f --- /dev/null +++ b/changedetectionio/blueprint/ui/templates/clear_all_history.html @@ -0,0 +1,49 @@ +{% extends 'base.html' %} {% block content %} +
+
+
+ +
+
+ This will remove version history (snapshots) for ALL watches, but keep + your list of URLs!
+ You may like to use the BACKUP link first.
+
+
+
+ + + Type in the word clear to confirm that you + understand. +
+
+
+ +
+
+
+ Cancel +
+
+
+
+
+ +{% endblock %} diff --git a/changedetectionio/blueprint/ui/views.py b/changedetectionio/blueprint/ui/views.py new file mode 100644 index 00000000..903a4c77 --- /dev/null +++ b/changedetectionio/blueprint/ui/views.py @@ -0,0 +1,220 @@ +from flask import Blueprint, request, redirect, url_for, flash, render_template, make_response, send_from_directory, abort +from flask_login import current_user +import os +import time +from copy import deepcopy + +from changedetectionio.store import ChangeDetectionStore +from changedetectionio.auth_decorator import login_optionally_required +from changedetectionio import html_tools + +def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMetaData): + views_blueprint = Blueprint('ui_views', __name__, template_folder="../ui/templates") + + @views_blueprint.route("/preview/", methods=['GET']) + @login_optionally_required + def preview_page(uuid): + content = [] + versions = [] + timestamp = None + + # More for testing, possible to return the first/only + if uuid == 'first': + uuid = list(datastore.data['watching'].keys()).pop() + + try: + watch = datastore.data['watching'][uuid] + except KeyError: + flash("No history found for the specified link, bad link?", "error") + return redirect(url_for('index')) + + system_uses_webdriver = datastore.data['settings']['application']['fetch_backend'] == 'html_webdriver' + extra_stylesheets = [url_for('static_content', group='styles', filename='diff.css')] + + is_html_webdriver = False + if (watch.get('fetch_backend') == 'system' and system_uses_webdriver) or watch.get('fetch_backend') == 'html_webdriver' or watch.get('fetch_backend', '').startswith('extra_browser_'): + is_html_webdriver = True + triggered_line_numbers = [] + if datastore.data['watching'][uuid].history_n == 0 and (watch.get_error_text() or watch.get_error_snapshot()): + flash("Preview unavailable - No fetch/check completed or triggers not reached", "error") + else: + # So prepare the latest preview or not + preferred_version = request.args.get('version') + versions = list(watch.history.keys()) + timestamp = versions[-1] + if preferred_version and preferred_version in versions: + timestamp = preferred_version + + try: + versions = list(watch.history.keys()) + content = watch.get_history_snapshot(timestamp) + + triggered_line_numbers = html_tools.strip_ignore_text(content=content, + wordlist=watch['trigger_text'], + mode='line numbers' + ) + + except Exception as e: + content.append({'line': f"File doesnt exist or unable to read timestamp {timestamp}", 'classes': ''}) + + output = render_template("preview.html", + content=content, + current_version=timestamp, + history_n=watch.history_n, + extra_stylesheets=extra_stylesheets, + extra_title=f" - Diff - {watch.label} @ {timestamp}", + triggered_line_numbers=triggered_line_numbers, + current_diff_url=watch['url'], + screenshot=watch.get_screenshot(), + watch=watch, + uuid=uuid, + is_html_webdriver=is_html_webdriver, + last_error=watch['last_error'], + last_error_text=watch.get_error_text(), + last_error_screenshot=watch.get_error_snapshot(), + versions=versions + ) + + return output + + @views_blueprint.route("/diff/", methods=['GET', 'POST']) + @login_optionally_required + def diff_history_page(uuid): + from changedetectionio import forms + + # More for testing, possible to return the first/only + if uuid == 'first': + uuid = list(datastore.data['watching'].keys()).pop() + + extra_stylesheets = [url_for('static_content', group='styles', filename='diff.css')] + try: + watch = datastore.data['watching'][uuid] + except KeyError: + flash("No history found for the specified link, bad link?", "error") + return redirect(url_for('index')) + + # For submission of requesting an extract + extract_form = forms.extractDataForm(request.form) + if request.method == 'POST': + if not extract_form.validate(): + flash("An error occurred, please see below.", "error") + + else: + extract_regex = request.form.get('extract_regex').strip() + output = watch.extract_regex_from_all_history(extract_regex) + if output: + watch_dir = os.path.join(datastore.datastore_path, uuid) + response = make_response(send_from_directory(directory=watch_dir, path=output, as_attachment=True)) + response.headers['Content-type'] = 'text/csv' + response.headers['Cache-Control'] = 'no-cache, no-store, must-revalidate' + response.headers['Pragma'] = 'no-cache' + response.headers['Expires'] = 0 + return response + + flash('Nothing matches that RegEx', 'error') + redirect(url_for('ui_views.diff_history_page', uuid=uuid)+'#extract') + + history = watch.history + dates = list(history.keys()) + + if len(dates) < 2: + flash("Not enough saved change detection snapshots to produce a report.", "error") + return redirect(url_for('index')) + + # Save the current newest history as the most recently viewed + datastore.set_last_viewed(uuid, time.time()) + + # Read as binary and force decode as UTF-8 + # Windows may fail decode in python if we just use 'r' mode (chardet decode exception) + from_version = request.args.get('from_version') + from_version_index = -2 # second newest + if from_version and from_version in dates: + from_version_index = dates.index(from_version) + else: + from_version = dates[from_version_index] + + try: + from_version_file_contents = watch.get_history_snapshot(dates[from_version_index]) + except Exception as e: + from_version_file_contents = f"Unable to read to-version at index {dates[from_version_index]}.\n" + + to_version = request.args.get('to_version') + to_version_index = -1 + if to_version and to_version in dates: + to_version_index = dates.index(to_version) + else: + to_version = dates[to_version_index] + + try: + to_version_file_contents = watch.get_history_snapshot(dates[to_version_index]) + except Exception as e: + to_version_file_contents = "Unable to read to-version at index{}.\n".format(dates[to_version_index]) + + screenshot_url = watch.get_screenshot() + + system_uses_webdriver = datastore.data['settings']['application']['fetch_backend'] == 'html_webdriver' + + is_html_webdriver = False + if (watch.get('fetch_backend') == 'system' and system_uses_webdriver) or watch.get('fetch_backend') == 'html_webdriver' or watch.get('fetch_backend', '').startswith('extra_browser_'): + is_html_webdriver = True + + password_enabled_and_share_is_off = False + if datastore.data['settings']['application'].get('password') or os.getenv("SALTED_PASS", False): + password_enabled_and_share_is_off = not datastore.data['settings']['application'].get('shared_diff_access') + + output = render_template("diff.html", + current_diff_url=watch['url'], + from_version=str(from_version), + to_version=str(to_version), + extra_stylesheets=extra_stylesheets, + extra_title=f" - Diff - {watch.label}", + extract_form=extract_form, + is_html_webdriver=is_html_webdriver, + last_error=watch['last_error'], + last_error_screenshot=watch.get_error_snapshot(), + last_error_text=watch.get_error_text(), + left_sticky=True, + newest=to_version_file_contents, + newest_version_timestamp=dates[-1], + password_enabled_and_share_is_off=password_enabled_and_share_is_off, + from_version_file_contents=from_version_file_contents, + to_version_file_contents=to_version_file_contents, + screenshot=screenshot_url, + uuid=uuid, + versions=dates, # All except current/last + watch_a=watch + ) + + return output + + @views_blueprint.route("/form/add/quickwatch", methods=['POST']) + @login_optionally_required + def form_quick_watch_add(): + from changedetectionio import forms + form = forms.quickWatchForm(request.form) + + if not form.validate(): + for widget, l in form.errors.items(): + flash(','.join(l), 'error') + return redirect(url_for('index')) + + url = request.form.get('url').strip() + if datastore.url_exists(url): + flash(f'Warning, URL {url} already exists', "notice") + + add_paused = request.form.get('edit_and_watch_submit_button') != None + processor = request.form.get('processor', 'text_json_diff') + new_uuid = datastore.add_watch(url=url, tag=request.form.get('tags').strip(), extras={'paused': add_paused, 'processor': processor}) + + if new_uuid: + if add_paused: + flash('Watch added in Paused state, saving will unpause.') + return redirect(url_for('ui.ui_edit.edit_page', uuid=new_uuid, unpause_on_save=1, tag=request.args.get('tag'))) + else: + # Straight into the queue. + update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': new_uuid})) + flash("Watch added.") + + return redirect(url_for('index', tag=request.args.get('tag',''))) + + return views_blueprint \ No newline at end of file diff --git a/changedetectionio/conditions/blueprint.py b/changedetectionio/conditions/blueprint.py index b9fb762e..5bd3bd39 100644 --- a/changedetectionio/conditions/blueprint.py +++ b/changedetectionio/conditions/blueprint.py @@ -46,6 +46,8 @@ def construct_blueprint(datastore): # Override the conditions in the temporary watch rule_json = request.args.get("rule") rule = json.loads(rule_json) if rule_json else None + + # Should be key/value of field, operator, value tmp_watch_data['conditions'] = [rule] tmp_watch_data['conditions_match_logic'] = "ALL" # Single rule, so use ALL diff --git a/changedetectionio/flask_app.py b/changedetectionio/flask_app.py index 6dc93546..232ad944 100644 --- a/changedetectionio/flask_app.py +++ b/changedetectionio/flask_app.py @@ -1,7 +1,5 @@ #!/usr/bin/env python3 -import datetime -from zoneinfo import ZoneInfo import flask_login import locale @@ -12,14 +10,9 @@ import threading import time import timeago -from .processors import find_processors, get_parent_module, get_custom_watch_obj_for_processor -from .safe_jinja import render as jinja_render from changedetectionio.strtobool import strtobool -from copy import deepcopy -from functools import wraps from threading import Event -from feedgen.feed import FeedGenerator from flask import ( Flask, abort, @@ -40,7 +33,7 @@ from flask_cors import CORS from flask_wtf import CSRFProtect from loguru import logger -from changedetectionio import html_tools, __version__ +from changedetectionio import __version__ from changedetectionio import queuedWatchMetaData from changedetectionio.api import api_v1 from .time_handler import is_within_schedule @@ -168,6 +161,9 @@ def _jinja2_filter_seconds_precise(timestamp): return format(int(time.time()-timestamp), ',d') +# Import login_optionally_required from auth_decorator +from changedetectionio.auth_decorator import login_optionally_required + # When nobody is logged in Flask-Login's current_user is set to an AnonymousUser object. class User(flask_login.UserMixin): id=None @@ -212,28 +208,6 @@ class User(flask_login.UserMixin): pass -def login_optionally_required(func): - @wraps(func) - def decorated_view(*args, **kwargs): - - has_password_enabled = datastore.data['settings']['application'].get('password') or os.getenv("SALTED_PASS", False) - - # Permitted - if request.endpoint == 'static_content' and request.view_args['group'] == 'styles': - return func(*args, **kwargs) - # Permitted - elif request.endpoint == 'diff_history_page' and datastore.data['settings']['application'].get('shared_diff_access'): - return func(*args, **kwargs) - elif request.method in flask_login.config.EXEMPT_METHODS: - return func(*args, **kwargs) - elif app.config.get('LOGIN_DISABLED'): - return func(*args, **kwargs) - elif has_password_enabled and not current_user.is_authenticated: - return app.login_manager.unauthorized() - - return func(*args, **kwargs) - - return decorated_view def changedetection_app(config=None, datastore_o=None): logger.trace("TRACE log is enabled") @@ -248,6 +222,30 @@ def changedetection_app(config=None, datastore_o=None): login_manager = flask_login.LoginManager(app) login_manager.login_view = 'login' app.secret_key = init_app_secret(config['datastore_path']) + + # Set up a request hook to check authentication for all routes + @app.before_request + def check_authentication(): + has_password_enabled = datastore.data['settings']['application'].get('password') or os.getenv("SALTED_PASS", False) + + if has_password_enabled and not flask_login.current_user.is_authenticated: + # Permitted + if request.endpoint and 'static_content' in request.endpoint and request.view_args and request.view_args.get('group') == 'styles': + return None + # Permitted + elif request.endpoint and 'login' in request.endpoint: + return None + elif request.endpoint and 'diff_history_page' in request.endpoint and datastore.data['settings']['application'].get('shared_diff_access'): + return None + elif request.method in flask_login.config.EXEMPT_METHODS: + return None + elif app.config.get('LOGIN_DISABLED'): + return None + # RSS access with token is allowed + elif request.endpoint and 'rss.feed' in request.endpoint: + return None + else: + return login_manager.unauthorized() watch_api.add_resource(api_v1.WatchSingleHistory, @@ -340,91 +338,6 @@ def changedetection_app(config=None, datastore_o=None): return None - @app.route("/rss", methods=['GET']) - def rss(): - now = time.time() - # Always requires token set - app_rss_token = datastore.data['settings']['application'].get('rss_access_token') - rss_url_token = request.args.get('token') - if rss_url_token != app_rss_token: - return "Access denied, bad token", 403 - - from . import diff - limit_tag = request.args.get('tag', '').lower().strip() - # Be sure limit_tag is a uuid - for uuid, tag in datastore.data['settings']['application'].get('tags', {}).items(): - if limit_tag == tag.get('title', '').lower().strip(): - limit_tag = uuid - - # Sort by last_changed and add the uuid which is usually the key.. - sorted_watches = [] - - # @todo needs a .itemsWithTag() or something - then we can use that in Jinaj2 and throw this away - for uuid, watch in datastore.data['watching'].items(): - # @todo tag notification_muted skip also (improve Watch model) - if datastore.data['settings']['application'].get('rss_hide_muted_watches') and watch.get('notification_muted'): - continue - if limit_tag and not limit_tag in watch['tags']: - continue - watch['uuid'] = uuid - sorted_watches.append(watch) - - sorted_watches.sort(key=lambda x: x.last_changed, reverse=False) - - fg = FeedGenerator() - fg.title('changedetection.io') - fg.description('Feed description') - fg.link(href='https://changedetection.io') - - for watch in sorted_watches: - - dates = list(watch.history.keys()) - # Re #521 - Don't bother processing this one if theres less than 2 snapshots, means we never had a change detected. - if len(dates) < 2: - continue - - if not watch.viewed: - # Re #239 - GUID needs to be individual for each event - # @todo In the future make this a configurable link back (see work on BASE_URL https://github.com/dgtlmoon/changedetection.io/pull/228) - guid = "{}/{}".format(watch['uuid'], watch.last_changed) - fe = fg.add_entry() - - # Include a link to the diff page, they will have to login here to see if password protection is enabled. - # Description is the page you watch, link takes you to the diff JS UI page - # Dict val base_url will get overriden with the env var if it is set. - ext_base_url = datastore.data['settings']['application'].get('active_base_url') - - # Because we are called via whatever web server, flask should figure out the right path ( - diff_link = {'href': url_for('diff_history_page', uuid=watch['uuid'], _external=True)} - - fe.link(link=diff_link) - - # @todo watch should be a getter - watch.get('title') (internally if URL else..) - - watch_title = watch.get('title') if watch.get('title') else watch.get('url') - fe.title(title=watch_title) - - html_diff = diff.render_diff(previous_version_file_contents=watch.get_history_snapshot(dates[-2]), - newest_version_file_contents=watch.get_history_snapshot(dates[-1]), - include_equal=False, - line_feed_sep="
") - - # @todo Make this configurable and also consider html-colored markup - # @todo User could decide if goes to the diff page, or to the watch link - rss_template = "\n

{{watch_title}}

\n

{{html_diff}}

\n\n" - content = jinja_render(template_str=rss_template, watch_title=watch_title, html_diff=html_diff, watch_url=watch.link) - - fe.content(content=content, type='CDATA') - - fe.guid(guid, permalink=False) - dt = datetime.datetime.fromtimestamp(int(watch.newest_history_key)) - dt = dt.replace(tzinfo=pytz.UTC) - fe.pubDate(dt) - - response = make_response(fg.rss_str()) - response.headers.set('Content-Type', 'application/rss+xml;charset=utf-8') - logger.trace(f"RSS generated in {time.time() - now:.3f}s") - return response @app.route("/", methods=['GET']) @login_optionally_required @@ -446,7 +359,7 @@ def changedetection_app(config=None, datastore_o=None): # Redirect for the old rss path which used the /?rss=true if request.args.get('rss'): - return redirect(url_for('rss', tag=active_tag_uuid)) + return redirect(url_for('rss.feed', tag=active_tag_uuid)) op = request.args.get('op') if op: @@ -526,771 +439,6 @@ def changedetection_app(config=None, datastore_o=None): return resp - - - # AJAX endpoint for sending a test - @app.route("/notification/send-test/", methods=['POST']) - @app.route("/notification/send-test", methods=['POST']) - @app.route("/notification/send-test/", methods=['POST']) - @login_optionally_required - def ajax_callback_send_notification_test(watch_uuid=None): - - # Watch_uuid could be unset in the case it`s used in tag editor, global settings - import apprise - import random - from .apprise_asset import asset - apobj = apprise.Apprise(asset=asset) - - # so that the custom endpoints are registered - from changedetectionio.apprise_plugin import apprise_custom_api_call_wrapper - is_global_settings_form = request.args.get('mode', '') == 'global-settings' - is_group_settings_form = request.args.get('mode', '') == 'group-settings' - - # Use an existing random one on the global/main settings form - if not watch_uuid and (is_global_settings_form or is_group_settings_form) \ - and datastore.data.get('watching'): - logger.debug(f"Send test notification - Choosing random Watch {watch_uuid}") - watch_uuid = random.choice(list(datastore.data['watching'].keys())) - - if not watch_uuid: - return make_response("Error: You must have atleast one watch configured for 'test notification' to work", 400) - - watch = datastore.data['watching'].get(watch_uuid) - - notification_urls = None - - if request.form.get('notification_urls'): - notification_urls = request.form['notification_urls'].strip().splitlines() - - if not notification_urls: - logger.debug("Test notification - Trying by group/tag in the edit form if available") - # On an edit page, we should also fire off to the tags if they have notifications - if request.form.get('tags') and request.form['tags'].strip(): - for k in request.form['tags'].split(','): - tag = datastore.tag_exists_by_name(k.strip()) - notification_urls = tag.get('notifications_urls') if tag and tag.get('notifications_urls') else None - - if not notification_urls and not is_global_settings_form and not is_group_settings_form: - # In the global settings, use only what is typed currently in the text box - logger.debug("Test notification - Trying by global system settings notifications") - if datastore.data['settings']['application'].get('notification_urls'): - notification_urls = datastore.data['settings']['application']['notification_urls'] - - - if not notification_urls: - return 'Error: No Notification URLs set/found' - - for n_url in notification_urls: - if len(n_url.strip()): - if not apobj.add(n_url): - return f'Error: {n_url} is not a valid AppRise URL.' - - try: - # use the same as when it is triggered, but then override it with the form test values - n_object = { - 'watch_url': request.form.get('window_url', "https://changedetection.io"), - 'notification_urls': notification_urls - } - - # Only use if present, if not set in n_object it should use the default system value - if 'notification_format' in request.form and request.form['notification_format'].strip(): - n_object['notification_format'] = request.form.get('notification_format', '').strip() - - if 'notification_title' in request.form and request.form['notification_title'].strip(): - n_object['notification_title'] = request.form.get('notification_title', '').strip() - elif datastore.data['settings']['application'].get('notification_title'): - n_object['notification_title'] = datastore.data['settings']['application'].get('notification_title') - else: - n_object['notification_title'] = "Test title" - - if 'notification_body' in request.form and request.form['notification_body'].strip(): - n_object['notification_body'] = request.form.get('notification_body', '').strip() - elif datastore.data['settings']['application'].get('notification_body'): - n_object['notification_body'] = datastore.data['settings']['application'].get('notification_body') - else: - n_object['notification_body'] = "Test body" - - n_object['as_async'] = False - n_object.update(watch.extra_notification_token_values()) - from .notification import process_notification - sent_obj = process_notification(n_object, datastore) - - except Exception as e: - e_str = str(e) - # Remove this text which is not important and floods the container - e_str = e_str.replace( - "DEBUG - .CustomNotifyPluginWrapper'>", - '') - - return make_response(e_str, 400) - - return 'OK - Sent test notifications' - - - @app.route("/clear_history/", methods=['GET']) - @login_optionally_required - def clear_watch_history(uuid): - try: - datastore.clear_watch_history(uuid) - except KeyError: - flash('Watch not found', 'error') - else: - flash("Cleared snapshot history for watch {}".format(uuid)) - - return redirect(url_for('index')) - - @app.route("/clear_history", methods=['GET', 'POST']) - @login_optionally_required - def clear_all_history(): - - if request.method == 'POST': - confirmtext = request.form.get('confirmtext') - - if confirmtext == 'clear': - changes_removed = 0 - for uuid in datastore.data['watching'].keys(): - datastore.clear_watch_history(uuid) - #TODO: KeyError not checked, as it is above - - flash("Cleared snapshot history for all watches") - else: - flash('Incorrect confirmation text.', 'error') - - return redirect(url_for('index')) - - output = render_template("clear_all_history.html") - return output - - def _watch_has_tag_options_set(watch): - """This should be fixed better so that Tag is some proper Model, a tag is just a Watch also""" - for tag_uuid, tag in datastore.data['settings']['application'].get('tags', {}).items(): - if tag_uuid in watch.get('tags', []) and (tag.get('include_filters') or tag.get('subtractive_selectors')): - return True - - @app.route("/edit/", methods=['GET', 'POST']) - @login_optionally_required - # https://stackoverflow.com/questions/42984453/wtforms-populate-form-with-data-if-data-exists - # https://wtforms.readthedocs.io/en/3.0.x/forms/#wtforms.form.Form.populate_obj ? - def edit_page(uuid): - from . import forms - from .blueprint.browser_steps.browser_steps import browser_step_ui_config - from . import processors - import importlib - - # More for testing, possible to return the first/only - if not datastore.data['watching'].keys(): - flash("No watches to edit", "error") - return redirect(url_for('index')) - - if uuid == 'first': - uuid = list(datastore.data['watching'].keys()).pop() - - if not uuid in datastore.data['watching']: - flash("No watch with the UUID %s found." % (uuid), "error") - return redirect(url_for('index')) - - switch_processor = request.args.get('switch_processor') - if switch_processor: - for p in processors.available_processors(): - if p[0] == switch_processor: - datastore.data['watching'][uuid]['processor'] = switch_processor - flash(f"Switched to mode - {p[1]}.") - datastore.clear_watch_history(uuid) - redirect(url_for('edit_page', uuid=uuid)) - - # be sure we update with a copy instead of accidently editing the live object by reference - default = deepcopy(datastore.data['watching'][uuid]) - - # Defaults for proxy choice - if datastore.proxy_list is not None: # When enabled - # @todo - # Radio needs '' not None, or incase that the chosen one no longer exists - if default['proxy'] is None or not any(default['proxy'] in tup for tup in datastore.proxy_list): - default['proxy'] = '' - # proxy_override set to the json/text list of the items - - # Does it use some custom form? does one exist? - processor_name = datastore.data['watching'][uuid].get('processor', '') - processor_classes = next((tpl for tpl in find_processors() if tpl[1] == processor_name), None) - if not processor_classes: - flash(f"Cannot load the edit form for processor/plugin '{processor_classes[1]}', plugin missing?", 'error') - return redirect(url_for('index')) - - parent_module = get_parent_module(processor_classes[0]) - - try: - # Get the parent of the "processor.py" go up one, get the form (kinda spaghetti but its reusing existing code) - forms_module = importlib.import_module(f"{parent_module.__name__}.forms") - # Access the 'processor_settings_form' class from the 'forms' module - form_class = getattr(forms_module, 'processor_settings_form') - except ModuleNotFoundError as e: - # .forms didnt exist - form_class = forms.processor_text_json_diff_form - except AttributeError as e: - # .forms exists but no useful form - form_class = forms.processor_text_json_diff_form - - form = form_class(formdata=request.form if request.method == 'POST' else None, - data=default, - extra_notification_tokens=default.extra_notification_token_values(), - default_system_settings=datastore.data['settings'] - ) - - # For the form widget tag UUID back to "string name" for the field - form.tags.datastore = datastore - - # Used by some forms that need to dig deeper - form.datastore = datastore - form.watch = default - - for p in datastore.extra_browsers: - form.fetch_backend.choices.append(p) - - form.fetch_backend.choices.append(("system", 'System settings default')) - - # form.browser_steps[0] can be assumed that we 'goto url' first - - if datastore.proxy_list is None: - # @todo - Couldn't get setattr() etc dynamic addition working, so remove it instead - del form.proxy - else: - form.proxy.choices = [('', 'Default')] - for p in datastore.proxy_list: - form.proxy.choices.append(tuple((p, datastore.proxy_list[p]['label']))) - - - if request.method == 'POST' and form.validate(): - - # If they changed processor, it makes sense to reset it. - if datastore.data['watching'][uuid].get('processor') != form.data.get('processor'): - datastore.data['watching'][uuid].clear_watch() - flash("Reset watch history due to change of processor") - - extra_update_obj = { - 'consecutive_filter_failures': 0, - 'last_error' : False - } - - if request.args.get('unpause_on_save'): - extra_update_obj['paused'] = False - - extra_update_obj['time_between_check'] = form.time_between_check.data - - # Ignore text - form_ignore_text = form.ignore_text.data - datastore.data['watching'][uuid]['ignore_text'] = form_ignore_text - - # Be sure proxy value is None - if datastore.proxy_list is not None and form.data['proxy'] == '': - extra_update_obj['proxy'] = None - - # Unsetting all filter_text methods should make it go back to default - # This particularly affects tests running - if 'filter_text_added' in form.data and not form.data.get('filter_text_added') \ - and 'filter_text_replaced' in form.data and not form.data.get('filter_text_replaced') \ - and 'filter_text_removed' in form.data and not form.data.get('filter_text_removed'): - extra_update_obj['filter_text_added'] = True - extra_update_obj['filter_text_replaced'] = True - extra_update_obj['filter_text_removed'] = True - - # Because wtforms doesn't support accessing other data in process_ , but we convert the CSV list of tags back to a list of UUIDs - tag_uuids = [] - if form.data.get('tags'): - # Sometimes in testing this can be list, dont know why - if type(form.data.get('tags')) == list: - extra_update_obj['tags'] = form.data.get('tags') - else: - for t in form.data.get('tags').split(','): - tag_uuids.append(datastore.add_tag(name=t)) - extra_update_obj['tags'] = tag_uuids - - datastore.data['watching'][uuid].update(form.data) - datastore.data['watching'][uuid].update(extra_update_obj) - - if not datastore.data['watching'][uuid].get('tags'): - # Force it to be a list, because form.data['tags'] will be string if nothing found - # And del(form.data['tags'] ) wont work either for some reason - datastore.data['watching'][uuid]['tags'] = [] - - # Recast it if need be to right data Watch handler - watch_class = get_custom_watch_obj_for_processor(form.data.get('processor')) - datastore.data['watching'][uuid] = watch_class(datastore_path=datastore_o.datastore_path, default=datastore.data['watching'][uuid]) - flash("Updated watch - unpaused!" if request.args.get('unpause_on_save') else "Updated watch.") - - # Re #286 - We wait for syncing new data to disk in another thread every 60 seconds - # But in the case something is added we should save straight away - datastore.needs_write_urgent = True - - # Do not queue on edit if its not within the time range - - # @todo maybe it should never queue anyway on edit... - is_in_schedule = True - watch = datastore.data['watching'].get(uuid) - - if watch.get('time_between_check_use_default'): - time_schedule_limit = datastore.data['settings']['requests'].get('time_schedule_limit', {}) - else: - time_schedule_limit = watch.get('time_schedule_limit') - - tz_name = time_schedule_limit.get('timezone') - if not tz_name: - tz_name = datastore.data['settings']['application'].get('timezone', 'UTC') - - if time_schedule_limit and time_schedule_limit.get('enabled'): - try: - is_in_schedule = is_within_schedule(time_schedule_limit=time_schedule_limit, - default_tz=tz_name - ) - except Exception as e: - logger.error( - f"{uuid} - Recheck scheduler, error handling timezone, check skipped - TZ name '{tz_name}' - {str(e)}") - return False - - ############################# - if not datastore.data['watching'][uuid].get('paused') and is_in_schedule: - # Queue the watch for immediate recheck, with a higher priority - update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid})) - - # Diff page [edit] link should go back to diff page - if request.args.get("next") and request.args.get("next") == 'diff': - return redirect(url_for('diff_history_page', uuid=uuid)) - - return redirect(url_for('index', tag=request.args.get("tag",''))) - - else: - if request.method == 'POST' and not form.validate(): - flash("An error occurred, please see below.", "error") - - visualselector_data_is_ready = datastore.visualselector_data_is_ready(uuid) - - - # JQ is difficult to install on windows and must be manually added (outside requirements.txt) - jq_support = True - try: - import jq - except ModuleNotFoundError: - jq_support = False - - watch = datastore.data['watching'].get(uuid) - - system_uses_webdriver = datastore.data['settings']['application']['fetch_backend'] == 'html_webdriver' - - watch_uses_webdriver = False - if (watch.get('fetch_backend') == 'system' and system_uses_webdriver) or watch.get('fetch_backend') == 'html_webdriver' or watch.get('fetch_backend', '').startswith('extra_browser_'): - watch_uses_webdriver = True - - from zoneinfo import available_timezones - - # Only works reliably with Playwright - - template_args = { - 'available_processors': processors.available_processors(), - 'available_timezones': sorted(available_timezones()), - 'browser_steps_config': browser_step_ui_config, - 'emailprefix': os.getenv('NOTIFICATION_MAIL_BUTTON_PREFIX', False), - 'extra_notification_token_placeholder_info': datastore.get_unique_notification_token_placeholders_available(), - 'extra_processor_config': form.extra_tab_content(), - 'extra_title': f" - Edit - {watch.label}", - 'form': form, - 'has_default_notification_urls': True if len(datastore.data['settings']['application']['notification_urls']) else False, - 'has_extra_headers_file': len(datastore.get_all_headers_in_textfile_for_watch(uuid=uuid)) > 0, - 'has_special_tag_options': _watch_has_tag_options_set(watch=watch), - 'watch_uses_webdriver': watch_uses_webdriver, - 'jq_support': jq_support, - 'playwright_enabled': os.getenv('PLAYWRIGHT_DRIVER_URL', False), - 'settings_application': datastore.data['settings']['application'], - 'timezone_default_config': datastore.data['settings']['application'].get('timezone'), - 'using_global_webdriver_wait': not default['webdriver_delay'], - 'uuid': uuid, - 'watch': watch - } - - included_content = None - if form.extra_form_content(): - # So that the extra panels can access _helpers.html etc, we set the environment to load from templates/ - # And then render the code from the module - from jinja2 import Environment, FileSystemLoader - import importlib.resources - templates_dir = str(importlib.resources.files("changedetectionio").joinpath('templates')) - env = Environment(loader=FileSystemLoader(templates_dir)) - template = env.from_string(form.extra_form_content()) - included_content = template.render(**template_args) - - output = render_template("edit.html", - extra_tab_content=form.extra_tab_content() if form.extra_tab_content() else None, - extra_form_content=included_content, - **template_args - ) - - return output - - @app.route("/settings", methods=['GET', "POST"]) - @login_optionally_required - def settings_page(): - from changedetectionio import forms - from datetime import datetime - from zoneinfo import available_timezones - - default = deepcopy(datastore.data['settings']) - if datastore.proxy_list is not None: - available_proxies = list(datastore.proxy_list.keys()) - # When enabled - system_proxy = datastore.data['settings']['requests']['proxy'] - # In the case it doesnt exist anymore - if not system_proxy in available_proxies: - system_proxy = None - - default['requests']['proxy'] = system_proxy if system_proxy is not None else available_proxies[0] - # Used by the form handler to keep or remove the proxy settings - default['proxy_list'] = available_proxies[0] - - - # Don't use form.data on POST so that it doesnt overrid the checkbox status from the POST status - form = forms.globalSettingsForm(formdata=request.form if request.method == 'POST' else None, - data=default, - extra_notification_tokens=datastore.get_unique_notification_tokens_available() - ) - - # Remove the last option 'System default' - form.application.form.notification_format.choices.pop() - - if datastore.proxy_list is None: - # @todo - Couldn't get setattr() etc dynamic addition working, so remove it instead - del form.requests.form.proxy - else: - form.requests.form.proxy.choices = [] - for p in datastore.proxy_list: - form.requests.form.proxy.choices.append(tuple((p, datastore.proxy_list[p]['label']))) - - - if request.method == 'POST': - # Password unset is a GET, but we can lock the session to a salted env password to always need the password - if form.application.form.data.get('removepassword_button', False): - # SALTED_PASS means the password is "locked" to what we set in the Env var - if not os.getenv("SALTED_PASS", False): - datastore.remove_password() - flash("Password protection removed.", 'notice') - flask_login.logout_user() - return redirect(url_for('settings_page')) - - if form.validate(): - # Don't set password to False when a password is set - should be only removed with the `removepassword` button - app_update = dict(deepcopy(form.data['application'])) - - # Never update password with '' or False (Added by wtforms when not in submission) - if 'password' in app_update and not app_update['password']: - del (app_update['password']) - - datastore.data['settings']['application'].update(app_update) - datastore.data['settings']['requests'].update(form.data['requests']) - - if not os.getenv("SALTED_PASS", False) and len(form.application.form.password.encrypted_password): - datastore.data['settings']['application']['password'] = form.application.form.password.encrypted_password - datastore.needs_write_urgent = True - flash("Password protection enabled.", 'notice') - flask_login.logout_user() - return redirect(url_for('index')) - - datastore.needs_write_urgent = True - flash("Settings updated.") - - else: - flash("An error occurred, please see below.", "error") - - # Convert to ISO 8601 format, all date/time relative events stored as UTC time - utc_time = datetime.now(ZoneInfo("UTC")).isoformat() - - output = render_template("settings.html", - api_key=datastore.data['settings']['application'].get('api_access_token'), - available_timezones=sorted(available_timezones()), - emailprefix=os.getenv('NOTIFICATION_MAIL_BUTTON_PREFIX', False), - extra_notification_token_placeholder_info=datastore.get_unique_notification_token_placeholders_available(), - form=form, - hide_remove_pass=os.getenv("SALTED_PASS", False), - min_system_recheck_seconds=int(os.getenv('MINIMUM_SECONDS_RECHECK_TIME', 3)), - settings_application=datastore.data['settings']['application'], - timezone_default_config=datastore.data['settings']['application'].get('timezone'), - utc_time=utc_time, - ) - - return output - - @app.route("/settings/reset-api-key", methods=['GET']) - @login_optionally_required - def settings_reset_api_key(): - import secrets - secret = secrets.token_hex(16) - datastore.data['settings']['application']['api_access_token'] = secret - datastore.needs_write_urgent = True - flash("API Key was regenerated.") - return redirect(url_for('settings_page')+'#api') - - @app.route("/import", methods=['GET', "POST"]) - @login_optionally_required - def import_page(): - remaining_urls = [] - from . import forms - - if request.method == 'POST': - - from .importer import import_url_list, import_distill_io_json - - # URL List import - if request.values.get('urls') and len(request.values.get('urls').strip()): - # Import and push into the queue for immediate update check - importer = import_url_list() - importer.run(data=request.values.get('urls'), flash=flash, datastore=datastore, processor=request.values.get('processor', 'text_json_diff')) - for uuid in importer.new_uuids: - update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid})) - - if len(importer.remaining_data) == 0: - return redirect(url_for('index')) - else: - remaining_urls = importer.remaining_data - - # Distill.io import - if request.values.get('distill-io') and len(request.values.get('distill-io').strip()): - # Import and push into the queue for immediate update check - d_importer = import_distill_io_json() - d_importer.run(data=request.values.get('distill-io'), flash=flash, datastore=datastore) - for uuid in d_importer.new_uuids: - update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid})) - - # XLSX importer - if request.files and request.files.get('xlsx_file'): - file = request.files['xlsx_file'] - from .importer import import_xlsx_wachete, import_xlsx_custom - - if request.values.get('file_mapping') == 'wachete': - w_importer = import_xlsx_wachete() - w_importer.run(data=file, flash=flash, datastore=datastore) - else: - w_importer = import_xlsx_custom() - # Building mapping of col # to col # type - map = {} - for i in range(10): - c = request.values.get(f"custom_xlsx[col_{i}]") - v = request.values.get(f"custom_xlsx[col_type_{i}]") - if c and v: - map[int(c)] = v - - w_importer.import_profile = map - w_importer.run(data=file, flash=flash, datastore=datastore) - - for uuid in w_importer.new_uuids: - update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid})) - - # Could be some remaining, or we could be on GET - form = forms.importForm(formdata=request.form if request.method == 'POST' else None) - output = render_template("import.html", - form=form, - import_url_list_remaining="\n".join(remaining_urls), - original_distill_json='' - ) - return output - - # Clear all statuses, so we do not see the 'unviewed' class - @app.route("/form/mark-all-viewed", methods=['GET']) - @login_optionally_required - def mark_all_viewed(): - - # Save the current newest history as the most recently viewed - with_errors = request.args.get('with_errors') == "1" - for watch_uuid, watch in datastore.data['watching'].items(): - if with_errors and not watch.get('last_error'): - continue - datastore.set_last_viewed(watch_uuid, int(time.time())) - - return redirect(url_for('index')) - - @app.route("/diff/", methods=['GET', 'POST']) - @login_optionally_required - def diff_history_page(uuid): - - from changedetectionio import forms - - # More for testing, possible to return the first/only - if uuid == 'first': - uuid = list(datastore.data['watching'].keys()).pop() - - extra_stylesheets = [url_for('static_content', group='styles', filename='diff.css')] - try: - watch = datastore.data['watching'][uuid] - except KeyError: - flash("No history found for the specified link, bad link?", "error") - return redirect(url_for('index')) - - # For submission of requesting an extract - extract_form = forms.extractDataForm(request.form) - if request.method == 'POST': - if not extract_form.validate(): - flash("An error occurred, please see below.", "error") - - else: - extract_regex = request.form.get('extract_regex').strip() - output = watch.extract_regex_from_all_history(extract_regex) - if output: - watch_dir = os.path.join(datastore_o.datastore_path, uuid) - response = make_response(send_from_directory(directory=watch_dir, path=output, as_attachment=True)) - response.headers['Content-type'] = 'text/csv' - response.headers['Cache-Control'] = 'no-cache, no-store, must-revalidate' - response.headers['Pragma'] = 'no-cache' - response.headers['Expires'] = 0 - return response - - - flash('Nothing matches that RegEx', 'error') - redirect(url_for('diff_history_page', uuid=uuid)+'#extract') - - history = watch.history - dates = list(history.keys()) - - if len(dates) < 2: - flash("Not enough saved change detection snapshots to produce a report.", "error") - return redirect(url_for('index')) - - # Save the current newest history as the most recently viewed - datastore.set_last_viewed(uuid, time.time()) - - # Read as binary and force decode as UTF-8 - # Windows may fail decode in python if we just use 'r' mode (chardet decode exception) - from_version = request.args.get('from_version') - from_version_index = -2 # second newest - if from_version and from_version in dates: - from_version_index = dates.index(from_version) - else: - from_version = dates[from_version_index] - - try: - from_version_file_contents = watch.get_history_snapshot(dates[from_version_index]) - except Exception as e: - from_version_file_contents = f"Unable to read to-version at index {dates[from_version_index]}.\n" - - to_version = request.args.get('to_version') - to_version_index = -1 - if to_version and to_version in dates: - to_version_index = dates.index(to_version) - else: - to_version = dates[to_version_index] - - try: - to_version_file_contents = watch.get_history_snapshot(dates[to_version_index]) - except Exception as e: - to_version_file_contents = "Unable to read to-version at index{}.\n".format(dates[to_version_index]) - - screenshot_url = watch.get_screenshot() - - system_uses_webdriver = datastore.data['settings']['application']['fetch_backend'] == 'html_webdriver' - - is_html_webdriver = False - if (watch.get('fetch_backend') == 'system' and system_uses_webdriver) or watch.get('fetch_backend') == 'html_webdriver' or watch.get('fetch_backend', '').startswith('extra_browser_'): - is_html_webdriver = True - - password_enabled_and_share_is_off = False - if datastore.data['settings']['application'].get('password') or os.getenv("SALTED_PASS", False): - password_enabled_and_share_is_off = not datastore.data['settings']['application'].get('shared_diff_access') - - output = render_template("diff.html", - current_diff_url=watch['url'], - from_version=str(from_version), - to_version=str(to_version), - extra_stylesheets=extra_stylesheets, - extra_title=f" - Diff - {watch.label}", - extract_form=extract_form, - is_html_webdriver=is_html_webdriver, - last_error=watch['last_error'], - last_error_screenshot=watch.get_error_snapshot(), - last_error_text=watch.get_error_text(), - left_sticky=True, - newest=to_version_file_contents, - newest_version_timestamp=dates[-1], - password_enabled_and_share_is_off=password_enabled_and_share_is_off, - from_version_file_contents=from_version_file_contents, - to_version_file_contents=to_version_file_contents, - screenshot=screenshot_url, - uuid=uuid, - versions=dates, # All except current/last - watch_a=watch - ) - - return output - - @app.route("/preview/", methods=['GET']) - @login_optionally_required - def preview_page(uuid): - content = [] - versions = [] - timestamp = None - - # More for testing, possible to return the first/only - if uuid == 'first': - uuid = list(datastore.data['watching'].keys()).pop() - - try: - watch = datastore.data['watching'][uuid] - except KeyError: - flash("No history found for the specified link, bad link?", "error") - return redirect(url_for('index')) - - system_uses_webdriver = datastore.data['settings']['application']['fetch_backend'] == 'html_webdriver' - extra_stylesheets = [url_for('static_content', group='styles', filename='diff.css')] - - is_html_webdriver = False - if (watch.get('fetch_backend') == 'system' and system_uses_webdriver) or watch.get('fetch_backend') == 'html_webdriver' or watch.get('fetch_backend', '').startswith('extra_browser_'): - is_html_webdriver = True - triggered_line_numbers = [] - if datastore.data['watching'][uuid].history_n == 0 and (watch.get_error_text() or watch.get_error_snapshot()): - flash("Preview unavailable - No fetch/check completed or triggers not reached", "error") - else: - # So prepare the latest preview or not - preferred_version = request.args.get('version') - versions = list(watch.history.keys()) - timestamp = versions[-1] - if preferred_version and preferred_version in versions: - timestamp = preferred_version - - try: - versions = list(watch.history.keys()) - content = watch.get_history_snapshot(timestamp) - - triggered_line_numbers = html_tools.strip_ignore_text(content=content, - wordlist=watch['trigger_text'], - mode='line numbers' - ) - - except Exception as e: - content.append({'line': f"File doesnt exist or unable to read timestamp {timestamp}", 'classes': ''}) - - output = render_template("preview.html", - content=content, - current_version=timestamp, - history_n=watch.history_n, - extra_stylesheets=extra_stylesheets, - extra_title=f" - Diff - {watch.label} @ {timestamp}", - triggered_line_numbers=triggered_line_numbers, - current_diff_url=watch['url'], - screenshot=watch.get_screenshot(), - watch=watch, - uuid=uuid, - is_html_webdriver=is_html_webdriver, - last_error=watch['last_error'], - last_error_text=watch.get_error_text(), - last_error_screenshot=watch.get_error_snapshot(), - versions=versions - ) - - - return output - - @app.route("/settings/notification-logs", methods=['GET']) - @login_optionally_required - def notification_logs(): - global notification_debug_log - output = render_template("notification-log.html", - logs=notification_debug_log if len(notification_debug_log) else ["Notification logs are empty - no notifications sent yet."]) - - return output - @app.route("/static//", methods=['GET']) def static_content(group, filename): from flask import make_response @@ -1351,329 +499,13 @@ def changedetection_app(config=None, datastore_o=None): except FileNotFoundError: abort(404) - @app.route("/edit//get-html", methods=['GET']) - @login_optionally_required - def watch_get_latest_html(uuid): - from io import BytesIO - from flask import send_file - import brotli - - watch = datastore.data['watching'].get(uuid) - if watch and watch.history.keys() and os.path.isdir(watch.watch_data_dir): - latest_filename = list(watch.history.keys())[-1] - html_fname = os.path.join(watch.watch_data_dir, f"{latest_filename}.html.br") - with open(html_fname, 'rb') as f: - if html_fname.endswith('.br'): - # Read and decompress the Brotli file - decompressed_data = brotli.decompress(f.read()) - else: - decompressed_data = f.read() - - buffer = BytesIO(decompressed_data) - - return send_file(buffer, as_attachment=True, download_name=f"{latest_filename}.html", mimetype='text/html') - - - # Return a 500 error - abort(500) - - # Ajax callback - @app.route("/edit//preview-rendered", methods=['POST']) - @login_optionally_required - def watch_get_preview_rendered(uuid): - '''For when viewing the "preview" of the rendered text from inside of Edit''' - from flask import jsonify - from .processors.text_json_diff import prepare_filter_prevew - result = prepare_filter_prevew(watch_uuid=uuid, form_data=request.form, datastore=datastore) - return jsonify(result) - - - @app.route("/form/add/quickwatch", methods=['POST']) - @login_optionally_required - def form_quick_watch_add(): - from changedetectionio import forms - form = forms.quickWatchForm(request.form) - - if not form.validate(): - for widget, l in form.errors.items(): - flash(','.join(l), 'error') - return redirect(url_for('index')) - - url = request.form.get('url').strip() - if datastore.url_exists(url): - flash(f'Warning, URL {url} already exists', "notice") - - add_paused = request.form.get('edit_and_watch_submit_button') != None - processor = request.form.get('processor', 'text_json_diff') - new_uuid = datastore.add_watch(url=url, tag=request.form.get('tags').strip(), extras={'paused': add_paused, 'processor': processor}) - - if new_uuid: - if add_paused: - flash('Watch added in Paused state, saving will unpause.') - return redirect(url_for('edit_page', uuid=new_uuid, unpause_on_save=1, tag=request.args.get('tag'))) - else: - # Straight into the queue. - update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': new_uuid})) - flash("Watch added.") - - return redirect(url_for('index', tag=request.args.get('tag',''))) - - - - @app.route("/api/delete", methods=['GET']) - @login_optionally_required - def form_delete(): - uuid = request.args.get('uuid') - - if uuid != 'all' and not uuid in datastore.data['watching'].keys(): - flash('The watch by UUID {} does not exist.'.format(uuid), 'error') - return redirect(url_for('index')) - - # More for testing, possible to return the first/only - if uuid == 'first': - uuid = list(datastore.data['watching'].keys()).pop() - datastore.delete(uuid) - flash('Deleted.') - - return redirect(url_for('index')) - - @app.route("/api/clone", methods=['GET']) - @login_optionally_required - def form_clone(): - uuid = request.args.get('uuid') - # More for testing, possible to return the first/only - if uuid == 'first': - uuid = list(datastore.data['watching'].keys()).pop() - - new_uuid = datastore.clone(uuid) - if new_uuid: - if not datastore.data['watching'].get(uuid).get('paused'): - update_q.put(queuedWatchMetaData.PrioritizedItem(priority=5, item={'uuid': new_uuid})) - flash('Cloned.') - - return redirect(url_for('index')) - - @app.route("/api/checknow", methods=['GET']) - @login_optionally_required - def form_watch_checknow(): - # Forced recheck will skip the 'skip if content is the same' rule (, 'reprocess_existing_data': True}))) - tag = request.args.get('tag') - uuid = request.args.get('uuid') - with_errors = request.args.get('with_errors') == "1" - - i = 0 - - running_uuids = [] - for t in running_update_threads: - running_uuids.append(t.current_uuid) - - if uuid: - if uuid not in running_uuids: - update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid})) - i = 1 - - elif tag: - # Items that have this current tag - for watch_uuid, watch in datastore.data['watching'].items(): - if tag in watch.get('tags', {}): - if with_errors and not watch.get('last_error'): - continue - if watch_uuid not in running_uuids and not datastore.data['watching'][watch_uuid]['paused']: - update_q.put( - queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': watch_uuid}) - ) - i += 1 - - else: - # No tag, no uuid, add everything. - for watch_uuid, watch in datastore.data['watching'].items(): - if watch_uuid not in running_uuids and not datastore.data['watching'][watch_uuid]['paused']: - if with_errors and not watch.get('last_error'): - continue - update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': watch_uuid})) - i += 1 - flash(f"{i} watches queued for rechecking.") - return redirect(url_for('index', tag=tag)) - - @app.route("/form/checkbox-operations", methods=['POST']) - @login_optionally_required - def form_watch_list_checkbox_operations(): - op = request.form['op'] - uuids = request.form.getlist('uuids') - - if (op == 'delete'): - for uuid in uuids: - uuid = uuid.strip() - if datastore.data['watching'].get(uuid): - datastore.delete(uuid.strip()) - flash("{} watches deleted".format(len(uuids))) - - elif (op == 'pause'): - for uuid in uuids: - uuid = uuid.strip() - if datastore.data['watching'].get(uuid): - datastore.data['watching'][uuid.strip()]['paused'] = True - flash("{} watches paused".format(len(uuids))) - - elif (op == 'unpause'): - for uuid in uuids: - uuid = uuid.strip() - if datastore.data['watching'].get(uuid): - datastore.data['watching'][uuid.strip()]['paused'] = False - flash("{} watches unpaused".format(len(uuids))) - - elif (op == 'mark-viewed'): - for uuid in uuids: - uuid = uuid.strip() - if datastore.data['watching'].get(uuid): - datastore.set_last_viewed(uuid, int(time.time())) - flash("{} watches updated".format(len(uuids))) - - elif (op == 'mute'): - for uuid in uuids: - uuid = uuid.strip() - if datastore.data['watching'].get(uuid): - datastore.data['watching'][uuid.strip()]['notification_muted'] = True - flash("{} watches muted".format(len(uuids))) - - elif (op == 'unmute'): - for uuid in uuids: - uuid = uuid.strip() - if datastore.data['watching'].get(uuid): - datastore.data['watching'][uuid.strip()]['notification_muted'] = False - flash("{} watches un-muted".format(len(uuids))) - - elif (op == 'recheck'): - for uuid in uuids: - uuid = uuid.strip() - if datastore.data['watching'].get(uuid): - # Recheck and require a full reprocessing - update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid})) - flash("{} watches queued for rechecking".format(len(uuids))) - - elif (op == 'clear-errors'): - for uuid in uuids: - uuid = uuid.strip() - if datastore.data['watching'].get(uuid): - datastore.data['watching'][uuid]["last_error"] = False - flash(f"{len(uuids)} watches errors cleared") - - elif (op == 'clear-history'): - for uuid in uuids: - uuid = uuid.strip() - if datastore.data['watching'].get(uuid): - datastore.clear_watch_history(uuid) - flash("{} watches cleared/reset.".format(len(uuids))) - - elif (op == 'notification-default'): - from changedetectionio.notification import ( - default_notification_format_for_watch - ) - for uuid in uuids: - uuid = uuid.strip() - if datastore.data['watching'].get(uuid): - datastore.data['watching'][uuid.strip()]['notification_title'] = None - datastore.data['watching'][uuid.strip()]['notification_body'] = None - datastore.data['watching'][uuid.strip()]['notification_urls'] = [] - datastore.data['watching'][uuid.strip()]['notification_format'] = default_notification_format_for_watch - flash("{} watches set to use default notification settings".format(len(uuids))) - - elif (op == 'assign-tag'): - op_extradata = request.form.get('op_extradata', '').strip() - if op_extradata: - tag_uuid = datastore.add_tag(name=op_extradata) - if op_extradata and tag_uuid: - for uuid in uuids: - uuid = uuid.strip() - if datastore.data['watching'].get(uuid): - # Bug in old versions caused by bad edit page/tag handler - if isinstance(datastore.data['watching'][uuid]['tags'], str): - datastore.data['watching'][uuid]['tags'] = [] - - datastore.data['watching'][uuid]['tags'].append(tag_uuid) - - flash(f"{len(uuids)} watches were tagged") - - return redirect(url_for('index')) - - @app.route("/api/share-url", methods=['GET']) - @login_optionally_required - def form_share_put_watch(): - """Given a watch UUID, upload the info and return a share-link - the share-link can be imported/added""" - import requests - import json - uuid = request.args.get('uuid') - - # more for testing - if uuid == 'first': - uuid = list(datastore.data['watching'].keys()).pop() - - # copy it to memory as trim off what we dont need (history) - watch = deepcopy(datastore.data['watching'][uuid]) - # For older versions that are not a @property - if (watch.get('history')): - del (watch['history']) - - # for safety/privacy - for k in list(watch.keys()): - if k.startswith('notification_'): - del watch[k] - - for r in['uuid', 'last_checked', 'last_changed']: - if watch.get(r): - del (watch[r]) - - # Add the global stuff which may have an impact - watch['ignore_text'] += datastore.data['settings']['application']['global_ignore_text'] - watch['subtractive_selectors'] += datastore.data['settings']['application']['global_subtractive_selectors'] - - watch_json = json.dumps(watch) - - try: - r = requests.request(method="POST", - data={'watch': watch_json}, - url="https://changedetection.io/share/share", - headers={'App-Guid': datastore.data['app_guid']}) - res = r.json() - - session['share-link'] = "https://changedetection.io/share/{}".format(res['share_key']) - - - except Exception as e: - logger.error(f"Error sharing -{str(e)}") - flash("Could not share, something went wrong while communicating with the share server - {}".format(str(e)), 'error') - - # https://changedetection.io/share/VrMv05wpXyQa - # in the browser - should give you a nice info page - wtf - # paste in etc - return redirect(url_for('index')) - - @app.route("/highlight_submit_ignore_url", methods=['POST']) - @login_optionally_required - def highlight_submit_ignore_url(): - import re - mode = request.form.get('mode') - selection = request.form.get('selection') - - uuid = request.args.get('uuid','') - if datastore.data["watching"].get(uuid): - if mode == 'exact': - for l in selection.splitlines(): - datastore.data["watching"][uuid]['ignore_text'].append(l.strip()) - elif mode == 'digit-regex': - for l in selection.splitlines(): - # Replace any series of numbers with a regex - s = re.escape(l.strip()) - s = re.sub(r'[0-9]+', r'\\d+', s) - datastore.data["watching"][uuid]['ignore_text'].append('/' + s + '/') - - return f"Click to preview" - import changedetectionio.blueprint.browser_steps as browser_steps app.register_blueprint(browser_steps.construct_blueprint(datastore), url_prefix='/browser-steps') + from changedetectionio.blueprint.imports import construct_blueprint as construct_import_blueprint + app.register_blueprint(construct_import_blueprint(datastore, update_q, queuedWatchMetaData), url_prefix='/imports') + import changedetectionio.blueprint.price_data_follower as price_data_follower app.register_blueprint(price_data_follower.construct_blueprint(datastore, update_q), url_prefix='/price_data_follower') @@ -1686,9 +518,19 @@ def changedetection_app(config=None, datastore_o=None): import changedetectionio.blueprint.backups as backups app.register_blueprint(backups.construct_blueprint(datastore), url_prefix='/backups') + import changedetectionio.blueprint.settings as settings + app.register_blueprint(settings.construct_blueprint(datastore), url_prefix='/settings') + import changedetectionio.conditions.blueprint as conditions app.register_blueprint(conditions.construct_blueprint(datastore), url_prefix='/conditions') + import changedetectionio.blueprint.rss as rss + app.register_blueprint(rss.construct_blueprint(datastore), url_prefix='/rss') + + import changedetectionio.blueprint.ui as ui + app.register_blueprint(ui.construct_blueprint(datastore, update_q, running_update_threads, queuedWatchMetaData)) + + # @todo handle ctrl break ticker_thread = threading.Thread(target=ticker_thread_check_time_launch_checks).start() threading.Thread(target=notification_runner).start() diff --git a/changedetectionio/model/Watch.py b/changedetectionio/model/Watch.py index de9e8aa2..c04af711 100644 --- a/changedetectionio/model/Watch.py +++ b/changedetectionio/model/Watch.py @@ -83,7 +83,7 @@ class model(watch_base): flash, Markup, url_for ) message = Markup('The URL {} is invalid and cannot be used, click to edit'.format( - url_for('edit_page', uuid=self.get('uuid')), self.get('url', ''))) + url_for('ui.ui_edit.edit_page', uuid=self.get('uuid')), self.get('url', ''))) flash(message, 'error') return '' diff --git a/changedetectionio/templates/_common_fields.html b/changedetectionio/templates/_common_fields.html index 9a1cd128..ebc27e08 100644 --- a/changedetectionio/templates/_common_fields.html +++ b/changedetectionio/templates/_common_fields.html @@ -28,7 +28,7 @@ {% if emailprefix %} Add email Add an email address {% endif %} - Notification debug logs + Notification debug logs
diff --git a/changedetectionio/templates/_helpers.html b/changedetectionio/templates/_helpers.html index 461eb22e..2ed75a30 100644 --- a/changedetectionio/templates/_helpers.html +++ b/changedetectionio/templates/_helpers.html @@ -199,7 +199,7 @@ {% else %} - Want to use a time schedule? First confirm/save your Time Zone Settings + Want to use a time schedule? First confirm/save your Time Zone Settings
{% endif %} diff --git a/changedetectionio/templates/base.html b/changedetectionio/templates/base.html index 842e648a..22fa3ccf 100644 --- a/changedetectionio/templates/base.html +++ b/changedetectionio/templates/base.html @@ -7,7 +7,7 @@ Change Detection{{extra_title}} {% if app_rss_token %} - + {% endif %} @@ -64,17 +64,17 @@ GROUPS
  • - SETTINGS + SETTINGS
  • - IMPORT + IMPORT
  • BACKUPS
    • {% else %}
  • - EDIT + EDIT
    • {% endif %} {% else %} @@ -144,7 +144,7 @@ {% endif %} {% if left_sticky %}
    - Show current snapshot
    + Show current snapshot
    Visualise triggers and ignored text
    {% endif %} diff --git a/changedetectionio/templates/clear_all_history.html b/changedetectionio/templates/clear_all_history.html index 01433bb0..fbbaa34f 100644 --- a/changedetectionio/templates/clear_all_history.html +++ b/changedetectionio/templates/clear_all_history.html @@ -3,7 +3,7 @@
    diff --git a/changedetectionio/templates/diff.html b/changedetectionio/templates/diff.html index ecf05acc..f7fdf868 100644 --- a/changedetectionio/templates/diff.html +++ b/changedetectionio/templates/diff.html @@ -7,7 +7,7 @@ const error_screenshot_url="{{url_for('static_content', group='screenshot', filename=uuid, error_screenshot=1) }}"; {% endif %} - const highlight_submit_ignore_url="{{url_for('highlight_submit_ignore_url', uuid=uuid)}}"; + const highlight_submit_ignore_url="{{url_for('ui.ui_edit.highlight_submit_ignore_url', uuid=uuid)}}"; @@ -125,7 +125,7 @@
    diff --git a/changedetectionio/templates/edit.html b/changedetectionio/templates/edit.html index 92c71967..3bda8e3f 100644 --- a/changedetectionio/templates/edit.html +++ b/changedetectionio/templates/edit.html @@ -20,7 +20,7 @@ {% if emailprefix %} const email_notification_prefix=JSON.parse('{{ emailprefix|tojson }}'); {% endif %} - const notification_base_url="{{url_for('ajax_callback_send_notification_test', watch_uuid=uuid)}}"; + const notification_base_url="{{url_for('ui.ui_notification.ajax_callback_send_notification_test', watch_uuid=uuid)}}"; const playwright_enabled={% if playwright_enabled %}true{% else %}false{% endif %}; const recheck_proxy_start_url="{{url_for('check_proxies.start_check', uuid=uuid)}}"; const proxy_recheck_status_url="{{url_for('check_proxies.get_recheck_status', uuid=uuid)}}"; @@ -62,7 +62,7 @@
    + action="{{ url_for('ui.ui_edit.edit_page', uuid=uuid, next = request.args.get('next'), unpause_on_save = request.args.get('unpause_on_save'), tag = request.args.get('tag')) }}" method="POST">
    @@ -274,7 +274,7 @@ Math: {{ 1 + 1 }}") }} {% if has_default_notification_urls %}
    Look out! - There are system-wide notification URLs enabled, this form will override notification settings for this watch only ‐ an empty Notification URL list here will still send notifications. + There are system-wide notification URLs enabled, this form will override notification settings for this watch only ‐ an empty Notification URL list here will still send notifications.
    {% endif %} Use system defaults @@ -480,7 +480,7 @@ keyword") }}
    - +
    diff --git a/changedetectionio/templates/preview.html b/changedetectionio/templates/preview.html index 6915da33..826fc041 100644 --- a/changedetectionio/templates/preview.html +++ b/changedetectionio/templates/preview.html @@ -7,7 +7,7 @@ {% if last_error_screenshot %} const error_screenshot_url = "{{url_for('static_content', group='screenshot', filename=uuid, error_screenshot=1) }}"; {% endif %} - const highlight_submit_ignore_url = "{{url_for('highlight_submit_ignore_url', uuid=uuid)}}"; + const highlight_submit_ignore_url = "{{url_for('ui.ui_edit.highlight_submit_ignore_url', uuid=uuid)}}"; diff --git a/changedetectionio/templates/settings.html b/changedetectionio/templates/settings.html index 752ff27e..2e651a01 100644 --- a/changedetectionio/templates/settings.html +++ b/changedetectionio/templates/settings.html @@ -4,7 +4,7 @@ {% from '_helpers.html' import render_field, render_checkbox_field, render_button, render_time_schedule_form %} {% from '_common_fields.html' import render_common_settings_form %}