diff --git a/changedetectionio/__init__.py b/changedetectionio/__init__.py index 5648cb53..4aa69713 100644 --- a/changedetectionio/__init__.py +++ b/changedetectionio/__init__.py @@ -810,10 +810,12 @@ def changedetection_app(config=None, datastore_o=None): return redirect(url_for('index')) - @app.route("/diff/", methods=['GET']) + @app.route("/diff/", methods=['GET', 'POST']) @login_required def diff_history_page(uuid): + from changedetectionio import forms + # More for testing, possible to return the first/only if uuid == 'first': uuid = list(datastore.data['watching'].keys()).pop() @@ -825,6 +827,28 @@ def changedetection_app(config=None, datastore_o=None): flash("No history found for the specified link, bad link?", "error") return redirect(url_for('index')) + # For submission of requesting an extract + extract_form = forms.extractDataForm(request.form) + if request.method == 'POST': + if not extract_form.validate(): + flash("An error occurred, please see below.", "error") + + else: + extract_regex = request.form.get('extract_regex').strip() + output = watch.extract_regex_from_all_history(extract_regex) + if output: + watch_dir = os.path.join(datastore_o.datastore_path, uuid) + response = make_response(send_from_directory(directory=watch_dir, path=output, as_attachment=True)) + response.headers['Content-type'] = 'text/csv' + response.headers['Cache-Control'] = 'no-cache, no-store, must-revalidate' + response.headers['Pragma'] = 'no-cache' + response.headers['Expires'] = 0 + return response + + + flash('Nothing matches that RegEx', 'error') + redirect(url_for('diff_history_page', uuid=uuid)+'#extract') + history = watch.history dates = list(history.keys()) @@ -867,23 +891,24 @@ def changedetection_app(config=None, datastore_o=None): watch.get('fetch_backend', None) is None and system_uses_webdriver) else False output = render_template("diff.html", - watch_a=watch, - newest=newest_version_file_contents, - previous=previous_version_file_contents, - extra_stylesheets=extra_stylesheets, - dark_mode=getDarkModeSetting(), - versions=dates[:-1], # All except current/last - uuid=uuid, - newest_version_timestamp=dates[-1], - current_previous_version=str(previous_version), current_diff_url=watch['url'], + current_previous_version=str(previous_version), + dark_mode=getDarkModeSetting(), + extra_stylesheets=extra_stylesheets, extra_title=" - Diff - {}".format(watch['title'] if watch['title'] else watch['url']), - left_sticky=True, - screenshot=screenshot_url, + extract_form=extract_form, is_html_webdriver=is_html_webdriver, last_error=watch['last_error'], + last_error_screenshot=watch.get_error_snapshot(), last_error_text=watch.get_error_text(), - last_error_screenshot=watch.get_error_snapshot() + left_sticky=True, + newest=newest_version_file_contents, + newest_version_timestamp=dates[-1], + previous=previous_version_file_contents, + screenshot=screenshot_url, + uuid=uuid, + versions=dates[:-1], # All except current/last + watch_a=watch ) return output diff --git a/changedetectionio/forms.py b/changedetectionio/forms.py index 116dfa03..88c846ae 100644 --- a/changedetectionio/forms.py +++ b/changedetectionio/forms.py @@ -462,3 +462,8 @@ class globalSettingsForm(Form): requests = FormField(globalSettingsRequestForm) application = FormField(globalSettingsApplicationForm) save_button = SubmitField('Save', render_kw={"class": "pure-button pure-button-primary"}) + + +class extractDataForm(Form): + extract_regex = StringField('RegEx to extract', validators=[validators.Length(min=1, message="Needs a RegEx")]) + extract_submit_button = SubmitField('Extract as CSV', render_kw={"class": "pure-button pure-button-primary"}) diff --git a/changedetectionio/model/Watch.py b/changedetectionio/model/Watch.py index 7c0ad045..c3a000b0 100644 --- a/changedetectionio/model/Watch.py +++ b/changedetectionio/model/Watch.py @@ -318,3 +318,47 @@ class model(dict): if os.path.isfile(fname): return fname return False + + def extract_regex_from_all_history(self, regex): + import csv + import re + import datetime + csv_output_filename = False + csv_writer = False + f = None + + # self.history will be keyed with the full path + for k, fname in self.history.items(): + if os.path.isfile(fname): + with open(fname, "r") as f: + contents = f.read() + res = re.findall(regex, contents, re.MULTILINE) + if res: + if not csv_writer: + # A file on the disk can be transferred much faster via flask than a string reply + csv_output_filename = 'report.csv' + f = open(os.path.join(self.watch_data_dir, csv_output_filename), 'w') + # @todo some headers in the future + #fieldnames = ['Epoch seconds', 'Date'] + csv_writer = csv.writer(f, + delimiter=',', + quotechar='"', + quoting=csv.QUOTE_MINIMAL, + #fieldnames=fieldnames + ) + csv_writer.writerow(['Epoch seconds', 'Date']) + # csv_writer.writeheader() + + date_str = datetime.datetime.fromtimestamp(int(k)).strftime('%Y-%m-%d %H:%M:%S') + for r in res: + row = [k, date_str] + if isinstance(r, str): + row.append(r) + else: + row+=r + csv_writer.writerow(row) + + if f: + f.close() + + return csv_output_filename diff --git a/changedetectionio/static/js/diff-overview.js b/changedetectionio/static/js/diff-overview.js index fa94316f..31c9bfdd 100644 --- a/changedetectionio/static/js/diff-overview.js +++ b/changedetectionio/static/js/diff-overview.js @@ -13,6 +13,8 @@ $(document).ready(function () { } else if (hash_name === '#error-screenshot') { $("img#error-screenshot-img").attr('src', error_screenshot_url); $("#settings").hide(); + } else if (hash_name === '#extract') { + $("#settings").hide(); } diff --git a/changedetectionio/static/js/toggle-theme.js b/changedetectionio/static/js/toggle-theme.js index 5147edd9..ca11ec99 100644 --- a/changedetectionio/static/js/toggle-theme.js +++ b/changedetectionio/static/js/toggle-theme.js @@ -19,6 +19,6 @@ $(document).ready(function () { }; const setCookieValue = (value) => { - document.cookie = `css_dark_mode=${value};max-age=31536000` + document.cookie = `css_dark_mode=${value};max-age=31536000;path=/` } }); diff --git a/changedetectionio/static/styles/diff.css b/changedetectionio/static/styles/diff.css index 394aa4fc..ec019a7f 100644 --- a/changedetectionio/static/styles/diff.css +++ b/changedetectionio/static/styles/diff.css @@ -18,6 +18,8 @@ --color-grey-850: #eee; --color-grey-900: #f2f2f2; --color-black: #000; + --color-dark-red: #a00; + --color-light-red: #dd0000; --color-background-page: var(--color-grey-100); --color-background-gradient-first: #5ad8f7; --color-background-gradient-second: #2f50af; @@ -27,9 +29,9 @@ --color-link: #1b98f8; --color-menu-accent: #ed5900; --color-background-code: var(--color-grey-850); - --color-error: #a00; + --color-error: var(--color-dark-red); --color-error-input: #ffebeb; - --color-error-list: #dd0000; + --color-error-list: var(--color-light-red); --color-table-background: var(--color-background); --color-table-stripe: var(--color-grey-900); --color-text-tab: var(--color-white); @@ -84,7 +86,9 @@ --color-text-menu-link-hover: var(--color-grey-300); --color-shadow-jump: var(--color-grey-500); --color-icon-github: var(--color-black); - --color-icon-github-hover: var(--color-grey-300); } + --color-icon-github-hover: var(--color-grey-300); + --color-watch-table-error: var(--color-dark-red); + --color-watch-table-row-text: var(--color-grey-100); } html[data-darkmode="true"] { --color-link: #59bdfb; @@ -114,9 +118,13 @@ html[data-darkmode="true"] { --color-background-snapshot-age: var(--color-grey-200); --color-shadow-jump: var(--color-grey-200); --color-icon-github: var(--color-white); - --color-icon-github-hover: var(--color-grey-700); } + --color-icon-github-hover: var(--color-grey-700); + --color-watch-table-error: var(--color-light-red); + --color-watch-table-row-text: var(--color-grey-800); } html[data-darkmode="true"] .watch-controls img { opacity: 0.4; } + html[data-darkmode="true"] .watch-table .unviewed { + color: #fff; } html[data-darkmode="true"] .icon-spread { filter: hue-rotate(-10deg) brightness(1.5); } html[data-darkmode="true"] .watch-table .title-col a[target="_blank"]::after, @@ -128,8 +136,9 @@ html[data-darkmode="true"] { padding: 2em; margin-left: 1em; margin-right: 1em; - border-radius: 5px; - font-size: 11px; } + border-radius: 5px; } + #diff-ui #text { + font-size: 11px; } #diff-ui table { table-layout: fixed; width: 100%; } diff --git a/changedetectionio/static/styles/scss/diff.scss b/changedetectionio/static/styles/scss/diff.scss index 7c219f20..19783b6f 100644 --- a/changedetectionio/static/styles/scss/diff.scss +++ b/changedetectionio/static/styles/scss/diff.scss @@ -7,7 +7,11 @@ margin-left: 1em; margin-right: 1em; border-radius: 5px; - font-size: 11px; + + // The first tab 'text' diff + #text { + font-size: 11px; + } table { table-layout: fixed; diff --git a/changedetectionio/static/styles/scss/parts/_variables.scss b/changedetectionio/static/styles/scss/parts/_variables.scss index ba6c8423..bccac914 100644 --- a/changedetectionio/static/styles/scss/parts/_variables.scss +++ b/changedetectionio/static/styles/scss/parts/_variables.scss @@ -19,6 +19,8 @@ --color-grey-850: #eee; --color-grey-900: #f2f2f2; --color-black: #000; + --color-dark-red: #a00; + --color-light-red: #dd0000; --color-background-page: var(--color-grey-100); --color-background-gradient-first: #5ad8f7; @@ -29,9 +31,9 @@ --color-link: #1b98f8; --color-menu-accent: #ed5900; --color-background-code: var(--color-grey-850); - --color-error: #a00; + --color-error: var(--color-dark-red); --color-error-input: #ffebeb; - --color-error-list: #dd0000; + --color-error-list: var(--color-light-red); --color-table-background: var(--color-background); --color-table-stripe: var(--color-grey-900); --color-text-tab: var(--color-white); @@ -96,6 +98,9 @@ --color-shadow-jump: var(--color-grey-500); --color-icon-github: var(--color-black); --color-icon-github-hover: var(--color-grey-300); + + --color-watch-table-error: var(--color-dark-red); + --color-watch-table-row-text: var(--color-grey-100); } html[data-darkmode="true"] { @@ -123,7 +128,6 @@ html[data-darkmode="true"] { --color-text-input-description: var(--color-grey-600); --color-text-input-placeholder: var(--color-grey-600); --color-text-watch-tag-list: #fa3e92; - --color-background-code: var(--color-grey-200); --color-background-tab: rgba(0, 0, 0, 0.2); @@ -133,6 +137,8 @@ html[data-darkmode="true"] { --color-shadow-jump: var(--color-grey-200); --color-icon-github: var(--color-white); --color-icon-github-hover: var(--color-grey-700); + --color-watch-table-error: var(--color-light-red); + --color-watch-table-row-text: var(--color-grey-800); // Anything that can't be manipulated through variables follows. .watch-controls { @@ -140,7 +146,9 @@ html[data-darkmode="true"] { opacity: 0.4; } } - + .watch-table .unviewed { + color: #fff; + } .icon-spread { filter: hue-rotate(-10deg) brightness(1.5); } diff --git a/changedetectionio/static/styles/scss/styles.scss b/changedetectionio/static/styles/scss/styles.scss index 82c11010..ebedf2da 100644 --- a/changedetectionio/static/styles/scss/styles.scss +++ b/changedetectionio/static/styles/scss/styles.scss @@ -121,12 +121,16 @@ code { width: 100%; font-size: 80%; - tr.unviewed { - font-weight: bold; + tr { + &.unviewed { + font-weight: bold; + } + color: var(--color-watch-table-row-text); } + .error { - color: var(--color-error); + color: var(--color-watch-table-error); } td { @@ -322,6 +326,12 @@ a.pure-button-selected { padding: 0.5rem 0 1rem 0; } +label { + &:hover { + cursor: pointer; + } +} + #notification-customisation { border: 1px solid var(--color-border-notification); padding: 0.5rem; diff --git a/changedetectionio/static/styles/styles.css b/changedetectionio/static/styles/styles.css index 3a7dbfd5..949ad0a3 100644 --- a/changedetectionio/static/styles/styles.css +++ b/changedetectionio/static/styles/styles.css @@ -21,6 +21,8 @@ --color-grey-850: #eee; --color-grey-900: #f2f2f2; --color-black: #000; + --color-dark-red: #a00; + --color-light-red: #dd0000; --color-background-page: var(--color-grey-100); --color-background-gradient-first: #5ad8f7; --color-background-gradient-second: #2f50af; @@ -30,9 +32,9 @@ --color-link: #1b98f8; --color-menu-accent: #ed5900; --color-background-code: var(--color-grey-850); - --color-error: #a00; + --color-error: var(--color-dark-red); --color-error-input: #ffebeb; - --color-error-list: #dd0000; + --color-error-list: var(--color-light-red); --color-table-background: var(--color-background); --color-table-stripe: var(--color-grey-900); --color-text-tab: var(--color-white); @@ -87,7 +89,9 @@ --color-text-menu-link-hover: var(--color-grey-300); --color-shadow-jump: var(--color-grey-500); --color-icon-github: var(--color-black); - --color-icon-github-hover: var(--color-grey-300); } + --color-icon-github-hover: var(--color-grey-300); + --color-watch-table-error: var(--color-dark-red); + --color-watch-table-row-text: var(--color-grey-100); } html[data-darkmode="true"] { --color-link: #59bdfb; @@ -117,9 +121,13 @@ html[data-darkmode="true"] { --color-background-snapshot-age: var(--color-grey-200); --color-shadow-jump: var(--color-grey-200); --color-icon-github: var(--color-white); - --color-icon-github-hover: var(--color-grey-700); } + --color-icon-github-hover: var(--color-grey-700); + --color-watch-table-error: var(--color-light-red); + --color-watch-table-row-text: var(--color-grey-800); } html[data-darkmode="true"] .watch-controls img { opacity: 0.4; } + html[data-darkmode="true"] .watch-table .unviewed { + color: #fff; } html[data-darkmode="true"] .icon-spread { filter: hue-rotate(-10deg) brightness(1.5); } html[data-darkmode="true"] .watch-table .title-col a[target="_blank"]::after, @@ -331,10 +339,12 @@ code { .watch-table { width: 100%; font-size: 80%; } - .watch-table tr.unviewed { - font-weight: bold; } + .watch-table tr { + color: var(--color-watch-table-row-text); } + .watch-table tr.unviewed { + font-weight: bold; } .watch-table .error { - color: var(--color-error); } + color: var(--color-watch-table-error); } .watch-table td { white-space: nowrap; } .watch-table td.title-col { @@ -470,6 +480,9 @@ a.pure-button-selected { .notifications-wrapper { padding: 0.5rem 0 1rem 0; } +label:hover { + cursor: pointer; } + #notification-customisation { border: 1px solid var(--color-border-notification); padding: 0.5rem; diff --git a/changedetectionio/templates/diff.html b/changedetectionio/templates/diff.html index d9b9195b..4c5d516b 100644 --- a/changedetectionio/templates/diff.html +++ b/changedetectionio/templates/diff.html @@ -1,5 +1,5 @@ {% extends 'base.html' %} - +{% from '_helpers.jinja' import render_field, render_checkbox_field, render_button %} {% block content %}