Extra validation

changedetectionio/__init__.py

@@ -810,10 +810,12 @@ def changedetection_app(config=None, datastore_o=None):
 
         return redirect(url_for('index'))
 
-    @app.route("/diff/<string:uuid>", methods=['GET'])
+    @app.route("/diff/<string:uuid>", methods=['GET', 'POST'])
     @login_required
     def diff_history_page(uuid):
 
+        from changedetectionio import forms
+
         # More for testing, possible to return the first/only
         if uuid == 'first':
             uuid = list(datastore.data['watching'].keys()).pop()
@@ -825,6 +827,28 @@ def changedetection_app(config=None, datastore_o=None):
             flash("No history found for the specified link, bad link?", "error")
             return redirect(url_for('index'))
 
+        # For submission of requesting an extract
+        extract_form = forms.extractDataForm(request.form)
+        if request.method == 'POST':
+            if not extract_form.validate():
+                flash("An error occurred, please see below.", "error")
+
+            else:
+                extract_regex = request.form.get('extract_regex').strip()
+                output = watch.extract_regex_from_all_history(extract_regex)
+                if output:
+                    watch_dir = os.path.join(datastore_o.datastore_path, uuid)
+                    response = make_response(send_from_directory(directory=watch_dir, path=output, as_attachment=True))
+                    response.headers['Content-type'] = 'text/csv'
+                    response.headers['Cache-Control'] = 'no-cache, no-store, must-revalidate'
+                    response.headers['Pragma'] = 'no-cache'
+                    response.headers['Expires'] = 0
+                    return response
+
+
+                flash('Nothing matches that RegEx', 'error')
+                redirect(url_for('diff_history_page', uuid=uuid)+'#extract')
+
         history = watch.history
         dates = list(history.keys())
 
@@ -867,23 +891,24 @@ def changedetection_app(config=None, datastore_o=None):
                     watch.get('fetch_backend', None) is None and system_uses_webdriver) else False
 
         output = render_template("diff.html",
-                                 watch_a=watch,
-                                 newest=newest_version_file_contents,
-                                 previous=previous_version_file_contents,
-                                 extra_stylesheets=extra_stylesheets,
-                                 dark_mode=getDarkModeSetting(),
-                                 versions=dates[:-1], # All except current/last
-                                 uuid=uuid,
-                                 newest_version_timestamp=dates[-1],
-                                 current_previous_version=str(previous_version),
                                  current_diff_url=watch['url'],
+                                 current_previous_version=str(previous_version),
+                                 dark_mode=getDarkModeSetting(),
+                                 extra_stylesheets=extra_stylesheets,
                                  extra_title=" - Diff - {}".format(watch['title'] if watch['title'] else watch['url']),
-                                 left_sticky=True,
-                                 screenshot=screenshot_url,
+                                 extract_form=extract_form,
                                  is_html_webdriver=is_html_webdriver,
                                  last_error=watch['last_error'],
+                                 last_error_screenshot=watch.get_error_snapshot(),
                                  last_error_text=watch.get_error_text(),
-                                 last_error_screenshot=watch.get_error_snapshot()
+                                 left_sticky=True,
+                                 newest=newest_version_file_contents,
+                                 newest_version_timestamp=dates[-1],
+                                 previous=previous_version_file_contents,
+                                 screenshot=screenshot_url,
+                                 uuid=uuid,
+                                 versions=dates[:-1], # All except current/last
+                                 watch_a=watch
                                  )
 
         return output

changedetectionio/forms.py

@@ -448,3 +448,8 @@ class globalSettingsForm(Form):
     requests = FormField(globalSettingsRequestForm)
     application = FormField(globalSettingsApplicationForm)
     save_button = SubmitField('Save', render_kw={"class": "pure-button pure-button-primary"})
+
+
+class extractDataForm(Form):
+    extract_regex = StringField('RegEx to extract', validators=[validators.Length(min=1, message="Needs a RegEx")])
+    extract_submit_button = SubmitField('Extract as CSV', render_kw={"class": "pure-button pure-button-primary"})

changedetectionio/model/Watch.py

@@ -318,3 +318,47 @@ class model(dict):
         if os.path.isfile(fname):
             return fname
         return False
+
+    def extract_regex_from_all_history(self, regex):
+        import csv
+        import re
+        import datetime
+        csv_output_filename = False
+        csv_writer = False
+        f = None
+
+        # self.history will be keyed with the full path
+        for k, fname in self.history.items():
+            if os.path.isfile(fname):
+                with open(fname, "r") as f:
+                    contents = f.read()
+                    res = re.findall(regex, contents, re.MULTILINE)
+                    if res:
+                        if not csv_writer:
+                            # A file on the disk can be transferred much faster via flask than a string reply
+                            csv_output_filename = 'report.csv'
+                            f = open(os.path.join(self.watch_data_dir, csv_output_filename), 'w')
+                            # @todo some headers in the future
+                            #fieldnames = ['Epoch seconds', 'Date']
+                            csv_writer = csv.writer(f,
+                                                    delimiter=',',
+                                                    quotechar='"',
+                                                    quoting=csv.QUOTE_MINIMAL,
+                                                    #fieldnames=fieldnames
+                                                    )
+                            csv_writer.writerow(['Epoch seconds', 'Date'])
+                            # csv_writer.writeheader()
+
+                        date_str = datetime.datetime.fromtimestamp(int(k)).strftime('%Y-%m-%d %H:%M:%S')
+                        for r in res:
+                            row = [k, date_str]
+                            if isinstance(r, str):
+                                row.append(r)
+                            else:
+                                row+=r
+                            csv_writer.writerow(row)
+
+        if f:
+            f.close()
+
+        return csv_output_filename

changedetectionio/static/js/diff-overview.js

@@ -13,6 +13,8 @@ $(document).ready(function () {
         } else if (hash_name === '#error-screenshot') {
             $("img#error-screenshot-img").attr('src', error_screenshot_url);
             $("#settings").hide();
+        } else if (hash_name === '#extract') {
+            $("#settings").hide();
         }
 
 

changedetectionio/static/js/toggle-theme.js

@@ -19,6 +19,6 @@ $(document).ready(function () {
   };
 
   const setCookieValue = (value) => {
-    document.cookie = `css_dark_mode=${value};max-age=31536000`
+    document.cookie = `css_dark_mode=${value};max-age=31536000;path=/`
   }
 });

changedetectionio/static/styles/diff.css

@@ -18,6 +18,8 @@
   --color-grey-850: #eee;
   --color-grey-900: #f2f2f2;
   --color-black: #000;
+  --color-dark-red: #a00;
+  --color-light-red: #dd0000;
   --color-background-page: var(--color-grey-100);
   --color-background-gradient-first: #5ad8f7;
   --color-background-gradient-second: #2f50af;
@@ -27,9 +29,9 @@
   --color-link: #1b98f8;
   --color-menu-accent: #ed5900;
   --color-background-code: var(--color-grey-850);
-  --color-error: #a00;
+  --color-error: var(--color-dark-red);
   --color-error-input: #ffebeb;
-  --color-error-list: #dd0000;
+  --color-error-list: var(--color-light-red);
   --color-table-background: var(--color-background);
   --color-table-stripe: var(--color-grey-900);
   --color-text-tab: var(--color-white);
@@ -84,7 +86,9 @@
   --color-text-menu-link-hover: var(--color-grey-300);
   --color-shadow-jump: var(--color-grey-500);
   --color-icon-github: var(--color-black);
-  --color-icon-github-hover: var(--color-grey-300); }
+  --color-icon-github-hover: var(--color-grey-300);
+  --color-watch-table-error: var(--color-dark-red);
+  --color-watch-table-row-text: var(--color-grey-100); }
 
 html[data-darkmode="true"] {
   --color-link: #59bdfb;
@@ -114,9 +118,13 @@ html[data-darkmode="true"] {
   --color-background-snapshot-age: var(--color-grey-200);
   --color-shadow-jump: var(--color-grey-200);
   --color-icon-github: var(--color-white);
-  --color-icon-github-hover: var(--color-grey-700); }
+  --color-icon-github-hover: var(--color-grey-700);
+  --color-watch-table-error: var(--color-light-red);
+  --color-watch-table-row-text: var(--color-grey-800); }
   html[data-darkmode="true"] .watch-controls img {
     opacity: 0.4; }
+  html[data-darkmode="true"] .watch-table .unviewed {
+    color: #fff; }
   html[data-darkmode="true"] .icon-spread {
     filter: hue-rotate(-10deg) brightness(1.5); }
   html[data-darkmode="true"] .watch-table .title-col a[target="_blank"]::after,
@@ -128,8 +136,9 @@ html[data-darkmode="true"] {
   padding: 2em;
   margin-left: 1em;
   margin-right: 1em;
-  border-radius: 5px;
-  font-size: 11px; }
+  border-radius: 5px; }
+  #diff-ui #text {
+    font-size: 11px; }
   #diff-ui table {
     table-layout: fixed;
     width: 100%; }

changedetectionio/static/styles/scss/diff.scss

@@ -7,7 +7,11 @@
   margin-left: 1em;
   margin-right: 1em;
   border-radius: 5px;
-  font-size: 11px;
+
+  // The first tab 'text' diff
+  #text {
+    font-size: 11px;
+  }
 
   table {
     table-layout: fixed;

changedetectionio/static/styles/scss/parts/_variables.scss

@@ -19,6 +19,8 @@
   --color-grey-850: #eee;
   --color-grey-900: #f2f2f2;
   --color-black: #000;
+  --color-dark-red: #a00;
+  --color-light-red: #dd0000;
 
   --color-background-page: var(--color-grey-100);
   --color-background-gradient-first: #5ad8f7;
@@ -29,9 +31,9 @@
   --color-link: #1b98f8;
   --color-menu-accent: #ed5900;
   --color-background-code: var(--color-grey-850);
-  --color-error: #a00;
+  --color-error: var(--color-dark-red);
   --color-error-input: #ffebeb;
-  --color-error-list: #dd0000;
+  --color-error-list: var(--color-light-red);
   --color-table-background: var(--color-background);
   --color-table-stripe: var(--color-grey-900);
   --color-text-tab: var(--color-white);
@@ -96,6 +98,9 @@
   --color-shadow-jump: var(--color-grey-500);
   --color-icon-github: var(--color-black);
   --color-icon-github-hover: var(--color-grey-300);
+
+  --color-watch-table-error: var(--color-dark-red);
+  --color-watch-table-row-text: var(--color-grey-100);
 }
 
 html[data-darkmode="true"] {
@@ -123,7 +128,6 @@ html[data-darkmode="true"] {
   --color-text-input-description: var(--color-grey-600);
   --color-text-input-placeholder: var(--color-grey-600);
   --color-text-watch-tag-list: #fa3e92;
-
   --color-background-code: var(--color-grey-200);
 
   --color-background-tab: rgba(0, 0, 0, 0.2);
@@ -133,6 +137,8 @@ html[data-darkmode="true"] {
   --color-shadow-jump: var(--color-grey-200);
   --color-icon-github: var(--color-white);
   --color-icon-github-hover: var(--color-grey-700);
+  --color-watch-table-error: var(--color-light-red);
+  --color-watch-table-row-text: var(--color-grey-800);
 
   // Anything that can't be manipulated through variables follows.
   .watch-controls {
@@ -140,7 +146,9 @@ html[data-darkmode="true"] {
       opacity: 0.4;
     }
   }
-
+  .watch-table .unviewed {
+    color: #fff;
+  }
   .icon-spread {
     filter: hue-rotate(-10deg) brightness(1.5);
   }

changedetectionio/static/styles/scss/styles.scss

@@ -121,12 +121,16 @@ code {
   width: 100%;
   font-size: 80%;
 
-  tr.unviewed {
-    font-weight: bold;
+  tr {
+    &.unviewed {
+      font-weight: bold;
+    }
+    color: var(--color-watch-table-row-text);
   }
 
+
   .error {
-    color: var(--color-error);
+    color: var(--color-watch-table-error);
   }
 
   td {
@@ -322,6 +326,12 @@ a.pure-button-selected {
   padding: 0.5rem 0 1rem 0;
 }
 
+label {
+ &:hover {
+   cursor: pointer;
+ }  
+}
+
 #notification-customisation {
   border: 1px solid var(--color-border-notification);
   padding: 0.5rem;

changedetectionio/static/styles/styles.css

@@ -21,6 +21,8 @@
   --color-grey-850: #eee;
   --color-grey-900: #f2f2f2;
   --color-black: #000;
+  --color-dark-red: #a00;
+  --color-light-red: #dd0000;
   --color-background-page: var(--color-grey-100);
   --color-background-gradient-first: #5ad8f7;
   --color-background-gradient-second: #2f50af;
@@ -30,9 +32,9 @@
   --color-link: #1b98f8;
   --color-menu-accent: #ed5900;
   --color-background-code: var(--color-grey-850);
-  --color-error: #a00;
+  --color-error: var(--color-dark-red);
   --color-error-input: #ffebeb;
-  --color-error-list: #dd0000;
+  --color-error-list: var(--color-light-red);
   --color-table-background: var(--color-background);
   --color-table-stripe: var(--color-grey-900);
   --color-text-tab: var(--color-white);
@@ -87,7 +89,9 @@
   --color-text-menu-link-hover: var(--color-grey-300);
   --color-shadow-jump: var(--color-grey-500);
   --color-icon-github: var(--color-black);
-  --color-icon-github-hover: var(--color-grey-300); }
+  --color-icon-github-hover: var(--color-grey-300);
+  --color-watch-table-error: var(--color-dark-red);
+  --color-watch-table-row-text: var(--color-grey-100); }
 
 html[data-darkmode="true"] {
   --color-link: #59bdfb;
@@ -117,9 +121,13 @@ html[data-darkmode="true"] {
   --color-background-snapshot-age: var(--color-grey-200);
   --color-shadow-jump: var(--color-grey-200);
   --color-icon-github: var(--color-white);
-  --color-icon-github-hover: var(--color-grey-700); }
+  --color-icon-github-hover: var(--color-grey-700);
+  --color-watch-table-error: var(--color-light-red);
+  --color-watch-table-row-text: var(--color-grey-800); }
   html[data-darkmode="true"] .watch-controls img {
     opacity: 0.4; }
+  html[data-darkmode="true"] .watch-table .unviewed {
+    color: #fff; }
   html[data-darkmode="true"] .icon-spread {
     filter: hue-rotate(-10deg) brightness(1.5); }
   html[data-darkmode="true"] .watch-table .title-col a[target="_blank"]::after,
@@ -331,10 +339,12 @@ code {
 .watch-table {
   width: 100%;
   font-size: 80%; }
-  .watch-table tr.unviewed {
-    font-weight: bold; }
+  .watch-table tr {
+    color: var(--color-watch-table-row-text); }
+    .watch-table tr.unviewed {
+      font-weight: bold; }
   .watch-table .error {
-    color: var(--color-error); }
+    color: var(--color-watch-table-error); }
   .watch-table td {
     white-space: nowrap; }
   .watch-table td.title-col {
@@ -470,6 +480,9 @@ a.pure-button-selected {
 .notifications-wrapper {
   padding: 0.5rem 0 1rem 0; }
 
+label:hover {
+  cursor: pointer; }
+
 #notification-customisation {
   border: 1px solid var(--color-border-notification);
   padding: 0.5rem;

changedetectionio/templates/base.html

@@ -86,7 +86,7 @@
             {% if dark_mode %}
             {% set darkClass = 'dark' %}
             {% endif %}
-            <button class="toggle-theme {{darkClass}}" type="button">
+            <button class="toggle-theme {{darkClass}}" type="button" title="Toggle Light/Dark Mode">
               <span class="visually-hidden">Toggle light/dark mode</span>
               <span class="icon-light">
                 {% include "svgs/light-mode-toggle-icon.svg" %}

changedetectionio/templates/diff.html

@@ -1,5 +1,5 @@
 {% extends 'base.html' %}
-
+{% from '_helpers.jinja' import render_field, render_checkbox_field, render_button %}
 {% block content %}
 <script>
     const screenshot_url="{{url_for('static_content', group='screenshot', filename=uuid)}}";
@@ -58,6 +58,7 @@
         {% if last_error_screenshot %}<li class="tab" id="error-screenshot-tab"><a href="#error-screenshot">Error Screenshot</a></li> {% endif %}
         <li class="tab" id=""><a href="#text">Text</a></li>
         <li class="tab" id="screenshot-tab"><a href="#screenshot">Screenshot</a></li>
+        <li class="tab" id="extract-tab"><a href="#extract">Extract Data</a></li>
     </ul>
 </div>
 
@@ -108,6 +109,37 @@
            <strong>Screenshot requires Playwright/WebDriver enabled</strong>
          {% endif %}
      </div>
+    <div class="tab-pane-inner" id="extract">
+        <form id="extract-data-form" class="pure-form pure-form-stacked edit-form"
+              action="{{ url_for('diff_history_page', uuid=uuid) }}#extract"
+              method="POST">
+            <input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
+
+            <p>This tool will extract text data from all of the watch history.</p>
+
+            <div class="pure-control-group">
+                {{ render_field(extract_form.extract_regex) }}
+                <span class="pure-form-message-inline">
+                    A <strong>RegEx</strong> is a pattern that identifies exactly which part inside of the text that you want to extract.<br/>
+
+                    <p>
+                        For example, to extract only the numbers from text &dash;</br>
+                        <strong>Raw text</strong>: <code>Temperature <span style="color: red">5.5</span>°C in Sydney</code></br>
+                        <strong>RegEx to extract:</strong> <code>Temperature ([0-9\.]+)</code><br/>
+                    </p>
+                    <p>
+                        <a href="https://RegExr.com/">Be sure to test your RegEx here.</a>
+                    </p>
+                    <p>
+                        Each RegEx group bracket <code>()</code> will be in its own column, the first column value is always the date.
+                    </p>
+                </span>
+            </div>
+            <div class="pure-control-group">
+                {{ render_button(extract_form.extract_submit_button) }}
+            </div>
+        </form>
+    </div>
 </div>
 
 <script>

changedetectionio/templates/edit.html

@@ -244,6 +244,7 @@ xpath://body/div/span[contains(@class, 'example-class')]",
                           <span class="pure-form-message-inline"><strong>Note!: //text() function does not work where the &lt;element&gt; contains &lt;![CDATA[]]&gt;</strong></span><br/>
                         {% endif %}
                         <span class="pure-form-message-inline">One rule per line, <i>any</i> rules that matches will be used.<br/>
+
                     <ul>
                         <li>CSS - Limit text to this CSS rule, only text matching this CSS rule is included.</li>
                         <li>JSON - Limit text to this JSON rule, using either <a href="https://pypi.org/project/jsonpath-ng/" target="new">JSONPath</a> or <a href="https://stedolan.github.io/jq/" target="new">jq</a> (if installed).

changedetectionio/tests/test_extract_csv.py

@@ -0,0 +1,70 @@
+#!/usr/bin/python3
+
+import time
+from flask import url_for
+from urllib.request import urlopen
+from .util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks
+
+sleep_time_for_fetch_thread = 3
+
+
+
+def test_check_extract_text_from_diff(client, live_server):
+    import time
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write("Now it's {} seconds since epoch, time flies!".format(str(time.time())))
+
+    live_server_setup(live_server)
+
+    # Add our URL to the import page
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": url_for('test_endpoint', _external=True)},
+        follow_redirects=True
+    )
+
+    assert b"1 Imported" in res.data
+    time.sleep(1)
+
+    # Load in 5 different numbers/changes
+    last_date=""
+    for n in range(5):
+        # Give the thread time to pick it up
+        print("Bumping snapshot and checking.. ", n)
+        last_date = str(time.time())
+        with open("test-datastore/endpoint-content.txt", "w") as f:
+            f.write("Now it's {} seconds since epoch, time flies!".format(last_date))
+
+        client.get(url_for("form_watch_checknow"), follow_redirects=True)
+        wait_for_all_checks(client)
+
+    res = client.post(
+        url_for("diff_history_page", uuid="first"),
+        data={"extract_regex": "Now it's ([0-9\.]+)",
+              "extract_submit_button": "Extract as CSV"},
+        follow_redirects=False
+    )
+
+    assert b'Nothing matches that RegEx' not in res.data
+    assert res.content_type == 'text/csv'
+
+    # Read the csv reply as stringio
+    from io import StringIO
+    import csv
+
+    f = StringIO(res.data.decode('utf-8'))
+    reader = csv.reader(f, delimiter=',')
+    output=[]
+
+    for row in reader:
+        output.append(row)
+
+    assert output[0][0] == 'Epoch seconds'
+
+    # Header line + 1 origin/first + 5 changes
+    assert(len(output) == 7)
+
+    # We expect to find the last bumped date in the changes in the last field of the spreadsheet
+    assert(output[6][2] == last_date)
+    # And nothing else, only that group () of the decimal and .
+    assert "time flies" not in output[6][2]