Re #3337 - Various fixes for 'Extract Data'

changedetectionio/blueprint/ui/views.py

@@ -93,12 +93,15 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMe
             return redirect(url_for('watchlist.index'))
 
         # For submission of requesting an extract
-        extract_form = forms.extractDataForm(request.form)
+        extract_form = forms.extractDataForm(formdata=request.form,
+                                             data={'extract_regex': request.form.get('extract_regex', '')}
+                                             )
         if not extract_form.validate():
             flash("An error occurred, please see below.", "error")
+            return _render_diff_template(uuid, extract_form)
 
         else:
-            extract_regex = request.form.get('extract_regex').strip()
+            extract_regex = request.form.get('extract_regex', '').strip()
             output = watch.extract_regex_from_all_history(extract_regex)
             if output:
                 watch_dir = os.path.join(datastore.datastore_path, uuid)
@@ -109,12 +112,11 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMe
                 response.headers['Expires'] = "0"
                 return response
 
-            flash('Nothing matches that RegEx', 'error')
+            flash('No matches found while scanning all of the watch history for that RegEx.', 'error')
-        redirect(url_for('ui_views.diff_history_page', uuid=uuid) + '#extract')
+        return redirect(url_for('ui.ui_views.diff_history_page', uuid=uuid) + '#extract')
 
-    @views_blueprint.route("/diff/<string:uuid>", methods=['GET'])
+    def _render_diff_template(uuid, extract_form=None):
-    @login_optionally_required
+        """Helper function to render the diff template with all required data"""
-    def diff_history_page(uuid):
         from changedetectionio import forms
 
         # More for testing, possible to return the first/only
@@ -128,8 +130,11 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMe
             flash("No history found for the specified link, bad link?", "error")
             return redirect(url_for('watchlist.index'))
 
-        # For submission of requesting an extract
+        # Use provided form or create a new one
-        extract_form = forms.extractDataForm(request.form)
+        if extract_form is None:
+            extract_form = forms.extractDataForm(formdata=request.form,
+                                                 data={'extract_regex': request.form.get('extract_regex', '')}
+                                                 )
 
         history = watch.history
         dates = list(history.keys())
@@ -170,7 +175,7 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMe
 
         datastore.set_last_viewed(uuid, time.time())
 
-        output = render_template("diff.html",
+        return render_template("diff.html",
                                  current_diff_url=watch['url'],
                                  from_version=str(from_version),
                                  to_version=str(to_version),
@@ -193,7 +198,10 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMe
                                  watch_a=watch
                                  )
 
-        return output
+    @views_blueprint.route("/diff/<string:uuid>", methods=['GET'])
+    @login_optionally_required
+    def diff_history_page(uuid):
+        return _render_diff_template(uuid)
 
     @views_blueprint.route("/form/add/quickwatch", methods=['POST'])
     @login_optionally_required

changedetectionio/forms.py

@@ -396,6 +396,19 @@ def validate_url(test_url):
         # This should be wtforms.validators.
         raise ValidationError('Watch protocol is not permitted by SAFE_PROTOCOL_REGEX or incorrect URL format')
 
+
+class ValidateSinglePythonRegexString(object):
+    def __init__(self, message=None):
+        self.message = message
+
+    def __call__(self, form, field):
+        try:
+            re.compile(field.data)
+        except re.error:
+            message = field.gettext('RegEx \'%s\' is not a valid regular expression.')
+            raise ValidationError(message % (field.data))
+
+
 class ValidateListRegex(object):
     """
     Validates that anything that looks like a regex passes as a regex
@@ -414,6 +427,7 @@ class ValidateListRegex(object):
                     message = field.gettext('RegEx \'%s\' is not a valid regular expression.')
                     raise ValidationError(message % (line))
 
+
 class ValidateCSSJSONXPATHInput(object):
     """
     Filter validation
@@ -791,5 +805,5 @@ class globalSettingsForm(Form):
 
 
 class extractDataForm(Form):
-    extract_regex = StringField('RegEx to extract', validators=[validators.Length(min=1, message="Needs a RegEx")])
+    extract_regex = StringField('RegEx to extract', validators=[validators.DataRequired(), ValidateSinglePythonRegexString()])
     extract_submit_button = SubmitField('Extract as CSV', render_kw={"class": "pure-button pure-button-primary"})

changedetectionio/model/Watch.py

@@ -639,7 +639,7 @@ class model(watch_base):
                     if res:
                         if not csv_writer:
                             # A file on the disk can be transferred much faster via flask than a string reply
-                            csv_output_filename = 'report.csv'
+                            csv_output_filename = f"report-{self.get('uuid')}.csv"
                             f = open(os.path.join(self.watch_data_dir, csv_output_filename), 'w')
                             # @todo some headers in the future
                             #fieldnames = ['Epoch seconds', 'Date']

changedetectionio/tests/test_extract_csv.py

@@ -46,7 +46,7 @@ def test_check_extract_text_from_diff(client, live_server, measure_memory_usage)
         follow_redirects=False
     )
 
-    assert b'Nothing matches that RegEx' not in res.data
+    assert b'No matches found while scanning all of the watch history for that RegEx.' not in res.data
     assert res.content_type == 'text/csv'
 
     # Read the csv reply as stringio