Merge branch 'master' into hours-day-schedule

changedetectionio/__init__.py

@@ -566,23 +566,12 @@ def changedetection_app(config=None, datastore_o=None):
             for p in datastore.proxy_list:
                 form.proxy.choices.append(tuple((p, datastore.proxy_list[p]['label'])))
 
-
         if request.method == 'POST' and form.validate():
             extra_update_obj = {}
 
             if request.args.get('unpause_on_save'):
                 extra_update_obj['paused'] = False
 
-            # Re #110, if they submit the same as the default value, set it to None, so we continue to follow the default
-            # Assume we use the default value, unless something relevant is different, then use the form value
-            # values could be None, 0 etc.
-            # Set to None unless the next for: says that something is different
-            extra_update_obj['time_between_check'] = dict.fromkeys(form.time_between_check.data)
-            for k, v in form.time_between_check.data.items():
-                if v and v != datastore.data['settings']['requests']['time_between_check'][k]:
-                    extra_update_obj['time_between_check'] = form.time_between_check.data
-                    using_default_check_time = False
-                    break
 
             # Use the default if its the same as system wide
             if form.fetch_backend.data == datastore.data['settings']['application']['fetch_backend']:
@@ -732,13 +721,19 @@ def changedetection_app(config=None, datastore_o=None):
             else:
                 flash("An error occurred, please see below.", "error")
 
+        import datetime
+        datetime = datetime.datetime.now(pytz.timezone(datastore.data['settings']['application'].get('timezone')))
+
         output = render_template("settings.html",
-                                 form=form,
-                                 current_base_url = datastore.data['settings']['application']['base_url'],
-                                 hide_remove_pass=os.getenv("SALTED_PASS", False),
                                  api_key=datastore.data['settings']['application'].get('api_access_token'),
+                                 current_base_url=datastore.data['settings']['application']['base_url'],
+                                 datetime=str(datetime),
                                  emailprefix=os.getenv('NOTIFICATION_MAIL_BUTTON_PREFIX', False),
-                                 settings_application=datastore.data['settings']['application'])
+                                 form=form,
+                                 hide_remove_pass=os.getenv("SALTED_PASS", False),
+                                 settings_application=datastore.data['settings']['application'],
+                                 timezone=datastore.data['settings']['application'].get('timezone')
+                                 )
 
         return output
 
@@ -1448,8 +1443,12 @@ def ticker_thread_check_time_launch_checks():
             seconds_since_last_recheck = now - watch['last_checked']
 
             if seconds_since_last_recheck >= (threshold + watch.jitter_seconds) and seconds_since_last_recheck >= recheck_time_minimum_seconds:
-                if not uuid in running_uuids and uuid not in [q_uuid for p,q_uuid in update_q.queue]:
 
+                if not watch.is_schedule_permitted:
+                    # Skip if the schedule (day of week and time) isnt permitted
+                    continue
+
+                if not uuid in running_uuids and uuid not in [q_uuid for p,q_uuid in update_q.queue]:
                     # Proxies can be set to have a limit on seconds between which they can be called
                     watch_proxy = datastore.get_preferred_proxy_for_watch(uuid=uuid)
                     if watch_proxy and watch_proxy in list(datastore.proxy_list.keys()):

changedetectionio/forms.py

@@ -1,5 +1,5 @@
 import re
-
+import pytz
 from wtforms import (
     BooleanField,
     Field,
@@ -8,9 +8,11 @@ from wtforms import (
     PasswordField,
     RadioField,
     SelectField,
+    SelectMultipleField,
     StringField,
     SubmitField,
     TextAreaField,
+    TimeField,
     fields,
     validators,
     widgets,
@@ -97,6 +99,44 @@ class TimeBetweenCheckForm(Form):
     seconds = IntegerField('Seconds', validators=[validators.Optional(), validators.NumberRange(min=0, message="Should contain zero or more seconds")])
     # @todo add total seconds minimum validatior = minimum_seconds_recheck_time
 
+class MultiCheckboxDayOfWeekField(SelectMultipleField):
+    widget = widgets.ListWidget(prefix_label=False)
+    option_widget = widgets.CheckboxInput()
+
+class TimeScheduleCheckLimitForm(Form):
+    # @todo must be a better python way todo this c/i list
+    c=[]
+    i=0
+    for d in ['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun']:
+        c.append((i, d))
+        i+=1
+    day_of_week = MultiCheckboxDayOfWeekField('',coerce=int, choices=c)
+    from_time = TimeField('From', validators=[validators.Optional()])
+    until_time = TimeField('Until', validators=[validators.Optional()])
+
+    def validate(self, **kwargs):
+        if not super().validate():
+            return False
+
+        result = True
+
+        f = self.data.get('from_time')
+        u = self.data.get('until_time')
+        if f and u:
+            import time
+            f = time.strptime(str(f), '%H:%M:%S')
+            u = time.strptime(str(u), '%H:%M:%S')
+            if f >= u:
+                #@todo doesnt present
+                self.from_time.errors.append('From time must be LESS than the until/end time')
+                result = False
+
+        if len(self.data.get('day_of_week', [])) == 0:
+            self.day_of_week.errors.append('No day selected')
+            result = False
+
+        return result
+
 # Separated by  key:value
 class StringDictKeyValue(StringField):
     widget = widgets.TextArea()
@@ -347,27 +387,22 @@ class watchForm(commonSettingsForm):
     url = fields.URLField('URL', validators=[validateURL()])
     tag = StringField('Group tag', [validators.Optional()], default='')
 
-    time_between_check = FormField(TimeBetweenCheckForm)
-
-    include_filters = StringListField('CSS/JSONPath/JQ/XPath Filters', [ValidateCSSJSONXPATHInput()], default='')
-
-    subtractive_selectors = StringListField('Remove elements', [ValidateCSSJSONXPATHInput(allow_xpath=False, allow_json=False)])
-
-    extract_text = StringListField('Extract text', [ValidateListRegex()])
-
-    title = StringField('Title', default='')
-
-    ignore_text = StringListField('Ignore text', [ValidateListRegex()])
-    headers = StringDictKeyValue('Request headers')
     body = TextAreaField('Request body', [validators.Optional()])
-    method = SelectField('Request method', choices=valid_method, default=default_method)
-    ignore_status_codes = BooleanField('Ignore status codes (process non-2xx status codes as normal)', default=False)
     check_unique_lines = BooleanField('Only trigger when new lines appear', default=False)
-    trigger_text = StringListField('Trigger/wait for text', [validators.Optional(), ValidateListRegex()])
+    extract_text = StringListField('Extract text', [ValidateListRegex()])
+    headers = StringDictKeyValue('Request headers')
+    ignore_status_codes = BooleanField('Ignore status codes (process non-2xx status codes as normal)', default=False)
+    ignore_text = StringListField('Ignore text', [ValidateListRegex()])
+    include_filters = StringListField('CSS/JSONPath/JQ/XPath Filters', [ValidateCSSJSONXPATHInput()], default='')
+    method = SelectField('Request method', choices=valid_method, default=default_method)
+    subtractive_selectors = StringListField('Remove elements', [ValidateCSSJSONXPATHInput(allow_xpath=False, allow_json=False)])
     text_should_not_be_present = StringListField('Block change-detection if text matches', [validators.Optional(), ValidateListRegex()])
-
+    time_between_check = FormField(TimeBetweenCheckForm)
+    time_schedule_check_limit = FormField(TimeScheduleCheckLimitForm)
+    time_use_system_default = BooleanField('Use system/default check time', default=False, validators=[validators.Optional()])
+    title = StringField('Title', default='')
+    trigger_text = StringListField('Trigger/wait for text', [validators.Optional(), ValidateListRegex()])
     webdriver_js_execute_code = TextAreaField('Execute JavaScript before change detection', render_kw={"rows": "5"}, validators=[validators.Optional()])
-
     save_button = SubmitField('Save', render_kw={"class": "pure-button pure-button-primary"})
 
     proxy = RadioField('Proxy')
@@ -389,10 +424,10 @@ class watchForm(commonSettingsForm):
 
         return result
 
-
 # datastore.data['settings']['requests']..
 class globalSettingsRequestForm(Form):
     time_between_check = FormField(TimeBetweenCheckForm)
+    time_schedule_check_limit = FormField(TimeScheduleCheckLimitForm)
     proxy = RadioField('Proxy')
     jitter_seconds = IntegerField('Random jitter seconds ± check',
                                   render_kw={"style": "width: 5em;"},
@@ -401,21 +436,21 @@ class globalSettingsRequestForm(Form):
 # datastore.data['settings']['application']..
 class globalSettingsApplicationForm(commonSettingsForm):
 
-    base_url = StringField('Base URL', validators=[validators.Optional()])
-    global_subtractive_selectors = StringListField('Remove elements', [ValidateCSSJSONXPATHInput(allow_xpath=False, allow_json=False)])
-    global_ignore_text = StringListField('Ignore Text', [ValidateListRegex()])
-    ignore_whitespace = BooleanField('Ignore whitespace')
-    removepassword_button = SubmitField('Remove password', render_kw={"class": "pure-button pure-button-primary"})
-    empty_pages_are_a_change =  BooleanField('Treat empty pages as a change?', default=False)
-    render_anchor_tag_content = BooleanField('Render anchor tag content', default=False)
-    fetch_backend = RadioField('Fetch Method', default="html_requests", choices=content_fetcher.available_fetchers(), validators=[ValidateContentFetcherIsReady()])
     api_access_token_enabled = BooleanField('API access token security check enabled', default=True, validators=[validators.Optional()])
-    password = SaltyPasswordField()
+    base_url = StringField('Base URL', validators=[validators.Optional()])
-
+    empty_pages_are_a_change =  BooleanField('Treat empty pages as a change?', default=False)
+    fetch_backend = RadioField('Fetch Method', default="html_requests", choices=content_fetcher.available_fetchers(), validators=[ValidateContentFetcherIsReady()])
     filter_failure_notification_threshold_attempts = IntegerField('Number of times the filter can be missing before sending a notification',
                                                                   render_kw={"style": "width: 5em;"},
                                                                   validators=[validators.NumberRange(min=0,
                                                                                                      message="Should contain zero or more attempts")])
+    global_ignore_text = StringListField('Ignore Text', [ValidateListRegex()])
+    global_subtractive_selectors = StringListField('Remove elements', [ValidateCSSJSONXPATHInput(allow_xpath=False, allow_json=False)])
+    ignore_whitespace = BooleanField('Ignore whitespace')
+    password = SaltyPasswordField()
+    removepassword_button = SubmitField('Remove password', render_kw={"class": "pure-button pure-button-primary"})
+    render_anchor_tag_content = BooleanField('Render anchor tag content', default=False)
+    timezone = SelectField('Timezone', choices=pytz.all_timezones)
 
 
 class globalSettingsForm(Form):

changedetectionio/model/App.py

@@ -17,29 +17,31 @@ class model(dict):
                 'requests': {
                     'timeout': int(getenv("DEFAULT_SETTINGS_REQUESTS_TIMEOUT", "45")),  # Default 45 seconds
                     'time_between_check': {'weeks': None, 'days': None, 'hours': 3, 'minutes': None, 'seconds': None},
+                    'time_schedule_check_limit': {'day_of_week': [0, 1, 2, 3, 4, 5, 6], 'time_from': '', 'time_until': ''},
                     'jitter_seconds': 0,
                     'workers': int(getenv("DEFAULT_SETTINGS_REQUESTS_WORKERS", "10")),  # Number of threads, lower is better for slow connections
                     'proxy': None # Preferred proxy connection
                 },
                 'application': {
+                    # Custom notification content
                     'api_access_token_enabled': True,
-                    'password': False,
                     'base_url' : None,
-                    'extract_title_as_title': False,
                     'empty_pages_are_a_change': False,
+                    'extract_title_as_title': False,
                     'fetch_backend': getenv("DEFAULT_FETCH_BACKEND", "html_requests"),
                     'filter_failure_notification_threshold_attempts': _FILTER_FAILURE_THRESHOLD_ATTEMPTS_DEFAULT,
                     'global_ignore_text': [], # List of text to ignore when calculating the comparison checksum
                     'global_subtractive_selectors': [],
                     'ignore_whitespace': True,
-                    'render_anchor_tag_content': False,
-                    'notification_urls': [], # Apprise URL list
-                    # Custom notification content
-                    'notification_title': default_notification_title,
                     'notification_body': default_notification_body,
                     'notification_format': default_notification_format,
+                    'notification_title': default_notification_title,
+                    'notification_urls': [], # Apprise URL list
+                    'password': False,
+                    'render_anchor_tag_content': False,
                     'schema_version' : 0,
-                    'webdriver_delay': None  # Extra delay in seconds before extracting text
+                    'timezone': 'UTC',
+                    'webdriver_delay': None,  # Extra delay in seconds before extracting text
                 }
             }
         }

changedetectionio/model/Watch.py

@@ -49,6 +49,7 @@ class model(dict):
             # Requires setting to None on submit if it's the same as the default
             # Should be all None by default, so we use the system default in this case.
             'time_between_check': {'weeks': None, 'days': None, 'hours': None, 'minutes': None, 'seconds': None},
+            'time_schedule_check_limit': {'day_of_week': [0, 1, 2, 3, 4, 5, 6], 'time_from': '', 'time_until': ''},
             'title': None,
             'trigger_text': [],  # List of text or regex to wait for until a change is detected
             'url': None,
@@ -228,6 +229,11 @@ class model(dict):
                 seconds += x * n
         return seconds
 
+    def is_schedule_permitted(self):
+        """According to the current day of week and time, is this watch queueable?"""
+
+        return True
+
     # Iterate over all history texts and see if something new exists
     def lines_contain_something_unique_compared_to_history(self, lines: list):
         local_lines = set([l.decode('utf-8').strip().lower() for l in lines])

changedetectionio/static/styles/styles.css

@@ -132,7 +132,7 @@ body:after, body:before {
 
 .fetch-error {
   padding-top: 1em;
-  font-size: 60%;
+  font-size: 80%;
   max-width: 400px;
   display: block; }
 
@@ -480,6 +480,22 @@ ul {
   .time-check-widget tr input[type="number"] {
     width: 5em; }
 
+.pure-control-group table label {
+  color: #333;
+  font-weight: normal; }
+
+.time-schedule-check-limit-widget tr {
+  display: inline-block; }
+
+.time-schedule-check-limit-widget li {
+  text-decoration: none; }
+
+.time-schedule-check-limit-widget ul {
+  padding-left: 0px; }
+  .time-schedule-check-limit-widget ul li {
+    display: inline-block;
+    width: 3em; }
+
 #selector-wrapper {
   height: 600px;
   overflow-y: scroll;

changedetectionio/static/styles/styles.scss

@@ -677,6 +677,29 @@ ul {
         }
     }
 }
+.pure-control-group table  label {
+  color: #333;
+  font-weight: normal;
+}
+
+.time-schedule-check-limit-widget {
+  tr {
+    display: inline-block;
+  }
+
+  li {
+    text-decoration: none;
+  }
+
+  ul {
+    padding-left: 0px;
+    li {
+      display: inline-block;
+      width: 3em;
+    }
+  }
+}
+
 
 #selector-wrapper {
  height: 600px;

changedetectionio/templates/edit.html

@@ -51,14 +51,15 @@
                         <span class="pure-form-message-inline">Organisational tag/group name used in the main listing page</span>
                     </div>
                     <div class="pure-control-group">
+                        {{ render_checkbox_field(form.time_use_system_default) }}
+                        <div style="opacity: 0.5">
                         {{ render_field(form.time_between_check, class="time-check-widget") }}
+                        {{ render_field(form.time_schedule_check_limit, class="time-schedule-check-limit-widget") }}
+@todo - add 'use default' checkbox
+                        </div>
                         {% if has_empty_checktime %}
-                        <span class="pure-form-message-inline">Currently using the <a
-                                href="{{ url_for('settings_page', uuid=uuid) }}">default global settings</a>, change to another value if you want to be specific.</span>
-                        {% else %}
-                        <span class="pure-form-message-inline">Set to blank to use the <a
-                                href="{{ url_for('settings_page', uuid=uuid) }}">default global settings</a>.</span>
                         {% endif %}
+
                     </div>
                     <div class="pure-control-group">
                         {{ render_checkbox_field(form.extract_title_as_title) }}

changedetectionio/templates/settings.html

@@ -21,6 +21,7 @@
             <li class="tab"><a href="#fetching">Fetching</a></li>
             <li class="tab"><a href="#filters">Global Filters</a></li>
             <li class="tab"><a href="#api">API</a></li>
+            <li class="tab"><a href="#date-time">Date &amp; Time</a></li>
         </ul>
     </div>
     <div class="box-wrap inner">
@@ -30,6 +31,7 @@
                 <fieldset>
                     <div class="pure-control-group">
                         {{ render_field(form.requests.form.time_between_check, class="time-check-widget") }}
+                        {{ render_field(form.requests.form.time_schedule_check_limit, class="time-schedule-check-limit-widget") }}
                         <span class="pure-form-message-inline">Default time for all watches, when the watch does not have a specific time setting.</span>
                     </div>
                     <div class="pure-control-group">
@@ -91,7 +93,6 @@
                     </div>
                 </fieldset>
             </div>
-
             <div class="tab-pane-inner" id="fetching">
                 <div class="pure-control-group inline-radio">
                     {{ render_field(form.application.form.fetch_backend, class="fetch-backend") }}
@@ -170,6 +171,19 @@ nav
                     </div>
                 </div>
             </div>
+            <div class="tab-pane-inner" id="date-time">
+                <fieldset>
+                    <div class="field-group">
+                        {{ render_field(form.application.form.timezone) }}
+                    </div>
+                    <div class="field-group">
+                        <p>
+                        <label>Local time</label> {{ datetime }}<br/>
+                        <label>Configured timezone:</label> {{ timezone }}<br/>
+                        </p>
+                    </div>
+                </fieldset>
+            </div>
 
             <div id="actions">
                 <div class="pure-control-group">