Make proxy configuration more consistent

changedetectionio/__init__.py

@@ -434,48 +434,21 @@ def changedetection_app(config=None, datastore_o=None):
     @login_required
     def scrub_page():
 
-        import re
-
         if request.method == 'POST':
             confirmtext = request.form.get('confirmtext')
-            limit_date = request.form.get('limit_date')
-            limit_timestamp = 0
-
-            # Re #149 - allow empty/0 timestamp limit
-            if len(limit_date):
-                try:
-                    limit_date = limit_date.replace('T', ' ')
-                    # I noticed chrome will show '/' but actually submit '-'
-                    limit_date = limit_date.replace('-', '/')
-                    # In the case that :ss seconds are supplied
-                    limit_date = re.sub(r'(\d\d:\d\d)(:\d\d)', '\\1', limit_date)
-
-                    str_to_dt = datetime.datetime.strptime(limit_date, '%Y/%m/%d %H:%M')
-                    limit_timestamp = int(str_to_dt.timestamp())
-
-                    if limit_timestamp > time.time():
-                        flash("Timestamp is in the future, cannot continue.", 'error')
-                        return redirect(url_for('scrub_page'))
-
-                except ValueError:
-                    flash('Incorrect date format, cannot continue.', 'error')
-                    return redirect(url_for('scrub_page'))
 
             if confirmtext == 'scrub':
                 changes_removed = 0
-                for uuid, watch in datastore.data['watching'].items():
-                    if limit_timestamp:
-                        changes_removed += datastore.scrub_watch(uuid, limit_timestamp=limit_timestamp)
-                    else:
-                        changes_removed += datastore.scrub_watch(uuid)
+                for uuid in datastore.data['watching'].keys():
+                    datastore.scrub_watch(uuid)
 
-                flash("Cleared snapshot history ({} snapshots removed)".format(changes_removed))
+                flash("Cleared all snapshot history")
             else:
                 flash('Incorrect confirmation text.', 'error')
 
             return redirect(url_for('index'))
 
-        output =  render_template("scrub.html")
+        output = render_template("scrub.html")
         return output
 
 

changedetectionio/content_fetcher.py

@@ -25,6 +25,8 @@ class Fetcher():
     # Will be needed in the future by the VisualSelector, always get this where possible.
     screenshot = False
     fetcher_description = "No description"
+    system_http_proxy = os.getenv('HTTP_PROXY')
+    system_https_proxy = os.getenv('HTTPS_PROXY')
 
     @abstractmethod
     def get_error(self):
@@ -80,17 +82,12 @@ class base_html_playwright(Fetcher):
     if os.getenv("PLAYWRIGHT_DRIVER_URL"):
         fetcher_description += " via '{}'".format(os.getenv("PLAYWRIGHT_DRIVER_URL"))
 
-#    try:
-#        from playwright.sync_api import sync_playwright
-#    except ModuleNotFoundError:
-#        fetcher_enabled = False
-
     browser_type = ''
     command_executor = ''
 
     # Configs for Proxy setup
     # In the ENV vars, is prefixed with "playwright_proxy_", so it is for example "playwright_proxy_server"
-    playwright_proxy_settings_mappings = ['server', 'bypass', 'username', 'password']
+    playwright_proxy_settings_mappings = ['bypass', 'server', 'username', 'password']
 
     proxy = None
 
@@ -130,8 +127,9 @@ class base_html_playwright(Fetcher):
             browser = browser_type.connect_over_cdp(self.command_executor, timeout=timeout * 1000)
 
             # Set user agent to prevent Cloudflare from blocking the browser
+            # Use the default one configured in the App.py model that's passed from fetch_site_status.py
             context = browser.new_context(
-                user_agent="Mozilla/5.0",
+                user_agent=request_headers['User-Agent'] if request_headers.get('User-Agent') else 'Mozilla/5.0',
                 proxy=self.proxy
             )
             page = context.new_page()
@@ -184,6 +182,12 @@ class base_html_webdriver(Fetcher):
             if v:
                 proxy_args[k] = v.strip('"')
 
+        # Map back standard HTTP_ and HTTPS_PROXY to webDriver httpProxy/sslProxy
+        if not proxy_args.get('webdriver_httpProxy') and self.system_http_proxy:
+            proxy_args['httpProxy'] = self.system_http_proxy
+        if not proxy_args.get('webdriver_sslProxy') and self.system_https_proxy:
+            proxy_args['httpsProxy'] = self.system_https_proxy
+
         if proxy_args:
             self.proxy = SeleniumProxy(raw=proxy_args)
 
@@ -259,11 +263,19 @@ class html_requests(Fetcher):
             request_method,
             ignore_status_codes=False):
 
+        # Map back standard HTTP_ and HTTPS_PROXY to requests http/https proxy
+        proxies={}
+        if self.system_http_proxy:
+            proxies['http'] = self.system_http_proxy
+        if self.system_https_proxy:
+            proxies['https'] = self.system_https_proxy
+
         r = requests.request(method=request_method,
                              data=request_body,
                              url=url,
                              headers=request_headers,
                              timeout=timeout,
+                             proxies=proxies,
                              verify=False)
 
         # If the response did not tell us what encoding format to expect, Then use chardet to override what `requests` thinks.

changedetectionio/static/js/settings.js

@@ -1,13 +0,0 @@
-window.addEventListener("load", (event) => {
-  // just an example for now
-  function toggleVisible(elem) {
-    // theres better ways todo this
-    var x = document.getElementById(elem);
-    if (x.style.display === "block") {
-      x.style.display = "none";
-    } else {
-      x.style.display = "block";
-    }
-  }
-});
-

changedetectionio/static/js/watch-settings.js

@@ -0,0 +1,14 @@
+$(document).ready(function() {
+    function toggle() {
+        if ($('input[name="fetch_backend"]:checked').val() != 'html_requests') {
+            $('#requests-override-options').hide();
+        } else {
+            $('#requests-override-options').show();
+        }
+    }
+    $('input[name="fetch_backend"]').click(function (e) {
+        toggle();
+    });
+    toggle();
+
+});

changedetectionio/store.py

@@ -145,6 +145,10 @@ class ChangeDetectionStore:
 
     def update_watch(self, uuid, update_obj):
 
+        # It's possible that the watch could be deleted before update
+        if not self.__data['watching'].get(uuid):
+            return
+
         with self.lock:
 
             # In python 3.9 we have the |= dict operator, but that still will lose data on nested structures...
@@ -256,46 +260,14 @@ class ChangeDetectionStore:
         return self.data['watching'][uuid].get(val)
 
     # Remove a watchs data but keep the entry (URL etc)
-    def scrub_watch(self, uuid, limit_timestamp = False):
+    def scrub_watch(self, uuid):
+        import pathlib
 
-        import hashlib
-        del_timestamps = []
+        self.__data['watching'][uuid].update({'history': {}, 'last_checked': 0, 'last_changed': 0, 'newest_history_key': 0, 'previous_md5': False})
+        self.needs_write_urgent = True
 
-        changes_removed = 0
-
-        for timestamp, path in self.data['watching'][uuid]['history'].items():
-            if not limit_timestamp or (limit_timestamp is not False and int(timestamp) > limit_timestamp):
-                self.unlink_history_file(path)
-                del_timestamps.append(timestamp)
-                changes_removed += 1
-
-        if not limit_timestamp:
-            self.data['watching'][uuid]['last_checked'] = 0
-            self.data['watching'][uuid]['last_changed'] = 0
-            self.data['watching'][uuid]['previous_md5'] = ""
-
-
-        for timestamp in del_timestamps:
-            del self.data['watching'][uuid]['history'][str(timestamp)]
-
-            # If there was a limitstamp, we need to reset some meta data about the entry
-            # This has to happen after we remove the others from the list
-            if limit_timestamp:
-                newest_key = self.get_newest_history_key(uuid)
-                if newest_key:
-                    self.data['watching'][uuid]['last_checked'] = int(newest_key)
-                    # @todo should be the original value if it was less than newest key
-                    self.data['watching'][uuid]['last_changed'] = int(newest_key)
-                    try:
-                        with open(self.data['watching'][uuid]['history'][str(newest_key)], "rb") as fp:
-                            content = fp.read()
-                        self.data['watching'][uuid]['previous_md5'] = hashlib.md5(content).hexdigest()
-                    except (FileNotFoundError, IOError):
-                        self.data['watching'][uuid]['previous_md5'] = ""
-                        pass
-
-        self.needs_write = True
-        return changes_removed
+        for item in pathlib.Path(self.datastore_path).rglob(uuid+"/*.txt"):
+            unlink(item)
 
     def add_watch(self, url, tag="", extras=None, write_to_disk_now=True):
         if extras is None:
@@ -449,10 +421,11 @@ class ChangeDetectionStore:
         import pathlib
 
         # Only in the sub-directories
-        for item in pathlib.Path(self.datastore_path).rglob("*/*txt"):
-            if not str(item) in index:
-                print ("Removing",item)
-                unlink(item)
+        for uuid in self.data['watching']:
+            for item in pathlib.Path(self.datastore_path).rglob(uuid+"/*.txt"):
+                if not str(item) in index:
+                    print ("Removing",item)
+                    unlink(item)
 
     # Run all updates
     # IMPORTANT - Each update could be run even when they have a new install and the schema is correct

changedetectionio/templates/edit.html

@@ -9,6 +9,7 @@
     const email_notification_prefix=JSON.parse('{{ emailprefix|tojson }}');
 {% endif %}
 </script>
+<script type="text/javascript" src="{{url_for('static_content', group='js', filename='watch-settings.js')}}" defer></script>
 <script type="text/javascript" src="{{url_for('static_content', group='js', filename='notifications.js')}}" defer></script>
 
 <div class="edit-form monospaced-textarea">
@@ -65,12 +66,10 @@
                         </span>
                     </div>
 
-                <hr/>
-                <fieldset class="pure-group">
-
-                    <span class="pure-form-message-inline">
+                <fieldset class="pure-group" id="requests-override-options">
+                    <div class="pure-form-message-inline">
                         <strong>Request override is currently only used by the <i>Basic fast Plaintext/HTTP Client</i> method.</strong>
-                    </span>
+                    </div>
                     <div class="pure-control-group">
                         {{ render_field(form.method) }}
                     </div>

changedetectionio/templates/scrub.html

@@ -7,7 +7,7 @@
         <input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
         <fieldset>
             <div class="pure-control-group">
-                This will remove all version snapshots/data, but keep your list of URLs. <br/>
+                This will remove ALL version snapshots/data, but keep your list of URLs. <br/>
                 You may like to use the <strong>BACKUP</strong> link first.<br/>
             </div>
             <br/>
@@ -17,12 +17,6 @@
                 <span class="pure-form-message-inline">Type in the word <strong>scrub</strong> to confirm that you understand!</span>
             </div>
             <br/>
-            <div class="pure-control-group">
-                <label for="confirmtext">Optional: Limit deletion of snapshots to snapshots <i>newer</i> than date/time</label>
-                <input type="datetime-local" id="limit_date" name="limit_date"  />
-                <span class="pure-form-message-inline">dd/mm/yyyy hh:mm (24 hour format)</span>
-            </div>
-            <br/>
             <div class="pure-control-group">
                 <button type="submit" class="pure-button pure-button-primary">Scrub!</button>
             </div>

changedetectionio/templates/settings.html

@@ -9,7 +9,6 @@
     const email_notification_prefix=JSON.parse('{{emailprefix|tojson}}');
 {% endif %}
 </script>
-<script type="text/javascript" src="{{url_for('static_content', group='js', filename='settings.js')}}" defer></script>
 <script type="text/javascript" src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
 <script type="text/javascript" src="{{url_for('static_content', group='js', filename='notifications.js')}}" defer></script>
 

docker-compose.yml

@@ -17,8 +17,8 @@ services:
   #       Alternative WebDriver/selenium URL, do not use "'s or 's!
   #      - WEBDRIVER_URL=http://browser-chrome:4444/wd/hub
   #
-  #       WebDriver proxy settings webdriver_proxyType, webdriver_ftpProxy, webdriver_httpProxy, webdriver_noProxy,
-  #                                webdriver_proxyAutoconfigUrl, webdriver_sslProxy, webdriver_autodetect,
+  #       WebDriver proxy settings webdriver_proxyType, webdriver_ftpProxy, webdriver_noProxy,
+  #                                webdriver_proxyAutoconfigUrl, webdriver_autodetect,
   #                                webdriver_socksProxy, webdriver_socksUsername, webdriver_socksVersion, webdriver_socksPassword
   #
   #             https://selenium-python.readthedocs.io/api.html#module-selenium.webdriver.common.proxy