Tidy up list icons a bit

.github/workflows/test-only.yml

@@ -60,6 +60,7 @@ jobs:
           cd changedetectionio
           ./run_proxy_tests.sh
           cd ..
+          
 
       - name: Test changedetection.io container starts+runs basically without error
         run: |
@@ -68,9 +69,6 @@ jobs:
           # Should return 0 (no error) when grep finds it
           curl -s http://localhost:5556 |grep -q checkbox-uuid
           curl -s http://localhost:5556/rss|grep -q rss-specification
-          # and IPv6
-          curl -s -g -6 "http://[::1]:5556"|grep -q checkbox-uuid
-          curl -s -g -6 "http://[::1]:5556/rss"|grep -q rss-specification
 
 #export WEBDRIVER_URL=http://localhost:4444/wd/hub
 #pytest tests/fetchers/test_content.py

Dockerfile

@@ -1,5 +1,5 @@
 # pip dependencies install stage
-FROM python:3.10-slim as builder
+FROM python:3.8-slim as builder
 
 # See `cryptography` pin comment in requirements.txt
 ARG CRYPTOGRAPHY_DONT_BUILD_RUST=1
@@ -29,16 +29,23 @@ RUN pip install --target=/dependencies playwright~=1.27.1 \
     || echo "WARN: Failed to install Playwright. The application can still run, but the Playwright option will be disabled."
 
 # Final image stage
-FROM python:3.10-slim
+FROM python:3.8-slim
 
+# See `cryptography` pin comment in requirements.txt
+ARG CRYPTOGRAPHY_DONT_BUILD_RUST=1
+
+# Re #93, #73, excluding rustc (adds another 430Mb~)
 RUN apt-get update && apt-get install -y --no-install-recommends \
-    libssl1.1 \
-    libxslt1.1 \
+    g++ \
+    gcc \
     # For pdftohtml
     poppler-utils \
-    zlib1g \
-    && apt-get clean && rm -rf /var/lib/apt/lists/*
-
+    libc-dev \
+    libffi-dev \
+    libjpeg-dev \
+    libssl-dev \
+    libxslt-dev \
+    zlib1g-dev
 
 # https://stackoverflow.com/questions/58701233/docker-logs-erroneously-appears-empty-until-container-stops
 ENV PYTHONUNBUFFERED=1

README.md

@@ -245,5 +245,5 @@ I offer commercial support, this software is depended on by network security, ae
 [test-shield]: https://github.com/dgtlmoon/changedetection.io/actions/workflows/test-only.yml/badge.svg?branch=master
 
 [license-shield]: https://img.shields.io/github/license/dgtlmoon/changedetection.io.svg?style=for-the-badge
-[release-link]: https://github.com/dgtlmoon/changedetection.io/releases
+[release-link]: https://github.com/dgtlmoon.com/changedetection.io/releases
 [docker-link]: https://hub.docker.com/r/dgtlmoon/changedetection.io

changedetection.py

@@ -7,7 +7,7 @@
 
 from changedetectionio import changedetection
 import multiprocessing
-import sys
+import signal
 import os
 
 def sigchld_handler(_signo, _stack_frame):
@@ -35,9 +35,6 @@ if __name__ == '__main__':
     try:
         while True:
             time.sleep(1)
-            if not parse_process.is_alive():
-                # Process died/crashed for some reason, exit with error set
-                sys.exit(1)
 
     except KeyboardInterrupt:
         #parse_process.terminate() not needed, because this process will issue it to the sub-process anyway

changedetectionio/__init__.py

@@ -36,7 +36,7 @@ from flask_wtf import CSRFProtect
 from changedetectionio import html_tools
 from changedetectionio.api import api_v1
 
-__version__ = '0.40.0.4'
+__version__ = '0.40.0.3'
 
 datastore = None
 
@@ -406,20 +406,17 @@ def changedetection_app(config=None, datastore_o=None):
 
         existing_tags = datastore.get_all_tags()
         form = forms.quickWatchForm(request.form)
-        output = render_template(
-            "watch-overview.html",
-                                 # Don't link to hosting when we're on the hosting environment
+        output = render_template("watch-overview.html",
+                                 form=form,
+                                 watches=sorted_watches,
+                                 tags=existing_tags,
                                  active_tag=limit_tag,
                                  app_rss_token=datastore.data['settings']['application']['rss_access_token'],
-                                 form=form,
-                                 guid=datastore.data['app_guid'],
-                                 has_proxies=datastore.proxy_list,
                                  has_unviewed=datastore.has_unviewed,
+                                 # Don't link to hosting when we're on the hosting environment
                                  hosted_sticky=os.getenv("SALTED_PASS", False) == False,
-                                 queued_uuids=[q_uuid.item['uuid'] for q_uuid in update_q.queue],
-                                 tags=existing_tags,
-                                 watches=sorted_watches
-                                 )
+                                 guid=datastore.data['app_guid'],
+                                 queued_uuids=[q_uuid.item['uuid'] for q_uuid in update_q.queue])
 
 
         if session.get('share-link'):
@@ -1218,7 +1215,7 @@ def changedetection_app(config=None, datastore_o=None):
                 if watch_uuid not in running_uuids and not datastore.data['watching'][watch_uuid]['paused']:
                     update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': watch_uuid, 'skip_when_checksum_same': False}))
                     i += 1
-        flash("{} watches queued for rechecking.".format(i))
+        flash("{} watches are queued for rechecking.".format(i))
         return redirect(url_for('index', tag=tag))
 
     @app.route("/form/checkbox-operations", methods=['POST'])
@@ -1239,6 +1236,7 @@ def changedetection_app(config=None, datastore_o=None):
                 uuid = uuid.strip()
                 if datastore.data['watching'].get(uuid):
                     datastore.data['watching'][uuid.strip()]['paused'] = True
+
             flash("{} watches paused".format(len(uuids)))
 
         elif (op == 'unpause'):
@@ -1268,8 +1266,8 @@ def changedetection_app(config=None, datastore_o=None):
                 if datastore.data['watching'].get(uuid):
                     # Recheck and require a full reprocessing
                     update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid, 'skip_when_checksum_same': False}))
-            flash("{} watches queued for rechecking".format(len(uuids)))
 
+            flash("{} watches un-muted".format(len(uuids)))
         elif (op == 'notification-default'):
             from changedetectionio.notification import (
                 default_notification_format_for_watch

changedetectionio/changedetection.py

@@ -3,14 +3,11 @@
 # Launch as a eventlet.wsgi server instance.
 
 from distutils.util import strtobool
-from json.decoder import JSONDecodeError
-
 import eventlet
 import eventlet.wsgi
 import getopt
 import os
 import signal
-import socket
 import sys
 
 from . import store, changedetection_app, content_fetcher
@@ -86,14 +83,8 @@ def main():
                 "Or use the -C parameter to create the directory.".format(app_config['datastore_path']), file=sys.stderr)
             sys.exit(2)
 
-    try:
-        datastore = store.ChangeDetectionStore(datastore_path=app_config['datastore_path'], version_tag=__version__)
-    except JSONDecodeError as e:
-        # Dont' start if the JSON DB looks corrupt
-        print ("ERROR: JSON DB or Proxy List JSON at '{}' appears to be corrupt, aborting".format(app_config['datastore_path']))
-        print(str(e))
-        return
 
+    datastore = store.ChangeDetectionStore(datastore_path=app_config['datastore_path'], version_tag=__version__)
     app = changedetection_app(app_config, datastore)
 
     signal.signal(signal.SIGTERM, sigterm_handler)
@@ -135,11 +126,11 @@ def main():
 
     if ssl_mode:
         # @todo finalise SSL config, but this should get you in the right direction if you need it.
-        eventlet.wsgi.server(eventlet.wrap_ssl(eventlet.listen((host, port), socket.AF_INET6),
+        eventlet.wsgi.server(eventlet.wrap_ssl(eventlet.listen((host, port)),
                                                certfile='cert.pem',
                                                keyfile='privkey.pem',
                                                server_side=True), app)
 
     else:
-        eventlet.wsgi.server(eventlet.listen((host, int(port)), socket.AF_INET6), app)
+        eventlet.wsgi.server(eventlet.listen((host, int(port))), app)
 

changedetectionio/content_fetcher.py

@@ -243,14 +243,6 @@ class base_html_playwright(Fetcher):
         if proxy_override:
             self.proxy = {'server': proxy_override}
 
-        if self.proxy:
-            # Playwright needs separate username and password values
-            from urllib.parse import urlparse
-            parsed = urlparse(self.proxy.get('server'))
-            if parsed.username:
-                self.proxy['username'] = parsed.username
-                self.proxy['password'] = parsed.password
-
     def screenshot_step(self, step_n=''):
 
         # There's a bug where we need to do it twice or it doesnt take the whole page, dont know why.
@@ -378,7 +370,7 @@ class base_html_playwright(Fetcher):
                 context.close()
                 browser.close()
                 print ("Content Fetcher > Content was empty")
-                raise EmptyReply(url=url, status_code=response.status)
+                raise EmptyReply(url=url, status_code=None)
 
             # Bug 2(?) Set the viewport size AFTER loading the page
             self.page.set_viewport_size({"width": 1280, "height": 1024})

changedetectionio/dev-docker/Dockerfile

@@ -0,0 +1,14 @@
+FROM python:3.8-slim
+
+# https://stackoverflow.com/questions/58701233/docker-logs-erroneously-appears-empty-until-container-stops
+ENV PYTHONUNBUFFERED=1
+
+WORKDIR /app
+
+RUN [ ! -d "/datastore" ] && mkdir /datastore
+
+COPY sleep.py /
+CMD [ "python", "/sleep.py" ]
+
+
+

changedetectionio/dev-docker/sleep.py

@@ -0,0 +1,7 @@
+import time
+
+print ("Sleep loop, you should run your script from the console")
+
+while True: 
+    # Wait for 5 seconds
+    time.sleep(2)

changedetectionio/forms.py

@@ -426,13 +426,6 @@ class watchForm(commonSettingsForm):
         return result
 
 
-class SingleExtraProxy(Form):
-
-    # maybe better to set some <script>var..
-    proxy_name = StringField('Name', [validators.Optional()], render_kw={"placeholder": "Name"})
-    proxy_url = StringField('Proxy URL', [validators.Optional()], render_kw={"placeholder": "http://user:pass@...:3128", "size":50})
-    # @todo do the validation here instead
-
 # datastore.data['settings']['requests']..
 class globalSettingsRequestForm(Form):
     time_between_check = FormField(TimeBetweenCheckForm)
@@ -440,15 +433,6 @@ class globalSettingsRequestForm(Form):
     jitter_seconds = IntegerField('Random jitter seconds ± check',
                                   render_kw={"style": "width: 5em;"},
                                   validators=[validators.NumberRange(min=0, message="Should contain zero or more seconds")])
-    extra_proxies = FieldList(FormField(SingleExtraProxy), min_entries=5)
-
-    def validate_extra_proxies(self, extra_validators=None):
-        for e in self.data['extra_proxies']:
-            if e.get('proxy_name') or e.get('proxy_url'):
-                if not e.get('proxy_name','').strip() or not e.get('proxy_url','').strip():
-                    self.extra_proxies.errors.append('Both a name, and a Proxy URL is required.')
-                    return False
-
 
 # datastore.data['settings']['application']..
 class globalSettingsApplicationForm(commonSettingsForm):

changedetectionio/model/App.py

@@ -15,12 +15,11 @@ class model(dict):
                 'headers': {
                 },
                 'requests': {
-                    'extra_proxies': [], # Configurable extra proxies via the UI
-                    'jitter_seconds': 0,
-                    'proxy': None, # Preferred proxy connection
-                    'time_between_check': {'weeks': None, 'days': None, 'hours': 3, 'minutes': None, 'seconds': None},
                     'timeout': int(getenv("DEFAULT_SETTINGS_REQUESTS_TIMEOUT", "45")),  # Default 45 seconds
+                    'time_between_check': {'weeks': None, 'days': None, 'hours': 3, 'minutes': None, 'seconds': None},
+                    'jitter_seconds': 0,
                     'workers': int(getenv("DEFAULT_SETTINGS_REQUESTS_WORKERS", "10")),  # Number of threads, lower is better for slow connections
+                    'proxy': None # Preferred proxy connection
                 },
                 'application': {
                     'api_access_token_enabled': True,

changedetectionio/run_proxy_tests.sh

@@ -8,15 +8,6 @@ set -e
 docker run --network changedet-network -d --name squid-one --hostname squid-one --rm -v `pwd`/tests/proxy_list/squid.conf:/etc/squid/conf.d/debian.conf ubuntu/squid:4.13-21.10_edge
 docker run --network changedet-network -d --name squid-two --hostname squid-two --rm -v `pwd`/tests/proxy_list/squid.conf:/etc/squid/conf.d/debian.conf ubuntu/squid:4.13-21.10_edge
 
-# Used for configuring a custom proxy URL via the UI
-docker run --network changedet-network -d \
-  --name squid-custom \
-  --hostname squid-custom \
-  --rm \
-  -v `pwd`/tests/proxy_list/squid-auth.conf:/etc/squid/conf.d/debian.conf \
-  -v `pwd`/tests/proxy_list/squid-passwords.txt:/etc/squid3/passwords \
-  ubuntu/squid:4.13-21.10_edge
-
 
 ## 2nd test actually choose the preferred proxy from proxies.json
 
@@ -41,21 +32,3 @@ then
   echo "Did not see a request to chosen.changedetection.io in the squid logs (while checking preferred proxy - squid two)"
   exit 1
 fi
-
-
-# Test the UI configurable proxies
-
-docker run --network changedet-network \
-  test-changedetectionio \
-  bash -c 'cd changedetectionio && pytest tests/proxy_list/test_select_custom_proxy.py'
-
-
-# Should see a request for one.changedetection.io in there
-docker logs squid-custom 2>/dev/null|grep "TCP_TUNNEL.200.*changedetection.io"
-if [ $? -ne 0 ]
-then
-  echo "Did not see a valid request to changedetection.io in the squid logs (while checking preferred proxy - squid two)"
-  exit 1
-fi
-
-docker kill squid-one squid-two squid-custom

changedetectionio/static/styles/scss/parts/_extra_proxies.scss

@@ -1,17 +0,0 @@
-ul#requests-extra_proxies {
-  list-style: none;
-  /* tidy up the table to look more "inline" */
-  li {
-    > label {
-      display: none;
-    }
-
-  }
-  /* each proxy entry is a `table` */
-  table {
-    tr {
-      display: inline;
-    }
-  }
-}
-

changedetectionio/static/styles/scss/styles.scss

@@ -2,11 +2,10 @@
  * -- BASE STYLES --
  */
 
-@import "parts/_arrows";
-@import "parts/_browser-steps";
-@import "parts/_extra_proxies";
-@import "parts/_spinners";
 @import "parts/_variables";
+@import "parts/_spinners";
+@import "parts/_browser-steps";
+@import "parts/_arrows";
 
 body {
   color: var(--color-text);

changedetectionio/static/styles/styles.css

@@ -1,139 +1,6 @@
 /*
  * -- BASE STYLES --
  */
-.arrow {
-  border: solid #1b98f8;
-  border-width: 0 2px 2px 0;
-  display: inline-block;
-  padding: 3px; }
-  .arrow.right {
-    transform: rotate(-45deg);
-    -webkit-transform: rotate(-45deg); }
-  .arrow.left {
-    transform: rotate(135deg);
-    -webkit-transform: rotate(135deg); }
-  .arrow.up, .arrow.asc {
-    transform: rotate(-135deg);
-    -webkit-transform: rotate(-135deg); }
-  .arrow.down, .arrow.desc {
-    transform: rotate(45deg);
-    -webkit-transform: rotate(45deg); }
-
-#browser_steps {
-  /* convert rows to horizontal cells */ }
-  #browser_steps th {
-    display: none; }
-  #browser_steps li {
-    list-style: decimal;
-    padding: 5px; }
-    #browser_steps li:not(:first-child):hover {
-      opacity: 1.0; }
-    #browser_steps li .control {
-      padding-left: 5px;
-      padding-right: 5px; }
-      #browser_steps li .control a {
-        font-size: 70%; }
-    #browser_steps li.empty {
-      padding: 0px;
-      opacity: 0.35; }
-      #browser_steps li.empty .control {
-        display: none; }
-    #browser_steps li:hover {
-      background: #eee; }
-    #browser_steps li > label {
-      display: none; }
-
-#browser-steps-fieldlist {
-  height: 100%;
-  overflow-y: scroll; }
-
-#browser-steps .flex-wrapper {
-  display: flex;
-  flex-flow: row;
-  height: 600px;
-  /*@todo make this dynamic */ }
-
-/*  this is duplicate :( */
-#browsersteps-selector-wrapper {
-  height: 100%;
-  width: 100%;
-  overflow-y: scroll;
-  position: relative;
-  /* nice tall skinny one */ }
-  #browsersteps-selector-wrapper > img {
-    position: absolute;
-    max-width: 100%; }
-  #browsersteps-selector-wrapper > canvas {
-    position: relative;
-    max-width: 100%; }
-    #browsersteps-selector-wrapper > canvas:hover {
-      cursor: pointer; }
-  #browsersteps-selector-wrapper .loader {
-    position: absolute;
-    left: 50%;
-    top: 50%;
-    transform: translate(-50%, -50%);
-    margin-left: -40px;
-    z-index: 100;
-    max-width: 350px;
-    text-align: center; }
-  #browsersteps-selector-wrapper .spinner, #browsersteps-selector-wrapper .spinner:after {
-    width: 80px;
-    height: 80px;
-    font-size: 3px; }
-  #browsersteps-selector-wrapper #browsersteps-click-start {
-    color: var(--color-grey-400); }
-    #browsersteps-selector-wrapper #browsersteps-click-start:hover {
-      cursor: pointer; }
-
-ul#requests-extra_proxies {
-  list-style: none;
-  /* tidy up the table to look more "inline" */
-  /* each proxy entry is a `table` */ }
-  ul#requests-extra_proxies li > label {
-    display: none; }
-  ul#requests-extra_proxies table tr {
-    display: inline; }
-
-/* spinner */
-.spinner,
-.spinner:after {
-  border-radius: 50%;
-  width: 10px;
-  height: 10px; }
-
-.spinner {
-  margin: 0px auto;
-  font-size: 3px;
-  vertical-align: middle;
-  display: inline-block;
-  text-indent: -9999em;
-  border-top: 1.1em solid rgba(38, 104, 237, 0.2);
-  border-right: 1.1em solid rgba(38, 104, 237, 0.2);
-  border-bottom: 1.1em solid rgba(38, 104, 237, 0.2);
-  border-left: 1.1em solid #2668ed;
-  -webkit-transform: translateZ(0);
-  -ms-transform: translateZ(0);
-  transform: translateZ(0);
-  -webkit-animation: load8 1.1s infinite linear;
-  animation: load8 1.1s infinite linear; }
-
-@-webkit-keyframes load8 {
-  0% {
-    -webkit-transform: rotate(0deg);
-    transform: rotate(0deg); }
-  100% {
-    -webkit-transform: rotate(360deg);
-    transform: rotate(360deg); } }
-
-@keyframes load8 {
-  0% {
-    -webkit-transform: rotate(0deg);
-    transform: rotate(0deg); }
-  100% {
-    -webkit-transform: rotate(360deg);
-    transform: rotate(360deg); } }
-
 /**
  * CSS custom properties (aka variables).
  */
@@ -271,6 +138,130 @@ html[data-darkmode="true"] {
     html[data-darkmode="true"] .watch-table .unviewed.error {
       color: var(--color-watch-table-error); }
 
+/* spinner */
+.spinner,
+.spinner:after {
+  border-radius: 50%;
+  width: 10px;
+  height: 10px; }
+
+.spinner {
+  margin: 0px auto;
+  font-size: 3px;
+  vertical-align: middle;
+  display: inline-block;
+  text-indent: -9999em;
+  border-top: 1.1em solid rgba(38, 104, 237, 0.2);
+  border-right: 1.1em solid rgba(38, 104, 237, 0.2);
+  border-bottom: 1.1em solid rgba(38, 104, 237, 0.2);
+  border-left: 1.1em solid #2668ed;
+  -webkit-transform: translateZ(0);
+  -ms-transform: translateZ(0);
+  transform: translateZ(0);
+  -webkit-animation: load8 1.1s infinite linear;
+  animation: load8 1.1s infinite linear; }
+
+@-webkit-keyframes load8 {
+  0% {
+    -webkit-transform: rotate(0deg);
+    transform: rotate(0deg); }
+  100% {
+    -webkit-transform: rotate(360deg);
+    transform: rotate(360deg); } }
+
+@keyframes load8 {
+  0% {
+    -webkit-transform: rotate(0deg);
+    transform: rotate(0deg); }
+  100% {
+    -webkit-transform: rotate(360deg);
+    transform: rotate(360deg); } }
+
+#browser_steps {
+  /* convert rows to horizontal cells */ }
+  #browser_steps th {
+    display: none; }
+  #browser_steps li {
+    list-style: decimal;
+    padding: 5px; }
+    #browser_steps li:not(:first-child):hover {
+      opacity: 1.0; }
+    #browser_steps li .control {
+      padding-left: 5px;
+      padding-right: 5px; }
+      #browser_steps li .control a {
+        font-size: 70%; }
+    #browser_steps li.empty {
+      padding: 0px;
+      opacity: 0.35; }
+      #browser_steps li.empty .control {
+        display: none; }
+    #browser_steps li:hover {
+      background: #eee; }
+    #browser_steps li > label {
+      display: none; }
+
+#browser-steps-fieldlist {
+  height: 100%;
+  overflow-y: scroll; }
+
+#browser-steps .flex-wrapper {
+  display: flex;
+  flex-flow: row;
+  height: 600px;
+  /*@todo make this dynamic */ }
+
+/*  this is duplicate :( */
+#browsersteps-selector-wrapper {
+  height: 100%;
+  width: 100%;
+  overflow-y: scroll;
+  position: relative;
+  /* nice tall skinny one */ }
+  #browsersteps-selector-wrapper > img {
+    position: absolute;
+    max-width: 100%; }
+  #browsersteps-selector-wrapper > canvas {
+    position: relative;
+    max-width: 100%; }
+    #browsersteps-selector-wrapper > canvas:hover {
+      cursor: pointer; }
+  #browsersteps-selector-wrapper .loader {
+    position: absolute;
+    left: 50%;
+    top: 50%;
+    transform: translate(-50%, -50%);
+    margin-left: -40px;
+    z-index: 100;
+    max-width: 350px;
+    text-align: center; }
+  #browsersteps-selector-wrapper .spinner, #browsersteps-selector-wrapper .spinner:after {
+    width: 80px;
+    height: 80px;
+    font-size: 3px; }
+  #browsersteps-selector-wrapper #browsersteps-click-start {
+    color: var(--color-grey-400); }
+    #browsersteps-selector-wrapper #browsersteps-click-start:hover {
+      cursor: pointer; }
+
+.arrow {
+  border: solid #1b98f8;
+  border-width: 0 2px 2px 0;
+  display: inline-block;
+  padding: 3px; }
+  .arrow.right {
+    transform: rotate(-45deg);
+    -webkit-transform: rotate(-45deg); }
+  .arrow.left {
+    transform: rotate(135deg);
+    -webkit-transform: rotate(135deg); }
+  .arrow.up, .arrow.asc {
+    transform: rotate(-135deg);
+    -webkit-transform: rotate(-135deg); }
+  .arrow.down, .arrow.desc {
+    transform: rotate(45deg);
+    -webkit-transform: rotate(45deg); }
+
 body {
   color: var(--color-text);
   background: var(--color-background-page); }

changedetectionio/store.py

@@ -36,6 +36,7 @@ class ChangeDetectionStore:
         self.datastore_path = datastore_path
         self.json_store_path = "{}/url-watches.json".format(self.datastore_path)
         self.needs_write = False
+        self.proxy_list = None
         self.start_time = time.time()
         self.stop_thread = False
         # Base definition for all watchers
@@ -77,10 +78,10 @@ class ChangeDetectionStore:
                     self.__data['watching'][uuid] = Watch.model(datastore_path=self.datastore_path, default=watch)
                     print("Watching:", uuid, self.__data['watching'][uuid]['url'])
 
-        # First time ran, Create the datastore.
-        except (FileNotFoundError):
+        # First time ran, doesnt exist.
+        except (FileNotFoundError, json.decoder.JSONDecodeError):
             if include_default_watches:
-                print("No JSON DB found at {}, creating JSON store at {}".format(self.json_store_path, self.datastore_path))
+                print("Creating JSON store at", self.datastore_path)
                 self.add_watch(url='https://news.ycombinator.com/',
                                tag='Tech news',
                                extras={'fetch_backend': 'html_requests'})
@@ -88,10 +89,8 @@ class ChangeDetectionStore:
                 self.add_watch(url='https://changedetection.io/CHANGELOG.txt',
                                tag='changedetection.io',
                                extras={'fetch_backend': 'html_requests'})
-        self.__data['version_tag'] = version_tag
 
-        # Just to test that proxies.json if it exists, doesnt throw a parsing error on startup
-        test_list = self.proxy_list
+        self.__data['version_tag'] = version_tag
 
         # Helper to remove password protection
         password_reset_lockfile = "{}/removepassword.lock".format(self.datastore_path)
@@ -117,6 +116,11 @@ class ChangeDetectionStore:
             secret = secrets.token_hex(16)
             self.__data['settings']['application']['api_access_token'] = secret
 
+        # Proxy list support - available as a selection in settings when text file is imported
+        proxy_list_file = "{}/proxies.json".format(self.datastore_path)
+        if path.isfile(proxy_list_file):
+            self.import_proxy_list(proxy_list_file)
+
         # Bump the update version by running updates
         self.run_updates()
 
@@ -459,30 +463,10 @@ class ChangeDetectionStore:
                     print ("Removing",item)
                     unlink(item)
 
-    @property
-    def proxy_list(self):
-        proxy_list = {}
-        proxy_list_file = os.path.join(self.datastore_path, 'proxies.json')
-
-        # Load from external config file
-        if path.isfile(proxy_list_file):
-            with open("{}/proxies.json".format(self.datastore_path)) as f:
-                proxy_list = json.load(f)
-
-        # Mapping from UI config if available
-        extras = self.data['settings']['requests'].get('extra_proxies')
-        if extras:
-            i=0
-            for proxy in extras:
-                i += 0
-                if proxy.get('proxy_name') and proxy.get('proxy_url'):
-                    k = "ui-" + str(i) + proxy.get('proxy_name')
-                    proxy_list[k] = {'label': proxy.get('proxy_name'), 'url': proxy.get('proxy_url')}
-
-
-        return proxy_list if len(proxy_list) else None
-
-
+    def import_proxy_list(self, filename):
+        with open(filename) as f:
+            self.proxy_list = json.load(f)
+            print ("Registered proxy list", list(self.proxy_list.keys()))
 
 
     def get_preferred_proxy_for_watch(self, uuid):
@@ -492,10 +476,11 @@ class ChangeDetectionStore:
         :return: proxy "key" id
         """
 
+        proxy_id = None
         if self.proxy_list is None:
             return None
 
-        # If it's a valid one
+        # If its a valid one
         watch = self.data['watching'].get(uuid)
 
         if watch.get('proxy') and watch.get('proxy') in list(self.proxy_list.keys()):
@@ -508,9 +493,8 @@ class ChangeDetectionStore:
             if self.proxy_list.get(system_proxy_id):
                 return system_proxy_id
 
-
-        # Fallback - Did not resolve anything, or doesnt exist, use the first available
-        if system_proxy_id is None or not self.proxy_list.get(system_proxy_id):
+        # Fallback - Did not resolve anything, use the first available
+        if system_proxy_id is None:
             first_default = list(self.proxy_list)[0]
             return first_default
 

changedetectionio/templates/settings.html

@@ -21,7 +21,6 @@
             <li class="tab"><a href="#fetching">Fetching</a></li>
             <li class="tab"><a href="#filters">Global Filters</a></li>
             <li class="tab"><a href="#api">API</a></li>
-            <li class="tab"><a href="#proxies">CAPTCHA &amp; Proxies</a></li>
         </ul>
     </div>
     <div class="box-wrap inner">
@@ -171,29 +170,14 @@ nav
                     </div>
                 </div>
             </div>
-            <div class="tab-pane-inner" id="proxies">
 
-                <p><strong>Tip</strong>: You can connect to websites using <a href="https://brightdata.grsm.io/n0r16zf7eivq">BrightData</a> proxies, their service <strong>WebUnlocker</strong> will solve most CAPTCHAs, whilst their <strong>Residential Proxies</strong> may help to avoid CAPTCHA altogether. </p>
-                <p>It may be easier to try <strong>WebUnlocker</strong> first, WebUnlocker also supports country selection.</p>
-                <p>
-                    When you have <a href="https://brightdata.grsm.io/n0r16zf7eivq">registered</a>, enabled the required services, visit the <A href="https://brightdata.com/cp/api_example?">API example page</A>, then select <strong>Python</strong>, set the country you wish to use, then copy+paste the example URL below<br/>
-                    The Proxy URL with BrightData should start with <code>http://brd-customer...</code>
-                </p>
-
-                <p>When you sign up using <a href="https://brightdata.grsm.io/n0r16zf7eivq">https://brightdata.grsm.io/n0r16zf7eivq</a> BrightData will match any first deposit up to $150</p>
-
-
-                <div class="pure-control-group">
-                    {{ render_field(form.requests.form.extra_proxies) }}
-                    <span class="pure-form-message-inline">"Name" will be used for selecting the proxy in the Watch Edit settings</span>
-                </div>
-            </div>
             <div id="actions">
                 <div class="pure-control-group">
                     {{ render_button(form.save_button) }}
                     <a href="{{url_for('index')}}" class="pure-button button-small button-cancel">Back</a>
                     <a href="{{url_for('clear_all_history')}}" class="pure-button button-small button-cancel">Clear Snapshot History</a>
                 </div>
+
             </div>
         </form>
     </div>

changedetectionio/templates/watch-overview.html

@@ -94,16 +94,7 @@
                     {%if watch.get_fetch_backend == "html_webdriver" %}<img class="status-icon" src="{{url_for('static_content', group='images', filename='Google-Chrome-icon.png')}}" title="Using a chrome browser" />{% endif %}
                     {%if watch.is_pdf  %}<img class="status-icon" src="{{url_for('static_content', group='images', filename='pdf-icon.svg')}}" title="Converting PDF to text" />{% endif %}
                     {% if watch.last_error is defined and watch.last_error != False %}
-                    <div class="fetch-error">{{ watch.last_error }}
-
-                        {% if '403' in watch.last_error %}
-                            {% if has_proxies %}
-                                <a href="{{ url_for('settings_page', uuid=watch.uuid) }}#proxies">Try other proxies/location</a>&nbsp;
-                            {% endif %}
-                            <a href="{{ url_for('settings_page', uuid=watch.uuid) }}#proxies">Try adding external proxies/locations</a>
-                        
-                        {% endif %}
-                    </div>
+                    <div class="fetch-error">{{ watch.last_error }}</div>
                     {% endif %}
                     {% if watch.last_notification_error is defined and watch.last_notification_error != False %}
                     <div class="fetch-error notification-error"><a href="{{url_for('notification_logs')}}">{{ watch.last_notification_error }}</a></div>

changedetectionio/tests/proxy_list/squid-auth.conf

@@ -1,48 +0,0 @@
-acl localnet src 0.0.0.1-0.255.255.255  # RFC 1122 "this" network (LAN)
-acl localnet src 10.0.0.0/8             # RFC 1918 local private network (LAN)
-acl localnet src 100.64.0.0/10          # RFC 6598 shared address space (CGN)
-acl localnet src 169.254.0.0/16         # RFC 3927 link-local (directly plugged) machines
-acl localnet src 172.16.0.0/12          # RFC 1918 local private network (LAN)
-acl localnet src 192.168.0.0/16         # RFC 1918 local private network (LAN)
-acl localnet src fc00::/7               # RFC 4193 local private network range
-acl localnet src fe80::/10              # RFC 4291 link-local (directly plugged) machines
-acl localnet src 159.65.224.174
-acl SSL_ports port 443
-acl Safe_ports port 80          # http
-acl Safe_ports port 21          # ftp
-acl Safe_ports port 443         # https
-acl Safe_ports port 70          # gopher
-acl Safe_ports port 210         # wais
-acl Safe_ports port 1025-65535  # unregistered ports
-acl Safe_ports port 280         # http-mgmt
-acl Safe_ports port 488         # gss-http
-acl Safe_ports port 591         # filemaker
-acl Safe_ports port 777         # multiling http
-acl CONNECT method CONNECT
-
-http_access deny !Safe_ports
-http_access deny CONNECT !SSL_ports
-#http_access allow localhost manager
-http_access deny manager
-#http_access allow localhost
-#http_access allow localnet
-
-auth_param basic program /usr/lib/squid3/basic_ncsa_auth /etc/squid3/passwords
-auth_param basic realm proxy
-acl authenticated proxy_auth REQUIRED
-http_access allow authenticated
-http_access deny all
-
-
-http_port 3128
-coredump_dir /var/spool/squid
-refresh_pattern ^ftp:           1440    20%     10080
-refresh_pattern ^gopher:        1440    0%      1440
-refresh_pattern -i (/cgi-bin/|\?) 0     0%      0
-refresh_pattern \/(Packages|Sources)(|\.bz2|\.gz|\.xz)$ 0 0% 0 refresh-ims
-refresh_pattern \/Release(|\.gpg)$ 0 0% 0 refresh-ims
-refresh_pattern \/InRelease$ 0 0% 0 refresh-ims
-refresh_pattern \/(Translation-.*)(|\.bz2|\.gz|\.xz)$ 0 0% 0 refresh-ims
-refresh_pattern .               0       20%     4320
-logfile_rotate 0
-

changedetectionio/tests/proxy_list/squid-passwords.txt

@@ -1 +0,0 @@
-test:$apr1$xvhFolTA$E/kz5/Rw1ewcyaSUdwqZs.

changedetectionio/tests/proxy_list/test_select_custom_proxy.py

@@ -1,50 +0,0 @@
-#!/usr/bin/python3
-
-import time
-from flask import url_for
-from ..util import live_server_setup, wait_for_all_checks
-
-# just make a request, we will grep in the docker logs to see it actually got called
-def test_select_custom(client, live_server):
-    live_server_setup(live_server)
-
-    # Goto settings, add our custom one
-    res = client.post(
-        url_for("settings_page"),
-        data={
-            "requests-time_between_check-minutes": 180,
-            "application-ignore_whitespace": "y",
-            "application-fetch_backend": "html_requests",
-            "requests-extra_proxies-0-proxy_name": "custom-test-proxy",
-            # test:awesome is set in tests/proxy_list/squid-passwords.txt
-            "requests-extra_proxies-0-proxy_url": "http://test:awesome@squid-custom:3128",
-        },
-        follow_redirects=True
-    )
-
-    assert b"Settings updated." in res.data
-
-    res = client.post(
-        url_for("import_page"),
-        # Because a URL wont show in squid/proxy logs due it being SSLed
-        # Use plain HTTP or a specific domain-name here
-        data={"urls": "https://changedetection.io/CHANGELOG.txt"},
-        follow_redirects=True
-    )
-
-    assert b"1 Imported" in res.data
-    wait_for_all_checks(client)
-
-    res = client.get(url_for("index"))
-    assert b'Proxy Authentication Required' not in res.data
-
-    res = client.get(
-        url_for("preview_page", uuid="first"),
-        follow_redirects=True
-    )
-    # We should see something via proxy
-    assert b'<div class=""> - 0.' in res.data
-
-    #
-    # Now we should see the request in the container logs for "squid-squid-custom" because it will be the only default
-

changedetectionio/tests/test_backend.py

@@ -67,7 +67,7 @@ def test_check_basic_change_detection_functionality(client, live_server):
 
     # Force recheck
     res = client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    assert b'1 watches queued for rechecking.' in res.data
+    assert b'1 watches are queued for rechecking.' in res.data
 
     wait_for_all_checks(client)
 

changedetectionio/tests/test_filter_failure_notification.py

@@ -1,7 +1,8 @@
 import os
 import time
+import re
 from flask import url_for
-from .util import set_original_response, live_server_setup, extract_UUID_from_client
+from .util import set_original_response, live_server_setup
 from changedetectionio.model import App
 
 
@@ -120,10 +121,6 @@ def run_filter_test(client, content_filter):
         notification = f.read()
     assert not 'CSS/xPath filter was not present in the page' in notification
 
-    # Re #1247 - All tokens got replaced
-    uuid = extract_UUID_from_client(client)
-    assert uuid in notification
-
     # cleanup for the next
     client.get(
         url_for("form_delete", uuid="all"),

changedetectionio/tests/test_source.py

@@ -40,7 +40,7 @@ def test_check_basic_change_detection_functionality_source(client, live_server):
 
     # Force recheck
     res = client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    assert b'1 watches queued for rechecking.' in res.data
+    assert b'1 watches are queued for rechecking.' in res.data
 
     time.sleep(5)
 
@@ -90,4 +90,4 @@ def test_check_ignore_elements(client, live_server):
     )
     assert b'foobar-detection' not in res.data
     assert b'<br' not in res.data
-    assert b'<p' in res.data
+    assert b'<p' in res.data

changedetectionio/update_worker.py

@@ -93,7 +93,7 @@ class update_worker(threading.Thread):
             return
 
         n_object = {'notification_title': 'Changedetection.io - Alert - CSS/xPath filter was not present in the page',
-                    'notification_body': "Your configured CSS/xPath filters of '{}' for {{{{watch_url}}}} did not appear on the page after {} attempts, did the page change layout?\n\nLink: {{{{base_url}}}}/edit/{{{{watch_uuid}}}}\n\nThanks - Your omniscient changedetection.io installation :)\n".format(
+                    'notification_body': "Your configured CSS/xPath filters of '{}' for {{watch_url}} did not appear on the page after {} attempts, did the page change layout?\n\nLink: {{base_url}}/edit/{{watch_uuid}}\n\nThanks - Your omniscient changedetection.io installation :)\n".format(
                         ", ".join(watch['include_filters']),
                         threshold),
                     'notification_format': 'text'}

docker-compose.yml

@@ -1,3 +1,4 @@
+version: '2'
 services:
     changedetection:
       image: ghcr.io/dgtlmoon/changedetection.io

requirements.txt

@@ -24,7 +24,7 @@ jsonpath-ng~=1.5.3
 # jq not available on Windows so must be installed manually
 
 # Notification library
-apprise~=1.2.1
+apprise~=1.2.0
 
 # apprise mqtt https://github.com/dgtlmoon/changedetection.io/issues/315
 paho-mqtt
@@ -62,4 +62,4 @@ pillow
 
 # Include pytest, so if theres a support issue we can ask them to run these tests on their setup
 pytest ~=6.2
-pytest-flask ~=1.2
+pytest-flask ~=1.2