fix curl request

.github/workflows/test-only.yml

@@ -60,7 +60,6 @@ jobs:
           cd changedetectionio
           ./run_proxy_tests.sh
           cd ..
-          
 
       - name: Test changedetection.io container starts+runs basically without error
         run: |
@@ -69,6 +68,9 @@ jobs:
           # Should return 0 (no error) when grep finds it
           curl -s http://localhost:5556 |grep -q checkbox-uuid
           curl -s http://localhost:5556/rss|grep -q rss-specification
+          # and IPv6
+          curl -s -g -6 "http://[::1]:5556"|grep -q checkbox-uuid
+          curl -s -g -6 "http://[::1]:5556/rss"|grep -q rss-specification
 
 #export WEBDRIVER_URL=http://localhost:4444/wd/hub
 #pytest tests/fetchers/test_content.py

changedetectionio/__init__.py

@@ -36,7 +36,7 @@ from flask_wtf import CSRFProtect
 from changedetectionio import html_tools
 from changedetectionio.api import api_v1
 
-__version__ = '0.40.0.3'
+__version__ = '0.40.0.4'
 
 datastore = None
 
@@ -406,17 +406,20 @@ def changedetection_app(config=None, datastore_o=None):
 
         existing_tags = datastore.get_all_tags()
         form = forms.quickWatchForm(request.form)
-        output = render_template("watch-overview.html",
-                                 form=form,
-                                 watches=sorted_watches,
-                                 tags=existing_tags,
+        output = render_template(
+            "watch-overview.html",
+                                 # Don't link to hosting when we're on the hosting environment
                                  active_tag=limit_tag,
                                  app_rss_token=datastore.data['settings']['application']['rss_access_token'],
-                                 has_unviewed=datastore.has_unviewed,
-                                 # Don't link to hosting when we're on the hosting environment
-                                 hosted_sticky=os.getenv("SALTED_PASS", False) == False,
+                                 form=form,
                                  guid=datastore.data['app_guid'],
-                                 queued_uuids=[q_uuid.item['uuid'] for q_uuid in update_q.queue])
+                                 has_proxies=datastore.proxy_list,
+                                 has_unviewed=datastore.has_unviewed,
+                                 hosted_sticky=os.getenv("SALTED_PASS", False) == False,
+                                 queued_uuids=[q_uuid.item['uuid'] for q_uuid in update_q.queue],
+                                 tags=existing_tags,
+                                 watches=sorted_watches
+                                 )
 
 
         if session.get('share-link'):

changedetectionio/changedetection.py

@@ -9,6 +9,7 @@ import getopt
 import os
 import signal
 import sys
+import socket
 
 from . import store, changedetection_app, content_fetcher
 from . import __version__
@@ -126,11 +127,11 @@ def main():
 
     if ssl_mode:
         # @todo finalise SSL config, but this should get you in the right direction if you need it.
-        eventlet.wsgi.server(eventlet.wrap_ssl(eventlet.listen((host, port)),
+        eventlet.wsgi.server(eventlet.wrap_ssl(eventlet.listen((host, port), socket.AF_INET6),
                                                certfile='cert.pem',
                                                keyfile='privkey.pem',
                                                server_side=True), app)
 
     else:
-        eventlet.wsgi.server(eventlet.listen((host, int(port))), app)
+        eventlet.wsgi.server(eventlet.listen((host, int(port)), socket.AF_INET6), app)
 

changedetectionio/forms.py

@@ -430,7 +430,7 @@ class SingleExtraProxy(Form):
 
     # maybe better to set some <script>var..
     proxy_name = StringField('Name', [validators.Optional()], render_kw={"placeholder": "Name"})
-    proxy_url = StringField('URL', [validators.Optional()], render_kw={"placeholder": "http://user:pass@...:3128"})
+    proxy_url = StringField('Proxy URL', [validators.Optional()], render_kw={"placeholder": "http://user:pass@...:3128", "size":50})
     # @todo do the validation here instead
 
 # datastore.data['settings']['requests']..

changedetectionio/run_proxy_tests.sh

@@ -11,7 +11,7 @@ docker run --network changedet-network -d --name squid-two --hostname squid-two
 # Used for configuring a custom proxy URL via the UI
 docker run --network changedet-network -d \
   --name squid-custom \
-  --hostname squid-squid-custom \
+  --hostname squid-custom \
   --rm \
   -v `pwd`/tests/proxy_list/squid-auth.conf:/etc/squid/conf.d/debian.conf \
   -v `pwd`/tests/proxy_list/squid-passwords.txt:/etc/squid3/passwords \
@@ -57,3 +57,5 @@ then
   echo "Did not see a valid request to changedetection.io in the squid logs (while checking preferred proxy - squid two)"
   exit 1
 fi
+
+docker kill squid-one squid-two squid-custom

changedetectionio/templates/settings.html

@@ -173,17 +173,19 @@ nav
             </div>
             <div class="tab-pane-inner" id="proxies">
 
-                <p><strong>Tip</strong>: You can connect <a href="https://brightdata.grsm.io/n0r16zf7eivq">BrightData <strong>WebUnlocker</strong></a> proxies to work around CAPTCHA.<br/>
-                    Simply <a href="https://brightdata.grsm.io/n0r16zf7eivq">register</a> and paste in the Proxy URL below.<br/>
-                    You can also add extra location/country proxies.</br>
+                <p><strong>Tip</strong>: You can connect to websites using <a href="https://brightdata.grsm.io/n0r16zf7eivq">BrightData</a> proxies, their service <strong>WebUnlocker</strong> will solve most CAPTCHAs, whilst their <strong>Residential Proxies</strong> may help to avoid CAPTCHA altogether. </p>
+                <p>It may be easier to try <strong>WebUnlocker</strong> first, WebUnlocker also supports country selection.</p>
+                <p>
+                    When you have <a href="https://brightdata.grsm.io/n0r16zf7eivq">registered</a>, enabled the required services, visit the <A href="https://brightdata.com/cp/api_example?">API example page</A>, then select <strong>Python</strong>, set the country you wish to use, then copy+paste the example URL below<br/>
+                    The Proxy URL with BrightData should start with <code>http://brd-customer...</code>
                 </p>
 
-                <p>Please use our referrer link for BrightData <a href="https://brightdata.grsm.io/n0r16zf7eivq">https://brightdata.grsm.io/n0r16zf7eivq</a></p>
+                <p>When you sign up using <a href="https://brightdata.grsm.io/n0r16zf7eivq">https://brightdata.grsm.io/n0r16zf7eivq</a> BrightData will match any first deposit up to $150</p>
 
 
                 <div class="pure-control-group">
                     {{ render_field(form.requests.form.extra_proxies) }}
-
+                    <span class="pure-form-message-inline">"Name" will be used for selecting the proxy in the Watch Edit settings</span>
                 </div>
             </div>
             <div id="actions">
@@ -192,7 +194,6 @@ nav
                     <a href="{{url_for('index')}}" class="pure-button button-small button-cancel">Back</a>
                     <a href="{{url_for('clear_all_history')}}" class="pure-button button-small button-cancel">Clear Snapshot History</a>
                 </div>
-
             </div>
         </form>
     </div>

changedetectionio/templates/watch-overview.html

@@ -94,7 +94,16 @@
                     {%if watch.get_fetch_backend == "html_webdriver" %}<img class="status-icon" src="{{url_for('static_content', group='images', filename='Google-Chrome-icon.png')}}" title="Using a chrome browser" />{% endif %}
                     {%if watch.is_pdf  %}<img class="status-icon" src="{{url_for('static_content', group='images', filename='pdf-icon.svg')}}" title="Converting PDF to text" />{% endif %}
                     {% if watch.last_error is defined and watch.last_error != False %}
-                    <div class="fetch-error">{{ watch.last_error }}</div>
+                    <div class="fetch-error">{{ watch.last_error }}
+
+                        {% if '403' in watch.last_error %}
+                            {% if has_proxies %}
+                                <a href="{{ url_for('settings_page', uuid=watch.uuid) }}#proxies">Try other proxies/location</a>&nbsp;
+                            {% endif %}
+                            <a href="{{ url_for('settings_page', uuid=watch.uuid) }}#proxies">Try adding external proxies/locations</a>
+                        
+                        {% endif %}
+                    </div>
                     {% endif %}
                     {% if watch.last_notification_error is defined and watch.last_notification_error != False %}
                     <div class="fetch-error notification-error"><a href="{{url_for('notification_logs')}}">{{ watch.last_notification_error }}</a></div>

changedetectionio/tests/proxy_list/test_select_custom_proxy.py

@@ -38,13 +38,12 @@ def test_select_custom(client, live_server):
     res = client.get(url_for("index"))
     assert b'Proxy Authentication Required' not in res.data
 
-
     res = client.get(
         url_for("preview_page", uuid="first"),
         follow_redirects=True
     )
     # We should see something via proxy
-    assert b'HEAD' in res.data
+    assert b'<div class=""> - 0.' in res.data
 
     #
     # Now we should see the request in the container logs for "squid-squid-custom" because it will be the only default

changedetectionio/tests/test_filter_failure_notification.py

@@ -1,8 +1,7 @@
 import os
 import time
-import re
 from flask import url_for
-from .util import set_original_response, live_server_setup
+from .util import set_original_response, live_server_setup, extract_UUID_from_client
 from changedetectionio.model import App
 
 
@@ -121,6 +120,10 @@ def run_filter_test(client, content_filter):
         notification = f.read()
     assert not 'CSS/xPath filter was not present in the page' in notification
 
+    # Re #1247 - All tokens got replaced
+    uuid = extract_UUID_from_client(client)
+    assert uuid in notification
+
     # cleanup for the next
     client.get(
         url_for("form_delete", uuid="all"),

changedetectionio/update_worker.py

@@ -93,7 +93,7 @@ class update_worker(threading.Thread):
             return
 
         n_object = {'notification_title': 'Changedetection.io - Alert - CSS/xPath filter was not present in the page',
-                    'notification_body': "Your configured CSS/xPath filters of '{}' for {{watch_url}} did not appear on the page after {} attempts, did the page change layout?\n\nLink: {{base_url}}/edit/{{watch_uuid}}\n\nThanks - Your omniscient changedetection.io installation :)\n".format(
+                    'notification_body': "Your configured CSS/xPath filters of '{}' for {{{{watch_url}}}} did not appear on the page after {} attempts, did the page change layout?\n\nLink: {{{{base_url}}}}/edit/{{{{watch_uuid}}}}\n\nThanks - Your omniscient changedetection.io installation :)\n".format(
                         ", ".join(watch['include_filters']),
                         threshold),
                     'notification_format': 'text'}