Python 3.11, will it chooch?

.github/workflows/containers.yml

@@ -50,7 +50,6 @@ jobs:
           python -m pip install --upgrade pip
           pip install flake8 pytest
           if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
-          if [ -f requirements-dev.txt ]; then pip install -r requirements-dev.txt; fi
 
       - name: Create release metadata
         run: |

.github/workflows/pypi.yml

@@ -19,12 +19,6 @@ jobs:
         with:
           python-version: 3.9
 
-#      - name: Install dependencies
-#        run: |
-#          python -m pip install --upgrade pip
-#          pip install flake8 pytest
-#          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
-#          if [ -f requirements-dev.txt ]; then pip install -r requirements-dev.txt; fi
 
       - name: Test that pip builds without error
         run: |

.github/workflows/test-only.yml

@@ -8,32 +8,68 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - name: Set up Python 3.9
+
+      # Mainly just for link/flake8
+      - name: Set up Python 3.10
         uses: actions/setup-python@v2
         with:
-          python-version: 3.9
+          python-version: '3.10'
-
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install flake8 pytest
-          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
-          if [ -f requirements-dev.txt ]; then pip install -r requirements-dev.txt; fi
 
       - name: Lint with flake8
         run: |
+          pip3 install flake8
           # stop the build if there are Python syntax errors or undefined names
           flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
           # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
           flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
 
-      - name: Unit tests
+      - name: Spin up ancillary testable services
         run: |
-          python3 -m unittest changedetectionio.tests.unit.test_notification_diff
           
-      - name: Test with pytest
+          docker network create changedet-network
+
+          # Selenium+browserless
+          docker run --network changedet-network -d --hostname selenium  -p 4444:4444 --rm --shm-size="2g"  selenium/standalone-chrome-debug:3.141.59
+          docker run --network changedet-network -d --hostname browserless -e "DEFAULT_LAUNCH_ARGS=[\"--window-size=1920,1080\"]" --rm  -p 3000:3000  --shm-size="2g"  browserless/chrome:1.53-chrome-stable
+
+      - name: Build changedetection.io container for testing
         run: |         
-          # Each test is totally isolated and performs its own cleanup/reset
+          # Build a changedetection.io container and start testing inside
-          cd changedetectionio; ./run_all_tests.sh
+          docker build . -t test-changedetectionio
+
+      - name: Test built container with pytest
+        run: |
+          
+          # Unit tests
+          docker run test-changedetectionio  bash -c 'python3 -m unittest changedetectionio.tests.unit.test_notification_diff'
+          
+          # All tests
+          docker run --network changedet-network  test-changedetectionio  bash -c 'cd changedetectionio && ./run_basic_tests.sh'
+
+      - name: Test built container selenium+browserless/playwright
+        run: |
+          
+          # Selenium fetch
+          docker run -e "WEBDRIVER_URL=http://selenium:4444/wd/hub" --network changedet-network  test-changedetectionio  bash -c 'cd changedetectionio;pytest tests/fetchers/test_content.py && pytest tests/test_errorhandling.py'
+          
+          # Playwright/Browserless fetch
+          docker run -e "PLAYWRIGHT_DRIVER_URL=ws://browserless:3000" --network changedet-network  test-changedetectionio  bash -c 'cd changedetectionio;pytest tests/fetchers/test_content.py && pytest tests/test_errorhandling.py && pytest tests/visualselector/test_fetch_data.py'
+
+      - name: Test proxy interaction
+        run: |
+          cd changedetectionio
+          ./run_proxy_tests.sh
+          cd ..
           
 
+      - name: Test changedetection.io container starts+runs basically without error
+        run: |
+          docker run -p 5556:5000 -d test-changedetectionio
+          sleep 3
+          # Should return 0 (no error) when grep finds it
+          curl -s http://localhost:5556 |grep -q checkbox-uuid
+          curl -s http://localhost:5556/rss|grep -q rss-specification
+
+#export WEBDRIVER_URL=http://localhost:4444/wd/hub
+#pytest tests/fetchers/test_content.py
+#pytest tests/test_errorhandling.py

CONTRIBUTING.md

@@ -7,9 +7,3 @@ Otherwise, it's always best to PR into the `dev` branch.
 Please be sure that all new functionality has a matching test!
 
 Use `pytest` to validate/test, you can run the existing tests as `pytest tests/test_notification.py` for example
-
-```
-pip3 install -r requirements-dev
-```
-
-this is from https://github.com/dgtlmoon/changedetection.io/blob/master/requirements-dev.txt

Dockerfile

@@ -1,7 +1,7 @@
 # pip dependencies install stage
-FROM python:3.8-slim as builder
+FROM python:3.11-slim as builder
 
-# rustc compiler would be needed on ARM type devices but theres an issue with some deps not building..
+# See `cryptography` pin comment in requirements.txt
 ARG CRYPTOGRAPHY_DONT_BUILD_RUST=1
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
@@ -29,22 +29,16 @@ RUN pip install --target=/dependencies playwright~=1.27.1 \
     || echo "WARN: Failed to install Playwright. The application can still run, but the Playwright option will be disabled."
 
 # Final image stage
-FROM python:3.8-slim
+FROM python:3.11-slim
 
-# Actual packages needed at runtime, usually due to the notification (apprise) backend
-# rustc compiler would be needed on ARM type devices but theres an issue with some deps not building..
-ARG CRYPTOGRAPHY_DONT_BUILD_RUST=1
-
-# Re #93, #73, excluding rustc (adds another 430Mb~)
 RUN apt-get update && apt-get install -y --no-install-recommends \
-    g++ \
+    libssl1.1 \
-    gcc \
+    libxslt1.1 \
-    libc-dev \
+    # For pdftohtml
-    libffi-dev \
+    poppler-utils \
-    libjpeg-dev \
+    zlib1g \
-    libssl-dev \
+    && apt-get clean && rm -rf /var/lib/apt/lists/*
-    libxslt-dev \
+
-    zlib1g-dev
 
 # https://stackoverflow.com/questions/58701233/docker-logs-erroneously-appears-empty-until-container-stops
 ENV PYTHONUNBUFFERED=1

MANIFEST.in

@@ -1,9 +1,10 @@
 recursive-include changedetectionio/api *
-recursive-include changedetectionio/templates *
+recursive-include changedetectionio/blueprint *
-recursive-include changedetectionio/static *
 recursive-include changedetectionio/model *
-recursive-include changedetectionio/tests *
 recursive-include changedetectionio/res *
+recursive-include changedetectionio/static *
+recursive-include changedetectionio/templates *
+recursive-include changedetectionio/tests *
 prune changedetectionio/static/package-lock.json
 prune changedetectionio/static/styles/node_modules
 prune changedetectionio/static/styles/package-lock.json

README.md

@@ -43,6 +43,7 @@ Requires Playwright to be enabled.
 
 - Products and services have a change in pricing
 - _Out of stock notification_ and _Back In stock notification_
+- Monitor and track PDF file changes, know when a PDF file has text changes.
 - Governmental department updates (changes are often only on their websites)
 - New software releases, security advisories when you're not on their mailing list.
 - Festivals with changes
@@ -68,6 +69,7 @@ _Need an actual Chrome runner with Javascript support? We support fetching via W
 - Lots of trigger filters, such as "Trigger on text", "Remove text by selector", "Ignore text", "Extract text", also using regular-expressions!
 - Target elements with xPath and CSS Selectors, Easily monitor complex JSON with JSONPath or jq
 - Switch between fast non-JS and Chrome JS based "fetchers"
+- Track changes in PDF files (Monitor text changed in the PDF, Also monitor PDF filesize and checksums)
 - Easily specify how often a site should be checked
 - Execute JS before extracting text (Good for logging in, see examples in the UI!)
 - Override Request Headers, Specify `POST` or `GET` and other methods
@@ -159,7 +161,7 @@ Just some examples
 
 <img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/docs/screenshot-notifications.png" style="max-width:100%;" alt="Self-hosted web page change monitoring notifications"  title="Self-hosted web page change monitoring notifications"  />
 
-Now you can also customise your notification content!
+Now you can also customise your notification content and use <a target="_new" href="https://jinja.palletsprojects.com/en/3.0.x/templates/">Jinja2 templating</a> for their title and body!
 
 ## JSON API Monitoring
 
@@ -187,11 +189,29 @@ When you enable a `json:` or `jq:` filter, you can even automatically extract an
 <html>
 ...
 <script type="application/ld+json">
-  {"@context":"http://schema.org","@type":"Product","name":"Nan Optipro Stage 1 Baby Formula  800g","price": 23.50 }
+
+{
+   "@context":"http://schema.org/",
+   "@type":"Product",
+   "offers":{
+      "@type":"Offer",
+      "availability":"http://schema.org/InStock",
+      "price":"3949.99",
+      "priceCurrency":"USD",
+      "url":"https://www.newegg.com/p/3D5-000D-001T1"
+   },
+   "description":"Cobratype King Cobra Hero Desktop Gaming PC",
+   "name":"Cobratype King Cobra Hero Desktop Gaming PC",
+   "sku":"3D5-000D-001T1",
+   "itemCondition":"NewCondition"
+}
 </script>
 ```  
 
-`json:$.price` or `jq:.price` would give `23.50`, or you can extract the whole structure
+`json:$..price` or `jq:..price` would give `3949.99`, or you can extract the whole structure (use a JSONpath test website to validate with)
+
+The application also supports notifying you that it can follow this information automatically
+
 
 ## Proxy Configuration
 

changedetectionio/__init__.py

@@ -10,6 +10,7 @@ import threading
 import time
 import timeago
 
+from changedetectionio import queuedWatchMetaData
 from copy import deepcopy
 from distutils.util import strtobool
 from feedgen.feed import FeedGenerator
@@ -35,7 +36,7 @@ from flask_wtf import CSRFProtect
 from changedetectionio import html_tools
 from changedetectionio.api import api_v1
 
-__version__ = '0.39.22.1'
+__version__ = '0.40.0.3'
 
 datastore = None
 
@@ -95,6 +96,12 @@ def init_app_secret(datastore_path):
 
     return secret
 
+
+@app.template_global()
+def get_darkmode_state():
+    css_dark_mode = request.cookies.get('css_dark_mode', 'false')
+    return 'true' if css_dark_mode and strtobool(css_dark_mode) else 'false'
+
 # We use the whole watch object from the store/JSON so we can see if there's some related status in terms of a thread
 # running or something similar.
 @app.template_filter('format_last_checked_time')
@@ -202,10 +209,6 @@ def changedetection_app(config=None, datastore_o=None):
     watch_api.add_resource(api_v1.SystemInfo, '/api/v1/systeminfo',
                            resource_class_kwargs={'datastore': datastore, 'update_q': update_q})
 
-    def getDarkModeSetting():
-      css_dark_mode = request.cookies.get('css_dark_mode')
-      return True if (css_dark_mode == 'true' or css_dark_mode == True) else False
-
     # Setup cors headers to allow all domains
     # https://flask-cors.readthedocs.io/en/latest/
     #    CORS(app)
@@ -402,10 +405,8 @@ def changedetection_app(config=None, datastore_o=None):
                 sorted_watches.append(watch)
 
         existing_tags = datastore.get_all_tags()
-
         form = forms.quickWatchForm(request.form)
         output = render_template("watch-overview.html",
-                                 dark_mode=getDarkModeSetting(),
                                  form=form,
                                  watches=sorted_watches,
                                  tags=existing_tags,
@@ -415,7 +416,7 @@ def changedetection_app(config=None, datastore_o=None):
                                  # Don't link to hosting when we're on the hosting environment
                                  hosted_sticky=os.getenv("SALTED_PASS", False) == False,
                                  guid=datastore.data['app_guid'],
-                                 queued_uuids=[uuid for p,uuid in update_q.queue])
+                                 queued_uuids=[q_uuid.item['uuid'] for q_uuid in update_q.queue])
 
 
         if session.get('share-link'):
@@ -595,25 +596,16 @@ def changedetection_app(config=None, datastore_o=None):
                     using_default_check_time = False
                     break
 
-            # Use the default if its the same as system wide
+            # Use the default if it's the same as system-wide.
             if form.fetch_backend.data == datastore.data['settings']['application']['fetch_backend']:
                 extra_update_obj['fetch_backend'] = None
 
 
+
              # Ignore text
             form_ignore_text = form.ignore_text.data
             datastore.data['watching'][uuid]['ignore_text'] = form_ignore_text
 
-            # Reset the previous_md5 so we process a new snapshot including stripping ignore text.
-            if form_ignore_text:
-                if len(datastore.data['watching'][uuid].history):
-                    extra_update_obj['previous_md5'] = get_current_checksum_include_ignore_text(uuid=uuid)
-
-            # Reset the previous_md5 so we process a new snapshot including stripping ignore text.
-            if form.include_filters.data != datastore.data['watching'][uuid].get('include_filters', []):
-                if len(datastore.data['watching'][uuid].history):
-                    extra_update_obj['previous_md5'] = get_current_checksum_include_ignore_text(uuid=uuid)
-
             # Be sure proxy value is None
             if datastore.proxy_list is not None and form.data['proxy'] == '':
                 extra_update_obj['proxy'] = None
@@ -631,7 +623,7 @@ def changedetection_app(config=None, datastore_o=None):
             datastore.needs_write_urgent = True
 
             # Queue the watch for immediate recheck, with a higher priority
-            update_q.put((1, uuid))
+            update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid, 'skip_when_checksum_same': False}))
 
             # Diff page [edit] link should go back to diff page
             if request.args.get("next") and request.args.get("next") == 'diff':
@@ -664,7 +656,6 @@ def changedetection_app(config=None, datastore_o=None):
                                      browser_steps_config=browser_step_ui_config,
                                      current_base_url=datastore.data['settings']['application']['base_url'],
                                      emailprefix=os.getenv('NOTIFICATION_MAIL_BUTTON_PREFIX', False),
-                                     dark_mode=getDarkModeSetting(),
                                      form=form,
                                      has_default_notification_urls=True if len(datastore.data['settings']['application']['notification_urls']) else False,
                                      has_empty_checktime=using_default_check_time,
@@ -752,7 +743,6 @@ def changedetection_app(config=None, datastore_o=None):
 
         output = render_template("settings.html",
                                  form=form,
-                                 dark_mode=getDarkModeSetting(),
                                  current_base_url = datastore.data['settings']['application']['base_url'],
                                  hide_remove_pass=os.getenv("SALTED_PASS", False),
                                  api_key=datastore.data['settings']['application'].get('api_access_token'),
@@ -774,7 +764,7 @@ def changedetection_app(config=None, datastore_o=None):
                 importer = import_url_list()
                 importer.run(data=request.values.get('urls'), flash=flash, datastore=datastore)
                 for uuid in importer.new_uuids:
-                    update_q.put((1, uuid))
+                    update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid, 'skip_when_checksum_same': True}))
 
                 if len(importer.remaining_data) == 0:
                     return redirect(url_for('index'))
@@ -787,13 +777,12 @@ def changedetection_app(config=None, datastore_o=None):
                 d_importer = import_distill_io_json()
                 d_importer.run(data=request.values.get('distill-io'), flash=flash, datastore=datastore)
                 for uuid in d_importer.new_uuids:
-                    update_q.put((1, uuid))
+                    update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid, 'skip_when_checksum_same': True}))
 
 
 
         # Could be some remaining, or we could be on GET
         output = render_template("import.html",
-                                 dark_mode=getDarkModeSetting(),
                                  import_url_list_remaining="\n".join(remaining_urls),
                                  original_distill_json=''
                                  )
@@ -810,10 +799,12 @@ def changedetection_app(config=None, datastore_o=None):
 
         return redirect(url_for('index'))
 
-    @app.route("/diff/<string:uuid>", methods=['GET'])
+    @app.route("/diff/<string:uuid>", methods=['GET', 'POST'])
     @login_required
     def diff_history_page(uuid):
 
+        from changedetectionio import forms
+
         # More for testing, possible to return the first/only
         if uuid == 'first':
             uuid = list(datastore.data['watching'].keys()).pop()
@@ -825,6 +816,28 @@ def changedetection_app(config=None, datastore_o=None):
             flash("No history found for the specified link, bad link?", "error")
             return redirect(url_for('index'))
 
+        # For submission of requesting an extract
+        extract_form = forms.extractDataForm(request.form)
+        if request.method == 'POST':
+            if not extract_form.validate():
+                flash("An error occurred, please see below.", "error")
+
+            else:
+                extract_regex = request.form.get('extract_regex').strip()
+                output = watch.extract_regex_from_all_history(extract_regex)
+                if output:
+                    watch_dir = os.path.join(datastore_o.datastore_path, uuid)
+                    response = make_response(send_from_directory(directory=watch_dir, path=output, as_attachment=True))
+                    response.headers['Content-type'] = 'text/csv'
+                    response.headers['Cache-Control'] = 'no-cache, no-store, must-revalidate'
+                    response.headers['Pragma'] = 'no-cache'
+                    response.headers['Expires'] = 0
+                    return response
+
+
+                flash('Nothing matches that RegEx', 'error')
+                redirect(url_for('diff_history_page', uuid=uuid)+'#extract')
+
         history = watch.history
         dates = list(history.keys())
 
@@ -867,23 +880,23 @@ def changedetection_app(config=None, datastore_o=None):
                     watch.get('fetch_backend', None) is None and system_uses_webdriver) else False
 
         output = render_template("diff.html",
-                                 watch_a=watch,
-                                 newest=newest_version_file_contents,
-                                 previous=previous_version_file_contents,
-                                 extra_stylesheets=extra_stylesheets,
-                                 dark_mode=getDarkModeSetting(),
-                                 versions=dates[:-1], # All except current/last
-                                 uuid=uuid,
-                                 newest_version_timestamp=dates[-1],
-                                 current_previous_version=str(previous_version),
                                  current_diff_url=watch['url'],
+                                 current_previous_version=str(previous_version),
+                                 extra_stylesheets=extra_stylesheets,
                                  extra_title=" - Diff - {}".format(watch['title'] if watch['title'] else watch['url']),
-                                 left_sticky=True,
+                                 extract_form=extract_form,
-                                 screenshot=screenshot_url,
                                  is_html_webdriver=is_html_webdriver,
                                  last_error=watch['last_error'],
+                                 last_error_screenshot=watch.get_error_snapshot(),
                                  last_error_text=watch.get_error_text(),
-                                 last_error_screenshot=watch.get_error_snapshot()
+                                 left_sticky=True,
+                                 newest=newest_version_file_contents,
+                                 newest_version_timestamp=dates[-1],
+                                 previous=previous_version_file_contents,
+                                 screenshot=screenshot_url,
+                                 uuid=uuid,
+                                 versions=dates[:-1], # All except current/last
+                                 watch_a=watch
                                  )
 
         return output
@@ -919,7 +932,6 @@ def changedetection_app(config=None, datastore_o=None):
                                      content=content,
                                      history_n=watch.history_n,
                                      extra_stylesheets=extra_stylesheets,
-                                     dark_mode=getDarkModeSetting(),
 #                                     current_diff_url=watch['url'],
                                      watch=watch,
                                      uuid=uuid,
@@ -966,7 +978,6 @@ def changedetection_app(config=None, datastore_o=None):
                                  content=content,
                                  history_n=watch.history_n,
                                  extra_stylesheets=extra_stylesheets,
-                                 dark_mode=getDarkModeSetting(),
                                  ignored_line_numbers=ignored_line_numbers,
                                  triggered_line_numbers=trigger_line_numbers,
                                  current_diff_url=watch['url'],
@@ -985,15 +996,10 @@ def changedetection_app(config=None, datastore_o=None):
     def notification_logs():
         global notification_debug_log
         output = render_template("notification-log.html",
-                                 dark_mode=getDarkModeSetting(),
                                  logs=notification_debug_log if len(notification_debug_log) else ["Notification logs are empty - no notifications sent yet."])
 
         return output
 
-    @app.route("/favicon.ico", methods=['GET'])
-    def favicon():
-        return send_from_directory("static/images", path="favicon.ico")
-
     # We're good but backups are even better!
     @app.route("/backup", methods=['GET'])
     @login_required
@@ -1136,7 +1142,7 @@ def changedetection_app(config=None, datastore_o=None):
 
         if not add_paused and new_uuid:
             # Straight into the queue.
-            update_q.put((1, new_uuid))
+            update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': new_uuid}))
             flash("Watch added.")
 
         if add_paused:
@@ -1173,7 +1179,7 @@ def changedetection_app(config=None, datastore_o=None):
             uuid = list(datastore.data['watching'].keys()).pop()
 
         new_uuid = datastore.clone(uuid)
-        update_q.put((5, new_uuid))
+        update_q.put(queuedWatchMetaData.PrioritizedItem(priority=5, item={'uuid': new_uuid, 'skip_when_checksum_same': True}))
         flash('Cloned.')
 
         return redirect(url_for('index'))
@@ -1181,7 +1187,7 @@ def changedetection_app(config=None, datastore_o=None):
     @app.route("/api/checknow", methods=['GET'])
     @login_required
     def form_watch_checknow():
-
+        # Forced recheck will skip the 'skip if content is the same' rule (, 'reprocess_existing_data': True})))
         tag = request.args.get('tag')
         uuid = request.args.get('uuid')
         i = 0
@@ -1190,11 +1196,9 @@ def changedetection_app(config=None, datastore_o=None):
         for t in running_update_threads:
             running_uuids.append(t.current_uuid)
 
-        # @todo check thread is running and skip
-
         if uuid:
             if uuid not in running_uuids:
-                update_q.put((1, uuid))
+                update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid, 'skip_when_checksum_same': False}))
             i = 1
 
         elif tag != None:
@@ -1202,14 +1206,14 @@ def changedetection_app(config=None, datastore_o=None):
             for watch_uuid, watch in datastore.data['watching'].items():
                 if (tag != None and tag in watch['tag']):
                     if watch_uuid not in running_uuids and not datastore.data['watching'][watch_uuid]['paused']:
-                        update_q.put((1, watch_uuid))
+                        update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': watch_uuid, 'skip_when_checksum_same': False}))
                         i += 1
 
         else:
             # No tag, no uuid, add everything.
             for watch_uuid, watch in datastore.data['watching'].items():
                 if watch_uuid not in running_uuids and not datastore.data['watching'][watch_uuid]['paused']:
-                    update_q.put((1, watch_uuid))
+                    update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': watch_uuid, 'skip_when_checksum_same': False}))
                     i += 1
         flash("{} watches are queued for rechecking.".format(i))
         return redirect(url_for('index', tag=tag))
@@ -1256,6 +1260,14 @@ def changedetection_app(config=None, datastore_o=None):
                     datastore.data['watching'][uuid.strip()]['notification_muted'] = False
             flash("{} watches un-muted".format(len(uuids)))
 
+        elif (op == 'recheck'):
+            for uuid in uuids:
+                uuid = uuid.strip()
+                if datastore.data['watching'].get(uuid):
+                    # Recheck and require a full reprocessing
+                    update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid, 'skip_when_checksum_same': False}))
+
+            flash("{} watches un-muted".format(len(uuids)))
         elif (op == 'notification-default'):
             from changedetectionio.notification import (
                 default_notification_format_for_watch
@@ -1328,6 +1340,10 @@ def changedetection_app(config=None, datastore_o=None):
     import changedetectionio.blueprint.browser_steps as browser_steps
     app.register_blueprint(browser_steps.construct_blueprint(datastore), url_prefix='/browser-steps')
 
+    import changedetectionio.blueprint.price_data_follower as price_data_follower
+    app.register_blueprint(price_data_follower.construct_blueprint(datastore, update_q), url_prefix='/price_data_follower')
+
+
     # @todo handle ctrl break
     ticker_thread = threading.Thread(target=ticker_thread_check_time_launch_checks).start()
     threading.Thread(target=notification_runner).start()
@@ -1433,7 +1449,11 @@ def ticker_thread_check_time_launch_checks():
         watch_uuid_list = []
         while True:
             try:
-                watch_uuid_list = datastore.data['watching'].keys()
+                # Get a list of watches sorted by last_checked, [1] because it gets passed a tuple
+                # This is so we examine the most over-due first
+                for k in sorted(datastore.data['watching'].items(), key=lambda item: item[1].get('last_checked',0)):
+                    watch_uuid_list.append(k[0])
+
             except RuntimeError as e:
                 # RuntimeError: dictionary changed size during iteration
                 time.sleep(0.1)
@@ -1473,7 +1493,7 @@ def ticker_thread_check_time_launch_checks():
             seconds_since_last_recheck = now - watch['last_checked']
 
             if seconds_since_last_recheck >= (threshold + watch.jitter_seconds) and seconds_since_last_recheck >= recheck_time_minimum_seconds:
-                if not uuid in running_uuids and uuid not in [q_uuid for p,q_uuid in update_q.queue]:
+                if not uuid in running_uuids and uuid not in [q_uuid.item['uuid'] for q_uuid in update_q.queue]:
 
                     # Proxies can be set to have a limit on seconds between which they can be called
                     watch_proxy = datastore.get_preferred_proxy_for_watch(uuid=uuid)
@@ -1504,8 +1524,9 @@ def ticker_thread_check_time_launch_checks():
                             priority,
                             watch.jitter_seconds,
                             now - watch['last_checked']))
+
                     # Into the queue with you
-                    update_q.put((priority, uuid))
+                    update_q.put(queuedWatchMetaData.PrioritizedItem(priority=priority, item={'uuid': uuid, 'skip_when_checksum_same': True}))
 
                     # Reset for next time
                     watch.jitter_seconds = 0

changedetectionio/api/api_v1.py

@@ -1,3 +1,4 @@
+from changedetectionio import queuedWatchMetaData
 from flask_restful import abort, Resource
 from flask import request, make_response
 import validators
@@ -24,7 +25,7 @@ class Watch(Resource):
             abort(404, message='No watch exists with the UUID of {}'.format(uuid))
 
         if request.args.get('recheck'):
-            self.update_q.put((1, uuid))
+            self.update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid, 'skip_when_checksum_same': True}))
             return "OK", 200
 
         # Return without history, get that via another API call
@@ -100,7 +101,7 @@ class CreateWatch(Resource):
         extras = {'title': json_data['title'].strip()} if json_data.get('title') else {}
 
         new_uuid = self.datastore.add_watch(url=json_data['url'].strip(), tag=tag, extras=extras)
-        self.update_q.put((1, new_uuid))
+        self.update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': new_uuid, 'skip_when_checksum_same': True}))
         return {'uuid': new_uuid}, 201
 
     # Return concise list of available watches and some very basic info
@@ -118,7 +119,7 @@ class CreateWatch(Resource):
 
         if request.args.get('recheck_all'):
             for uuid in self.datastore.data['watching'].keys():
-                self.update_q.put((1, uuid))
+                self.update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid, 'skip_when_checksum_same': True}))
             return {'status': "OK"}, 200
 
         return list, 200

changedetectionio/blueprint/browser_steps/browser_steps.py

@@ -75,15 +75,13 @@ class steppable_browser_interface():
     def action_goto_url(self, url, optional_value):
         # self.page.set_viewport_size({"width": 1280, "height": 5000})
         now = time.time()
-        response = self.page.goto(url, timeout=0, wait_until='domcontentloaded')
+        response = self.page.goto(url, timeout=0, wait_until='commit')
-        print("Time to goto URL", time.time() - now)
 
         # Wait_until = commit
         # - `'commit'` - consider operation to be finished when network response is received and the document started loading.
         # Better to not use any smarts from Playwright and just wait an arbitrary number of seconds
         # This seemed to solve nearly all 'TimeoutErrors'
-        extra_wait = int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5))
+        print("Time to goto URL ", time.time() - now)
-        self.page.wait_for_timeout(extra_wait * 1000)
 
     def action_click_element_containing_text(self, selector=None, value=''):
         if not len(value.strip()):

changedetectionio/blueprint/price_data_follower/__init__.py

@@ -0,0 +1,33 @@
+
+from distutils.util import strtobool
+from flask import Blueprint, flash, redirect, url_for
+from flask_login import login_required
+from changedetectionio.store import ChangeDetectionStore
+from changedetectionio import queuedWatchMetaData
+from queue import PriorityQueue
+
+PRICE_DATA_TRACK_ACCEPT = 'accepted'
+PRICE_DATA_TRACK_REJECT = 'rejected'
+
+def construct_blueprint(datastore: ChangeDetectionStore, update_q: PriorityQueue):
+
+    price_data_follower_blueprint = Blueprint('price_data_follower', __name__)
+
+    @login_required
+    @price_data_follower_blueprint.route("/<string:uuid>/accept", methods=['GET'])
+    def accept(uuid):
+        datastore.data['watching'][uuid]['track_ldjson_price_data'] = PRICE_DATA_TRACK_ACCEPT
+        update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid, 'skip_when_checksum_same': False}))
+        return redirect(url_for("form_watch_checknow", uuid=uuid))
+
+
+    @login_required
+    @price_data_follower_blueprint.route("/<string:uuid>/reject", methods=['GET'])
+    def reject(uuid):
+        datastore.data['watching'][uuid]['track_ldjson_price_data'] = PRICE_DATA_TRACK_REJECT
+        return redirect(url_for("index"))
+
+
+    return price_data_follower_blueprint
+
+

changedetectionio/content_fetcher.py

@@ -1,3 +1,4 @@
+import hashlib
 from abc import abstractmethod
 import chardet
 import json
@@ -23,6 +24,9 @@ class Non200ErrorCodeReceived(Exception):
             self.page_text = html_tools.html_to_text(page_html)
         return
 
+class checksumFromPreviousCheckWasTheSame(Exception):
+    def __init__(self):
+        return
 
 class JSActionExceptions(Exception):
     def __init__(self, status_code, url, screenshot, message=''):
@@ -39,7 +43,7 @@ class BrowserStepsStepTimout(Exception):
 
 
 class PageUnloadable(Exception):
-    def __init__(self, status_code, url, screenshot=False, message=False):
+    def __init__(self, status_code, url, message, screenshot=False):
         # Set this so we can use it in other parts of the app
         self.status_code = status_code
         self.url = url
@@ -113,7 +117,8 @@ class Fetcher():
             request_body,
             request_method,
             ignore_status_codes=False,
-            current_include_filters=None):
+            current_include_filters=None,
+            is_binary=False):
         # Should set self.error, self.status_code and self.content
         pass
 
@@ -238,6 +243,14 @@ class base_html_playwright(Fetcher):
         if proxy_override:
             self.proxy = {'server': proxy_override}
 
+        if self.proxy:
+            # Playwright needs separate username and password values
+            from urllib.parse import urlparse
+            parsed = urlparse(self.proxy.get('server'))
+            if parsed.username:
+                self.proxy['username'] = parsed.username
+                self.proxy['password'] = parsed.password
+
     def screenshot_step(self, step_n=''):
 
         # There's a bug where we need to do it twice or it doesnt take the whole page, dont know why.
@@ -264,7 +277,8 @@ class base_html_playwright(Fetcher):
             request_body,
             request_method,
             ignore_status_codes=False,
-            current_include_filters=None):
+            current_include_filters=None,
+            is_binary=False):
 
         from playwright.sync_api import sync_playwright
         import playwright._impl._api_types
@@ -286,6 +300,8 @@ class base_html_playwright(Fetcher):
                 proxy=self.proxy,
                 # This is needed to enable JavaScript execution on GitHub and others
                 bypass_csp=True,
+                # Can't think why we need the service workers for our use case?
+                service_workers='block',
                 # Should never be needed
                 accept_downloads=False
             )
@@ -294,24 +310,34 @@ class base_html_playwright(Fetcher):
             if len(request_headers):
                 context.set_extra_http_headers(request_headers)
 
-            try:
                 self.page.set_default_navigation_timeout(90000)
                 self.page.set_default_timeout(90000)
 
                 # Listen for all console events and handle errors
                 self.page.on("console", lambda msg: print(f"Playwright console: Watch URL: {url} {msg.type}: {msg.text} {msg.args}"))
 
-                # Bug - never set viewport size BEFORE page.goto
+            # Goto page
-
+            try:
-
-                # Waits for the next navigation. Using Python context manager
-                # prevents a race condition between clicking and waiting for a navigation.
-                with self.page.expect_navigation():
-                    response = self.page.goto(url, wait_until='load')
                 # Wait_until = commit
                 # - `'commit'` - consider operation to be finished when network response is received and the document started loading.
                 # Better to not use any smarts from Playwright and just wait an arbitrary number of seconds
                 # This seemed to solve nearly all 'TimeoutErrors'
+                response = self.page.goto(url, wait_until='commit')
+            except playwright._impl._api_types.Error as e:
+                # Retry once - https://github.com/browserless/chrome/issues/2485
+                # Sometimes errors related to invalid cert's and other can be random
+                print ("Content Fetcher > retrying request got error - ", str(e))
+                time.sleep(1)
+                response = self.page.goto(url, wait_until='commit')
+
+            except Exception as e:
+                print ("Content Fetcher > Other exception when page.goto", str(e))
+                context.close()
+                browser.close()
+                raise PageUnloadable(url=url, status_code=None, message=str(e))
+
+            # Execute any browser steps
+            try:
                 extra_wait = int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)) + self.render_extract_delay
                 self.page.wait_for_timeout(extra_wait * 1000)
 
@@ -324,17 +350,15 @@ class base_html_playwright(Fetcher):
                 # This can be ok, we will try to grab what we could retrieve
                 pass
             except Exception as e:
-                print ("other exception when page.goto")
+                print ("Content Fetcher > Other exception when executing custom JS code", str(e))
-                print (str(e))
                 context.close()
                 browser.close()
-                raise PageUnloadable(url=url, status_code=None)
+                raise PageUnloadable(url=url, status_code=None, message=str(e))
-
 
             if response is None:
                 context.close()
                 browser.close()
-                print ("response object was none")
+                print ("Content Fetcher > Response object was none")
                 raise EmptyReply(url=url, status_code=None)
 
             # Bug 2(?) Set the viewport size AFTER loading the page
@@ -353,8 +377,8 @@ class base_html_playwright(Fetcher):
             if len(self.page.content().strip()) == 0:
                 context.close()
                 browser.close()
-                print ("Content was empty")
+                print ("Content Fetcher > Content was empty")
-                raise EmptyReply(url=url, status_code=None)
+                raise EmptyReply(url=url, status_code=response.status)
 
             # Bug 2(?) Set the viewport size AFTER loading the page
             self.page.set_viewport_size({"width": 1280, "height": 1024})
@@ -440,7 +464,8 @@ class base_html_webdriver(Fetcher):
             request_body,
             request_method,
             ignore_status_codes=False,
-            current_include_filters=None):
+            current_include_filters=None,
+            is_binary=False):
 
         from selenium import webdriver
         from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
@@ -498,7 +523,7 @@ class base_html_webdriver(Fetcher):
             try:
                 self.driver.quit()
             except Exception as e:
-                print("Exception in chrome shutdown/quit" + str(e))
+                print("Content Fetcher > Exception in chrome shutdown/quit" + str(e))
 
 
 # "html_requests" is listed as the default fetcher in store.py!
@@ -515,7 +540,8 @@ class html_requests(Fetcher):
             request_body,
             request_method,
             ignore_status_codes=False,
-            current_include_filters=None):
+            current_include_filters=None,
+            is_binary=False):
 
         # Make requests use a more modern looking user-agent
         if not 'User-Agent' in request_headers:
@@ -545,6 +571,8 @@ class html_requests(Fetcher):
         # For example - some sites don't tell us it's utf-8, but return utf-8 content
         # This seems to not occur when using webdriver/selenium, it seems to detect the text encoding more reliably.
         # https://github.com/psf/requests/issues/1604 good info about requests encoding detection
+        if not is_binary:
+            # Don't run this for PDF (and requests identified as binary) takes a _long_ time
             if not r.headers.get('content-type') or not 'charset=' in r.headers.get('content-type'):
                 encoding = chardet.detect(r.content)['encoding']
                 if encoding:
@@ -560,8 +588,14 @@ class html_requests(Fetcher):
             raise Non200ErrorCodeReceived(url=url, status_code=r.status_code, page_html=r.text)
 
         self.status_code = r.status_code
+        if is_binary:
+            # Binary files just return their checksum until we add something smarter
+            self.content = hashlib.md5(r.content).hexdigest()
+        else:
             self.content = r.text
+
         self.headers = r.headers
+        self.raw_content = r.content
 
 
 # Decide which is the 'real' HTML webdriver, this is more a system wide config

changedetectionio/dev-docker/Dockerfile

@@ -1,14 +0,0 @@
-FROM python:3.8-slim
-
-# https://stackoverflow.com/questions/58701233/docker-logs-erroneously-appears-empty-until-container-stops
-ENV PYTHONUNBUFFERED=1
-
-WORKDIR /app
-
-RUN [ ! -d "/datastore" ] && mkdir /datastore
-
-COPY sleep.py /
-CMD [ "python", "/sleep.py" ]
-
-
-

changedetectionio/dev-docker/sleep.py

@@ -1,7 +0,0 @@
-import time
-
-print ("Sleep loop, you should run your script from the console")
-
-while True: 
-    # Wait for 5 seconds
-    time.sleep(2)

changedetectionio/fetch_site_status.py

@@ -1,11 +1,13 @@
 import hashlib
+import json
 import logging
 import os
 import re
-import time
 import urllib3
 
 from changedetectionio import content_fetcher, html_tools
+from changedetectionio.blueprint.price_data_follower import PRICE_DATA_TRACK_ACCEPT, PRICE_DATA_TRACK_REJECT
+from copy import deepcopy
 
 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 
@@ -14,6 +16,10 @@ class FilterNotFoundInResponse(ValueError):
     def __init__(self, msg):
         ValueError.__init__(self, msg)
 
+class PDFToHTMLToolNotFound(ValueError):
+    def __init__(self, msg):
+        ValueError.__init__(self, msg)
+
 
 # Some common stuff here that can be moved to a base class
 # (set_proxy_from_list)
@@ -38,8 +44,7 @@ class perform_site_check():
 
         return regex
 
-    def run(self, uuid):
+    def run(self, uuid, skip_when_checksum_same=True):
-        from copy import deepcopy
         changed_detected = False
         screenshot = False  # as bytes
         stripped_text_from_html = ""
@@ -86,7 +91,7 @@ class perform_site_check():
             is_source = True
 
         # Pluggable content fetcher
-        prefer_backend = watch.get('fetch_backend')
+        prefer_backend = watch.get_fetch_backend
         if hasattr(content_fetcher, prefer_backend):
             klass = getattr(content_fetcher, prefer_backend)
         else:
@@ -116,12 +121,26 @@ class perform_site_check():
         if watch.get('webdriver_js_execute_code') is not None and watch.get('webdriver_js_execute_code').strip():
             fetcher.webdriver_js_execute_code = watch.get('webdriver_js_execute_code')
 
-        fetcher.run(url, timeout, request_headers, request_body, request_method, ignore_status_codes, watch.get('include_filters'))
+        # requests for PDF's, images etc should be passwd the is_binary flag
+        is_binary = watch.is_pdf
+
+        fetcher.run(url, timeout, request_headers, request_body, request_method, ignore_status_codes, watch.get('include_filters'), is_binary=is_binary)
         fetcher.quit()
 
         self.screenshot = fetcher.screenshot
         self.xpath_data = fetcher.xpath_data
 
+        # Track the content type
+        update_obj['content_type'] = fetcher.headers.get('Content-Type', '')
+
+        # Watches added automatically in the queue manager will skip if its the same checksum as the previous run
+        # Saves a lot of CPU
+        update_obj['previous_md5_before_filters'] = hashlib.md5(fetcher.content.encode('utf-8')).hexdigest()
+        if skip_when_checksum_same:
+            if update_obj['previous_md5_before_filters'] == watch.get('previous_md5_before_filters'):
+                raise content_fetcher.checksumFromPreviousCheckWasTheSame()
+
+
         # Fetching complete, now filters
         # @todo move to class / maybe inside of fetcher abstract base?
 
@@ -140,7 +159,32 @@ class perform_site_check():
             is_html = False
             is_json = False
 
-        include_filters_rule = watch.get('include_filters', [])
+        if watch.is_pdf or 'application/pdf' in fetcher.headers.get('Content-Type', '').lower():
+            from shutil import which
+            tool = os.getenv("PDF_TO_HTML_TOOL", "pdftohtml")
+            if not which(tool):
+                raise PDFToHTMLToolNotFound("Command-line `{}` tool was not found in system PATH, was it installed?".format(tool))
+
+            import subprocess
+            proc = subprocess.Popen(
+                [tool, '-stdout', '-', '-s', 'out.pdf', '-i'],
+                stdout=subprocess.PIPE,
+                stdin=subprocess.PIPE)
+            proc.stdin.write(fetcher.raw_content)
+            proc.stdin.close()
+            fetcher.content = proc.stdout.read().decode('utf-8')
+            proc.wait(timeout=60)
+
+            # Add a little metadata so we know if the file changes (like if an image changes, but the text is the same
+            # @todo may cause problems with non-UTF8?
+            metadata = "<p>Added by changedetection.io: Document checksum - {} Filesize - {} bytes</p>".format(
+                hashlib.md5(fetcher.raw_content).hexdigest().upper(),
+                len(fetcher.content))
+
+            fetcher.content = fetcher.content.replace('</body>', metadata + '</body>')
+
+
+        include_filters_rule = deepcopy(watch.get('include_filters', []))
         # include_filters_rule = watch['include_filters']
         subtractive_selectors = watch.get(
             "subtractive_selectors", []
@@ -148,6 +192,10 @@ class perform_site_check():
             "global_subtractive_selectors", []
         )
 
+        # Inject a virtual LD+JSON price tracker rule
+        if watch.get('track_ldjson_price_data', '') == PRICE_DATA_TRACK_ACCEPT:
+            include_filters_rule.append(html_tools.LD_JSON_PRODUCT_OFFER_SELECTOR)
+
         has_filter_rule = include_filters_rule and len("".join(include_filters_rule).strip())
         has_subtractive_selectors = subtractive_selectors and len(subtractive_selectors[0].strip())
 
@@ -155,6 +203,14 @@ class perform_site_check():
             include_filters_rule.append("json:$")
             has_filter_rule = True
 
+        if is_json:
+            # Sort the JSON so we dont get false alerts when the content is just re-ordered
+            try:
+                fetcher.content = json.dumps(json.loads(fetcher.content), sort_keys=True)
+            except Exception as e:
+                # Might have just been a snippet, or otherwise bad JSON, continue
+                pass
+
         if has_filter_rule:
             json_filter_prefixes = ['json:', 'jq:']
             for filter in include_filters_rule:
@@ -162,6 +218,8 @@ class perform_site_check():
                     stripped_text_from_html += html_tools.extract_json_as_string(content=fetcher.content, json_filter=filter)
                     is_html = False
 
+
+
         if is_html or is_source:
 
             # CSS Filter, extract the HTML that matches and feed that into the existing inscriptis::get_text
@@ -173,9 +231,13 @@ class perform_site_check():
                 # Don't run get_text or xpath/css filters on plaintext
                 stripped_text_from_html = html_content
             else:
+                # Does it have some ld+json price data? used for easier monitoring
+                update_obj['has_ldjson_price_data'] = html_tools.has_ldjson_product_info(fetcher.content)
+
                 # Then we assume HTML
                 if has_filter_rule:
                     html_content = ""
+
                     for filter_rule in include_filters_rule:
                         # For HTML/XML we offer xpath as an option, just start a regular xPath "/.."
                         if filter_rule[0] == '/' or filter_rule.startswith('xpath:'):

changedetectionio/forms.py

@@ -193,7 +193,7 @@ class ValidateAppRiseServers(object):
                 message = field.gettext('\'%s\' is not a valid AppRise URL.' % (server_url))
                 raise ValidationError(message)
 
-class ValidateTokensList(object):
+class ValidateJinja2Template(object):
     """
     Validates that a {token} is from a valid set
     """
@@ -202,11 +202,24 @@ class ValidateTokensList(object):
 
     def __call__(self, form, field):
         from changedetectionio import notification
-        regex = re.compile('{.*?}')
+
-        for p in re.findall(regex, field.data):
+        from jinja2 import Environment, BaseLoader, TemplateSyntaxError
-            if not p.strip('{}') in notification.valid_tokens:
+        from jinja2.meta import find_undeclared_variables
-                message = field.gettext('Token \'%s\' is not a valid token.')
+
-                raise ValidationError(message % (p))
+
+        try:
+            jinja2_env = Environment(loader=BaseLoader)
+            jinja2_env.globals.update(notification.valid_tokens)
+            rendered = jinja2_env.from_string(field.data).render()
+        except TemplateSyntaxError as e:
+            raise ValidationError(f"This is not a valid Jinja2 template: {e}") from e
+
+        ast = jinja2_env.parse(field.data)
+        undefined = ", ".join(find_undeclared_variables(ast))
+        if undefined:
+            raise ValidationError(
+                f"The following tokens used in the notification are not valid: {undefined}"
+            )
 
 class validateURL(object):
 
@@ -225,6 +238,7 @@ class validateURL(object):
             message = field.gettext('\'%s\' is not a valid URL.' % (field.data.strip()))
             raise ValidationError(message)
 
+
 class ValidateListRegex(object):
     """
     Validates that anything that looks like a regex passes as a regex
@@ -333,11 +347,11 @@ class quickWatchForm(Form):
 
 # Common to a single watch and the global settings
 class commonSettingsForm(Form):
-    notification_urls = StringListField('Notification URL list', validators=[validators.Optional(), ValidateAppRiseServers()])
+    notification_urls = StringListField('Notification URL List', validators=[validators.Optional(), ValidateAppRiseServers()])
-    notification_title = StringField('Notification title', validators=[validators.Optional(), ValidateTokensList()])
+    notification_title = StringField('Notification Title', default='ChangeDetection.io Notification - {{ watch_url }}', validators=[validators.Optional(), ValidateJinja2Template()])
-    notification_body = TextAreaField('Notification body', validators=[validators.Optional(), ValidateTokensList()])
+    notification_body = TextAreaField('Notification Body', default='{{ watch_url }} had a change.', validators=[validators.Optional(), ValidateJinja2Template()])
     notification_format = SelectField('Notification format', choices=valid_notification_formats.keys())
-    fetch_backend = RadioField(u'Fetch method', choices=content_fetcher.available_fetchers(), validators=[ValidateContentFetcherIsReady()])
+    fetch_backend = RadioField(u'Fetch Method', choices=content_fetcher.available_fetchers(), validators=[ValidateContentFetcherIsReady()])
     extract_title_as_title = BooleanField('Extract <title> from document and use as watch title', default=False)
     webdriver_delay = IntegerField('Wait seconds before extracting text', validators=[validators.Optional(), validators.NumberRange(min=1,
                                                                                                                                     message="Should contain one or more seconds")])
@@ -412,6 +426,13 @@ class watchForm(commonSettingsForm):
         return result
 
 
+class SingleExtraProxy(Form):
+
+    # maybe better to set some <script>var..
+    proxy_name = StringField('Name', [validators.Optional()], render_kw={"placeholder": "Name"})
+    proxy_url = StringField('Proxy URL', [validators.Optional()], render_kw={"placeholder": "http://user:pass@...:3128", "size":50})
+    # @todo do the validation here instead
+
 # datastore.data['settings']['requests']..
 class globalSettingsRequestForm(Form):
     time_between_check = FormField(TimeBetweenCheckForm)
@@ -419,6 +440,15 @@ class globalSettingsRequestForm(Form):
     jitter_seconds = IntegerField('Random jitter seconds ± check',
                                   render_kw={"style": "width: 5em;"},
                                   validators=[validators.NumberRange(min=0, message="Should contain zero or more seconds")])
+    extra_proxies = FieldList(FormField(SingleExtraProxy), min_entries=5)
+
+    def validate_extra_proxies(self, extra_validators=None):
+        for e in self.data['extra_proxies']:
+            if e.get('proxy_name') or e.get('proxy_url'):
+                if not e.get('proxy_name','').strip() or not e.get('proxy_url','').strip():
+                    self.extra_proxies.errors.append('Both a name, and a Proxy URL is required.')
+                    return False
+
 
 # datastore.data['settings']['application']..
 class globalSettingsApplicationForm(commonSettingsForm):
@@ -448,3 +478,8 @@ class globalSettingsForm(Form):
     requests = FormField(globalSettingsRequestForm)
     application = FormField(globalSettingsApplicationForm)
     save_button = SubmitField('Save', render_kw={"class": "pure-button pure-button-primary"})
+
+
+class extractDataForm(Form):
+    extract_regex = StringField('RegEx to extract', validators=[validators.Length(min=1, message="Needs a RegEx")])
+    extract_submit_button = SubmitField('Extract as CSV', render_kw={"class": "pure-button pure-button-primary"})

changedetectionio/html_tools.py

@@ -10,6 +10,10 @@ import re
 # HTML added to be sure each result matching a filter (.example) gets converted to a new line by Inscriptis
 TEXT_FILTER_LIST_LINE_SUFFIX = "<br/>"
 
+# 'price' , 'lowPrice', 'highPrice' are usually under here
+# all of those may or may not appear on different websites
+LD_JSON_PRODUCT_OFFER_SELECTOR = "json:$..offers"
+
 class JSONNotFound(ValueError):
     def __init__(self, msg):
         ValueError.__init__(self, msg)
@@ -127,8 +131,10 @@ def _get_stripped_text_from_json_match(match):
 
     return stripped_text_from_html
 
-def extract_json_as_string(content, json_filter):
+# content - json
-
+# json_filter - ie json:$..price
+# ensure_is_ldjson_info_type - str "product", optional, "@type == product" (I dont know how to do that as a json selector)
+def extract_json_as_string(content, json_filter, ensure_is_ldjson_info_type=None):
     stripped_text_from_html = False
 
     # Try to parse/filter out the JSON, if we get some parser error, then maybe it's embedded <script type=ldjson>
@@ -139,8 +145,13 @@ def extract_json_as_string(content, json_filter):
         # Foreach <script json></script> blob.. just return the first that matches json_filter
         s = []
         soup = BeautifulSoup(content, 'html.parser')
+
+        if ensure_is_ldjson_info_type:
+            bs_result = soup.findAll('script', {"type": "application/ld+json"})
+        else:
             bs_result = soup.findAll('script')
 
+
         if not bs_result:
             raise JSONNotFound("No parsable JSON found in this document")
 
@@ -156,7 +167,14 @@ def extract_json_as_string(content, json_filter):
                 continue
             else:
                 stripped_text_from_html = _parse_json(json_data, json_filter)
-                if stripped_text_from_html:
+                if ensure_is_ldjson_info_type:
+                    # Could sometimes be list, string or something else random
+                    if isinstance(json_data, dict):
+                        # If it has LD JSON 'key' @type, and @type is 'product', and something was found for the search
+                        # (Some sites have multiple of the same ld+json @type='product', but some have the review part, some have the 'price' part)
+                        if json_data.get('@type', False) and json_data.get('@type','').lower() == ensure_is_ldjson_info_type.lower() and stripped_text_from_html:
+                            break
+                elif stripped_text_from_html:
                     break
 
     if not stripped_text_from_html:
@@ -243,6 +261,18 @@ def html_to_text(html_content: str, render_anchor_tag_content=False) -> str:
 
     return text_content
 
+
+# Does LD+JSON exist with a @type=='product' and a .price set anywhere?
+def has_ldjson_product_info(content):
+    try:
+        pricing_data = extract_json_as_string(content=content, json_filter=LD_JSON_PRODUCT_OFFER_SELECTOR, ensure_is_ldjson_info_type="product")
+    except JSONNotFound as e:
+        # Totally fine
+        return False
+    x=bool(pricing_data)
+    return x
+
+
 def workarounds_for_obfuscations(content):
     """
     Some sites are using sneaky tactics to make prices and other information un-renderable by Inscriptis

changedetectionio/model/App.py

@@ -15,11 +15,12 @@ class model(dict):
                 'headers': {
                 },
                 'requests': {
-                    'timeout': int(getenv("DEFAULT_SETTINGS_REQUESTS_TIMEOUT", "45")),  # Default 45 seconds
+                    'extra_proxies': [], # Configurable extra proxies via the UI
-                    'time_between_check': {'weeks': None, 'days': None, 'hours': 3, 'minutes': None, 'seconds': None},
                     'jitter_seconds': 0,
+                    'proxy': None, # Preferred proxy connection
+                    'time_between_check': {'weeks': None, 'days': None, 'hours': 3, 'minutes': None, 'seconds': None},
+                    'timeout': int(getenv("DEFAULT_SETTINGS_REQUESTS_TIMEOUT", "45")),  # Default 45 seconds
                     'workers': int(getenv("DEFAULT_SETTINGS_REQUESTS_WORKERS", "10")),  # Number of threads, lower is better for slow connections
-                    'proxy': None # Preferred proxy connection
                 },
                 'application': {
                     'api_access_token_enabled': True,
@@ -27,7 +28,6 @@ class model(dict):
                     'base_url' : None,
                     'extract_title_as_title': False,
                     'empty_pages_are_a_change': False,
-                    'css_dark_mode': False,
                     'fetch_backend': getenv("DEFAULT_FETCH_BACKEND", "html_requests"),
                     'filter_failure_notification_threshold_attempts': _FILTER_FAILURE_THRESHOLD_ATTEMPTS_DEFAULT,
                     'global_ignore_text': [], # List of text to ignore when calculating the comparison checksum

changedetectionio/model/Watch.py

@@ -26,6 +26,8 @@ class model(dict):
         'extract_title_as_title': False,
         'fetch_backend': None,
         'filter_failure_notification_send': strtobool(os.getenv('FILTER_FAILURE_NOTIFICATION_SEND_DEFAULT', 'True')),
+        'has_ldjson_price_data': None,
+        'track_ldjson_price_data': None,
         'headers': {},  # Extra headers to send
         'ignore_text': [],  # List of text to ignore when calculating the comparison checksum
         'include_filters': [],
@@ -42,6 +44,7 @@ class model(dict):
         'notification_urls': [],  # List of URLs to add to the notification Queue (Usually AppRise)
         'paused': False,
         'previous_md5': False,
+        'previous_md5_before_filters': False,  # Used for skipping changedetection entirely
         'proxy': None,  # Preferred proxy connection
         'subtractive_selectors': [],
         'tag': None,
@@ -111,6 +114,24 @@ class model(dict):
 
         return ready_url
 
+    @property
+    def get_fetch_backend(self):
+        """
+        Like just using the `fetch_backend` key but there could be some logic
+        :return:
+        """
+        # Maybe also if is_image etc?
+        # This is because chrome/playwright wont render the PDF in the browser and we will just fetch it and use pdf2html to see the text.
+        if self.is_pdf:
+            return 'html_requests'
+
+        return self.get('fetch_backend')
+
+    @property
+    def is_pdf(self):
+        # content_type field is set in the future
+        return '.pdf' in self.get('url', '').lower() or 'pdf' in self.get('content_type', '').lower()
+
     @property
     def label(self):
         # Used for sorting
@@ -318,3 +339,47 @@ class model(dict):
         if os.path.isfile(fname):
             return fname
         return False
+
+    def extract_regex_from_all_history(self, regex):
+        import csv
+        import re
+        import datetime
+        csv_output_filename = False
+        csv_writer = False
+        f = None
+
+        # self.history will be keyed with the full path
+        for k, fname in self.history.items():
+            if os.path.isfile(fname):
+                with open(fname, "r") as f:
+                    contents = f.read()
+                    res = re.findall(regex, contents, re.MULTILINE)
+                    if res:
+                        if not csv_writer:
+                            # A file on the disk can be transferred much faster via flask than a string reply
+                            csv_output_filename = 'report.csv'
+                            f = open(os.path.join(self.watch_data_dir, csv_output_filename), 'w')
+                            # @todo some headers in the future
+                            #fieldnames = ['Epoch seconds', 'Date']
+                            csv_writer = csv.writer(f,
+                                                    delimiter=',',
+                                                    quotechar='"',
+                                                    quoting=csv.QUOTE_MINIMAL,
+                                                    #fieldnames=fieldnames
+                                                    )
+                            csv_writer.writerow(['Epoch seconds', 'Date'])
+                            # csv_writer.writeheader()
+
+                        date_str = datetime.datetime.fromtimestamp(int(k)).strftime('%Y-%m-%d %H:%M:%S')
+                        for r in res:
+                            row = [k, date_str]
+                            if isinstance(r, str):
+                                row.append(r)
+                            else:
+                                row+=r
+                            csv_writer.writerow(row)
+
+        if f:
+            f.close()
+
+        return csv_output_filename

changedetectionio/notification.py

@@ -1,5 +1,7 @@
 import apprise
+from jinja2 import Environment, BaseLoader
 from apprise import NotifyFormat
+import json
 
 valid_tokens = {
     'base_url': '',
@@ -16,8 +18,8 @@ valid_tokens = {
 
 default_notification_format_for_watch = 'System default'
 default_notification_format = 'Text'
-default_notification_body = '{watch_url} had a change.\n---\n{diff}\n---\n'
+default_notification_body = '{{watch_url}} had a change.\n---\n{{diff}}\n---\n'
-default_notification_title = 'ChangeDetection.io Notification - {watch_url}'
+default_notification_title = 'ChangeDetection.io Notification - {{watch_url}}'
 
 valid_notification_formats = {
     'Text': NotifyFormat.TEXT,
@@ -27,24 +29,66 @@ valid_notification_formats = {
     default_notification_format_for_watch: default_notification_format_for_watch
 }
 
-def process_notification(n_object, datastore):
+# include the decorator
+from apprise.decorators import notify
 
-    # Get the notification body from datastore
+@notify(on="delete")
-    n_body = n_object.get('notification_body', default_notification_body)
+@notify(on="deletes")
-    n_title = n_object.get('notification_title', default_notification_title)
+@notify(on="get")
-    n_format = valid_notification_formats.get(
+@notify(on="gets")
-        n_object['notification_format'],
+@notify(on="post")
-        valid_notification_formats[default_notification_format],
+@notify(on="posts")
-    )
+@notify(on="put")
+@notify(on="puts")
+def apprise_custom_api_call_wrapper(body, title, notify_type, *args, **kwargs):
+    import requests
+    url = kwargs['meta'].get('url')
+
+    if url.startswith('post'):
+        r = requests.post
+    elif url.startswith('get'):
+        r = requests.get
+    elif url.startswith('put'):
+        r = requests.put
+    elif url.startswith('delete'):
+        r = requests.delete
+
+    url = url.replace('post://', 'http://')
+    url = url.replace('posts://', 'https://')
+    url = url.replace('put://', 'http://')
+    url = url.replace('puts://', 'https://')
+    url = url.replace('get://', 'http://')
+    url = url.replace('gets://', 'https://')
+    url = url.replace('put://', 'http://')
+    url = url.replace('puts://', 'https://')
+    url = url.replace('delete://', 'http://')
+    url = url.replace('deletes://', 'https://')
+
+    # Try to auto-guess if it's JSON
+    headers = {}
+    try:
+        json.loads(body)
+        headers = {'Content-Type': 'application/json; charset=utf-8'}
+    except ValueError as e:
+        pass
+
+
+    r(url, headers=headers, data=body)
+
+
+def process_notification(n_object, datastore):
 
     # Insert variables into the notification content
     notification_parameters = create_notification_parameters(n_object, datastore)
 
-    for n_k in notification_parameters:
+    # Get the notification body from datastore
-        token = '{' + n_k + '}'
+    jinja2_env = Environment(loader=BaseLoader)
-        val = notification_parameters[n_k]
+    n_body = jinja2_env.from_string(n_object.get('notification_body', default_notification_body)).render(**notification_parameters)
-        n_title = n_title.replace(token, val)
+    n_title = jinja2_env.from_string(n_object.get('notification_title', default_notification_title)).render(**notification_parameters)
-        n_body = n_body.replace(token, val)
+    n_format = valid_notification_formats.get(
+        n_object['notification_format'],
+        valid_notification_formats[default_notification_format],
+    )
     
     # https://github.com/caronc/apprise/wiki/Development_LogCapture
     # Anything higher than or equal to WARNING (which covers things like Connection errors)
@@ -53,6 +97,7 @@ def process_notification(n_object, datastore):
     sent_objs=[]
     from .apprise_asset import asset
     for url in n_object['notification_urls']:
+        url = jinja2_env.from_string(url).render(**notification_parameters)
         apobj = apprise.Apprise(debug=True, asset=asset)
         url = url.strip()
         if len(url):
@@ -66,7 +111,12 @@ def process_notification(n_object, datastore):
 
                 # So if no avatar_url is specified, add one so it can be correctly calculated into the total payload
                 k = '?' if not '?' in url else '&'
-                if not 'avatar_url' in url and not url.startswith('mail'):
+                if not 'avatar_url' in url \
+                        and not url.startswith('mail') \
+                        and not url.startswith('post') \
+                        and not url.startswith('get') \
+                        and not url.startswith('delete') \
+                        and not url.startswith('put'):
                     url += k + 'avatar_url=https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/changedetectionio/static/images/avatar-256x256.png'
 
                 if url.startswith('tgram://'):
@@ -144,7 +194,7 @@ def create_notification_parameters(n_object, datastore):
 
     watch_url = n_object['watch_url']
 
-    # Re #148 - Some people have just {base_url} in the body or title, but this may break some notification services
+    # Re #148 - Some people have just {{ base_url }} in the body or title, but this may break some notification services
     #           like 'Join', so it's always best to atleast set something obvious so that they are not broken.
     if base_url == '':
         base_url = "<base-url-env-var-not-set>"

changedetectionio/queuedWatchMetaData.py

@@ -0,0 +1,10 @@
+from dataclasses import dataclass, field
+from typing import Any
+
+# So that we can queue some metadata in `item`
+# https://docs.python.org/3/library/queue.html#queue.PriorityQueue
+#
+@dataclass(order=True)
+class PrioritizedItem:
+    priority: int
+    item: Any=field(compare=False)

changedetectionio/res/xpath_element_scraper.js

@@ -1,3 +1,6 @@
+// Copyright (C) 2021 Leigh Morresi (dgtlmoon@gmail.com)
+// All rights reserved.
+
 // @file Scrape the page looking for elements of concern (%ELEMENTS%)
 // http://matatk.agrip.org.uk/tests/position-and-width/
 // https://stackoverflow.com/questions/26813480/when-is-element-getboundingclientrect-guaranteed-to-be-updated-accurate
@@ -81,8 +84,16 @@ var bbox;
 for (var i = 0; i < elements.length; i++) {
     bbox = elements[i].getBoundingClientRect();
 
-    // Forget really small ones
+    // Exclude items that are not interactable or visible
-    if (bbox['width'] < 10 && bbox['height'] < 10) {
+    if(elements[i].style.opacity === "0") {
+        continue
+    }
+    if(elements[i].style.display === "none" || elements[i].style.pointerEvents === "none" ) {
+        continue
+    }
+
+    // Skip really small ones, and where width or height ==0
+    if (bbox['width'] * bbox['height'] < 100) {
         continue;
     }
 
@@ -138,7 +149,6 @@ for (var i = 0; i < elements.length; i++) {
 
 }
 
-
 // Inject the current one set in the include_filters, which may be a CSS rule
 // used for displaying the current one in VisualSelector, where its not one we generated.
 if (include_filters.length) {
@@ -166,9 +176,22 @@ if (include_filters.length) {
         }
 
         if (q) {
+            // #1231 - IN the case XPath attribute filter is applied, we will have to traverse up and find the element.
+            if (q.hasOwnProperty('getBoundingClientRect')) {
                 bbox = q.getBoundingClientRect();
                 console.log("xpath_element_scraper: Got filter element, scroll from top was " + scroll_y)
             } else {
+                try {
+                    // Try and see we can find its ownerElement
+                    bbox = q.ownerElement.getBoundingClientRect();
+                    console.log("xpath_element_scraper: Got filter by ownerElement element, scroll from top was " + scroll_y)
+                } catch (e) {
+                    console.log("xpath_element_scraper: error looking up ownerElement")
+                }
+            }
+        }
+        
+        if(!q) {
             console.log("xpath_element_scraper: filter element " + f + " was not found");
         }
 
@@ -184,5 +207,9 @@ if (include_filters.length) {
     }
 }
 
+// Sort the elements so we find the smallest one first, in other words, we find the smallest one matching in that area
+// so that we dont select the wrapping element by mistake and be unable to select what we want
+size_pos.sort((a, b) => (a.width*a.height > b.width*b.height) ? 1 : -1)
+
 // Window.width required for proper scaling in the frontend
 return {'size_pos': size_pos, 'browser_width': window.innerWidth};

changedetectionio/run_all_tests.sh

@@ -1,104 +0,0 @@
-#!/bin/bash
-
-
-# live_server will throw errors even with live_server_scope=function if I have the live_server setup in different functions
-# and I like to restart the server for each test (and have the test cleanup after each test)
-# merge request welcome :)
-
-
-# exit when any command fails
-set -e
-
-SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
-
-find tests/test_*py -type f|while read test_name
-do
-  echo "TEST RUNNING $test_name"
-  pytest $test_name
-done
-
-echo "RUNNING WITH BASE_URL SET"
-
-# Now re-run some tests with BASE_URL enabled
-# Re #65 - Ability to include a link back to the installation, in the notification.
-export BASE_URL="https://really-unique-domain.io"
-pytest tests/test_notification.py
-
-
-# Re-run with HIDE_REFERER set - could affect login
-export HIDE_REFERER=True
-pytest tests/test_access_control.py
-
-
-# Now for the selenium and playwright/browserless fetchers
-# Note - this is not UI functional tests - just checking that each one can fetch the content
-
-echo "TESTING WEBDRIVER FETCH > SELENIUM/WEBDRIVER..."
-docker run -d --name $$-test_selenium  -p 4444:4444 --rm --shm-size="2g"  selenium/standalone-chrome-debug:3.141.59
-# takes a while to spin up
-sleep 5
-export WEBDRIVER_URL=http://localhost:4444/wd/hub
-pytest tests/fetchers/test_content.py
-pytest tests/test_errorhandling.py
-unset WEBDRIVER_URL
-docker kill $$-test_selenium
-
-echo "TESTING WEBDRIVER FETCH > PLAYWRIGHT/BROWSERLESS..."
-# Not all platforms support playwright (not ARM/rPI), so it's not packaged in requirements.txt
-PLAYWRIGHT_VERSION=$(grep -i -E "RUN pip install.+" "$SCRIPT_DIR/../Dockerfile" | grep --only-matching -i -E "playwright[=><~+]+[0-9\.]+")
-echo "using $PLAYWRIGHT_VERSION"
-pip3 install "$PLAYWRIGHT_VERSION"
-docker run -d --name $$-test_browserless -e "DEFAULT_LAUNCH_ARGS=[\"--window-size=1920,1080\"]" --rm  -p 3000:3000  --shm-size="2g"  browserless/chrome:1.53-chrome-stable
-# takes a while to spin up
-sleep 5
-export PLAYWRIGHT_DRIVER_URL=ws://127.0.0.1:3000
-pytest tests/fetchers/test_content.py
-pytest tests/test_errorhandling.py
-pytest tests/visualselector/test_fetch_data.py
-
-unset PLAYWRIGHT_DRIVER_URL
-docker kill $$-test_browserless
-
-# Test proxy list handling, starting two squids on different ports
-# Each squid adds a different header to the response, which is the main thing we test for.
-docker run -d --name $$-squid-one --rm -v `pwd`/tests/proxy_list/squid.conf:/etc/squid/conf.d/debian.conf -p 3128:3128 ubuntu/squid:4.13-21.10_edge
-docker run -d --name $$-squid-two --rm -v `pwd`/tests/proxy_list/squid.conf:/etc/squid/conf.d/debian.conf -p 3129:3128 ubuntu/squid:4.13-21.10_edge
-
-
-# So, basic HTTP as env var test
-export HTTP_PROXY=http://localhost:3128
-export HTTPS_PROXY=http://localhost:3128
-pytest tests/proxy_list/test_proxy.py
-docker logs $$-squid-one 2>/dev/null|grep one.changedetection.io
-if [ $? -ne 0 ]
-then
-  echo "Did not see a request to one.changedetection.io in the squid logs (while checking env vars HTTP_PROXY/HTTPS_PROXY)"
-fi
-unset HTTP_PROXY
-unset HTTPS_PROXY
-
-
-# 2nd test actually choose the preferred proxy from proxies.json
-cp tests/proxy_list/proxies.json-example ./test-datastore/proxies.json
-# Makes a watch use a preferred proxy
-pytest tests/proxy_list/test_multiple_proxy.py
-
-# Should be a request in the default "first" squid
-docker logs $$-squid-one 2>/dev/null|grep chosen.changedetection.io
-if [ $? -ne 0 ]
-then
-  echo "Did not see a request to chosen.changedetection.io in the squid logs (while checking preferred proxy)"
-fi
-
-# And one in the 'second' squid (user selects this as preferred)
-docker logs $$-squid-two 2>/dev/null|grep chosen.changedetection.io
-if [ $? -ne 0 ]
-then
-  echo "Did not see a request to chosen.changedetection.io in the squid logs (while checking preferred proxy)"
-fi
-
-# @todo - test system override proxy selection and watch defaults, setup a 3rd squid?
-docker kill $$-squid-one
-docker kill $$-squid-two
-
-

changedetectionio/run_basic_tests.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+
+# live_server will throw errors even with live_server_scope=function if I have the live_server setup in different functions
+# and I like to restart the server for each test (and have the test cleanup after each test)
+# merge request welcome :)
+
+
+# exit when any command fails
+set -e
+
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+find tests/test_*py -type f|while read test_name
+do
+  echo "TEST RUNNING $test_name"
+  pytest $test_name
+done
+
+echo "RUNNING WITH BASE_URL SET"
+
+# Now re-run some tests with BASE_URL enabled
+# Re #65 - Ability to include a link back to the installation, in the notification.
+export BASE_URL="https://really-unique-domain.io"
+pytest tests/test_notification.py
+
+
+# Re-run with HIDE_REFERER set - could affect login
+export HIDE_REFERER=True
+pytest tests/test_access_control.py

changedetectionio/run_proxy_tests.sh

@@ -0,0 +1,59 @@
+#!/bin/bash
+
+# exit when any command fails
+set -e
+
+# Test proxy list handling, starting two squids on different ports
+# Each squid adds a different header to the response, which is the main thing we test for.
+docker run --network changedet-network -d --name squid-one --hostname squid-one --rm -v `pwd`/tests/proxy_list/squid.conf:/etc/squid/conf.d/debian.conf ubuntu/squid:4.13-21.10_edge
+docker run --network changedet-network -d --name squid-two --hostname squid-two --rm -v `pwd`/tests/proxy_list/squid.conf:/etc/squid/conf.d/debian.conf ubuntu/squid:4.13-21.10_edge
+
+# Used for configuring a custom proxy URL via the UI
+docker run --network changedet-network -d \
+  --name squid-custom \
+  --hostname squid-squid-custom \
+  --rm \
+  -v `pwd`/tests/proxy_list/squid-auth.conf:/etc/squid/conf.d/debian.conf \
+  -v `pwd`/tests/proxy_list/squid-passwords.txt:/etc/squid3/passwords \
+  ubuntu/squid:4.13-21.10_edge
+
+
+## 2nd test actually choose the preferred proxy from proxies.json
+
+docker run --network changedet-network \
+  -v `pwd`/tests/proxy_list/proxies.json-example:/app/changedetectionio/test-datastore/proxies.json \
+  test-changedetectionio \
+  bash -c 'cd changedetectionio && pytest tests/proxy_list/test_multiple_proxy.py'
+
+
+## Should be a request in the default "first" squid
+docker logs squid-one 2>/dev/null|grep chosen.changedetection.io
+if [ $? -ne 0 ]
+then
+  echo "Did not see a request to chosen.changedetection.io in the squid logs (while checking preferred proxy - squid one)"
+  exit 1
+fi
+
+# And one in the 'second' squid (user selects this as preferred)
+docker logs squid-two 2>/dev/null|grep chosen.changedetection.io
+if [ $? -ne 0 ]
+then
+  echo "Did not see a request to chosen.changedetection.io in the squid logs (while checking preferred proxy - squid two)"
+  exit 1
+fi
+
+
+# Test the UI configurable proxies
+
+docker run --network changedet-network \
+  test-changedetectionio \
+  bash -c 'cd changedetectionio && pytest tests/proxy_list/test_select_custom_proxy.py'
+
+
+# Should see a request for one.changedetection.io in there
+docker logs squid-custom 2>/dev/null|grep "TCP_TUNNEL.200.*changedetection.io"
+if [ $? -ne 0 ]
+then
+  echo "Did not see a valid request to changedetection.io in the squid logs (while checking preferred proxy - squid two)"
+  exit 1
+fi

changedetectionio/static/images/pdf-icon.svg

@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<svg xmlns="http://www.w3.org/2000/svg" width="75.320129mm" height="92.604164mm" viewBox="0 0 75.320129 92.604164">
+  <g transform="translate(53.548057 -183.975276) scale(1.4843)">
+    <path fill="#ff2116" d="M-29.632812 123.94727c-3.551967 0-6.44336 2.89347-6.44336 6.44531v49.49804c0 3.55185 2.891393 6.44532 6.44336 6.44532H8.2167969c3.5519661 0 6.4433591-2.89335 6.4433591-6.44532v-40.70117s.101353-1.19181-.416015-2.35156c-.484969-1.08711-1.275391-1.84375-1.275391-1.84375a1.0584391 1.0584391 0 0 0-.0059-.008l-9.3906254-9.21094a1.0584391 1.0584391 0 0 0-.015625-.0156s-.8017392-.76344-1.9902344-1.27344c-1.39939552-.6005-2.8417968-.53711-2.8417968-.53711l.021484-.002z" color="#000" font-family="sans-serif" overflow="visible" paint-order="markers fill stroke" style="line-height:normal;font-variant-ligatures:normal;font-variant-position:normal;font-variant-caps:normal;font-variant-numeric:normal;font-variant-alternates:normal;font-feature-settings:normal;text-indent:0;text-align:start;text-decoration-line:none;text-decoration-style:solid;text-decoration-color:#000000;text-transform:none;text-orientation:mixed;white-space:normal;shape-padding:0;isolation:auto;mix-blend-mode:normal;solid-color:#000000;solid-opacity:1"/>
+    <path fill="#f5f5f5" d="M-29.632812 126.06445h28.3789058a1.0584391 1.0584391 0 0 0 .021484 0s1.13480448.011 1.96484378.36719c.79889772.34282 1.36536982.86176 1.36914062.86524.0000125.00001.00391.004.00391.004l9.3671868 9.18945s.564354.59582.837891 1.20899c.220779.49491.234375 1.40039.234375 1.40039a1.0584391 1.0584391 0 0 0-.002.0449v40.74609c0 2.41592-1.910258 4.32813-4.3261717 4.32813H-29.632812c-2.415914 0-4.326172-1.91209-4.326172-4.32813v-49.49804c0-2.41603 1.910258-4.32813 4.326172-4.32813z" color="#000" font-family="sans-serif" overflow="visible" paint-order="markers fill stroke" style="line-height:normal;font-variant-ligatures:normal;font-variant-position:normal;font-variant-caps:normal;font-variant-numeric:normal;font-variant-alternates:normal;font-feature-settings:normal;text-indent:0;text-align:start;text-decoration-line:none;text-decoration-style:solid;text-decoration-color:#000000;text-transform:none;text-orientation:mixed;white-space:normal;shape-padding:0;isolation:auto;mix-blend-mode:normal;solid-color:#000000;solid-opacity:1"/>
+    <path fill="#ff2116" d="M-23.40766 161.09299c-1.45669-1.45669.11934-3.45839 4.39648-5.58397l2.69124-1.33743 1.04845-2.29399c.57665-1.26169 1.43729-3.32036 1.91254-4.5748l.8641-2.28082-.59546-1.68793c-.73217-2.07547-.99326-5.19438-.52872-6.31588.62923-1.51909 2.69029-1.36323 3.50626.26515.63727 1.27176.57212 3.57488-.18329 6.47946l-.6193 2.38125.5455.92604c.30003.50932 1.1764 1.71867 1.9475 2.68743l1.44924 1.80272 1.8033728-.23533c5.72900399-.74758 7.6912472.523 7.6912472 2.34476 0 2.29921-4.4984914 2.48899-8.2760865-.16423-.8499666-.59698-1.4336605-1.19001-1.4336605-1.19001s-2.3665326.48178-3.531704.79583c-1.202707.32417-1.80274.52719-3.564509 1.12186 0 0-.61814.89767-1.02094 1.55026-1.49858 2.4279-3.24833 4.43998-4.49793 5.1723-1.3991.81993-2.86584.87582-3.60433.13733zm2.28605-.81668c.81883-.50607 2.47616-2.46625 3.62341-4.28553l.46449-.73658-2.11497 1.06339c-3.26655 1.64239-4.76093 3.19033-3.98386 4.12664.43653.52598.95874.48237 2.01093-.16792zm21.21809-5.95578c.80089-.56097.68463-1.69142-.22082-2.1472-.70466-.35471-1.2726074-.42759-3.1031574-.40057-1.1249.0767-2.9337647.3034-3.2403347.37237 0 0 .993716.68678 1.434896.93922.58731.33544 2.0145161.95811 3.0565161 1.27706 1.02785.31461 1.6224.28144 2.0729-.0409zm-8.53152-3.54594c-.4847-.50952-1.30889-1.57296-1.83152-2.3632-.68353-.89643-1.02629-1.52887-1.02629-1.52887s-.4996 1.60694-.90948 2.57394l-1.27876 3.16076-.37075.71695s1.971043-.64627 2.97389-.90822c1.0621668-.27744 3.21787-.70134 3.21787-.70134zm-2.74938-11.02573c.12363-1.0375.1761-2.07346-.15724-2.59587-.9246-1.01077-2.04057-.16787-1.85154 2.23517.0636.8084.26443 2.19033.53292 3.04209l.48817 1.54863.34358-1.16638c.18897-.64151.47882-2.02015.64411-3.06364z"/>
+    <path fill="#2c2c2c" d="M-20.930423 167.83862h2.364986q1.133514 0 1.840213.2169.706698.20991 1.189489.9446.482795.72769.482795 1.75625 0 .94459-.391832 1.6233-.391833.67871-1.056548.97958-.65772.30087-2.02913.30087h-.818651v3.72941h-1.581322zm1.581322 1.22447v3.33058h.783664q1.049552 0 1.44838-.39184.405826-.39183.405826-1.27345 0-.65772-.265887-1.06355-.265884-.41282-.587747-.50378-.314866-.098-1.000572-.098zm5.50664-1.22447h2.148082q1.560333 0 2.4909318.55276.9375993.55276 1.4133973 1.6443.482791 1.09153.482791 2.42096 0 1.3994-.4338151 2.49793-.4268149 1.09153-1.3154348 1.76324-.8816233.67172-2.5189212.67172h-2.267031zm1.581326 1.26645v7.018h.657715q1.378411 0 2.001144-.9516.6227329-.95858.6227329-2.5539 0-3.5125-2.6238769-3.5125zm6.4722254-1.26645h5.30372941v1.26645H-4.2075842v2.85478h2.9807225v1.26646h-2.9807225v4.16322h-1.5813254z" font-family="Franklin Gothic Medium Cond" letter-spacing="0" style="line-height:125%;-inkscape-font-specification:'Franklin Gothic Medium Cond'" word-spacing="4.26000023"/>
+  </g>
+</svg>

changedetectionio/static/images/price-tag-icon.svg

@@ -0,0 +1,2 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<svg width="83.39" height="89.648" enable-background="new 0 0 122.406 122.881" version="1.1" viewBox="0 0 83.39 89.648" xml:space="preserve" xmlns="http://www.w3.org/2000/svg"><g transform="translate(5e-4 -33.234)"><path d="m44.239 42.946-39.111 39.896 34.908 34.91 39.09-39.876-1.149-34.931zm-0.91791 42.273c0.979-0.979 1.507-1.99 1.577-3.027 0.077-1.043-0.248-2.424-0.967-4.135-0.725-1.717-1.348-3.346-1.87-4.885s-0.814-3.014-0.897-4.432c-0.07-1.42 0.134-2.768 0.624-4.045 0.477-1.279 1.348-2.545 2.607-3.804 2.099-2.099 4.535-3.123 7.314-3.065 2.773 0.063 5.457 1.158 8.04 3.294l2.881 3.034c1.946 2.607 2.799 5.33 2.557 8.166-0.235 2.83-1.532 5.426-3.893 7.785l-6.296-6.297c1.291-1.291 2.035-2.531 2.238-3.727 0.191-1.197-0.165-2.252-1.081-3.168-0.821-0.82-1.717-1.195-2.69-1.139-0.967 0.064-1.908 0.547-2.817 1.457-0.922 0.922-1.393 1.914-1.412 2.977s0.306 2.416 0.973 4.064c0.661 1.652 1.24 3.25 1.736 4.801 0.496 1.553 0.782 3.035 0.858 4.445 0.076 1.426-0.127 2.787-0.591 4.104-0.477 1.316-1.336 2.596-2.588 3.848-2.125 2.125-4.522 3.186-7.212 3.18s-5.311-1.063-7.855-3.16l-3.747 3.746-2.964-2.965 3.766-3.764c-2.423-2.996-3.568-5.998-3.447-9.02 0.127-3.014 1.476-5.813 4.045-8.383l6.278 6.277c-1.412 1.412-2.175 2.799-2.277 4.16-0.108 1.367 0.414 2.627 1.571 3.783 0.839 0.84 1.755 1.26 2.741 1.242 0.985-0.017 1.92-0.47 2.798-1.347zm21.127-46.435h17.457c-0.0269 2.2368 0.69936 16.025 0.69936 16.025l0.785 23.858c0.019 0.609-0.221 1.164-0.619 1.564l5e-3 4e-3 -41.236 42.022c-0.82213 0.8378-2.175 0.83-3.004 0l-37.913-37.91c-0.83-0.83-0.83-2.176 0-3.006l41.236-42.021c0.39287-0.42671 1.502-0.53568 1.502-0.53568zm18.011 11.59c-59.392-29.687-29.696-14.843 0 0z"/></g></svg>

changedetectionio/static/js/diff-overview.js

@@ -13,6 +13,8 @@ $(document).ready(function () {
         } else if (hash_name === '#error-screenshot') {
             $("img#error-screenshot-img").attr('src', error_screenshot_url);
             $("#settings").hide();
+        } else if (hash_name === '#extract') {
+            $("#settings").hide();
         }
 
 

changedetectionio/static/js/toggle-theme.js

@@ -19,6 +19,6 @@ $(document).ready(function () {
   };
 
   const setCookieValue = (value) => {
-    document.cookie = `css_dark_mode=${value};max-age=31536000`
+    document.cookie = `css_dark_mode=${value};max-age=31536000;path=/`
   }
 });

changedetectionio/static/js/visual-selector.js

@@ -1,4 +1,5 @@
-// Horrible proof of concept code :)
+// Copyright (C) 2021 Leigh Morresi (dgtlmoon@gmail.com)
+// All rights reserved.
 // yes - this is really a hack, if you are a front-ender and want to help, please get in touch!
 
 $(document).ready(function () {
@@ -177,9 +178,10 @@ $(document).ready(function () {
             // Basically, find the most 'deepest'
             var found = 0;
             ctx.fillStyle = 'rgba(205,0,0,0.35)';
-            for (var i = selector_data['size_pos'].length; i !== 0; i--) {
+            // Will be sorted by smallest width*height first
+            for (var i = 0; i <= selector_data['size_pos'].length; i++) {
                 // draw all of them? let them choose somehow?
-                var sel = selector_data['size_pos'][i - 1];
+                var sel = selector_data['size_pos'][i];
                 // If we are in a bounding-box
                 if (e.offsetY > sel.top * y_scale && e.offsetY < sel.top * y_scale + sel.height * y_scale
                     &&
@@ -195,7 +197,7 @@ $(document).ready(function () {
                     // no need to keep digging
                     // @todo or, O to go out/up, I to go in
                     // or double click to go up/out the selector?
-                    current_selected_i = i - 1;
+                    current_selected_i = i;
                     found += 1;
                     break;
                 }

changedetectionio/static/styles/.dockerignore

@@ -0,0 +1,3 @@
+node_modules
+package-lock.json
+

changedetectionio/static/styles/diff.css

@@ -18,6 +18,8 @@
   --color-grey-850: #eee;
   --color-grey-900: #f2f2f2;
   --color-black: #000;
+  --color-dark-red: #a00;
+  --color-light-red: #dd0000;
   --color-background-page: var(--color-grey-100);
   --color-background-gradient-first: #5ad8f7;
   --color-background-gradient-second: #2f50af;
@@ -27,9 +29,9 @@
   --color-link: #1b98f8;
   --color-menu-accent: #ed5900;
   --color-background-code: var(--color-grey-850);
-  --color-error: #a00;
+  --color-error: var(--color-dark-red);
   --color-error-input: #ffebeb;
-  --color-error-list: #dd0000;
+  --color-error-list: var(--color-light-red);
   --color-table-background: var(--color-background);
   --color-table-stripe: var(--color-grey-900);
   --color-text-tab: var(--color-white);
@@ -84,7 +86,9 @@
   --color-text-menu-link-hover: var(--color-grey-300);
   --color-shadow-jump: var(--color-grey-500);
   --color-icon-github: var(--color-black);
-  --color-icon-github-hover: var(--color-grey-300); }
+  --color-icon-github-hover: var(--color-grey-300);
+  --color-watch-table-error: var(--color-dark-red);
+  --color-watch-table-row-text: var(--color-grey-100); }
 
 html[data-darkmode="true"] {
   --color-link: #59bdfb;
@@ -114,21 +118,30 @@ html[data-darkmode="true"] {
   --color-background-snapshot-age: var(--color-grey-200);
   --color-shadow-jump: var(--color-grey-200);
   --color-icon-github: var(--color-white);
-  --color-icon-github-hover: var(--color-grey-700); }
+  --color-icon-github-hover: var(--color-grey-700);
-  html[data-darkmode="true"] .watch-controls img {
+  --color-watch-table-error: var(--color-light-red);
-    opacity: 0.4; }
+  --color-watch-table-row-text: var(--color-grey-800); }
   html[data-darkmode="true"] .icon-spread {
     filter: hue-rotate(-10deg) brightness(1.5); }
   html[data-darkmode="true"] .watch-table .title-col a[target="_blank"]::after,
   html[data-darkmode="true"] .watch-table .current-diff-url::after {
     filter: invert(0.5) hue-rotate(10deg) brightness(2); }
+  html[data-darkmode="true"] .watch-table .watch-controls .state-off img {
+    opacity: 0.3; }
+  html[data-darkmode="true"] .watch-table .watch-controls .state-on img {
+    opacity: 1.0; }
+  html[data-darkmode="true"] .watch-table .unviewed {
+    color: #fff; }
+    html[data-darkmode="true"] .watch-table .unviewed.error {
+      color: var(--color-watch-table-error); }
 
 #diff-ui {
   background: var(--color-background);
   padding: 2em;
   margin-left: 1em;
   margin-right: 1em;
-  border-radius: 5px;
+  border-radius: 5px; }
+  #diff-ui #text {
     font-size: 11px; }
   #diff-ui table {
     table-layout: fixed;

changedetectionio/static/styles/scss/diff.scss

@@ -7,7 +7,11 @@
   margin-left: 1em;
   margin-right: 1em;
   border-radius: 5px;
+
+  // The first tab 'text' diff
+  #text {
     font-size: 11px;
+  }
 
   table {
     table-layout: fixed;

changedetectionio/static/styles/scss/parts/_extra_proxies.scss

@@ -0,0 +1,17 @@
+ul#requests-extra_proxies {
+  list-style: none;
+  /* tidy up the table to look more "inline" */
+  li {
+    > label {
+      display: none;
+    }
+
+  }
+  /* each proxy entry is a `table` */
+  table {
+    tr {
+      display: inline;
+    }
+  }
+}
+

changedetectionio/static/styles/scss/parts/_variables.scss

@@ -19,6 +19,8 @@
   --color-grey-850: #eee;
   --color-grey-900: #f2f2f2;
   --color-black: #000;
+  --color-dark-red: #a00;
+  --color-light-red: #dd0000;
 
   --color-background-page: var(--color-grey-100);
   --color-background-gradient-first: #5ad8f7;
@@ -29,9 +31,9 @@
   --color-link: #1b98f8;
   --color-menu-accent: #ed5900;
   --color-background-code: var(--color-grey-850);
-  --color-error: #a00;
+  --color-error: var(--color-dark-red);
   --color-error-input: #ffebeb;
-  --color-error-list: #dd0000;
+  --color-error-list: var(--color-light-red);
   --color-table-background: var(--color-background);
   --color-table-stripe: var(--color-grey-900);
   --color-text-tab: var(--color-white);
@@ -96,6 +98,9 @@
   --color-shadow-jump: var(--color-grey-500);
   --color-icon-github: var(--color-black);
   --color-icon-github-hover: var(--color-grey-300);
+
+  --color-watch-table-error: var(--color-dark-red);
+  --color-watch-table-row-text: var(--color-grey-100);
 }
 
 html[data-darkmode="true"] {
@@ -123,7 +128,6 @@ html[data-darkmode="true"] {
   --color-text-input-description: var(--color-grey-600);
   --color-text-input-placeholder: var(--color-grey-600);
   --color-text-watch-tag-list: #fa3e92;
-
   --color-background-code: var(--color-grey-200);
 
   --color-background-tab: rgba(0, 0, 0, 0.2);
@@ -133,13 +137,9 @@ html[data-darkmode="true"] {
   --color-shadow-jump: var(--color-grey-200);
   --color-icon-github: var(--color-white);
   --color-icon-github-hover: var(--color-grey-700);
+  --color-watch-table-error: var(--color-light-red);
+  --color-watch-table-row-text: var(--color-grey-800);
 
-  // Anything that can't be manipulated through variables follows.
-  .watch-controls {
-    img {
-      opacity: 0.4;
-    }
-  }
 
   .icon-spread {
     filter: hue-rotate(-10deg) brightness(1.5);
@@ -151,5 +151,25 @@ html[data-darkmode="true"] {
     .current-diff-url::after {
       filter: invert(.5) hue-rotate(10deg) brightness(2);
     }
+
+    .watch-controls {
+      .state-off {
+        img {
+          opacity: 0.3;
+        }
+      }
+      .state-on {
+        img {
+          opacity: 1.0;
+        }
+      }
+    }
+
+    .unviewed {
+      color: #fff;
+      &.error {
+        color: var(--color-watch-table-error);
+      }
+    }
   }
 }

changedetectionio/static/styles/scss/styles.scss

@@ -2,10 +2,11 @@
  * -- BASE STYLES --
  */
 
-@import "parts/_variables";
-@import "parts/_spinners";
-@import "parts/_browser-steps";
 @import "parts/_arrows";
+@import "parts/_browser-steps";
+@import "parts/_extra_proxies";
+@import "parts/_spinners";
+@import "parts/_variables";
 
 body {
   color: var(--color-text);
@@ -22,6 +23,13 @@ body {
   width: 1px;
 }
 
+// Row icons like chrome, pdf, share, etc
+.status-icon {
+  display: inline-block;
+  height: 1rem;
+  vertical-align: middle;
+}
+
 .pure-table-even {
   background: var(--color-background);
 }
@@ -121,22 +129,25 @@ code {
   width: 100%;
   font-size: 80%;
 
-  tr.unviewed {
+  tr {
+    &.unviewed {
       font-weight: bold;
     }
-
+    &.error {
-  .error {
+      color: var(--color-watch-table-error);
-    color: var(--color-error);
     }
+    color: var(--color-watch-table-row-text);
+  }
+
 
   td {
     white-space: nowrap;
-  }
+    &.title-col {
-
-  td.title-col {
       word-break: break-all;
       white-space: normal;
     }
+  }
+
 
   th {
     white-space: nowrap;
@@ -322,6 +333,12 @@ a.pure-button-selected {
   padding: 0.5rem 0 1rem 0;
 }
 
+label {
+ &:hover {
+   cursor: pointer;
+ }  
+}
+
 #notification-customisation {
   border: 1px solid var(--color-border-notification);
   padding: 0.5rem;
@@ -868,6 +885,9 @@ body.full-width {
   .pure-form-message-inline {
     padding-left: 0;
     color: var(--color-text-input-description);
+    code {
+      font-size: .875em;
+    }
   }
 }
 
@@ -997,3 +1017,30 @@ ul {
   border-radius: 5px;
   color: var(--color-warning);
 }
+
+/* automatic price following helpers */
+.tracking-ldjson-price-data {
+  background-color: var(--color-background-button-green);
+  color: #000;
+  padding: 3px;
+  border-radius: 3px;
+  white-space: nowrap;
+}
+
+.ldjson-price-track-offer {
+  a.pure-button {
+    border-radius: 3px;
+    padding: 3px;
+    background-color: var(--color-background-button-green);
+  }
+
+  font-weight: bold;
+  font-style: italic;
+}
+
+.price-follow-tag-icon {
+  display: inline-block;
+  height: 0.8rem;
+  vertical-align: middle;
+}
+

changedetectionio/static/styles/styles.css

@@ -1,169 +1,23 @@
 /*
  * -- BASE STYLES --
  */
-/**
+.arrow {
- * CSS custom properties (aka variables).
+  border: solid #1b98f8;
- */
+  border-width: 0 2px 2px 0;
-:root {
-  --color-white: #fff;
-  --color-grey-50: #111;
-  --color-grey-100: #262626;
-  --color-grey-200: #333;
-  --color-grey-300: #444;
-  --color-grey-325: #555;
-  --color-grey-350: #565d64;
-  --color-grey-400: #666;
-  --color-grey-500: #777;
-  --color-grey-600: #999;
-  --color-grey-700: #cbcbcb;
-  --color-grey-750: #ddd;
-  --color-grey-800: #e0e0e0;
-  --color-grey-850: #eee;
-  --color-grey-900: #f2f2f2;
-  --color-black: #000;
-  --color-background-page: var(--color-grey-100);
-  --color-background-gradient-first: #5ad8f7;
-  --color-background-gradient-second: #2f50af;
-  --color-background-gradient-third: #9150bf;
-  --color-background: var(--color-white);
-  --color-text: var(--color-grey-200);
-  --color-link: #1b98f8;
-  --color-menu-accent: #ed5900;
-  --color-background-code: var(--color-grey-850);
-  --color-error: #a00;
-  --color-error-input: #ffebeb;
-  --color-error-list: #dd0000;
-  --color-table-background: var(--color-background);
-  --color-table-stripe: var(--color-grey-900);
-  --color-text-tab: var(--color-white);
-  --color-background-tab: rgba(255, 255, 255, 0.2);
-  --color-background-tab-hover: rgba(255, 255, 255, 0.5);
-  --color-text-tab-active: #222;
-  --color-api-key: #0078e7;
-  --color-background-button-primary: #0078e7;
-  --color-background-button-green: #42dd53;
-  --color-background-button-red: #dd4242;
-  --color-background-button-success: rgb(28, 184, 65);
-  --color-background-button-error: rgb(202, 60, 60);
-  --color-text-button-error: var(--color-white);
-  --color-background-button-warning: rgb(202, 60, 60);
-  --color-text-button-warning: var(--color-white);
-  --color-background-button-secondary: rgb(66, 184, 221);
-  --color-background-button-cancel: rgb(200, 200, 200);
-  --color-text-button: var(--color-white);
-  --color-background-button-tag: rgb(99, 99, 99);
-  --color-background-snapshot-age: #dfdfdf;
-  --color-error-text-snapshot-age: var(--color-white);
-  --color-error-background-snapshot-age: #ff0000;
-  --color-background-button-tag-active: #9c9c9c;
-  --color-text-messages: var(--color-white);
-  --color-background-messages-message: rgba(255, 255, 255, .2);
-  --color-background-messages-error: rgba(255, 1, 1, .5);
-  --color-background-messages-notice: rgba(255, 255, 255, .5);
-  --color-border-notification: #ccc;
-  --color-background-checkbox-operations: rgba(0, 0, 0, 0.05);
-  --color-warning: #ff3300;
-  --color-border-warning: var(--color-warning);
-  --color-text-legend: var(--color-white);
-  --color-link-new-version: #e07171;
-  --color-last-checked: #bbb;
-  --color-text-footer: #444;
-  --color-border-watch-table-cell: #eee;
-  --color-text-watch-tag-list: #e70069;
-  --color-background-new-watch-form: rgba(0, 0, 0, 0.05);
-  --color-background-new-watch-input: var(--color-white);
-  --color-text-new-watch-input: var(--color-text);
-  --color-border-input: var(--color-grey-500);
-  --color-shadow-input: var(--color-grey-400);
-  --color-background-input: var(--color-white);
-  --color-text-input: var(--color-text);
-  --color-text-input-description: var(--color-grey-500);
-  --color-text-input-placeholder: var(--color-grey-600);
-  --color-background-table-thead: var(--color-grey-800);
-  --color-border-table-cell: var(--color-grey-700);
-  --color-text-menu-heading: var(--color-grey-350);
-  --color-text-menu-link: var(--color-grey-500);
-  --color-background-menu-link-hover: var(--color-grey-850);
-  --color-text-menu-link-hover: var(--color-grey-300);
-  --color-shadow-jump: var(--color-grey-500);
-  --color-icon-github: var(--color-black);
-  --color-icon-github-hover: var(--color-grey-300); }
-
-html[data-darkmode="true"] {
-  --color-link: #59bdfb;
-  --color-text: var(--color-white);
-  --color-background-gradient-first: #3f90a5;
-  --color-background-gradient-second: #1e316c;
-  --color-background-gradient-third: #4d2c64;
-  --color-background-new-watch-input: var(--color-grey-100);
-  --color-text-new-watch-input: var(--color-text);
-  --color-background-table-thead: var(--color-grey-200);
-  --color-table-background: var(--color-grey-300);
-  --color-table-stripe: var(--color-grey-325);
-  --color-background: var(--color-grey-300);
-  --color-text-menu-heading: var(--color-grey-850);
-  --color-text-menu-link: var(--color-grey-800);
-  --color-border-table-cell: var(--color-grey-400);
-  --color-text-tab-active: var(--color-text);
-  --color-border-input: var(--color-grey-400);
-  --color-shadow-input: var(--color-grey-50);
-  --color-background-input: var(--color-grey-350);
-  --color-text-input-description: var(--color-grey-600);
-  --color-text-input-placeholder: var(--color-grey-600);
-  --color-text-watch-tag-list: #fa3e92;
-  --color-background-code: var(--color-grey-200);
-  --color-background-tab: rgba(0, 0, 0, 0.2);
-  --color-background-tab-hover: rgba(0, 0, 0, 0.5);
-  --color-background-snapshot-age: var(--color-grey-200);
-  --color-shadow-jump: var(--color-grey-200);
-  --color-icon-github: var(--color-white);
-  --color-icon-github-hover: var(--color-grey-700); }
-  html[data-darkmode="true"] .watch-controls img {
-    opacity: 0.4; }
-  html[data-darkmode="true"] .icon-spread {
-    filter: hue-rotate(-10deg) brightness(1.5); }
-  html[data-darkmode="true"] .watch-table .title-col a[target="_blank"]::after,
-  html[data-darkmode="true"] .watch-table .current-diff-url::after {
-    filter: invert(0.5) hue-rotate(10deg) brightness(2); }
-
-/* spinner */
-.spinner,
-.spinner:after {
-  border-radius: 50%;
-  width: 10px;
-  height: 10px; }
-
-.spinner {
-  margin: 0px auto;
-  font-size: 3px;
-  vertical-align: middle;
   display: inline-block;
-  text-indent: -9999em;
+  padding: 3px; }
-  border-top: 1.1em solid rgba(38, 104, 237, 0.2);
+  .arrow.right {
-  border-right: 1.1em solid rgba(38, 104, 237, 0.2);
+    transform: rotate(-45deg);
-  border-bottom: 1.1em solid rgba(38, 104, 237, 0.2);
+    -webkit-transform: rotate(-45deg); }
-  border-left: 1.1em solid #2668ed;
+  .arrow.left {
-  -webkit-transform: translateZ(0);
+    transform: rotate(135deg);
-  -ms-transform: translateZ(0);
+    -webkit-transform: rotate(135deg); }
-  transform: translateZ(0);
+  .arrow.up, .arrow.asc {
-  -webkit-animation: load8 1.1s infinite linear;
+    transform: rotate(-135deg);
-  animation: load8 1.1s infinite linear; }
+    -webkit-transform: rotate(-135deg); }
-
+  .arrow.down, .arrow.desc {
-@-webkit-keyframes load8 {
+    transform: rotate(45deg);
-  0% {
+    -webkit-transform: rotate(45deg); }
-    -webkit-transform: rotate(0deg);
-    transform: rotate(0deg); }
-  100% {
-    -webkit-transform: rotate(360deg);
-    transform: rotate(360deg); } }
-
-@keyframes load8 {
-  0% {
-    -webkit-transform: rotate(0deg);
-    transform: rotate(0deg); }
-  100% {
-    -webkit-transform: rotate(360deg);
-    transform: rotate(360deg); } }
 
 #browser_steps {
   /* convert rows to horizontal cells */ }
@@ -232,23 +86,190 @@ html[data-darkmode="true"] {
     #browsersteps-selector-wrapper #browsersteps-click-start:hover {
       cursor: pointer; }
 
-.arrow {
+ul#requests-extra_proxies {
-  border: solid #1b98f8;
+  list-style: none;
-  border-width: 0 2px 2px 0;
+  /* tidy up the table to look more "inline" */
+  /* each proxy entry is a `table` */ }
+  ul#requests-extra_proxies li > label {
+    display: none; }
+  ul#requests-extra_proxies table tr {
+    display: inline; }
+
+/* spinner */
+.spinner,
+.spinner:after {
+  border-radius: 50%;
+  width: 10px;
+  height: 10px; }
+
+.spinner {
+  margin: 0px auto;
+  font-size: 3px;
+  vertical-align: middle;
   display: inline-block;
-  padding: 3px; }
+  text-indent: -9999em;
-  .arrow.right {
+  border-top: 1.1em solid rgba(38, 104, 237, 0.2);
-    transform: rotate(-45deg);
+  border-right: 1.1em solid rgba(38, 104, 237, 0.2);
-    -webkit-transform: rotate(-45deg); }
+  border-bottom: 1.1em solid rgba(38, 104, 237, 0.2);
-  .arrow.left {
+  border-left: 1.1em solid #2668ed;
-    transform: rotate(135deg);
+  -webkit-transform: translateZ(0);
-    -webkit-transform: rotate(135deg); }
+  -ms-transform: translateZ(0);
-  .arrow.up, .arrow.asc {
+  transform: translateZ(0);
-    transform: rotate(-135deg);
+  -webkit-animation: load8 1.1s infinite linear;
-    -webkit-transform: rotate(-135deg); }
+  animation: load8 1.1s infinite linear; }
-  .arrow.down, .arrow.desc {
+
-    transform: rotate(45deg);
+@-webkit-keyframes load8 {
-    -webkit-transform: rotate(45deg); }
+  0% {
+    -webkit-transform: rotate(0deg);
+    transform: rotate(0deg); }
+  100% {
+    -webkit-transform: rotate(360deg);
+    transform: rotate(360deg); } }
+
+@keyframes load8 {
+  0% {
+    -webkit-transform: rotate(0deg);
+    transform: rotate(0deg); }
+  100% {
+    -webkit-transform: rotate(360deg);
+    transform: rotate(360deg); } }
+
+/**
+ * CSS custom properties (aka variables).
+ */
+:root {
+  --color-white: #fff;
+  --color-grey-50: #111;
+  --color-grey-100: #262626;
+  --color-grey-200: #333;
+  --color-grey-300: #444;
+  --color-grey-325: #555;
+  --color-grey-350: #565d64;
+  --color-grey-400: #666;
+  --color-grey-500: #777;
+  --color-grey-600: #999;
+  --color-grey-700: #cbcbcb;
+  --color-grey-750: #ddd;
+  --color-grey-800: #e0e0e0;
+  --color-grey-850: #eee;
+  --color-grey-900: #f2f2f2;
+  --color-black: #000;
+  --color-dark-red: #a00;
+  --color-light-red: #dd0000;
+  --color-background-page: var(--color-grey-100);
+  --color-background-gradient-first: #5ad8f7;
+  --color-background-gradient-second: #2f50af;
+  --color-background-gradient-third: #9150bf;
+  --color-background: var(--color-white);
+  --color-text: var(--color-grey-200);
+  --color-link: #1b98f8;
+  --color-menu-accent: #ed5900;
+  --color-background-code: var(--color-grey-850);
+  --color-error: var(--color-dark-red);
+  --color-error-input: #ffebeb;
+  --color-error-list: var(--color-light-red);
+  --color-table-background: var(--color-background);
+  --color-table-stripe: var(--color-grey-900);
+  --color-text-tab: var(--color-white);
+  --color-background-tab: rgba(255, 255, 255, 0.2);
+  --color-background-tab-hover: rgba(255, 255, 255, 0.5);
+  --color-text-tab-active: #222;
+  --color-api-key: #0078e7;
+  --color-background-button-primary: #0078e7;
+  --color-background-button-green: #42dd53;
+  --color-background-button-red: #dd4242;
+  --color-background-button-success: rgb(28, 184, 65);
+  --color-background-button-error: rgb(202, 60, 60);
+  --color-text-button-error: var(--color-white);
+  --color-background-button-warning: rgb(202, 60, 60);
+  --color-text-button-warning: var(--color-white);
+  --color-background-button-secondary: rgb(66, 184, 221);
+  --color-background-button-cancel: rgb(200, 200, 200);
+  --color-text-button: var(--color-white);
+  --color-background-button-tag: rgb(99, 99, 99);
+  --color-background-snapshot-age: #dfdfdf;
+  --color-error-text-snapshot-age: var(--color-white);
+  --color-error-background-snapshot-age: #ff0000;
+  --color-background-button-tag-active: #9c9c9c;
+  --color-text-messages: var(--color-white);
+  --color-background-messages-message: rgba(255, 255, 255, .2);
+  --color-background-messages-error: rgba(255, 1, 1, .5);
+  --color-background-messages-notice: rgba(255, 255, 255, .5);
+  --color-border-notification: #ccc;
+  --color-background-checkbox-operations: rgba(0, 0, 0, 0.05);
+  --color-warning: #ff3300;
+  --color-border-warning: var(--color-warning);
+  --color-text-legend: var(--color-white);
+  --color-link-new-version: #e07171;
+  --color-last-checked: #bbb;
+  --color-text-footer: #444;
+  --color-border-watch-table-cell: #eee;
+  --color-text-watch-tag-list: #e70069;
+  --color-background-new-watch-form: rgba(0, 0, 0, 0.05);
+  --color-background-new-watch-input: var(--color-white);
+  --color-text-new-watch-input: var(--color-text);
+  --color-border-input: var(--color-grey-500);
+  --color-shadow-input: var(--color-grey-400);
+  --color-background-input: var(--color-white);
+  --color-text-input: var(--color-text);
+  --color-text-input-description: var(--color-grey-500);
+  --color-text-input-placeholder: var(--color-grey-600);
+  --color-background-table-thead: var(--color-grey-800);
+  --color-border-table-cell: var(--color-grey-700);
+  --color-text-menu-heading: var(--color-grey-350);
+  --color-text-menu-link: var(--color-grey-500);
+  --color-background-menu-link-hover: var(--color-grey-850);
+  --color-text-menu-link-hover: var(--color-grey-300);
+  --color-shadow-jump: var(--color-grey-500);
+  --color-icon-github: var(--color-black);
+  --color-icon-github-hover: var(--color-grey-300);
+  --color-watch-table-error: var(--color-dark-red);
+  --color-watch-table-row-text: var(--color-grey-100); }
+
+html[data-darkmode="true"] {
+  --color-link: #59bdfb;
+  --color-text: var(--color-white);
+  --color-background-gradient-first: #3f90a5;
+  --color-background-gradient-second: #1e316c;
+  --color-background-gradient-third: #4d2c64;
+  --color-background-new-watch-input: var(--color-grey-100);
+  --color-text-new-watch-input: var(--color-text);
+  --color-background-table-thead: var(--color-grey-200);
+  --color-table-background: var(--color-grey-300);
+  --color-table-stripe: var(--color-grey-325);
+  --color-background: var(--color-grey-300);
+  --color-text-menu-heading: var(--color-grey-850);
+  --color-text-menu-link: var(--color-grey-800);
+  --color-border-table-cell: var(--color-grey-400);
+  --color-text-tab-active: var(--color-text);
+  --color-border-input: var(--color-grey-400);
+  --color-shadow-input: var(--color-grey-50);
+  --color-background-input: var(--color-grey-350);
+  --color-text-input-description: var(--color-grey-600);
+  --color-text-input-placeholder: var(--color-grey-600);
+  --color-text-watch-tag-list: #fa3e92;
+  --color-background-code: var(--color-grey-200);
+  --color-background-tab: rgba(0, 0, 0, 0.2);
+  --color-background-tab-hover: rgba(0, 0, 0, 0.5);
+  --color-background-snapshot-age: var(--color-grey-200);
+  --color-shadow-jump: var(--color-grey-200);
+  --color-icon-github: var(--color-white);
+  --color-icon-github-hover: var(--color-grey-700);
+  --color-watch-table-error: var(--color-light-red);
+  --color-watch-table-row-text: var(--color-grey-800); }
+  html[data-darkmode="true"] .icon-spread {
+    filter: hue-rotate(-10deg) brightness(1.5); }
+  html[data-darkmode="true"] .watch-table .title-col a[target="_blank"]::after,
+  html[data-darkmode="true"] .watch-table .current-diff-url::after {
+    filter: invert(0.5) hue-rotate(10deg) brightness(2); }
+  html[data-darkmode="true"] .watch-table .watch-controls .state-off img {
+    opacity: 0.3; }
+  html[data-darkmode="true"] .watch-table .watch-controls .state-on img {
+    opacity: 1.0; }
+  html[data-darkmode="true"] .watch-table .unviewed {
+    color: #fff; }
+    html[data-darkmode="true"] .watch-table .unviewed.error {
+      color: var(--color-watch-table-error); }
 
 body {
   color: var(--color-text);
@@ -263,6 +284,11 @@ body {
   white-space: nowrap;
   width: 1px; }
 
+.status-icon {
+  display: inline-block;
+  height: 1rem;
+  vertical-align: middle; }
+
 .pure-table-even {
   background: var(--color-background); }
 
@@ -331,10 +357,12 @@ code {
 .watch-table {
   width: 100%;
   font-size: 80%; }
+  .watch-table tr {
+    color: var(--color-watch-table-row-text); }
     .watch-table tr.unviewed {
       font-weight: bold; }
-  .watch-table .error {
+    .watch-table tr.error {
-    color: var(--color-error); }
+      color: var(--color-watch-table-error); }
   .watch-table td {
     white-space: nowrap; }
     .watch-table td.title-col {
@@ -470,6 +498,9 @@ a.pure-button-selected {
 .notifications-wrapper {
   padding: 0.5rem 0 1rem 0; }
 
+label:hover {
+  cursor: pointer; }
+
 #notification-customisation {
   border: 1px solid var(--color-border-notification);
   padding: 0.5rem;
@@ -836,6 +867,8 @@ body.full-width .edit-form {
   .edit-form .pure-form-message-inline {
     padding-left: 0;
     color: var(--color-text-input-description); }
+    .edit-form .pure-form-message-inline code {
+      font-size: .875em; }
 
 ul {
   padding-left: 1em;
@@ -926,3 +959,24 @@ ul {
     display: inline;
     height: 26px;
     vertical-align: middle; }
+
+/* automatic price following helpers */
+.tracking-ldjson-price-data {
+  background-color: var(--color-background-button-green);
+  color: #000;
+  padding: 3px;
+  border-radius: 3px;
+  white-space: nowrap; }
+
+.ldjson-price-track-offer {
+  font-weight: bold;
+  font-style: italic; }
+  .ldjson-price-track-offer a.pure-button {
+    border-radius: 3px;
+    padding: 3px;
+    background-color: var(--color-background-button-green); }
+
+.price-follow-tag-icon {
+  display: inline-block;
+  height: 0.8rem;
+  vertical-align: middle; }

changedetectionio/store.py

@@ -36,7 +36,6 @@ class ChangeDetectionStore:
         self.datastore_path = datastore_path
         self.json_store_path = "{}/url-watches.json".format(self.datastore_path)
         self.needs_write = False
-        self.proxy_list = None
         self.start_time = time.time()
         self.stop_thread = False
         # Base definition for all watchers
@@ -116,11 +115,6 @@ class ChangeDetectionStore:
             secret = secrets.token_hex(16)
             self.__data['settings']['application']['api_access_token'] = secret
 
-        # Proxy list support - available as a selection in settings when text file is imported
-        proxy_list_file = "{}/proxies.json".format(self.datastore_path)
-        if path.isfile(proxy_list_file):
-            self.import_proxy_list(proxy_list_file)
-
         # Bump the update version by running updates
         self.run_updates()
 
@@ -250,12 +244,15 @@ class ChangeDetectionStore:
     def clear_watch_history(self, uuid):
         import pathlib
 
-        self.__data['watching'][uuid].update(
+        self.__data['watching'][uuid].update({
-            {'last_checked': 0,
+                'last_checked': 0,
+                'has_ldjson_price_data': None,
+                'last_error': False,
+                'last_notification_error': False,
                 'last_viewed': 0,
                 'previous_md5': False,
-             'last_notification_error': False,
+                'track_ldjson_price_data': None,
-             'last_error': False})
+            })
 
         # JSON Data, Screenshots, Textfiles (history index and snapshots), HTML in the future etc
         for item in pathlib.Path(os.path.join(self.datastore_path, uuid)).rglob("*.*"):
@@ -460,10 +457,30 @@ class ChangeDetectionStore:
                     print ("Removing",item)
                     unlink(item)
 
-    def import_proxy_list(self, filename):
+    @property
-        with open(filename) as f:
+    def proxy_list(self):
-            self.proxy_list = json.load(f)
+        proxy_list = {}
-            print ("Registered proxy list", list(self.proxy_list.keys()))
+        proxy_list_file = os.path.join(self.datastore_path, 'proxies.json')
+
+        # Load from external config file
+        if path.isfile(proxy_list_file):
+            with open("{}/proxies.json".format(self.datastore_path)) as f:
+                proxy_list = json.load(f)
+
+        # Mapping from UI config if available
+        extras = self.data['settings']['requests'].get('extra_proxies')
+        if extras:
+            i=0
+            for proxy in extras:
+                i += 0
+                if proxy.get('proxy_name') and proxy.get('proxy_url'):
+                    k = "ui-" + str(i) + proxy.get('proxy_name')
+                    proxy_list[k] = {'label': proxy.get('proxy_name'), 'url': proxy.get('proxy_url')}
+
+
+        return proxy_list if len(proxy_list) else None
+
+
 
 
     def get_preferred_proxy_for_watch(self, uuid):
@@ -473,11 +490,10 @@ class ChangeDetectionStore:
         :return: proxy "key" id
         """
 
-        proxy_id = None
         if self.proxy_list is None:
             return None
 
-        # If its a valid one
+        # If it's a valid one
         watch = self.data['watching'].get(uuid)
 
         if watch.get('proxy') and watch.get('proxy') in list(self.proxy_list.keys()):
@@ -490,8 +506,9 @@ class ChangeDetectionStore:
             if self.proxy_list.get(system_proxy_id):
                 return system_proxy_id
 
-        # Fallback - Did not resolve anything, use the first available
+
-        if system_proxy_id is None:
+        # Fallback - Did not resolve anything, or doesnt exist, use the first available
+        if system_proxy_id is None or not self.proxy_list.get(system_proxy_id):
             first_default = list(self.proxy_list)[0]
             return first_default
 
@@ -622,3 +639,43 @@ class ChangeDetectionStore:
             except:
                 continue
         return
+
+    # Convert old static notification tokens to jinja2 tokens
+    def update_9(self):
+        # Each watch
+        import re
+        # only { } not {{ or }}
+        r = r'(?<!{){(?!{)(\w+)(?<!})}(?!})'
+        for uuid, watch in self.data['watching'].items():
+            try:
+                n_body = watch.get('notification_body', '')
+                if n_body:
+                    watch['notification_body'] = re.sub(r, r'{{\1}}', n_body)
+
+                n_title = watch.get('notification_title')
+                if n_title:
+                    watch['notification_title'] = re.sub(r, r'{{\1}}', n_title)
+
+                n_urls = watch.get('notification_urls')
+                if n_urls:
+                    for i, url in enumerate(n_urls):
+                        watch['notification_urls'][i] = re.sub(r, r'{{\1}}', url)
+
+            except:
+                continue
+
+        # System wide
+        n_body = self.data['settings']['application'].get('notification_body')
+        if n_body:
+            self.data['settings']['application']['notification_body'] = re.sub(r, r'{{\1}}', n_body)
+
+        n_title = self.data['settings']['application'].get('notification_title')
+        if n_body:
+            self.data['settings']['application']['notification_title'] = re.sub(r, r'{{\1}}', n_title)
+
+        n_urls =  self.data['settings']['application'].get('notification_urls')
+        if n_urls:
+            for i, url in enumerate(n_urls):
+                self.data['settings']['application']['notification_urls'][i] = re.sub(r, r'{{\1}}', url)
+
+        return

changedetectionio/templates/_common_fields.jinja

@@ -16,6 +16,7 @@
                                 <li><code>discord://</code> only supports a maximum <strong>2,000 characters</strong> of notification text, including the title.</li>
                                 <li><code>tgram://</code> bots cant send messages to other bots, so you should specify chat ID of non-bot user.</li>
                                 <li><code>tgram://</code> only supports very limited HTML and can fail when extra tags are sent, <a href="https://core.telegram.org/bots/api#html-style">read more here</a> (or use plaintext/markdown format)</li>
+                                <li><code>gets://</code>, <code>posts://</code>, <code>puts://</code>, <code>deletes://</code> for direct API calls (or omit the "<code>s</code>" for non-SSL ie <code>get://</code>)</li>
                               </ul>
                             </div>
                             <div class="notifications-wrapper">
@@ -41,8 +42,9 @@
                                 <span class="pure-form-message-inline">Format for all notifications</span>
                             </div>
                             <div class="pure-controls">
-                            <span class="pure-form-message-inline">
+                            <p class="pure-form-message-inline">
-                                These tokens can be used in the notification body and title to customise the notification text.
+                                You can use <a target="_new" href="https://jinja.palletsprojects.com/en/3.0.x/templates/">Jinja2</a> templating in the notification title, body and URL.
+                            </p>
 
                                 <table class="pure-table" id="token-table">
                                     <thead>
@@ -53,52 +55,49 @@
                                     </thead>
                                     <tbody>
                                     <tr>
-                                        <td><code>{base_url}</code></td>
+                                        <td><code>{{ '{{ base_url }}' }}</code></td>
                                         <td>The URL of the changedetection.io instance you are running.</td>
                                     </tr>
                                     <tr>
-                                        <td><code>{watch_url}</code></td>
+                                        <td><code>{{ '{{ watch_url }}' }}</code></td>
                                         <td>The URL being watched.</td>
                                     </tr>
                                     <tr>
-                                        <td><code>{watch_uuid}</code></td>
+                                        <td><code>{{ '{{ watch_uuid }}' }}</code></td>
                                         <td>The UUID of the watch.</td>
                                     </tr>
                                     <tr>
-                                        <td><code>{watch_title}</code></td>
+                                        <td><code>{{ '{{ watch_title }}' }}</code></td>
                                         <td>The title of the watch.</td>
                                     </tr>
                                     <tr>
-                                        <td><code>{watch_tag}</code></td>
+                                        <td><code>{{ '{{ watch_tag }}' }}</code></td>
-                                        <td>The tag of the watch.</td>
+                                        <td>The watch label / tag</td>
                                     </tr>
                                     <tr>
-                                        <td><code>{preview_url}</code></td>
+                                        <td><code>{{ '{{ preview_url }}' }}</code></td>
                                         <td>The URL of the preview page generated by changedetection.io.</td>
                                     </tr>
                                     <tr>
-                                        <td><code>{diff}</code></td>
+                                        <td><code>{{ '{{ diff_url }}' }}</code></td>
                                         <td>The diff output - differences only</td>
                                     </tr>
                                     <tr>
-                                        <td><code>{diff_full}</code></td>
+                                        <td><code>{{ '{{ diff_full }}' }}</code></td>
                                         <td>The diff output - full difference output</td>
                                     </tr>
                                     <tr>
-                                        <td><code>{diff_url}</code></td>
+                                        <td><code>{{ '{{ current_snapshot }}' }}</code></td>
-                                        <td>The URL of the diff page generated by changedetection.io.</td>
-                                    </tr>
-                                    <tr>
-                                        <td><code>{current_snapshot}</code></td>
                                         <td>The current snapshot value, useful when combined with JSON or CSS filters
                                         </td>
                                     </tr>
                                     </tbody>
                                 </table>
-                                <br/>
+                                <div class="pure-form-message-inline">
-                                URLs generated by changedetection.io (such as <code>{diff_url}</code>) require the <code>BASE_URL</code> environment variable set.<br/>
+                                    <br>
+                                    URLs generated by changedetection.io (such as <code>{{ '{{ diff_url }}' }}</code>) require the <code>BASE_URL</code> environment variable set.<br/>
                                     Your <code>BASE_URL</code> var is currently "{{settings_application['current_base_url']}}"
-                            </span>
+                                </div>
                             </div>
                         </div>
 {% endmacro %}

changedetectionio/templates/base.html

@@ -1,5 +1,5 @@
 <!DOCTYPE html>
-<html lang="en" data-darkmode="{{ dark_mode|lower }}">
+<html lang="en" data-darkmode="{{ get_darkmode_state() }}">
 
   <head>
     <meta charset="utf-8"/>
@@ -86,7 +86,7 @@
             {% if dark_mode %}
             {% set darkClass = 'dark' %}
             {% endif %}
-            <button class="toggle-theme {{darkClass}}" type="button">
+            <button class="toggle-theme {{darkClass}}" type="button" title="Toggle Light/Dark Mode">
               <span class="visually-hidden">Toggle light/dark mode</span>
               <span class="icon-light">
                 {% include "svgs/light-mode-toggle-icon.svg" %}

changedetectionio/templates/diff.html

@@ -1,5 +1,5 @@
 {% extends 'base.html' %}
-
+{% from '_helpers.jinja' import render_field, render_checkbox_field, render_button %}
 {% block content %}
 <script>
     const screenshot_url="{{url_for('static_content', group='screenshot', filename=uuid)}}";
@@ -58,6 +58,7 @@
         {% if last_error_screenshot %}<li class="tab" id="error-screenshot-tab"><a href="#error-screenshot">Error Screenshot</a></li> {% endif %}
         <li class="tab" id=""><a href="#text">Text</a></li>
         <li class="tab" id="screenshot-tab"><a href="#screenshot">Screenshot</a></li>
+        <li class="tab" id="extract-tab"><a href="#extract">Extract Data</a></li>
     </ul>
 </div>
 
@@ -108,6 +109,37 @@
            <strong>Screenshot requires Playwright/WebDriver enabled</strong>
          {% endif %}
      </div>
+    <div class="tab-pane-inner" id="extract">
+        <form id="extract-data-form" class="pure-form pure-form-stacked edit-form"
+              action="{{ url_for('diff_history_page', uuid=uuid) }}#extract"
+              method="POST">
+            <input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
+
+            <p>This tool will extract text data from all of the watch history.</p>
+
+            <div class="pure-control-group">
+                {{ render_field(extract_form.extract_regex) }}
+                <span class="pure-form-message-inline">
+                    A <strong>RegEx</strong> is a pattern that identifies exactly which part inside of the text that you want to extract.<br/>
+
+                    <p>
+                        For example, to extract only the numbers from text &dash;</br>
+                        <strong>Raw text</strong>: <code>Temperature <span style="color: red">5.5</span>°C in Sydney</code></br>
+                        <strong>RegEx to extract:</strong> <code>Temperature <span style="color: red">([0-9\.]+)</span></code><br/>
+                    </p>
+                    <p>
+                        <a href="https://RegExr.com/">Be sure to test your RegEx here.</a>
+                    </p>
+                    <p>
+                        Each RegEx group bracket <code>()</code> will be in its own column, the first column value is always the date.
+                    </p>
+                </span>
+            </div>
+            <div class="pure-control-group">
+                {{ render_button(extract_form.extract_submit_button) }}
+            </div>
+        </form>
+    </div>
 </div>
 
 <script>

changedetectionio/templates/edit.html

@@ -1,11 +1,8 @@
-{% extends 'base.html' %} {% block content %} {% from '_helpers.jinja' import
+{% extends 'base.html' %}
-render_field, render_checkbox_field, render_button %} {% from
+{% block content %}
-'_common_fields.jinja' import render_common_settings_form %}
+{% from '_helpers.jinja' import render_field, render_checkbox_field, render_button %}
-<script
+{% from '_common_fields.jinja' import render_common_settings_form %}
-  type="text/javascript"
+<script type="text/javascript" src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
-  src="{{url_for('static_content', group='js', filename='tabs.js')}}"
-  defer
-></script>
 <script>
     const notification_base_url="{{url_for('ajax_callback_send_notification_test')}}";
     const watch_visual_selector_data_url="{{url_for('static_content', group='visual_selector_data', filename=uuid)}}";
@@ -20,111 +17,56 @@ render_field, render_checkbox_field, render_button %} {% from
     const browser_steps_sync_url="{{url_for('browser_steps.browsersteps_ui_update', uuid=uuid)}}";
 </script>
 
-<script
+<script type="text/javascript" src="{{url_for('static_content', group='js', filename='watch-settings.js')}}" defer></script>
-  type="text/javascript"
+<script type="text/javascript" src="{{url_for('static_content', group='js', filename='limit.js')}}" defer></script>
-  src="{{url_for('static_content', group='js', filename='watch-settings.js')}}"
+<script type="text/javascript" src="{{url_for('static_content', group='js', filename='notifications.js')}}" defer></script>
-  defer
+<script type="text/javascript" src="{{url_for('static_content', group='js', filename='visual-selector.js')}}" defer></script>
-></script>
-<script
-  type="text/javascript"
-  src="{{url_for('static_content', group='js', filename='limit.js')}}"
-  defer
-></script>
-<script
-  type="text/javascript"
-  src="{{url_for('static_content', group='js', filename='notifications.js')}}"
-  defer
-></script>
-<script
-  type="text/javascript"
-  src="{{url_for('static_content', group='js', filename='visual-selector.js')}}"
-  defer
-></script>
 {% if playwright_enabled %}
-<script
+<script type="text/javascript" src="{{url_for('static_content', group='js', filename='browser-steps.js')}}" defer></script>
-  type="text/javascript"
-  src="{{url_for('static_content', group='js', filename='browser-steps.js')}}"
-  defer
-></script>
 {% endif %}
 
 <div class="edit-form monospaced-textarea">
+
     <div class="tabs collapsable">
         <ul>
             <li class="tab" id=""><a href="#general">General</a></li>
             <li class="tab"><a href="#request">Request</a></li>
             {% if playwright_enabled %}
-      <li class="tab">
+            <li class="tab"><a id="browsersteps-tab" href="#browser-steps">Browser Steps</a></li>
-        <a id="browsersteps-tab" href="#browser-steps">Browser Steps</a>
-      </li>
             {% endif %}
-      <li class="tab">
+            <li class="tab"><a id="visualselector-tab" href="#visualselector">Visual Filter Selector</a></li>
-        <a id="visualselector-tab" href="#visualselector"
+            <li class="tab"><a href="#filters-and-triggers">Filters &amp; Triggers</a></li>
-          >Visual Filter Selector</a
-        >
-      </li>
-      <li class="tab">
-        <a href="#filters-and-triggers">Filters &amp; Triggers</a>
-      </li>
             <li class="tab"><a href="#notifications">Notifications</a></li>
         </ul>
     </div>
 
     <div class="box-wrap inner">
-    <form
+        <form class="pure-form pure-form-stacked"
-      class="pure-form pure-form-stacked"
+              action="{{ url_for('edit_page', uuid=uuid, next = request.args.get('next'), unpause_on_save = request.args.get('unpause_on_save')) }}" method="POST">
-      action="{{ url_for('edit_page', uuid=uuid, next = request.args.get('next'), unpause_on_save = request.args.get('unpause_on_save')) }}"
-      method="POST"
-    >
              <input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
 
             <div class="tab-pane-inner" id="general">
                 <fieldset>
                     <div class="pure-control-group">
-            {{ render_field(form.url, placeholder="https://...", required=true,
+                        {{ render_field(form.url, placeholder="https://...", required=true, class="m-d") }}
-            class="m-d") }}
+                        <span class="pure-form-message-inline">Some sites use JavaScript to create the content, for this you should <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Fetching-pages-with-WebDriver">use the Chrome/WebDriver Fetcher</a></span><br/>
-            <span class="pure-form-message-inline"
+                        <span class="pure-form-message-inline">You can use variables in the URL, perfect for inserting the current date and other logic, <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Handling-variables-in-the-watched-URL">help and examples here</a></span><br/>
-              >Some sites use JavaScript to create the content, for this you
-              should
-              <a
-                href="https://github.com/dgtlmoon/changedetection.io/wiki/Fetching-pages-with-WebDriver"
-                >use the Chrome/WebDriver Fetcher</a
-              ></span
-            ><br />
-            <span class="pure-form-message-inline"
-              >You can use variables in the URL, perfect for inserting the
-              current date and other logic,
-              <a
-                href="https://github.com/dgtlmoon/changedetection.io/wiki/Handling-variables-in-the-watched-URL"
-                >help and examples here</a
-              ></span
-            ><br />
                     </div>
                     <div class="pure-control-group">
                         {{ render_field(form.title, class="m-d") }}
                     </div>
                     <div class="pure-control-group">
                         {{ render_field(form.tag) }}
-            <span class="pure-form-message-inline"
+                        <span class="pure-form-message-inline">Organisational tag/group name used in the main listing page</span>
-              >Organisational tag/group name used in the main listing page</span
-            >
                     </div>
                     <div class="pure-control-group">
-            {{ render_field(form.time_between_check, class="time-check-widget")
+                        {{ render_field(form.time_between_check, class="time-check-widget") }}
-            }} {% if has_empty_checktime %}
+                        {% if has_empty_checktime %}
-            <span class="pure-form-message-inline"
+                        <span class="pure-form-message-inline">Currently using the <a
-              >Currently using the
+                                href="{{ url_for('settings_page', uuid=uuid) }}">default global settings</a>, change to another value if you want to be specific.</span>
-              <a href="{{ url_for('settings_page', uuid=uuid) }}"
-                >default global settings</a
-              >, change to another value if you want to be specific.</span
-            >
                         {% else %}
-            <span class="pure-form-message-inline"
+                        <span class="pure-form-message-inline">Set to blank to use the <a
-              >Set to blank to use the
+                                href="{{ url_for('settings_page', uuid=uuid) }}">default global settings</a>.</span>
-              <a href="{{ url_for('settings_page', uuid=uuid) }}"
-                >default global settings</a
-              >.</span
-            >
                         {% endif %}
                     </div>
                     <div class="pure-control-group">
@@ -133,9 +75,7 @@ render_field, render_checkbox_field, render_button %} {% from
                     <div class="pure-control-group">
                         {{ render_checkbox_field(form.filter_failure_notification_send) }}
                         <span class="pure-form-message-inline">
-              Sends a notification when the filter can no longer be seen on the
+                         Sends a notification when the filter can no longer be seen on the page, good for knowing when the page changed and your filter will not work anymore.
-              page, good for knowing when the page changed and your filter will
-              not work anymore.
                         </span>
                     </div>
                 </fieldset>
@@ -145,20 +85,9 @@ render_field, render_checkbox_field, render_button %} {% from
                     <div class="pure-control-group inline-radio">
                         {{ render_field(form.fetch_backend, class="fetch-backend") }}
                         <span class="pure-form-message-inline">
-            <p>
+                            <p>Use the <strong>Basic</strong> method (default) where your watched site doesn't need Javascript to render.</p>
-              Use the <strong>Basic</strong> method (default) where your watched
+                            <p>The <strong>Chrome/Javascript</strong> method requires a network connection to a running WebDriver+Chrome server, set by the ENV var 'WEBDRIVER_URL'. </p>
-              site doesn't need Javascript to render.
+                            Tip: <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Proxy-configuration#brightdata-proxy-support">Connect using BrightData Proxies, find out more here.</a>
-            </p>
-            <p>
-              The <strong>Chrome/Javascript</strong> method requires a network
-              connection to a running WebDriver+Chrome server, set by the ENV
-              var 'WEBDRIVER_URL'.
-            </p>
-            Tip:
-            <a
-              href="https://github.com/dgtlmoon/changedetection.io/wiki/Proxy-configuration#brightdata-proxy-support"
-              >Connect using BrightData Proxies, find out more here.</a
-            >
                         </span>
                     </div>
                 {% if form.proxy %}
@@ -176,38 +105,25 @@ render_field, render_checkbox_field, render_button %} {% from
                     <div class="pure-control-group">
                         {{ render_field(form.webdriver_delay) }}
                         <div class="pure-form-message-inline">
-              <strong
+                            <strong>If you're having trouble waiting for the page to be fully rendered (text missing etc), try increasing the 'wait' time here.</strong>
-                >If you're having trouble waiting for the page to be fully
-                rendered (text missing etc), try increasing the 'wait' time
-                here.</strong
-              >
                             <br/>
-              This will wait <i>n</i> seconds before extracting the text. {% if
+                            This will wait <i>n</i> seconds before extracting the text.
-              using_global_webdriver_wait %} <br /><strong
+                            {% if using_global_webdriver_wait %}
-                >Using the current global default settings</strong
+                            <br/><strong>Using the current global default settings</strong>
-              >
                             {% endif %}
                         </div>
                     </div>
                     <div class="pure-control-group">
                         {{ render_field(form.webdriver_js_execute_code) }}
                         <div class="pure-form-message-inline">
-              Run this code before performing change detection, handy for
+                            Run this code before performing change detection, handy for filling in fields and other actions <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Run-JavaScript-before-change-detection">More help and examples here</a>
-              filling in fields and other actions
-              <a
-                href="https://github.com/dgtlmoon/changedetection.io/wiki/Run-JavaScript-before-change-detection"
-                >More help and examples here</a
-              >
                         </div>
                     </div>
                 </fieldset>
                 <fieldset class="pure-group" id="requests-override-options">
                     {% if not playwright_enabled %}
                         <div class="pure-form-message-inline">
-            <strong
+                            <strong>Request override is currently only used by the <i>Basic fast Plaintext/HTTP Client</i> method.</strong>
-              >Request override is currently only used by the
-              <i>Basic fast Plaintext/HTTP Client</i> method.</strong
-            >
                         </div>
                     {% endif %}
                     <div class="pure-control-group" id="request-method">
@@ -215,20 +131,22 @@ render_field, render_checkbox_field, render_button %} {% from
                     </div>
                     <div class="pure-control-group" id="request-headers">
 {{ render_field(form.headers, rows=5, placeholder="Example
-Cookie: foobar User-Agent: wonderbra 1.0") }}
+Cookie: foobar
+User-Agent: wonderbra 1.0") }}
                     </div>
                     <div class="pure-control-group" id="request-body">
                                         {{ render_field(form.body, rows=5, placeholder="Example
-            {\"name\":\"John\", \"age\":30, \"car\":null }") }}
+{
+   \"name\":\"John\",
+   \"age\":30,
+   \"car\":null
+}") }}
                     </div>
                 </fieldset>
             </div>
             {% if playwright_enabled %}
             <div class="tab-pane-inner" id="browser-steps">
-        <img
+                <img class="beta-logo" src="{{url_for('static_content', group='images', filename='beta-logo.png')}}">
-          class="beta-logo"
-          src="{{url_for('static_content', group='images', filename='beta-logo.png')}}"
-        />
                 <fieldset>
                     <div class="pure-control-group">
                         <!--
@@ -238,58 +156,29 @@ Cookie: foobar User-Agent: wonderbra 1.0") }}
                         -->
 
                         <!---  Do this later -->
-            <div class="checkbox" style="display: none">
+                        <div class="checkbox" style="display: none;">
-              <input type="checkbox" id="include_text_elements" />
+                            <input type=checkbox id="include_text_elements" > <label for="include_text_elements">Turn on text finder</label>
-              <label for="include_text_elements">Turn on text finder</label>
                         </div>
 
-            <div id="loading-status-text" style="display: none">
+                        <div id="loading-status-text" style="display: none;">Please wait, first browser step can take a little time to load..<div class="spinner"></div></div>
-              Please wait, first browser step can take a little time to load..
-              <div class="spinner"></div>
-            </div>
                         <div class="flex-wrapper" >
-              <div
+
-                id="browser-steps-ui"
+                            <div id="browser-steps-ui" class="noselect"  style="width: 100%; background-color: #eee; border-radius: 5px;">
-                class="noselect"
+
-                style="width: 100%; background-color: #eee; border-radius: 5px"
+                                <div class="noselect"  id="browsersteps-selector-wrapper" style="width: 100%">
-              >
-                <div
-                  class="noselect"
-                  id="browsersteps-selector-wrapper"
-                  style="width: 100%"
-                >
                                     <span class="loader" >
                                         <span id="browsersteps-click-start">
                                             <h2 >Click here to Start</h2>
                                             Please allow 10-15 seconds for the browser to connect.
                                         </span>
-                    <div class="spinner" style="display: none"></div>
+                                        <div class="spinner"  style="display: none;"></div>
                                     </span>
-                  <img
+                                    <img  class="noselect" id="browsersteps-img" src="" style="max-width: 100%; width: 100%;" />
-                    class="noselect"
+                                    <canvas  class="noselect" id="browsersteps-selector-canvas" style="max-width: 100%; width: 100%;"></canvas>
-                    id="browsersteps-img"
-                    src=""
-                    style="max-width: 100%; width: 100%"
-                  />
-                  <canvas
-                    class="noselect"
-                    id="browsersteps-selector-canvas"
-                    style="max-width: 100%; width: 100%"
-                  ></canvas>
                                 </div>
                             </div>
-              <div
+                            <div id="browser-steps-fieldlist" style="padding-left: 1em;  width: 350px; font-size: 80%;" >
-                id="browser-steps-fieldlist"
+                                <span id="browserless-seconds-remaining">Loading</span> <span style="font-size: 80%;"> (<a target=_new href="https://github.com/dgtlmoon/changedetection.io/pull/478/files#diff-1a79d924d1840c485238e66772391268a89c95b781d69091384cf1ea1ac146c9R4">?</a>) </span>
-                style="padding-left: 1em; width: 350px; font-size: 80%"
-              >
-                <span id="browserless-seconds-remaining">Loading</span>
-                <span style="font-size: 80%">
-                  (<a
-                    target="_new"
-                    href="https://github.com/dgtlmoon/changedetection.io/pull/478/files#diff-1a79d924d1840c485238e66772391268a89c95b781d69091384cf1ea1ac146c9R4"
-                    >?</a
-                  >)
-                </span>
                                 {{ render_field(form.browser_steps) }}
                             </div>
                         </div>
@@ -307,46 +196,20 @@ Cookie: foobar User-Agent: wonderbra 1.0") }}
                     <div class="pure-control-group inline-radio">
                       {{ render_checkbox_field(form.notification_screenshot) }}
                         <span class="pure-form-message-inline">
-              <strong>Use with caution!</strong> This will easily fill up your
+                            <strong>Use with caution!</strong> This will easily fill up your email storage quota or flood other storages.
-              email storage quota or flood other storages.
                         </span>
                     </div>
                     {% endif %}
                     <div class="field-group" id="notification-field-group">
                         {% if has_default_notification_urls %}
                         <div class="inline-warning">
-              <img
+                            <img class="inline-warning-icon" src="{{url_for('static_content', group='images', filename='notice.svg')}}" alt="Look out!" title="Lookout!"/>
-                class="inline-warning-icon"
+                            There are <a href="{{ url_for('settings_page')}}#notifications">system-wide notification URLs enabled</a>, this form will override notification settings for this watch only &dash; an empty Notification URL list here will still send notifications.
-                src="{{url_for('static_content', group='images', filename='notice.svg')}}"
-                alt="Look out!"
-                title="Lookout!"
-              />
-              There are
-              <a href="{{ url_for('settings_page')}}#notifications"
-                >system-wide notification URLs enabled</a
-              >, this form will override notification settings for this watch
-              only &dash; an empty Notification URL list here will still send
-              notifications.
                         </div>
                         {% endif %}
-            <a
+                        <a href="#notifications" id="notification-setting-reset-to-default" class="pure-button button-xsmall" style="right: 20px; top: 20px; position: absolute; background-color: #5f42dd; border-radius: 4px; font-size: 70%; color: #fff">Use system defaults</a>
-              href="#notifications"
-              id="notification-setting-reset-to-default"
-              class="pure-button button-xsmall"
-              style="
-                right: 20px;
-                top: 20px;
-                position: absolute;
-                background-color: #5f42dd;
-                border-radius: 4px;
-                font-size: 70%;
-                color: #fff;
-              "
-              >Use system defaults</a
-            >
 
-            {{ render_common_settings_form(form, emailprefix,
+                        {{ render_common_settings_form(form, emailprefix, settings_application) }}
-            settings_application) }}
                     </div>
                 </fieldset>
             </div>
@@ -359,226 +222,121 @@ Cookie: foobar User-Agent: wonderbra 1.0") }}
                                     Use the preview page to see your filters and triggers highlighted.
                                 </li>
                                 <li>
-              Some sites use JavaScript to create the content, for this you
+                                    Some sites use JavaScript to create the content, for this you should <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Fetching-pages-with-WebDriver">use the Chrome/WebDriver Fetcher</a>
-              should
-              <a
-                href="https://github.com/dgtlmoon/changedetection.io/wiki/Fetching-pages-with-WebDriver"
-                >use the Chrome/WebDriver Fetcher</a
-              >
                                 </li>
                             </ul>
                     </div>
                     <fieldset>
                         <div class="pure-control-group">
                             {{ render_checkbox_field(form.check_unique_lines) }}
-            <span class="pure-form-message-inline"
+                            <span class="pure-form-message-inline">Good for websites that just move the content around, and you want to know when NEW content is added, compares new lines against all history for this watch.</span>
-              >Good for websites that just move the content around, and you want
-              to know when NEW content is added, compares new lines against all
-              history for this watch.</span
-            >
                         </div>
                     </fieldset>
                     <div class="pure-control-group">
-          {% set field = render_field(form.include_filters, rows=5,
+                        {% set field = render_field(form.include_filters,
+                            rows=5,
                             placeholder="#example
-xpath://body/div/span[contains(@class, 'example-class')]", class="m-d") %} {{ field }} {% if '/text()' in
+xpath://body/div/span[contains(@class, 'example-class')]",
-          field %}
+                            class="m-d")
-          <span class="pure-form-message-inline"
+                        %}
-            ><strong
+                        {{ field }}
-              >Note!: //text() function does not work where the &lt;element&gt;
+                        {% if '/text()' in  field %}
-              contains &lt;![CDATA[]]&gt;</strong
+                          <span class="pure-form-message-inline"><strong>Note!: //text() function does not work where the &lt;element&gt; contains &lt;![CDATA[]]&gt;</strong></span><br/>
-            ></span
-          ><br />
                         {% endif %}
-          <span class="pure-form-message-inline"
+                        <span class="pure-form-message-inline">One rule per line, <i>any</i> rules that matches will be used.<br/>
-            >One rule per line, <i>any</i> rules that matches will be used.<br />
+
                     <ul>
-              <li>
+                        <li>CSS - Limit text to this CSS rule, only text matching this CSS rule is included.</li>
-                CSS - Limit text to this CSS rule, only text matching this CSS
+                        <li>JSON - Limit text to this JSON rule, using either <a href="https://pypi.org/project/jsonpath-ng/" target="new">JSONPath</a> or <a href="https://stedolan.github.io/jq/" target="new">jq</a> (if installed).
-                rule is included.
-              </li>
-              <li>
-                JSON - Limit text to this JSON rule, using either
-                <a href="https://pypi.org/project/jsonpath-ng/" target="new"
-                  >JSONPath</a
-                >
-                or
-                <a href="https://stedolan.github.io/jq/" target="new">jq</a> (if
-                installed).
                             <ul>
-                  <li>
+                                <li>JSONPath: Prefix with <code>json:</code>, use <code>json:$</code> to force re-formatting if required,  <a href="https://jsonpath.com/" target="new">test your JSONPath here</a>.</li>
-                    JSONPath: Prefix with <code>json:</code>, use
-                    <code>json:$</code> to force re-formatting if required,
-                    <a href="https://jsonpath.com/" target="new"
-                      >test your JSONPath here</a
-                    >.
-                  </li>
                                 {% if jq_support %}
-                  <li>
+                                <li>jq: Prefix with <code>jq:</code> and <a href="https://jqplay.org/" target="new">test your jq here</a>. Using <a href="https://stedolan.github.io/jq/" target="new">jq</a> allows for complex filtering and processing of JSON data with built-in functions, regex, filtering, and more. See examples and documentation <a href="https://stedolan.github.io/jq/manual/" target="new">here</a>.</li>
-                    jq: Prefix with <code>jq:</code> and
-                    <a href="https://jqplay.org/" target="new"
-                      >test your jq here</a
-                    >. Using
-                    <a href="https://stedolan.github.io/jq/" target="new">jq</a>
-                    allows for complex filtering and processing of JSON data
-                    with built-in functions, regex, filtering, and more. See
-                    examples and documentation
-                    <a href="https://stedolan.github.io/jq/manual/" target="new"
-                      >here</a
-                    >.
-                  </li>
                                 {% else %}
                                 <li>jq support not installed</li>
                                 {% endif %}
                             </ul>
                         </li>
-              <li>
+                        <li>XPath - Limit text to this XPath rule, simply start with a forward-slash,
-                XPath - Limit text to this XPath rule, simply start with a
-                forward-slash,
                             <ul>
-                  <li>
+                                <li>Example:  <code>//*[contains(@class, 'sametext')]</code> or <code>xpath://*[contains(@class, 'sametext')]</code>, <a
-                    Example: <code>//*[contains(@class, 'sametext')]</code> or
+                                href="http://xpather.com/" target="new">test your XPath here</a></li>
-                    <code>xpath://*[contains(@class, 'sametext')]</code>,
+                                <li>Example: Get all titles from an RSS feed <code>//title/text()</code></li>
-                    <a href="http://xpather.com/" target="new"
-                      >test your XPath here</a
-                    >
-                  </li>
-                  <li>
-                    Example: Get all titles from an RSS feed
-                    <code>//title/text()</code>
-                  </li>
                             </ul>
                             </li>
                     </ul>
-            Please be sure that you thoroughly understand how to write CSS,
+                    Please be sure that you thoroughly understand how to write CSS, JSONPath, XPath{% if jq_support %}, or jq selector{%endif%} rules before filing an issue on GitHub! <a
-            JSONPath, XPath{% if jq_support %}, or jq selector{%endif%} rules
+                                href="https://github.com/dgtlmoon/changedetection.io/wiki/CSS-Selector-help">here for more CSS selector help</a>.<br/>
-            before filing an issue on GitHub!
-            <a
-              href="https://github.com/dgtlmoon/changedetection.io/wiki/CSS-Selector-help"
-              >here for more CSS selector help</a
-            >.<br />
                 </span>
                     </div>
                     <div class="pure-control-group">
-          {{ render_field(form.subtractive_selectors, rows=5,
+                      {{ render_field(form.subtractive_selectors, rows=5, placeholder="header
-          placeholder="header footer nav .stockticker") }}
+footer
+nav
+.stockticker") }}
                       <span class="pure-form-message-inline">
                         <ul>
-              <li>
+                          <li> Remove HTML element(s) by CSS selector before text conversion. </li>
-                Remove HTML element(s) by CSS selector before text conversion.
+                          <li> Add multiple elements or CSS selectors per line to ignore multiple parts of the HTML. </li>
-              </li>
-              <li>
-                Add multiple elements or CSS selectors per line to ignore
-                multiple parts of the HTML.
-              </li>
                         </ul>
                       </span>
                     </div>
                 <fieldset class="pure-group">
-          {{ render_field(form.ignore_text, rows=5, placeholder="Some text to ignore in a line /some.regex\d{2}/ for case-INsensitive regex ") }}
+                    {{ render_field(form.ignore_text, rows=5, placeholder="Some text to ignore in a line
+/some.regex\d{2}/ for case-INsensitive regex
+                    ") }}
                     <span class="pure-form-message-inline">
                         <ul>
-              <li>
+                            <li>Each line processed separately, any line matching will be ignored (removed before creating the checksum)</li>
-                Each line processed separately, any line matching will be
+                            <li>Regular Expression support, wrap the entire line in forward slash <code>/regex/</code></li>
-                ignored (removed before creating the checksum)
+                            <li>Changing this will affect the comparison checksum which may trigger an alert</li>
-              </li>
-              <li>
-                Regular Expression support, wrap the entire line in forward
-                slash <code>/regex/</code>
-              </li>
-              <li>
-                Changing this will affect the comparison checksum which may
-                trigger an alert
-              </li>
                             <li>Use the preview/show current tab to see ignores</li>
                         </ul>
                 </span>
+
             </fieldset>
                 <fieldset>
                     <div class="pure-control-group">
-            {{ render_field(form.trigger_text, rows=5, placeholder="Some text to wait for in a line /some.regex\d{2}/ for case-INsensitive regex ")
+                        {{ render_field(form.trigger_text, rows=5, placeholder="Some text to wait for in a line
-            }}
+/some.regex\d{2}/ for case-INsensitive regex
+                    ") }}
                         <span class="pure-form-message-inline">
                     <ul>
-                <li>
+                        <li>Text to wait for before triggering a change/notification, all text and regex are tested <i>case-insensitive</i>.</li>
-                  Text to wait for before triggering a change/notification, all
+                        <li>Trigger text is processed from the result-text that comes out of any CSS/JSON Filters for this watch</li>
-                  text and regex are tested <i>case-insensitive</i>.
+                        <li>Each line is processed separately (think of each line as "OR")</li>
-                </li>
+                        <li>Note: Wrap in forward slash / to use regex  example: <code>/foo\d/</code></li>
-                <li>
-                  Trigger text is processed from the result-text that comes out
-                  of any CSS/JSON Filters for this watch
-                </li>
-                <li>
-                  Each line is processed separately (think of each line as "OR")
-                </li>
-                <li>
-                  Note: Wrap in forward slash / to use regex example:
-                  <code>/foo\d/</code>
-                </li>
                     </ul>
                         </span>
                     </div>
                 </fieldset>
                 <fieldset>
                     <div class="pure-control-group">
-            {{ render_field(form.text_should_not_be_present, rows=5,
+                        {{ render_field(form.text_should_not_be_present, rows=5, placeholder="For example: Out of stock
-            placeholder="For example:
-            
-Out of stock
 Sold out
 Not in stock
 Unavailable") }}
                         <span class="pure-form-message-inline">
                             <ul>
-                <li>
+                                <li>Block change-detection while this text is on the page, all text and regex are tested <i>case-insensitive</i>, good for waiting for when a product is available again</li>
-                  Block change-detection while this text is on the page, all
+                                <li>Block text is processed from the result-text that comes out of any CSS/JSON Filters for this watch</li>
-                  text and regex are tested <i>case-insensitive</i>, good for
+                                <li>All lines here must not exist (think of each line as "OR")</li>
-                  waiting for when a product is available again
+                                <li>Note: Wrap in forward slash / to use regex  example: <code>/foo\d/</code></li>
-                </li>
-                <li>
-                  Block text is processed from the result-text that comes out of
-                  any CSS/JSON Filters for this watch
-                </li>
-                <li>
-                  All lines here must not exist (think of each line as "OR")
-                </li>
-                <li>
-                  Note: Wrap in forward slash / to use regex example:
-                  <code>/foo\d/</code>
-                </li>
                             </ul>
                         </span>
                     </div>
                 </fieldset>
                 <fieldset>
                     <div class="pure-control-group">
-            {{ render_field(form.extract_text, rows=5, placeholder="\d+ online")
+                        {{ render_field(form.extract_text, rows=5, placeholder="\d+ online") }}
-            }}
                         <span class="pure-form-message-inline">
                     <ul>
-                <li>
+                        <li>Extracts text in the final output (line by line) after other filters using regular expressions;
-                  Extracts text in the final output (line by line) after other
-                  filters using regular expressions;
                             <ul>
-                    <li>
+                                <li>Regular expression &dash; example <code>/reports.+?2022/i</code></li>
-                      Regular expression &dash; example
+                                <li>Use <code>//(?aiLmsux))</code> type flags (more <a href="https://docs.python.org/3/library/re.html#index-15">information here</a>)<br/></li>
-                      <code>/reports.+?2022/i</code>
+                                <li>Keyword example &dash; example <code>Out of stock</code></li>
-                    </li>
+                                <li>Use groups to extract just that text &dash; example <code>/reports.+?(\d+)/i</code> returns a list of years only</li>
-                    <li>
-                      Use <code>//(?aiLmsux))</code> type flags (more
-                      <a
-                        href="https://docs.python.org/3/library/re.html#index-15"
-                        >information here</a
-                      >)<br />
-                    </li>
-                    <li>
-                      Keyword example &dash; example <code>Out of stock</code>
-                    </li>
-                    <li>
-                      Use groups to extract just that text &dash; example
-                      <code>/reports.+?(\d+)/i</code> returns a list of years
-                      only
-                    </li>
                             </ul>
                         </li>
                         <li>One line per regular-expression/ string match</li>
@@ -589,30 +347,18 @@ Unavailable") }}
             </div>
 
             <div class="tab-pane-inner visual-selector-ui" id="visualselector">
-        <img
+                <img class="beta-logo" src="{{url_for('static_content', group='images', filename='beta-logo.png')}}">
-          class="beta-logo"
-          src="{{url_for('static_content', group='images', filename='beta-logo.png')}}"
-        />
 
                 <fieldset>
                     <div class="pure-control-group">
                         {% if visualselector_enabled %}
                             <span class="pure-form-message-inline">
-              The Visual Selector tool lets you select the <i>text</i> elements
+                                The Visual Selector tool lets you select the <i>text</i> elements that will be used for the change detection &dash; after the <i>Browser Steps</i> has completed.<br/><br/>
-              that will be used for the change detection &dash; after the
-              <i>Browser Steps</i> has completed.<br /><br />
                             </span>
 
                             <div id="selector-header">
-              <a
+                                <a id="clear-selector" class="pure-button button-secondary button-xsmall" style="font-size: 70%">Clear selection</a>
-                id="clear-selector"
+                                <i class="fetching-update-notice" style="font-size: 80%;">One moment, fetching screenshot and element information..</i>
-                class="pure-button button-secondary button-xsmall"
-                style="font-size: 70%"
-                >Clear selection</a
-              >
-              <i class="fetching-update-notice" style="font-size: 80%"
-                >One moment, fetching screenshot and element information..</i
-              >
                             </div>
                             <div id="selector-wrapper" style="display: none">
                                 <!-- request the screenshot and get the element offset info ready -->
@@ -621,27 +367,12 @@ Unavailable") }}
                                 <img id="selector-background" />
                                 <canvas id="selector-canvas"></canvas>
                             </div>
-            <div id="selector-current-xpath" style="overflow-x: hidden">
+                            <div id="selector-current-xpath" style="overflow-x: hidden"><strong>Currently:</strong>&nbsp;<span class="text">Loading...</span></div>
-              <strong>Currently:</strong>&nbsp;<span class="text"
-                >Loading...</span
-              >
-            </div>
                         {% else %}
                             <span class="pure-form-message-inline">
-              <p>
+                                <p>Sorry, this functionality only works with Playwright/Chrome enabled watches.</p>
-                Sorry, this functionality only works with Playwright/Chrome
+                                <p>Enable the Playwright Chrome fetcher, or alternatively try our <a href="https://lemonade.changedetection.io/start">very affordable subscription based service</a>.</p>
-                enabled watches.
+                                <p>This is because Selenium/WebDriver can not extract full page screenshots reliably.</p>
-              </p>
-              <p>
-                Enable the Playwright Chrome fetcher, or alternatively try our
-                <a href="https://lemonade.changedetection.io/start"
-                  >very affordable subscription based service</a
-                >.
-              </p>
-              <p>
-                This is because Selenium/WebDriver can not extract full page
-                screenshots reliably.
-              </p>
                             </span>
                         {% endif %}
                     </div>
@@ -651,19 +382,12 @@ Unavailable") }}
             <div id="actions">
                 <div class="pure-control-group">
                     {{ render_button(form.save_button) }}
-          <a
+                    <a href="{{url_for('form_delete', uuid=uuid)}}"
-            href="{{url_for('form_delete', uuid=uuid)}}"
+                       class="pure-button button-small button-error ">Delete</a>
-            class="pure-button button-warning"
+                    <a href="{{url_for('clear_watch_history', uuid=uuid)}}"
-            >Delete</a
+                       class="pure-button button-small button-error ">Clear History</a>
-          >
+                    <a href="{{url_for('form_clone', uuid=uuid)}}"
-          <a
+                       class="pure-button button-small ">Create Copy</a>
-            href="{{url_for('clear_watch_history', uuid=uuid)}}"
-            class="pure-button button-warning"
-            >Clear History</a
-          >
-          <a href="{{url_for('form_clone', uuid=uuid)}}" class="pure-button"
-            >Create Copy</a
-          >
                 </div>
             </div>
         </form>

changedetectionio/templates/settings.html

@@ -1,28 +1,18 @@
-{% extends 'base.html' %} {% block content %} {% from '_helpers.jinja' import
+{% extends 'base.html' %}
-render_field, render_checkbox_field, render_button %} {% from
+
-'_common_fields.jinja' import render_common_settings_form %}
+{% block content %}
+{% from '_helpers.jinja' import render_field, render_checkbox_field, render_button %}
+{% from '_common_fields.jinja' import render_common_settings_form %}
 <script>
     const notification_base_url="{{url_for('ajax_callback_send_notification_test')}}";
 {% if emailprefix %}
     const email_notification_prefix=JSON.parse('{{emailprefix|tojson}}');
 {% endif %}
 </script>
-<script
+<script type="text/javascript" src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
-  type="text/javascript"
+<script type="text/javascript" src="{{url_for('static_content', group='js', filename='notifications.js')}}" defer></script>
-  src="{{url_for('static_content', group='js', filename='tabs.js')}}"
-  defer
-></script>
-<script
-  type="text/javascript"
-  src="{{url_for('static_content', group='js', filename='notifications.js')}}"
-  defer
-></script>
 
-<script
+<script type="text/javascript" src="{{url_for('static_content', group='js', filename='global-settings.js')}}" defer></script>
-  type="text/javascript"
-  src="{{url_for('static_content', group='js', filename='global-settings.js')}}"
-  defer
-></script>
 <div class="edit-form">
     <div class="tabs collapsable">
         <ul>
@@ -31,94 +21,62 @@ render_field, render_checkbox_field, render_button %} {% from
             <li class="tab"><a href="#fetching">Fetching</a></li>
             <li class="tab"><a href="#filters">Global Filters</a></li>
             <li class="tab"><a href="#api">API</a></li>
+            <li class="tab"><a href="#proxies">CAPTCHA &amp; Proxies</a></li>
         </ul>
     </div>
     <div class="box-wrap inner">
-    <form
+        <form class="pure-form pure-form-stacked settings" action="{{url_for('settings_page')}}" method="POST">
-      class="pure-form pure-form-stacked settings"
-      action="{{url_for('settings_page')}}"
-      method="POST"
-    >
             <input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
             <div class="tab-pane-inner" id="general">
                 <fieldset>
                     <div class="pure-control-group">
-            {{ render_field(form.requests.form.time_between_check,
+                        {{ render_field(form.requests.form.time_between_check, class="time-check-widget") }}
-            class="time-check-widget") }}
+                        <span class="pure-form-message-inline">Default time for all watches, when the watch does not have a specific time setting.</span>
-            <span class="pure-form-message-inline"
-              >Default time for all watches, when the watch does not have a
-              specific time setting.</span
-            >
                     </div>
                     <div class="pure-control-group">
-            {{ render_field(form.requests.form.jitter_seconds,
+                        {{ render_field(form.requests.form.jitter_seconds, class="jitter_seconds") }}
-            class="jitter_seconds") }}
+                        <span class="pure-form-message-inline">Example - 3 seconds random jitter could trigger up to 3 seconds earlier or up to 3 seconds later</span>
-            <span class="pure-form-message-inline"
-              >Example - 3 seconds random jitter could trigger up to 3 seconds
-              earlier or up to 3 seconds later</span
-            >
                     </div>
                     <div class="pure-control-group">
-            {{
+                        {{ render_field(form.application.form.filter_failure_notification_threshold_attempts, class="filter_failure_notification_threshold_attempts") }}
-            render_field(form.application.form.filter_failure_notification_threshold_attempts,
+                        <span class="pure-form-message-inline">After this many consecutive times that the CSS/xPath filter is missing, send a notification
-            class="filter_failure_notification_threshold_attempts") }}
-            <span class="pure-form-message-inline"
-              >After this many consecutive times that the CSS/xPath filter is
-              missing, send a notification
                             <br/>
                         Set to <strong>0</strong> to disable
                         </span>
                     </div>
                     <div class="pure-control-group">
-            {% if not hide_remove_pass %} {% if current_user.is_authenticated %}
+                        {% if not hide_remove_pass %}
-            {{ render_button(form.application.form.removepassword_button) }} {%
+                            {% if current_user.is_authenticated %}
-            else %} {{ render_field(form.application.form.password) }}
+                                {{ render_button(form.application.form.removepassword_button) }}
-            <span class="pure-form-message-inline"
+                            {% else %}
-              >Password protection for your changedetection.io
+                            {{ render_field(form.application.form.password) }}
-              application.</span
+                            <span class="pure-form-message-inline">Password protection for your changedetection.io application.</span>
-            >
+                            {% endif %}
-            {% endif %} {% else %}
+                        {% else %}
                             <span class="pure-form-message-inline">Password is locked.</span>
                         {% endif %}
                     </div>
 
                     <div class="pure-control-group">
-            {{ render_field(form.application.form.base_url,
+                        {{ render_field(form.application.form.base_url, placeholder="http://yoursite.com:5000/",
-            placeholder="http://yoursite.com:5000/", class="m-d") }}
+                        class="m-d") }}
                         <span class="pure-form-message-inline">
-              Base URL used for the <code>{base_url}</code> token in
+                            Base URL used for the <code>{{ '{{ base_url }}' }}</code> token in notifications and RSS links.<br/>Default value is the ENV var 'BASE_URL' (Currently "{{settings_application['current_base_url']}}"),
-              notifications and RSS links.<br />Default value is the ENV var
+                            <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Configurable-BASE_URL-setting">read more here</a>.
-              'BASE_URL' (Currently
-              "{{settings_application['current_base_url']}}"),
-              <a
-                href="https://github.com/dgtlmoon/changedetection.io/wiki/Configurable-BASE_URL-setting"
-                >read more here</a
-              >.
                         </span>
                     </div>
 
                     <div class="pure-control-group">
-            {{
+                        {{ render_checkbox_field(form.application.form.extract_title_as_title) }}
-            render_checkbox_field(form.application.form.extract_title_as_title)
+                        <span class="pure-form-message-inline">Note: This will automatically apply to all existing watches.</span>
-            }}
-            <span class="pure-form-message-inline"
-              >Note: This will automatically apply to all existing
-              watches.</span
-            >
                     </div>
                     <div class="pure-control-group">
-            {{
+                        {{ render_checkbox_field(form.application.form.empty_pages_are_a_change) }}
-            render_checkbox_field(form.application.form.empty_pages_are_a_change)
+                        <span class="pure-form-message-inline">When a page contains HTML, but no renderable text appears (empty page), is this considered a change?</span>
-            }}
-            <span class="pure-form-message-inline"
-              >When a page contains HTML, but no renderable text appears (empty
-              page), is this considered a change?</span
-            >
                     </div>
                 {% if form.requests.proxy %}
                     <div class="pure-control-group inline-radio">
-            {{ render_field(form.requests.form.proxy,
+                        {{ render_field(form.requests.form.proxy, class="fetch-backend-proxy") }}
-            class="fetch-backend-proxy") }}
                         <span class="pure-form-message-inline">
                         Choose a default proxy for all watches
                         </span>
@@ -130,41 +88,24 @@ render_field, render_checkbox_field, render_button %} {% from
             <div class="tab-pane-inner" id="notifications">
                 <fieldset>
                     <div class="field-group">
-            {{ render_common_settings_form(form.application.form, emailprefix,
+                        {{ render_common_settings_form(form.application.form, emailprefix, settings_application) }}
-            settings_application) }}
                     </div>
                 </fieldset>
             </div>
 
             <div class="tab-pane-inner" id="fetching">
                 <div class="pure-control-group inline-radio">
-          {{ render_field(form.application.form.fetch_backend,
+                    {{ render_field(form.application.form.fetch_backend, class="fetch-backend") }}
-          class="fetch-backend") }}
                     <span class="pure-form-message-inline">
-            <p>
+                        <p>Use the <strong>Basic</strong> method (default) where your watched sites don't need Javascript to render.</p>
-              Use the <strong>Basic</strong> method (default) where your watched
+                        <p>The <strong>Chrome/Javascript</strong> method requires a network connection to a running WebDriver+Chrome server, set by the ENV var 'WEBDRIVER_URL'. </p>
-              sites don't need Javascript to render.
-            </p>
-            <p>
-              The <strong>Chrome/Javascript</strong> method requires a network
-              connection to a running WebDriver+Chrome server, set by the ENV
-              var 'WEBDRIVER_URL'.
-            </p>
                     </span>
                     <br/>
-          Tip:
+                    Tip: <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Proxy-configuration#brightdata-proxy-support">Connect using BrightData Proxies, find out more here.</a>
-          <a
-            href="https://github.com/dgtlmoon/changedetection.io/wiki/Proxy-configuration#brightdata-proxy-support"
-            >Connect using BrightData Proxies, find out more here.</a
-          >
                 </div>
                 <fieldset class="pure-group" id="webdriver-override-options">
                     <div class="pure-form-message-inline">
-            <strong
+                        <strong>If you're having trouble waiting for the page to be fully rendered (text missing etc), try increasing the 'wait' time here.</strong>
-              >If you're having trouble waiting for the page to be fully
-              rendered (text missing etc), try increasing the 'wait' time
-              here.</strong
-            >
                         <br/>
                         This will wait <i>n</i> seconds before extracting the text.
                     </div>
@@ -175,71 +116,43 @@ render_field, render_checkbox_field, render_button %} {% from
             </div>
 
             <div class="tab-pane-inner" id="filters">
+
                     <fieldset class="pure-group">
                     {{ render_checkbox_field(form.application.form.ignore_whitespace) }}
-          <span class="pure-form-message-inline"
+                    <span class="pure-form-message-inline">Ignore whitespace, tabs and new-lines/line-feeds when considering if a change was detected.<br/>
-            >Ignore whitespace, tabs and new-lines/line-feeds when considering
+                    <i>Note:</i> Changing this will change the status of your existing watches, possibly trigger alerts etc.
-            if a change was detected.<br />
-            <i>Note:</i> Changing this will change the status of your existing
-            watches, possibly trigger alerts etc.
                     </span>
                     </fieldset>
                 <fieldset class="pure-group">
-          {{
+                    {{ render_checkbox_field(form.application.form.render_anchor_tag_content) }}
-          render_checkbox_field(form.application.form.render_anchor_tag_content)
+                    <span class="pure-form-message-inline">Render anchor tag content, default disabled, when enabled renders links as <code>(link text)[https://somesite.com]</code>
-          }}
-          <span class="pure-form-message-inline"
-            >Render anchor tag content, default disabled, when enabled renders
-            links as <code>(link text)[https://somesite.com]</code>
                         <br/>
-            <i>Note:</i> Changing this could affect the content of your existing
+                    <i>Note:</i> Changing this could affect the content of your existing watches, possibly trigger alerts etc.
-            watches, possibly trigger alerts etc.
                     </span>
                     </fieldset>
                     <fieldset class="pure-group">
-          {{ render_field(form.application.form.global_subtractive_selectors,
+                      {{ render_field(form.application.form.global_subtractive_selectors, rows=5, placeholder="header
-          rows=5, placeholder="header
 footer
 nav
 .stockticker") }}
                       <span class="pure-form-message-inline">
                         <ul>
-              <li>
+                          <li> Remove HTML element(s) by CSS selector before text conversion. </li>
-                Remove HTML element(s) by CSS selector before text conversion.
+                          <li> Add multiple elements or CSS selectors per line to ignore multiple parts of the HTML. </li>
-              </li>
-              <li>
-                Add multiple elements or CSS selectors per line to ignore
-                multiple parts of the HTML.
-              </li>
                         </ul>
                       </span>
                     </fieldset>
                     <fieldset class="pure-group">
-          {{ render_field(form.application.form.global_ignore_text, rows=5,
+                    {{ render_field(form.application.form.global_ignore_text, rows=5, placeholder="Some text to ignore in a line
-          placeholder="Some text to ignore in a line /some.regex\d{2}/ for
+/some.regex\d{2}/ for case-INsensitive regex
-          case-INsensitive regex ") }}
+                    ") }}
-          <span class="pure-form-message-inline"
+                    <span class="pure-form-message-inline">Note: This is applied globally in addition to the per-watch rules.</span><br/>
-            >Note: This is applied globally in addition to the per-watch
-            rules.</span
-          ><br />
                     <span class="pure-form-message-inline">
                         <ul>
-              <li>
+                            <li>Note: This is applied globally in addition to the per-watch rules.</li>
-                Note: This is applied globally in addition to the per-watch
+                            <li>Each line processed separately, any line matching will be ignored (removed before creating the checksum)</li>
-                rules.
+                            <li>Regular Expression support, wrap the entire line in forward slash <code>/regex/</code></li>
-              </li>
+                            <li>Changing this will affect the comparison checksum which may trigger an alert</li>
-              <li>
-                Each line processed separately, any line matching will be
-                ignored (removed before creating the checksum)
-              </li>
-              <li>
-                Regular Expression support, wrap the entire line in forward
-                slash <code>/regex/</code>
-              </li>
-              <li>
-                Changing this will affect the comparison checksum which may
-                trigger an alert
-              </li>
                             <li>Use the preview/show current tab to see ignores</li>
                         </ul>
                      </span>
@@ -247,41 +160,41 @@ nav
            </div>
 
             <div class="tab-pane-inner" id="api">
-        <p>
+
-          Drive your changedetection.io via API, More about
+                <p>Drive your changedetection.io via API, More about <a href="https://github.com/dgtlmoon/changedetection.io/wiki/API-Reference">API access here</a></p>
-          <a
-            href="https://github.com/dgtlmoon/changedetection.io/wiki/API-Reference"
-            >API access here</a
-          >
-        </p>
 
                 <div class="pure-control-group">
-          {{
+                    {{ render_checkbox_field(form.application.form.api_access_token_enabled) }}
-          render_checkbox_field(form.application.form.api_access_token_enabled)
+                    <div class="pure-form-message-inline">Restrict API access limit by using <code>x-api-key</code> header</div><br/>
-          }}
+                    <div class="pure-form-message-inline"><br/>API Key <span id="api-key">{{api_key}}</span>
-          <div class="pure-form-message-inline">
+                        <span style="display:none;" id="api-key-copy" >copy</span>
-            Restrict API access limit by using <code>x-api-key</code> header
-          </div>
-          <br />
-          <div class="pure-form-message-inline">
-            <br />API Key <span id="api-key">{{api_key}}</span>
-            <span style="display: none" id="api-key-copy">copy</span>
                     </div>
                 </div>
             </div>
+            <div class="tab-pane-inner" id="proxies">
 
+                <p><strong>Tip</strong>: You can connect to websites using <a href="https://brightdata.grsm.io/n0r16zf7eivq">BrightData</a> proxies, their service <strong>WebUnlocker</strong> will solve most CAPTCHAs, whilst their <strong>Residential Proxies</strong> may help to avoid CAPTCHA altogether. </p>
+                <p>It may be easier to try <strong>WebUnlocker</strong> first, WebUnlocker also supports country selection.</p>
+                <p>
+                    When you have <a href="https://brightdata.grsm.io/n0r16zf7eivq">registered</a>, enabled the required services, visit the <A href="https://brightdata.com/cp/api_example?">API example page</A>, then select <strong>Python</strong>, set the country you wish to use, then copy+paste the example URL below<br/>
+                    The Proxy URL with BrightData should start with <code>http://brd-customer...</code>
+                </p>
+
+                <p>When you sign up using <a href="https://brightdata.grsm.io/n0r16zf7eivq">https://brightdata.grsm.io/n0r16zf7eivq</a> BrightData will match any first deposit up to $150</p>
+
+
+                <div class="pure-control-group">
+                    {{ render_field(form.requests.form.extra_proxies) }}
+                    <div class="pure-form-message-inline">"Name" will be used for selecting the proxy in the Watch Edit settings</span>
+                </div>
+            </div>
             <div id="actions">
                 <div class="pure-control-group">
                     {{ render_button(form.save_button) }}
-          <a href="{{url_for('index')}}" class="pure-button button-cancel"
+                    <a href="{{url_for('index')}}" class="pure-button button-small button-cancel">Back</a>
-            >Back</a
+                    <a href="{{url_for('clear_all_history')}}" class="pure-button button-small button-cancel">Clear Snapshot History</a>
-          >
-          <a
-            href="{{url_for('clear_all_history')}}"
-            class="pure-button button-cancel"
-            >Clear Snapshot History</a
-          >
                 </div>
+
             </div>
         </form>
     </div>

changedetectionio/templates/watch-overview.html

@@ -14,7 +14,7 @@
             <div id="watch-add-wrapper-zone">
                 <div>
                     {{ render_simple_field(form.url, placeholder="https://...", required=true) }}
-                    {{ render_simple_field(form.tag, value=active_tag if active_tag else '', placeholder="watch group") }}
+                    {{ render_simple_field(form.tag, value=active_tag if active_tag else '', placeholder="watch label / tag") }}
                 </div>
                 <div>
                     {{ render_simple_field(form.watch_submit_button, title="Watch this URL!" ) }}
@@ -32,6 +32,7 @@
         <button class="pure-button button-secondary button-xsmall" style="font-size: 70%"  name="op" value="unpause">UnPause</button>
         <button class="pure-button button-secondary button-xsmall" style="font-size: 70%"  name="op" value="mute">Mute</button>
         <button class="pure-button button-secondary button-xsmall" style="font-size: 70%"  name="op" value="unmute">UnMute</button>
+        <button class="pure-button button-secondary button-xsmall" style="font-size: 70%" name="op" value="recheck">Recheck</button>
         <button class="pure-button button-secondary button-xsmall" style="font-size: 70%" name="op" value="notification-default">Use default notification</button>
         <button class="pure-button button-secondary button-xsmall" style="background: #dd4242; font-size: 70%" name="op" value="delete">Delete</button>
     </div>
@@ -88,16 +89,22 @@
                 </td>
                 <td class="title-col inline">{{watch.title if watch.title is not none and watch.title|length > 0 else watch.url}}
                     <a class="external" target="_blank" rel="noopener" href="{{ watch.link.replace('source:','') }}"></a>
-                    <a class="link-spread" href="{{url_for('form_share_put_watch', uuid=watch.uuid)}}"><img style="height: 1em;display:inline-block;" src="{{url_for('static_content', group='images', filename='spread.svg')}}" class="icon icon-spread" /></a>
+                    <a class="link-spread" href="{{url_for('form_share_put_watch', uuid=watch.uuid)}}"><img class="status-icon"  src="{{url_for('static_content', group='images', filename='spread.svg')}}" class="status-icon icon icon-spread" title="Create a link to share watch config with others" /></a>
-
-                    {%if watch.fetch_backend == "html_webdriver" %}<img style="height: 1em; display:inline-block;" src="{{url_for('static_content', group='images', filename='Google-Chrome-icon.png')}}" />{% endif %}
 
+                    {%if watch.get_fetch_backend == "html_webdriver" %}<img class="status-icon" src="{{url_for('static_content', group='images', filename='Google-Chrome-icon.png')}}" title="Using a chrome browser" />{% endif %}
+                    {%if watch.is_pdf  %}<img class="status-icon" src="{{url_for('static_content', group='images', filename='pdf-icon.svg')}}" title="Converting PDF to text" />{% endif %}
                     {% if watch.last_error is defined and watch.last_error != False %}
                     <div class="fetch-error">{{ watch.last_error }}</div>
                     {% endif %}
                     {% if watch.last_notification_error is defined and watch.last_notification_error != False %}
                     <div class="fetch-error notification-error"><a href="{{url_for('notification_logs')}}">{{ watch.last_notification_error }}</a></div>
                     {% endif %}
+                    {% if watch['has_ldjson_price_data'] and not watch['track_ldjson_price_data']  %}
+                    <div class="ldjson-price-track-offer">Embedded price data detected, follow only price data? <a href="{{url_for('price_data_follower.accept', uuid=watch.uuid)}}" class="pure-button button-xsmall">Yes</a> <a href="{{url_for('price_data_follower.reject', uuid=watch.uuid)}}" class="">No</a></div>
+                    {% endif %}
+                    {% if watch['track_ldjson_price_data'] == 'accepted' %}
+                    <span class="tracking-ldjson-price-data" title="Automatically following embedded price information"><img src="{{url_for('static_content', group='images', filename='price-tag-icon.svg')}}"  class="status-icon price-follow-tag-icon"/> Price</span>
+                    {% endif %}
                     {% if not active_tag %}
                     <span class="watch-tag-list">{{ watch.tag}}</span>
                     {% endif %}

changedetectionio/tests/proxy_list/proxies.json-example

@@ -1,10 +1,10 @@
 {
   "proxy-one": {
-    "label": "One",
+    "label": "Proxy One",
-    "url": "http://127.0.0.1:3128"
+    "url": "http://squid-one:3128"
   },
   "proxy-two": {
-    "label": "two",
+    "label": "Proxy Two",
-    "url": "http://127.0.0.1:3129"
+    "url": "http://squid-two:3128"
   }
 }

changedetectionio/tests/proxy_list/squid-auth.conf

@@ -0,0 +1,48 @@
+acl localnet src 0.0.0.1-0.255.255.255  # RFC 1122 "this" network (LAN)
+acl localnet src 10.0.0.0/8             # RFC 1918 local private network (LAN)
+acl localnet src 100.64.0.0/10          # RFC 6598 shared address space (CGN)
+acl localnet src 169.254.0.0/16         # RFC 3927 link-local (directly plugged) machines
+acl localnet src 172.16.0.0/12          # RFC 1918 local private network (LAN)
+acl localnet src 192.168.0.0/16         # RFC 1918 local private network (LAN)
+acl localnet src fc00::/7               # RFC 4193 local private network range
+acl localnet src fe80::/10              # RFC 4291 link-local (directly plugged) machines
+acl localnet src 159.65.224.174
+acl SSL_ports port 443
+acl Safe_ports port 80          # http
+acl Safe_ports port 21          # ftp
+acl Safe_ports port 443         # https
+acl Safe_ports port 70          # gopher
+acl Safe_ports port 210         # wais
+acl Safe_ports port 1025-65535  # unregistered ports
+acl Safe_ports port 280         # http-mgmt
+acl Safe_ports port 488         # gss-http
+acl Safe_ports port 591         # filemaker
+acl Safe_ports port 777         # multiling http
+acl CONNECT method CONNECT
+
+http_access deny !Safe_ports
+http_access deny CONNECT !SSL_ports
+#http_access allow localhost manager
+http_access deny manager
+#http_access allow localhost
+#http_access allow localnet
+
+auth_param basic program /usr/lib/squid3/basic_ncsa_auth /etc/squid3/passwords
+auth_param basic realm proxy
+acl authenticated proxy_auth REQUIRED
+http_access allow authenticated
+http_access deny all
+
+
+http_port 3128
+coredump_dir /var/spool/squid
+refresh_pattern ^ftp:           1440    20%     10080
+refresh_pattern ^gopher:        1440    0%      1440
+refresh_pattern -i (/cgi-bin/|\?) 0     0%      0
+refresh_pattern \/(Packages|Sources)(|\.bz2|\.gz|\.xz)$ 0 0% 0 refresh-ims
+refresh_pattern \/Release(|\.gpg)$ 0 0% 0 refresh-ims
+refresh_pattern \/InRelease$ 0 0% 0 refresh-ims
+refresh_pattern \/(Translation-.*)(|\.bz2|\.gz|\.xz)$ 0 0% 0 refresh-ims
+refresh_pattern .               0       20%     4320
+logfile_rotate 0
+

changedetectionio/tests/proxy_list/squid-passwords.txt

@@ -0,0 +1 @@
+test:$apr1$xvhFolTA$E/kz5/Rw1ewcyaSUdwqZs.

changedetectionio/tests/proxy_list/test_select_custom_proxy.py

@@ -0,0 +1,51 @@
+#!/usr/bin/python3
+
+import time
+from flask import url_for
+from ..util import live_server_setup, wait_for_all_checks
+
+# just make a request, we will grep in the docker logs to see it actually got called
+def test_select_custom(client, live_server):
+    live_server_setup(live_server)
+
+    # Goto settings, add our custom one
+    res = client.post(
+        url_for("settings_page"),
+        data={
+            "requests-time_between_check-minutes": 180,
+            "application-ignore_whitespace": "y",
+            "application-fetch_backend": "html_requests",
+            "requests-extra_proxies-0-proxy_name": "custom-test-proxy",
+            # test:awesome is set in tests/proxy_list/squid-passwords.txt
+            "requests-extra_proxies-0-proxy_url": "http://test:awesome@squid-custom:3128",
+        },
+        follow_redirects=True
+    )
+
+    assert b"Settings updated." in res.data
+
+    res = client.post(
+        url_for("import_page"),
+        # Because a URL wont show in squid/proxy logs due it being SSLed
+        # Use plain HTTP or a specific domain-name here
+        data={"urls": "https://changedetection.io/CHANGELOG.txt"},
+        follow_redirects=True
+    )
+
+    assert b"1 Imported" in res.data
+    wait_for_all_checks(client)
+
+    res = client.get(url_for("index"))
+    assert b'Proxy Authentication Required' not in res.data
+
+
+    res = client.get(
+        url_for("preview_page", uuid="first"),
+        follow_redirects=True
+    )
+    # We should see something via proxy
+    assert b'HEAD' in res.data
+
+    #
+    # Now we should see the request in the container logs for "squid-squid-custom" because it will be the only default
+

changedetectionio/tests/test_automatic_follow_ldjson_price.py

@@ -0,0 +1,146 @@
+#!/usr/bin/python3
+
+import time
+from flask import url_for
+from .util import live_server_setup, extract_UUID_from_client, extract_api_key_from_UI
+
+def set_response_with_ldjson():
+    test_return_data = """<html>
+       <body>
+     Some initial text</br>
+     <p>Which is across multiple lines</p>
+     </br>
+     So let's see what happens.  </br>
+     <div class="sametext">Some text thats the same</div>
+     <div class="changetext">Some text that will change</div>
+     <script type="application/ld+json">
+        {
+           "@context":"https://schema.org/",
+           "@type":"Product",
+           "@id":"https://www.some-virtual-phone-shop.com/celular-iphone-14/p",
+           "name":"Celular Iphone 14 Pro Max 256Gb E Sim A16 Bionic",
+           "brand":{
+              "@type":"Brand",
+              "name":"APPLE"
+           },
+           "image":"https://www.some-virtual-phone-shop.com/15509426/image.jpg",
+           "description":"You dont need it",
+           "mpn":"111111",
+           "sku":"22222",
+           "offers":{
+              "@type":"AggregateOffer",
+              "lowPrice":8097000,
+              "highPrice":8099900,
+              "priceCurrency":"COP",
+              "offers":[
+                 {
+                    "@type":"Offer",
+                    "price":8097000,
+                    "priceCurrency":"COP",
+                    "availability":"http://schema.org/InStock",
+                    "sku":"102375961",
+                    "itemCondition":"http://schema.org/NewCondition",
+                    "seller":{
+                       "@type":"Organization",
+                       "name":"ajax"
+                    }
+                 }
+              ],
+              "offerCount":1
+           }
+        }
+       </script>
+     </body>
+     </html>
+"""
+
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write(test_return_data)
+    return None
+
+def set_response_without_ldjson():
+    test_return_data = """<html>
+       <body>
+     Some initial text</br>
+     <p>Which is across multiple lines</p>
+     </br>
+     So let's see what happens.  </br>
+     <div class="sametext">Some text thats the same</div>
+     <div class="changetext">Some text that will change</div>     
+     </body>
+     </html>
+"""
+
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write(test_return_data)
+    return None
+
+# actually only really used by the distll.io importer, but could be handy too
+def test_check_ldjson_price_autodetect(client, live_server):
+    live_server_setup(live_server)
+
+    # Give the endpoint time to spin up
+    time.sleep(1)
+
+    set_response_with_ldjson()
+
+    # Add our URL to the import page
+    test_url = url_for('test_endpoint', _external=True)
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+    time.sleep(3)
+
+    # Should get a notice that it's available
+    res = client.get(url_for("index"))
+    assert b'ldjson-price-track-offer' in res.data
+
+    # Accept it
+    uuid = extract_UUID_from_client(client)
+
+    client.get(url_for('price_data_follower.accept', uuid=uuid, follow_redirects=True))
+    time.sleep(2)
+
+    # Trigger a check
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    time.sleep(2)
+    # Offer should be gone
+    res = client.get(url_for("index"))
+    assert b'Embedded price data' not in res.data
+    assert b'tracking-ldjson-price-data' in res.data
+
+    # and last snapshop (via API) should be just the price
+    api_key = extract_api_key_from_UI(client)
+    res = client.get(
+        url_for("watchsinglehistory", uuid=uuid, timestamp='latest'),
+        headers={'x-api-key': api_key},
+    )
+
+    # Should see this (dont know where the whitespace came from)
+    assert b'"highPrice": 8099900' in res.data
+    # And not this cause its not the ld-json
+    assert b"So let's see what happens" not in res.data
+
+    client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+
+    ##########################################################################################
+    # And we shouldnt see the offer
+    set_response_without_ldjson()
+
+    # Add our URL to the import page
+    test_url = url_for('test_endpoint', _external=True)
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+    time.sleep(3)
+    res = client.get(url_for("index"))
+    assert b'ldjson-price-track-offer' not in res.data
+    
+    ##########################################################################################
+    client.get(url_for("form_delete", uuid="all"), follow_redirects=True)

changedetectionio/tests/test_element_removal.py

@@ -121,7 +121,7 @@ def test_element_removal_full(client, live_server):
         url_for("import_page"), data={"urls": test_url}, follow_redirects=True
     )
     assert b"1 Imported" in res.data
-
+    time.sleep(1)
     # Goto the edit page, add the filter data
     # Not sure why \r needs to be added - absent of the #changetext this is not necessary
     subtractive_selectors_data = "header\r\nfooter\r\nnav\r\n#changetext"

changedetectionio/tests/test_encoding.py

@@ -38,9 +38,6 @@ def test_check_encoding_detection(client, live_server):
         follow_redirects=True
     )
 
-    # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
-
     # Give the thread time to pick it up
     time.sleep(2)
 

changedetectionio/tests/test_errorhandling.py

@@ -77,7 +77,8 @@ def test_DNS_errors(client, live_server):
     time.sleep(3)
 
     res = client.get(url_for("index"))
-    assert b'Name or service not known' in res.data
+    found_name_resolution_error = b"Temporary failure in name resolution" in res.data or b"Name or service not known" in res.data
+    assert found_name_resolution_error
     # Should always record that we tried
     assert bytes("just now".encode('utf-8')) in res.data
 

changedetectionio/tests/test_extract_csv.py

@@ -0,0 +1,70 @@
+#!/usr/bin/python3
+
+import time
+from flask import url_for
+from urllib.request import urlopen
+from .util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks
+
+sleep_time_for_fetch_thread = 3
+
+
+
+def test_check_extract_text_from_diff(client, live_server):
+    import time
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write("Now it's {} seconds since epoch, time flies!".format(str(time.time())))
+
+    live_server_setup(live_server)
+
+    # Add our URL to the import page
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": url_for('test_endpoint', _external=True)},
+        follow_redirects=True
+    )
+
+    assert b"1 Imported" in res.data
+    time.sleep(1)
+
+    # Load in 5 different numbers/changes
+    last_date=""
+    for n in range(5):
+        # Give the thread time to pick it up
+        print("Bumping snapshot and checking.. ", n)
+        last_date = str(time.time())
+        with open("test-datastore/endpoint-content.txt", "w") as f:
+            f.write("Now it's {} seconds since epoch, time flies!".format(last_date))
+
+        client.get(url_for("form_watch_checknow"), follow_redirects=True)
+        wait_for_all_checks(client)
+
+    res = client.post(
+        url_for("diff_history_page", uuid="first"),
+        data={"extract_regex": "Now it's ([0-9\.]+)",
+              "extract_submit_button": "Extract as CSV"},
+        follow_redirects=False
+    )
+
+    assert b'Nothing matches that RegEx' not in res.data
+    assert res.content_type == 'text/csv'
+
+    # Read the csv reply as stringio
+    from io import StringIO
+    import csv
+
+    f = StringIO(res.data.decode('utf-8'))
+    reader = csv.reader(f, delimiter=',')
+    output=[]
+
+    for row in reader:
+        output.append(row)
+
+    assert output[0][0] == 'Epoch seconds'
+
+    # Header line + 1 origin/first + 5 changes
+    assert(len(output) == 7)
+
+    # We expect to find the last bumped date in the changes in the last field of the spreadsheet
+    assert(output[6][2] == last_date)
+    # And nothing else, only that group () of the decimal and .
+    assert "time flies" not in output[6][2]

changedetectionio/tests/test_filter_exist_changes.py

@@ -73,17 +73,17 @@ def test_filter_doesnt_exist_then_exists_should_get_notification(client, live_se
 
     # Just a regular notification setting, this will be used by the special 'filter not found' notification
     notification_form_data = {"notification_urls": notification_url,
-                              "notification_title": "New ChangeDetection.io Notification - {watch_url}",
+                              "notification_title": "New ChangeDetection.io Notification - {{watch_url}}",
-                              "notification_body": "BASE URL: {base_url}\n"
+                              "notification_body": "BASE URL: {{base_url}}\n"
-                                                   "Watch URL: {watch_url}\n"
+                                                   "Watch URL: {{watch_url}}\n"
-                                                   "Watch UUID: {watch_uuid}\n"
+                                                   "Watch UUID: {{watch_uuid}}\n"
-                                                   "Watch title: {watch_title}\n"
+                                                   "Watch title: {{watch_title}}\n"
-                                                   "Watch tag: {watch_tag}\n"
+                                                   "Watch tag: {{watch_tag}}\n"
-                                                   "Preview: {preview_url}\n"
+                                                   "Preview: {{preview_url}}\n"
-                                                   "Diff URL: {diff_url}\n"
+                                                   "Diff URL: {{diff_url}}\n"
-                                                   "Snapshot: {current_snapshot}\n"
+                                                   "Snapshot: {{current_snapshot}}\n"
-                                                   "Diff: {diff}\n"
+                                                   "Diff: {{diff}}\n"
-                                                   "Diff Full: {diff_full}\n"
+                                                   "Diff Full: {{diff_full}}\n"
                                                    ":-)",
                               "notification_format": "Text"}
 

changedetectionio/tests/test_filter_failure_notification.py

@@ -56,17 +56,17 @@ def run_filter_test(client, content_filter):
 
     # Just a regular notification setting, this will be used by the special 'filter not found' notification
     notification_form_data = {"notification_urls": notification_url,
-                              "notification_title": "New ChangeDetection.io Notification - {watch_url}",
+                              "notification_title": "New ChangeDetection.io Notification - {{watch_url}}",
-                              "notification_body": "BASE URL: {base_url}\n"
+                              "notification_body": "BASE URL: {{base_url}}\n"
-                                                   "Watch URL: {watch_url}\n"
+                                                   "Watch URL: {{watch_url}}\n"
-                                                   "Watch UUID: {watch_uuid}\n"
+                                                   "Watch UUID: {{watch_uuid}}\n"
-                                                   "Watch title: {watch_title}\n"
+                                                   "Watch title: {{watch_title}}\n"
-                                                   "Watch tag: {watch_tag}\n"
+                                                   "Watch tag: {{watch_tag}}\n"
-                                                   "Preview: {preview_url}\n"
+                                                   "Preview: {{preview_url}}\n"
-                                                   "Diff URL: {diff_url}\n"
+                                                   "Diff URL: {{diff_url}}\n"
-                                                   "Snapshot: {current_snapshot}\n"
+                                                   "Snapshot: {{current_snapshot}}\n"
-                                                   "Diff: {diff}\n"
+                                                   "Diff: {{diff}}\n"
-                                                   "Diff Full: {diff_full}\n"
+                                                   "Diff Full: {{diff_full}}\n"
                                                    ":-)",
                               "notification_format": "Text"}
 
@@ -84,6 +84,7 @@ def run_filter_test(client, content_filter):
         data=notification_form_data,
         follow_redirects=True
     )
+
     assert b"Updated watch." in res.data
     time.sleep(3)
 

changedetectionio/tests/test_ignore_text.py

@@ -101,9 +101,6 @@ def test_check_ignore_text_functionality(client, live_server):
     )
     assert b"1 Imported" in res.data
 
-    # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
-
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
 
@@ -199,9 +196,6 @@ def test_check_global_ignore_text_functionality(client, live_server):
     )
     assert b"1 Imported" in res.data
 
-    # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
-
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
 

changedetectionio/tests/test_ignorestatuscode.py

@@ -69,8 +69,6 @@ def test_normal_page_check_works_with_ignore_status_code(client, live_server):
     assert b"1 Imported" in res.data
 
     time.sleep(sleep_time_for_fetch_thread)
-    # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
 
     set_some_changed_response()
     time.sleep(sleep_time_for_fetch_thread)
@@ -104,9 +102,6 @@ def test_403_page_check_works_with_ignore_status_code(client, live_server):
     )
     assert b"1 Imported" in res.data
 
-    # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
-
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
 
@@ -119,11 +114,9 @@ def test_403_page_check_works_with_ignore_status_code(client, live_server):
     )
     assert b"Updated watch." in res.data
 
-    # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
-
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
+
     #  Make a change
     set_some_changed_response()
 

changedetectionio/tests/test_jsonpath_jq_selector.py

@@ -394,6 +394,48 @@ def check_json_ext_filter(json_filter, client, live_server):
     res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
 
+def test_ignore_json_order(client, live_server):
+    # A change in order shouldn't trigger a notification
+
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write('{"hello" : 123, "world": 123}')
+
+
+    # Add our URL to the import page
+    test_url = url_for('test_endpoint', content_type="application/json", _external=True)
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+    time.sleep(2)
+
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write('{"world" : 123, "hello": 123}')
+
+    # Trigger a check
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    time.sleep(2)
+
+    res = client.get(url_for("index"))
+    assert b'unviewed' not in res.data
+
+    # Just to be sure it still works
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write('{"world" : 123, "hello": 124}')
+
+    # Trigger a check
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    time.sleep(2)
+
+    res = client.get(url_for("index"))
+    assert b'unviewed' in res.data
+
+    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    assert b'Deleted' in res.data
+
 def test_check_jsonpath_ext_filter(client, live_server):
     check_json_ext_filter('json:$[?(@.status==Sold)]', client, live_server)
 

changedetectionio/tests/test_notification.py

@@ -90,17 +90,17 @@ def test_check_notification(client, live_server):
     print (">>>> Notification URL: "+notification_url)
 
     notification_form_data = {"notification_urls": notification_url,
-                              "notification_title": "New ChangeDetection.io Notification - {watch_url}",
+                              "notification_title": "New ChangeDetection.io Notification - {{watch_url}}",
-                              "notification_body": "BASE URL: {base_url}\n"
+                              "notification_body": "BASE URL: {{base_url}}\n"
-                                                   "Watch URL: {watch_url}\n"
+                                                   "Watch URL: {{watch_url}}\n"
-                                                   "Watch UUID: {watch_uuid}\n"
+                                                   "Watch UUID: {{watch_uuid}}\n"
-                                                   "Watch title: {watch_title}\n"
+                                                   "Watch title: {{watch_title}}\n"
-                                                   "Watch tag: {watch_tag}\n"
+                                                   "Watch tag: {{watch_tag}}\n"
-                                                   "Preview: {preview_url}\n"
+                                                   "Preview: {{preview_url}}\n"
-                                                   "Diff URL: {diff_url}\n"
+                                                   "Diff URL: {{diff_url}}\n"
-                                                   "Snapshot: {current_snapshot}\n"
+                                                   "Snapshot: {{current_snapshot}}\n"
-                                                   "Diff: {diff}\n"
+                                                   "Diff: {{diff}}\n"
-                                                   "Diff Full: {diff_full}\n"
+                                                   "Diff Full: {{diff_full}}\n"
                                                    ":-)",
                               "notification_screenshot": True,
                               "notification_format": "Text"}
@@ -179,7 +179,6 @@ def test_check_notification(client, live_server):
         logging.debug(">>> Skipping BASE_URL check")
 
 
-
     # This should insert the {current_snapshot}
     set_more_modified_response()
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
@@ -237,10 +236,10 @@ def test_check_notification(client, live_server):
         follow_redirects=True
     )
 
-
 def test_notification_validation(client, live_server):
-    #live_server_setup(live_server)
+
-    time.sleep(3)
+    time.sleep(1)
+
     # re #242 - when you edited an existing new entry, it would not correctly show the notification settings
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)
@@ -268,21 +267,6 @@ def test_notification_validation(client, live_server):
 #    )
 #    assert b"Notification Body and Title is required when a Notification URL is used" in res.data
 
-    # Now adding a wrong token should give us an error
-    res = client.post(
-        url_for("settings_page"),
-        data={"application-notification_title": "New ChangeDetection.io Notification - {watch_url}",
-              "application-notification_body": "Rubbish: {rubbish}\n",
-              "application-notification_format": "Text",
-              "application-notification_urls": "json://localhost/foobar",
-              "requests-time_between_check-minutes": 180,
-              "fetch_backend": "html_requests"
-              },
-        follow_redirects=True
-    )
-
-    assert bytes("is not a valid token".encode('utf-8')) in res.data
-
     # cleanup for the next
     client.get(
         url_for("form_delete", uuid="all"),
@@ -290,3 +274,58 @@ def test_notification_validation(client, live_server):
     )
 
 
+
+def test_notification_custom_endpoint_and_jinja2(client, live_server):
+    time.sleep(1)
+
+    # test_endpoint - that sends the contents of a file
+    # test_notification_endpoint - that takes a POST and writes it to file (test-datastore/notification.txt)
+
+    # CUSTOM JSON BODY CHECK for POST://
+    set_original_response()
+    test_notification_url = url_for('test_notification_endpoint', _external=True).replace('http://', 'post://')+"?xxx={{ watch_url }}"
+
+    res = client.post(
+        url_for("settings_page"),
+        data={"application-notification_title": "New ChangeDetection.io Notification - {{ watch_url }}",
+              "application-notification_body": '{ "url" : "{{ watch_url }}", "secret": 444 }',
+              # https://github.com/caronc/apprise/wiki/Notify_Custom_JSON#get-parameter-manipulation
+              "application-notification_urls": test_notification_url,
+              "application-minutes_between_check": 180,
+              "application-fetch_backend": "html_requests"
+              },
+        follow_redirects=True
+    )
+    assert b'Settings updated' in res.data
+
+    # Add a watch and trigger a HTTP POST
+    test_url = url_for('test_endpoint', _external=True)
+    res = client.post(
+        url_for("form_quick_watch_add"),
+        data={"url": test_url, "tag": 'nice one'},
+        follow_redirects=True
+    )
+
+    assert b"Watch added" in res.data
+
+    time.sleep(2)
+    set_modified_response()
+
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    time.sleep(2)
+
+
+    with open("test-datastore/notification.txt", 'r') as f:
+        x=f.read()
+        j = json.loads(x)
+        assert j['url'].startswith('http://localhost')
+        assert j['secret'] == 444
+
+    # URL check, this will always be converted to lowercase
+    assert os.path.isfile("test-datastore/notification-url.txt")
+    with open("test-datastore/notification-url.txt", 'r') as f:
+        notification_url = f.read()
+        assert 'xxx=http' in notification_url
+
+    os.unlink("test-datastore/notification-url.txt")
+

changedetectionio/tests/test_notification_errors.py

@@ -11,23 +11,23 @@ def test_check_notification_error_handling(client, live_server):
     set_original_response()
 
     # Give the endpoint time to spin up
-    time.sleep(3)
+    time.sleep(2)
 
-    # use a different URL so that it doesnt interfere with the actual check until we are ready
+    # Set a URL and fetch it, then set a notification URL which is going to give errors
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
         url_for("form_quick_watch_add"),
-        data={"url": "https://changedetection.io/CHANGELOG.txt", "tag": ''},
+        data={"url": test_url, "tag": ''},
         follow_redirects=True
     )
     assert b"Watch added" in res.data
 
-    time.sleep(10)
+    time.sleep(2)
+    set_modified_response()
 
-    # Check we capture the failure, we can just use trigger_check = y here
     res = client.post(
         url_for("edit_page", uuid="first"),
-        data={"notification_urls": "jsons://broken-url.changedetection.io/test",
+        data={"notification_urls": "jsons://broken-url-xxxxxxxx123/test",
               "notification_title": "xxx",
               "notification_body": "xxxxx",
               "notification_format": "Text",
@@ -36,15 +36,14 @@ def test_check_notification_error_handling(client, live_server):
               "title": "",
               "headers": "",
               "time_between_check-minutes": "180",
-              "fetch_backend": "html_requests",
+              "fetch_backend": "html_requests"},
-              "trigger_check": "y"},
         follow_redirects=True
     )
     assert b"Updated watch." in res.data
 
     found=False
     for i in range(1, 10):
-        time.sleep(1)
+
         logging.debug("Fetching watch overview....")
         res = client.get(
             url_for("index"))
@@ -53,6 +52,7 @@ def test_check_notification_error_handling(client, live_server):
             found=True
             break
 
+        time.sleep(1)
 
     assert found
 
@@ -60,7 +60,7 @@ def test_check_notification_error_handling(client, live_server):
     # The error should show in the notification logs
     res = client.get(
         url_for("notification_logs"))
-    assert bytes("Name or service not known".encode('utf-8')) in res.data
+    found_name_resolution_error = b"Temporary failure in name resolution" in res.data or b"Name or service not known" in res.data
+    assert found_name_resolution_error
 
-
+    client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
-    # And it should be listed on the watch overview

changedetectionio/tests/test_pdf.py

@@ -0,0 +1,40 @@
+#!/usr/bin/python3
+
+import time
+from flask import url_for
+from .util import set_original_response, set_modified_response, live_server_setup
+
+sleep_time_for_fetch_thread = 3
+
+# `subtractive_selectors` should still work in `source:` type requests
+def test_fetch_pdf(client, live_server):
+    import shutil
+    shutil.copy("tests/test.pdf", "test-datastore/endpoint-test.pdf")
+
+    live_server_setup(live_server)
+    test_url = url_for('test_pdf_endpoint', _external=True)
+    # Add our URL to the import page
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+
+    assert b"1 Imported" in res.data
+
+    time.sleep(sleep_time_for_fetch_thread)
+    res = client.get(
+        url_for("preview_page", uuid="first"),
+        follow_redirects=True
+    )
+
+    assert b'PDF-1.5' not in res.data
+    assert b'hello world' in res.data
+
+    # So we know if the file changes in other ways
+    import hashlib
+    md5 = hashlib.md5(open("test-datastore/endpoint-test.pdf", 'rb').read()).hexdigest().upper()
+    # We should have one
+    assert len(md5) >0
+    # And it's going to be in the document
+    assert b'Document checksum - '+bytes(str(md5).encode('utf-8')) in res.data

changedetectionio/tests/test_request.py

@@ -20,6 +20,8 @@ def test_headers_in_request(client, live_server):
     )
     assert b"1 Imported" in res.data
 
+    time.sleep(1)
+
     res = client.post(
         url_for("import_page"),
         data={"urls": test_url},
@@ -174,6 +176,7 @@ def test_method_in_request(client, live_server):
     )
     assert b"1 Imported" in res.data
 
+    time.sleep(2)
     res = client.post(
         url_for("import_page"),
         data={"urls": test_url},
@@ -181,6 +184,8 @@ def test_method_in_request(client, live_server):
     )
     assert b"1 Imported" in res.data
 
+    time.sleep(2)
+
     # Attempt to add a method which is not valid
     res = client.post(
         url_for("edit_page", uuid="first"),
@@ -206,7 +211,7 @@ def test_method_in_request(client, live_server):
     assert b"Updated watch." in res.data
 
     # Give the thread time to pick up the first version
-    time.sleep(5)
+    time.sleep(2)
 
     # The service should echo back the request verb
     res = client.get(
@@ -217,7 +222,7 @@ def test_method_in_request(client, live_server):
     # The test call service will return the verb as the body
     assert b"PATCH" in res.data
 
-    time.sleep(5)
+    time.sleep(2)
 
     watches_with_method = 0
     with open('test-datastore/url-watches.json') as f:

changedetectionio/tests/util.py

@@ -149,6 +149,9 @@ def live_server_setup(live_server):
             if data != None:
                 f.write(data)
 
+        with open("test-datastore/notification-url.txt", "w") as f:
+            f.write(request.url)
+
         print("\n>> Test notification endpoint was hit.\n", data)
         return "Text was set"
 
@@ -165,5 +168,15 @@ def live_server_setup(live_server):
     def test_return_query():
         return request.query_string
 
+
+    @live_server.app.route('/endpoint-test.pdf')
+    def test_pdf_endpoint():
+
+        # Tried using a global var here but didn't seem to work, so reading from a file instead.
+        with open("test-datastore/endpoint-test.pdf", "rb") as f:
+            resp = make_response(f.read(), 200)
+            resp.headers['Content-Type'] = 'application/pdf'
+            return resp
+
     live_server.start()
 

changedetectionio/update_worker.py

@@ -4,6 +4,7 @@ import queue
 import time
 
 from changedetectionio import content_fetcher
+from changedetectionio import queuedWatchMetaData
 from changedetectionio.fetch_site_status import FilterNotFoundInResponse
 
 # A single update worker
@@ -157,11 +158,12 @@ class update_worker(threading.Thread):
         while not self.app.config.exit.is_set():
 
             try:
-                priority, uuid = self.q.get(block=False)
+                queued_item_data = self.q.get(block=False)
             except queue.Empty:
                 pass
 
             else:
+                uuid = queued_item_data.item.get('uuid')
                 self.current_uuid = uuid
 
                 if uuid in list(self.datastore.data['watching'].keys()):
@@ -171,11 +173,11 @@ class update_worker(threading.Thread):
                     update_obj= {}
                     xpath_data = False
                     process_changedetection_results = True
-                    print("> Processing UUID {} Priority {} URL {}".format(uuid, priority, self.datastore.data['watching'][uuid]['url']))
+                    print("> Processing UUID {} Priority {} URL {}".format(uuid, queued_item_data.priority, self.datastore.data['watching'][uuid]['url']))
                     now = time.time()
 
                     try:
-                        changed_detected, update_obj, contents = update_handler.run(uuid)
+                        changed_detected, update_obj, contents = update_handler.run(uuid, skip_when_checksum_same=queued_item_data.item.get('skip_when_checksum_same'))
                         # Re #342
                         # In Python 3, all strings are sequences of Unicode characters. There is a bytes type that holds raw bytes.
                         # We then convert/.decode('utf-8') for the notification etc
@@ -241,6 +243,10 @@ class update_worker(threading.Thread):
 
                         process_changedetection_results = True
 
+                    except content_fetcher.checksumFromPreviousCheckWasTheSame as e:
+                        # Yes fine, so nothing todo
+                        pass
+
                     except content_fetcher.BrowserStepsStepTimout as e:
 
                         if not self.datastore.data['watching'].get(uuid):

requirements-dev.txt

@@ -1,2 +0,0 @@
-pytest ~=6.2
-pytest-flask ~=1.2

requirements.txt

@@ -29,8 +29,9 @@ apprise~=1.2.0
 # apprise mqtt https://github.com/dgtlmoon/changedetection.io/issues/315
 paho-mqtt
 
-# Pinned version of cryptography otherwise
+# This mainly affects some ARM builds, which unlike the other builds ignores "ARG CRYPTOGRAPHY_DONT_BUILD_RUST=1"
-# ERROR: Could not build wheels for cryptography which use PEP 517 and cannot be installed directly
+# so without this pinning, the newer versions on ARM will forcefully try to build rust, which results in "rust compiler not found"
+# (introduced once apprise became a dep)
 cryptography~=3.4
 
 # Used for CSS filtering
@@ -58,3 +59,7 @@ jq~=1.3 ;python_version >= "3.8" and sys_platform == "linux"
 # Any current modern version, required so far for screenshot PNG->JPEG conversion but will be used more in the future
 pillow
 # playwright is installed at Dockerfile build time because it's not available on all platforms
+
+# Include pytest, so if theres a support issue we can ask them to run these tests on their setup
+pytest ~=6.2
+pytest-flask ~=1.2

runtime.txt

@@ -1 +1 @@
-python-3.8.12
+python-3.9.15