0.45

.dockerignore

@@ -1,2 +1,18 @@
 .git
 .github
+changedetectionio/processors/__pycache__
+changedetectionio/api/__pycache__
+changedetectionio/model/__pycache__
+changedetectionio/blueprint/price_data_follower/__pycache__
+changedetectionio/blueprint/tags/__pycache__
+changedetectionio/blueprint/__pycache__
+changedetectionio/blueprint/browser_steps/__pycache__
+changedetectionio/fetchers/__pycache__
+changedetectionio/tests/visualselector/__pycache__
+changedetectionio/tests/restock/__pycache__
+changedetectionio/tests/__pycache__
+changedetectionio/tests/fetchers/__pycache__
+changedetectionio/tests/unit/__pycache__
+changedetectionio/tests/proxy_list/__pycache__
+changedetectionio/__pycache__
+

.github/workflows/containers.yml

@@ -95,7 +95,7 @@ jobs:
           push: true
           tags: |
             ${{ secrets.DOCKER_HUB_USERNAME }}/changedetection.io:dev,ghcr.io/${{ github.repository }}:dev
-          platforms: linux/amd64,linux/arm64,linux/arm/v6,linux/arm/v7
+          platforms: linux/amd64,linux/arm64,linux/arm/v6,linux/arm/v7,linux/arm/v8
           cache-from: type=local,src=/tmp/.buildx-cache
           cache-to: type=local,dest=/tmp/.buildx-cache
 # Looks like this was disabled
@@ -115,7 +115,7 @@ jobs:
             ghcr.io/dgtlmoon/changedetection.io:${{ github.event.release.tag_name }}
             ${{ secrets.DOCKER_HUB_USERNAME }}/changedetection.io:latest
             ghcr.io/dgtlmoon/changedetection.io:latest
-          platforms: linux/amd64,linux/arm64,linux/arm/v6,linux/arm/v7
+          platforms: linux/amd64,linux/arm64,linux/arm/v6,linux/arm/v7,linux/arm/v8
           cache-from: type=local,src=/tmp/.buildx-cache
           cache-to: type=local,dest=/tmp/.buildx-cache
 # Looks like this was disabled

.github/workflows/test-container-build.yml

@@ -62,7 +62,7 @@ jobs:
           with:
             context: ./
             file: ./Dockerfile
-            platforms: linux/arm/v7,linux/arm/v6,linux/amd64,linux/arm64,
+            platforms: linux/amd64,linux/arm64,linux/arm/v6,linux/arm/v7,linux/arm/v8
             cache-from: type=local,src=/tmp/.buildx-cache
             cache-to: type=local,dest=/tmp/.buildx-cache
 

.github/workflows/test-only.yml

@@ -36,10 +36,16 @@ jobs:
         run: |         
           # Build a changedetection.io container and start testing inside
           docker build . -t test-changedetectionio
+          # Debug info
+          docker run test-changedetectionio  bash -c 'pip list'
+
+      - name: Spin up ancillary SMTP+Echo message test server
+        run: |
+          # Debug SMTP server/echo message back server
+          docker run --network changedet-network -d -p 11025:11025 -p 11080:11080  --hostname mailserver test-changedetectionio  bash -c 'python changedetectionio/tests/smtp/smtp-test-server.py' 
 
       - name: Test built container with pytest
         run: |
-          
           # Unit tests
           docker run test-changedetectionio  bash -c 'python3 -m unittest changedetectionio.tests.unit.test_notification_diff'
           
@@ -55,9 +61,19 @@ jobs:
           # Playwright/Browserless fetch
           docker run --rm -e "PLAYWRIGHT_DRIVER_URL=ws://browserless:3000" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio;pytest tests/fetchers/test_content.py && pytest tests/test_errorhandling.py && pytest tests/visualselector/test_fetch_data.py'
           
+          # Settings headers playwright tests - Call back in from Browserless, check headers
+          docker run --name "changedet" --hostname changedet --rm -e "FLASK_SERVER_NAME=changedet" -e "PLAYWRIGHT_DRIVER_URL=ws://browserless:3000?dumpio=true" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0  --live-server-port=5004 tests/test_request.py'
+          docker run --name "changedet" --hostname changedet --rm -e "FLASK_SERVER_NAME=changedet" -e "WEBDRIVER_URL=http://selenium:4444/wd/hub" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0  --live-server-port=5004 tests/test_request.py'
+          docker run --name "changedet" --hostname changedet --rm -e "FLASK_SERVER_NAME=changedet" -e "USE_EXPERIMENTAL_PUPPETEER_FETCH=yes" -e "PLAYWRIGHT_DRIVER_URL=ws://browserless:3000?dumpio=true" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0  --live-server-port=5004 tests/test_request.py'          
+          
           # restock detection via playwright - added name=changedet here so that playwright/browserless can connect to it
           docker run --rm --name "changedet" -e "FLASK_SERVER_NAME=changedet" -e "PLAYWRIGHT_DRIVER_URL=ws://browserless:3000" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-port=5004 --live-server-host=0.0.0.0 tests/restock/test_restock.py'
 
+      - name: Test SMTP notification mime types
+        run: |
+          # SMTP content types - needs the 'Debug SMTP server/echo message back server' container from above
+          docker run --rm  --network changedet-network test-changedetectionio bash -c 'cd changedetectionio;pytest tests/smtp/test_notification_smtp.py'
+
       - name: Test with puppeteer fetcher and disk cache
         run: |
           docker run --rm -e "PUPPETEER_DISK_CACHE=/tmp/data/" -e "USE_EXPERIMENTAL_PUPPETEER_FETCH=yes" -e "PLAYWRIGHT_DRIVER_URL=ws://browserless:3000" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio;pytest tests/fetchers/test_content.py && pytest tests/test_errorhandling.py && pytest tests/visualselector/test_fetch_data.py'

Dockerfile

@@ -1,5 +1,5 @@
 # pip dependencies install stage
-FROM python:3.10-slim as builder
+FROM python:3.10-slim-bullseye as builder
 
 # See `cryptography` pin comment in requirements.txt
 ARG CRYPTOGRAPHY_DONT_BUILD_RUST=1
@@ -29,7 +29,7 @@ RUN pip install --target=/dependencies playwright~=1.27.1 \
     || echo "WARN: Failed to install Playwright. The application can still run, but the Playwright option will be disabled."
 
 # Final image stage
-FROM python:3.10-slim
+FROM python:3.10-slim-bullseye
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
     libssl1.1 \

MANIFEST.in

@@ -13,3 +13,6 @@ include changedetection.py
 global-exclude *.pyc
 global-exclude node_modules
 global-exclude venv
+
+global-exclude test-datastore
+global-exclude changedetection.io*dist-info

README-pip.md

@@ -2,10 +2,10 @@
 
 Live your data-life pro-actively, track website content changes and receive notifications via Discord, Email, Slack, Telegram and 70+ more
 
-[<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/docs/screenshot.png" style="max-width:100%;" alt="Self-hosted web page change monitoring"  title="Self-hosted web page change monitoring"  />](https://lemonade.changedetection.io/start?src=pip)
+[<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/docs/screenshot.png" style="max-width:100%;" alt="Self-hosted web page change monitoring"  title="Self-hosted web page change monitoring"  />](https://changedetection.io)
 
 
-[**Don't have time? Let us host it for you! try our extremely affordable subscription use our proxies and support!**](https://lemonade.changedetection.io/start) 
+[**Don't have time? Let us host it for you! try our extremely affordable subscription use our proxies and support!**](https://changedetection.io) 
 
 
 #### Example use cases
@@ -54,5 +54,5 @@ $ changedetection.io -d /path/to/empty/data/dir -p 5000
 
 Then visit http://127.0.0.1:5000 , You should now be able to access the UI.
 
-See https://github.com/dgtlmoon/changedetection.io for more information.
+See https://changedetection.io for more information.
 

README.md

@@ -5,13 +5,13 @@
 _Live your data-life pro-actively._ 
 
 
-[<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/docs/screenshot.png" style="max-width:100%;" alt="Self-hosted web page change monitoring"  title="Self-hosted web page change monitoring"  />](https://lemonade.changedetection.io/start?src=github)
+[<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/docs/screenshot.png" style="max-width:100%;" alt="Self-hosted web page change monitoring"  title="Self-hosted web page change monitoring"  />](https://changedetection.io?src=github)
 
 [![Release Version][release-shield]][release-link] [![Docker Pulls][docker-pulls]][docker-link] [![License][license-shield]](LICENSE.md)
 
 ![changedetection.io](https://github.com/dgtlmoon/changedetection.io/actions/workflows/test-only.yml/badge.svg?branch=master)
 
-[**Don't have time? Let us host it for you! try our $8.99/month subscription - use our proxies and support!**](https://lemonade.changedetection.io/start) , _half the price of other website change monitoring services and comes with unlimited watches & checks!_
+[**Don't have time? Let us host it for you! try our $8.99/month subscription - use our proxies and support!**](https://changedetection.io) , _half the price of other website change monitoring services!_
 
 - Chrome browser included.
 - Super fast, no registration needed setup.
@@ -22,11 +22,11 @@ _Live your data-life pro-actively._
 
 Available when connected to a <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Playwright-content-fetcher">playwright content fetcher</a> (included as part of our subscription service)
 
-[<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/docs/visualselector-anim.gif" style="max-width:100%;" alt="Self-hosted web page change monitoring context difference "  title="Self-hosted web page change monitoring context difference " />](https://lemonade.changedetection.io/start?src=github)
+[<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/docs/visualselector-anim.gif" style="max-width:100%;" alt="Self-hosted web page change monitoring context difference "  title="Self-hosted web page change monitoring context difference " />](https://changedetection.io?src=github)
 
 ### Easily see what changed, examine by word, line, or individual character.
 
-[<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/docs/screenshot-diff.png" style="max-width:100%;" alt="Self-hosted web page change monitoring context difference "  title="Self-hosted web page change monitoring context difference " />](https://lemonade.changedetection.io/start?src=github)
+[<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/docs/screenshot-diff.png" style="max-width:100%;" alt="Self-hosted web page change monitoring context difference "  title="Self-hosted web page change monitoring context difference " />](https://changedetection.io?src=github)
 
 
 ### Perform interactive browser steps
@@ -35,7 +35,7 @@ Fill in text boxes, click buttons and more, setup your changedetection scenario.
 
 Using the **Browser Steps** configuration, add basic steps before performing change detection, such as logging into websites, adding a product to a cart, accept cookie logins, entering dates and refining searches.
 
-[<img src="docs/browsersteps-anim.gif" style="max-width:100%;" alt="Self-hosted web page change monitoring context difference "  title="Website change detection with interactive browser steps, login, cookies etc" />](https://lemonade.changedetection.io/start?src=github)
+[<img src="docs/browsersteps-anim.gif" style="max-width:100%;" alt="Self-hosted web page change monitoring context difference "  title="Website change detection with interactive browser steps, login, cookies etc" />](https://changedetection.io?src=github)
 
 After **Browser Steps** have been run, then visit the **Visual Selector** tab to refine the content you're interested in.
 Requires Playwright to be enabled.
@@ -66,6 +66,8 @@ Requires Playwright to be enabled.
 - Proactively search for jobs, get notified when companies update their careers page, search job portals for keywords.
 - Get alerts when new job positions are open on Bamboo HR and other job platforms
 - Website defacement monitoring
+- Pokémon Card Restock Tracker / Pokémon TCG Tracker
+- RegTech - stay ahead of regulatory changes, regulatory compliance
 
 _Need an actual Chrome runner with Javascript support? We support fetching via WebDriver and Playwright!</a>_
 
@@ -84,6 +86,8 @@ _Need an actual Chrome runner with Javascript support? We support fetching via W
 
 We [recommend and use Bright Data](https://brightdata.grsm.io/n0r16zf7eivq) global proxy services, Bright Data will match any first deposit up to $100 using our signup link.
 
+[OxyLabs](https://oxylabs.io) is also an excellent proxy provider and well worth using, they offer Residental, ISP, Rotating and many other proxy types to suit your project. 
+
 Please :star: star :star: this project and help it grow! https://github.com/dgtlmoon/changedetection.io/
 
 ## Installation
@@ -144,7 +148,7 @@ See the wiki for more information https://github.com/dgtlmoon/changedetection.io
 ## Filters
 
 XPath, JSONPath, jq, and CSS support comes baked in! You can be as specific as you need, use XPath exported from various XPath element query creation tools. 
-(We support LXML `re:test`, `re:math` and `re:replace`.)
+(We support LXML `re:test`, `re:match` and `re:replace`.)
 
 ## Notifications
 
@@ -182,7 +186,7 @@ This will re-parse the JSON and apply formatting to the text, making it super ea
 
 ### JSONPath or jq?
 
-For more complex parsing, filtering, and modifying of JSON data, jq is recommended due to the built-in operators and functions. Refer to the [documentation](https://stedolan.github.io/jq/manual/) for more specifc information on jq.
+For more complex parsing, filtering, and modifying of JSON data, jq is recommended due to the built-in operators and functions. Refer to the [documentation](https://stedolan.github.io/jq/manual/) for more specific information on jq.
 
 One big advantage of `jq` is that you can use logic in your JSON filter, such as filters to only show items that have a value greater than/less than etc.
 
@@ -237,7 +241,7 @@ Supports managing the website watch list [via our API](https://changedetection.i
 Do you use changedetection.io to make money? does it save you time or money? Does it make your life easier? less stressful? Remember, we write this software when we should be doing actual paid work, we have to buy food and pay rent just like you.
 
 
-Firstly, consider taking out a [change detection monthly subscription - unlimited checks and watches](https://lemonade.changedetection.io/start) , even if you don't use it, you still get the warm fuzzy feeling of helping out the project. (And who knows, you might just use it!)
+Firstly, consider taking out a [change detection monthly subscription - unlimited checks and watches](https://changedetection.io?src=github) , even if you don't use it, you still get the warm fuzzy feeling of helping out the project. (And who knows, you might just use it!)
 
 Or directly donate an amount PayPal [![Donate](https://img.shields.io/badge/Donate-PayPal-green.svg)](https://www.paypal.com/donate/?hosted_button_id=7CP6HR9ZCNDYJ)
 

changedetectionio/__init__.py

@@ -38,7 +38,7 @@ from flask_paginate import Pagination, get_page_parameter
 from changedetectionio import html_tools
 from changedetectionio.api import api_v1
 
-__version__ = '0.41.1'
+__version__ = '0.45'
 
 datastore = None
 
@@ -124,6 +124,15 @@ def _jinja2_filter_datetimestamp(timestamp, format="%Y-%m-%d %H:%M:%S"):
 
     return timeago.format(timestamp, time.time())
 
+
+@app.template_filter('pagination_slice')
+def _jinja2_filter_pagination_slice(arr, skip):
+    per_page = datastore.data['settings']['application'].get('pager_size', 50)
+    if per_page:
+        return arr[skip:skip + per_page]
+
+    return arr
+
 @app.template_filter('format_seconds_ago')
 def _jinja2_filter_seconds_precise(timestamp):
     if timestamp == False:
@@ -308,25 +317,21 @@ def changedetection_app(config=None, datastore_o=None):
             return "Access denied, bad token", 403
 
         from . import diff
-        limit_tag = request.args.get('tag')
+        limit_tag = request.args.get('tag', '').lower().strip()
+        # Be sure limit_tag is a uuid
+        for uuid, tag in datastore.data['settings']['application'].get('tags', {}).items():
+            if limit_tag == tag.get('title', '').lower().strip():
+                limit_tag = uuid
 
         # Sort by last_changed and add the uuid which is usually the key..
         sorted_watches = []
 
         # @todo needs a .itemsWithTag() or something - then we can use that in Jinaj2 and throw this away
         for uuid, watch in datastore.data['watching'].items():
-
-            if limit_tag != None:
-                # Support for comma separated list of tags.
-                for tag_in_watch in watch['tag'].split(','):
-                    tag_in_watch = tag_in_watch.strip()
-                    if tag_in_watch == limit_tag:
-                        watch['uuid'] = uuid
-                        sorted_watches.append(watch)
-
-            else:
-                watch['uuid'] = uuid
-                sorted_watches.append(watch)
+            if limit_tag and not limit_tag in watch['tags']:
+                    continue
+            watch['uuid'] = uuid
+            sorted_watches.append(watch)
 
         sorted_watches.sort(key=lambda x: x.last_changed, reverse=False)
 
@@ -350,11 +355,13 @@ def changedetection_app(config=None, datastore_o=None):
 
                 # Include a link to the diff page, they will have to login here to see if password protection is enabled.
                 # Description is the page you watch, link takes you to the diff JS UI page
-                base_url = datastore.data['settings']['application']['base_url']
-                if base_url == '':
-                    base_url = "<base-url-env-var-not-set>"
-
-                diff_link = {'href': "{}{}".format(base_url, url_for('diff_history_page', uuid=watch['uuid']))}
+                # Dict val base_url will get overriden with the env var if it is set.
+                ext_base_url = datastore.data['settings']['application'].get('base_url')
+                if ext_base_url:
+                    # Go with overriden value
+                    diff_link = {'href': "{}{}".format(ext_base_url, url_for('diff_history_page', uuid=watch['uuid'], _external=False))}
+                else:
+                    diff_link = {'href': url_for('diff_history_page', uuid=watch['uuid'], _external=True)}
 
                 fe.link(link=diff_link)
 
@@ -383,9 +390,17 @@ def changedetection_app(config=None, datastore_o=None):
     @app.route("/", methods=['GET'])
     @login_optionally_required
     def index():
+        global datastore
         from changedetectionio import forms
 
-        limit_tag = request.args.get('tag')
+        limit_tag = request.args.get('tag', '').lower().strip()
+
+        # Be sure limit_tag is a uuid
+        for uuid, tag in datastore.data['settings']['application'].get('tags', {}).items():
+            if limit_tag == tag.get('title', '').lower().strip():
+                limit_tag = uuid
+
+
         # Redirect for the old rss path which used the /?rss=true
         if request.args.get('rss'):
             return redirect(url_for('rss', tag=limit_tag))
@@ -403,33 +418,32 @@ def changedetection_app(config=None, datastore_o=None):
 
         # Sort by last_changed and add the uuid which is usually the key..
         sorted_watches = []
+        search_q = request.args.get('q').strip().lower() if request.args.get('q') else False
         for uuid, watch in datastore.data['watching'].items():
-
-            if limit_tag != None:
-                # Support for comma separated list of tags.
-                if watch['tag'] is None:
+            if limit_tag and not limit_tag in watch['tags']:
                     continue
-                for tag_in_watch in watch['tag'].split(','):
-                    tag_in_watch = tag_in_watch.strip()
-                    if tag_in_watch == limit_tag:
-                        watch['uuid'] = uuid
-                        sorted_watches.append(watch)
 
+            if search_q:
+                if (watch.get('title') and search_q in watch.get('title').lower()) or search_q in watch.get('url', '').lower():
+                    sorted_watches.append(watch)
             else:
-                watch['uuid'] = uuid
                 sorted_watches.append(watch)
 
-        existing_tags = datastore.get_all_tags()
         form = forms.quickWatchForm(request.form)
         page = request.args.get(get_page_parameter(), type=int, default=1)
-        total_count = len(sorted_watches) if sorted_watches else len(datastore.data['watching'])
-        pagination = Pagination(page=page, total=total_count, per_page=int(os.getenv('pagination_per_page', 50)), css_framework = "semantic")
+        total_count = len(sorted_watches)
+
+        pagination = Pagination(page=page,
+                                total=total_count,
+                                per_page=datastore.data['settings']['application'].get('pager_size', 50), css_framework="semantic")
+
 
         output = render_template(
             "watch-overview.html",
                                  # Don't link to hosting when we're on the hosting environment
                                  active_tag=limit_tag,
                                  app_rss_token=datastore.data['settings']['application']['rss_access_token'],
+                                 datastore=datastore,
                                  form=form,
                                  guid=datastore.data['app_guid'],
                                  has_proxies=datastore.proxy_list,
@@ -437,10 +451,11 @@ def changedetection_app(config=None, datastore_o=None):
                                  hosted_sticky=os.getenv("SALTED_PASS", False) == False,
                                  pagination=pagination,
                                  queued_uuids=[q_uuid.item['uuid'] for q_uuid in update_q.queue],
+                                 search_q=request.args.get('q','').strip(),
                                  sort_attribute=request.args.get('sort') if request.args.get('sort') else request.cookies.get('sort'),
                                  sort_order=request.args.get('order') if request.args.get('order') else request.cookies.get('order'),
                                  system_default_fetcher=datastore.data['settings']['application'].get('fetch_backend'),
-                                 tags=existing_tags,
+                                 tags=datastore.data['settings']['application'].get('tags'),
                                  watches=sorted_watches
                                  )
 
@@ -583,9 +598,13 @@ def changedetection_app(config=None, datastore_o=None):
 
         # proxy_override set to the json/text list of the items
         form = forms.watchForm(formdata=request.form if request.method == 'POST' else None,
-                               data=default,
+                               data=default
                                )
 
+        # For the form widget tag uuid lookup
+        form.tags.datastore = datastore # in _value
+
+
         form.fetch_backend.choices.append(("system", 'System settings default'))
 
         # form.browser_steps[0] can be assumed that we 'goto url' first
@@ -636,6 +655,16 @@ def changedetection_app(config=None, datastore_o=None):
                 extra_update_obj['filter_text_replaced'] = True
                 extra_update_obj['filter_text_removed'] = True
 
+            # Because wtforms doesn't support accessing other data in process_ , but we convert the CSV list of tags back to a list of UUIDs
+            tag_uuids = []
+            if form.data.get('tags'):
+                # Sometimes in testing this can be list, dont know why
+                if type(form.data.get('tags')) == list:
+                    extra_update_obj['tags'] = form.data.get('tags')
+                else:
+                    for t in form.data.get('tags').split(','):
+                        tag_uuids.append(datastore.add_tag(name=t))
+                    extra_update_obj['tags'] = tag_uuids
 
             datastore.data['watching'][uuid].update(form.data)
             datastore.data['watching'][uuid].update(extra_update_obj)
@@ -690,6 +719,7 @@ def changedetection_app(config=None, datastore_o=None):
                                      form=form,
                                      has_default_notification_urls=True if len(datastore.data['settings']['application']['notification_urls']) else False,
                                      has_empty_checktime=using_default_check_time,
+                                     has_extra_headers_file=len(datastore.get_all_headers_in_textfile_for_watch(uuid=uuid)) > 0,
                                      is_html_webdriver=is_html_webdriver,
                                      jq_support=jq_support,
                                      playwright_enabled=os.getenv('PLAYWRIGHT_DRIVER_URL', False),
@@ -1086,8 +1116,8 @@ def changedetection_app(config=None, datastore_o=None):
                 os.path.join(datastore_o.datastore_path, list_with_tags_file), "w"
             ) as f:
                 for uuid in datastore.data["watching"]:
-                    url = datastore.data["watching"][uuid]["url"]
-                    tag = datastore.data["watching"][uuid]["tag"]
+                    url = datastore.data["watching"][uuid].get('url')
+                    tag = datastore.data["watching"][uuid].get('tags', {})
                     f.write("{} {}\r\n".format(url, tag))
 
             # Add it to the Zip
@@ -1175,7 +1205,7 @@ def changedetection_app(config=None, datastore_o=None):
 
         add_paused = request.form.get('edit_and_watch_submit_button') != None
         processor = request.form.get('processor', 'text_json_diff')
-        new_uuid = datastore.add_watch(url=url, tag=request.form.get('tag').strip(), extras={'paused': add_paused, 'processor': processor})
+        new_uuid = datastore.add_watch(url=url, tag=request.form.get('tags').strip(), extras={'paused': add_paused, 'processor': processor})
 
         if new_uuid:
             if add_paused:
@@ -1243,9 +1273,11 @@ def changedetection_app(config=None, datastore_o=None):
         elif tag != None:
             # Items that have this current tag
             for watch_uuid, watch in datastore.data['watching'].items():
-                if (tag != None and tag in watch['tag']):
+                if (tag != None and tag in watch.get('tags', {})):
                     if watch_uuid not in running_uuids and not datastore.data['watching'][watch_uuid]['paused']:
-                        update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': watch_uuid, 'skip_when_checksum_same': False}))
+                        update_q.put(
+                            queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': watch_uuid, 'skip_when_checksum_same': False})
+                        )
                         i += 1
 
         else:
@@ -1333,6 +1365,18 @@ def changedetection_app(config=None, datastore_o=None):
                     datastore.data['watching'][uuid.strip()]['notification_format'] = default_notification_format_for_watch
             flash("{} watches set to use default notification settings".format(len(uuids)))
 
+        elif (op == 'assign-tag'):
+            op_extradata = request.form.get('op_extradata', '').strip()
+            if op_extradata:
+                tag_uuid = datastore.add_tag(name=op_extradata)
+                if op_extradata and tag_uuid:
+                    for uuid in uuids:
+                        uuid = uuid.strip()
+                        if datastore.data['watching'].get(uuid):
+                            datastore.data['watching'][uuid]['tags'].append(tag_uuid)
+
+            flash("{} watches assigned tag".format(len(uuids)))
+
         return redirect(url_for('index'))
 
     @app.route("/api/share-url", methods=['GET'])
@@ -1342,7 +1386,6 @@ def changedetection_app(config=None, datastore_o=None):
            the share-link can be imported/added"""
         import requests
         import json
-        tag = request.args.get('tag')
         uuid = request.args.get('uuid')
 
         # more for testing
@@ -1389,12 +1432,39 @@ def changedetection_app(config=None, datastore_o=None):
         # paste in etc
         return redirect(url_for('index'))
 
+    @app.route("/highlight_submit_ignore_url", methods=['POST'])
+    def highlight_submit_ignore_url():
+        import re
+        mode = request.form.get('mode')
+        selection = request.form.get('selection')
+
+        uuid = request.args.get('uuid','')
+        if datastore.data["watching"].get(uuid):
+            if mode == 'exact':
+                for l in selection.splitlines():
+                    datastore.data["watching"][uuid]['ignore_text'].append(l.strip())
+            elif mode == 'digit-regex':
+                for l in selection.splitlines():
+                    # Replace any series of numbers with a regex
+                    s = re.escape(l.strip())
+                    s = re.sub(r'[0-9]+', r'\\d+', s)
+                    datastore.data["watching"][uuid]['ignore_text'].append('/' + s + '/')
+
+        return f"<a href={url_for('preview_page', uuid=uuid)}>Click to preview</a>"
+
+
     import changedetectionio.blueprint.browser_steps as browser_steps
     app.register_blueprint(browser_steps.construct_blueprint(datastore), url_prefix='/browser-steps')
 
     import changedetectionio.blueprint.price_data_follower as price_data_follower
     app.register_blueprint(price_data_follower.construct_blueprint(datastore, update_q), url_prefix='/price_data_follower')
 
+    import changedetectionio.blueprint.tags as tags
+    app.register_blueprint(tags.construct_blueprint(datastore), url_prefix='/tags')
+
+    import changedetectionio.blueprint.check_proxies as check_proxies
+    app.register_blueprint(check_proxies.construct_blueprint(datastore=datastore), url_prefix='/check_proxy')
+
 
     # @todo handle ctrl break
     ticker_thread = threading.Thread(target=ticker_thread_check_time_launch_checks).start()
@@ -1434,6 +1504,7 @@ def check_for_new_version():
         # Check daily
         app.config.exit.wait(86400)
 
+
 def notification_runner():
     global notification_debug_log
     from datetime import datetime

changedetectionio/api/api_v1.py

@@ -1,3 +1,6 @@
+import os
+from distutils.util import strtobool
+
 from flask_expects_json import expects_json
 from changedetectionio import queuedWatchMetaData
 from flask_restful import abort, Resource
@@ -33,7 +36,7 @@ class Watch(Resource):
     @auth.check_token
     def get(self, uuid):
         """
-        @api {get} /api/v1/watch/:uuid Get a single watch data
+        @api {get} /api/v1/watch/:uuid Single watch - get data, recheck, pause, mute.
         @apiDescription Retrieve watch information and set muted/paused status
         @apiExample {curl} Example usage:
             curl http://localhost:4000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091  -H"x-api-key:813031b16330fe25e3780cf0325daa45"
@@ -209,7 +212,9 @@ class CreateWatch(Resource):
         json_data = request.get_json()
         url = json_data['url'].strip()
 
-        if not validators.url(json_data['url'].strip()):
+        # If hosts that only contain alphanumerics are allowed ("localhost" for example)
+        allow_simplehost = not strtobool(os.getenv('BLOCK_SIMPLEHOSTS', 'False'))
+        if not validators.url(url, simple_host=allow_simplehost):
             return "Invalid or unsupported URL", 400
 
         if json_data.get('proxy'):
@@ -218,9 +223,16 @@ class CreateWatch(Resource):
                 return "Invalid proxy choice, currently supported proxies are '{}'".format(', '.join(plist)), 400
 
         extras = copy.deepcopy(json_data)
+
+        # Because we renamed 'tag' to 'tags' but don't want to change the API (can do this in v2 of the API)
+        tags = None
+        if extras.get('tag'):
+            tags = extras.get('tag')
+            del extras['tag']
+
         del extras['url']
 
-        new_uuid = self.datastore.add_watch(url=url, extras=extras)
+        new_uuid = self.datastore.add_watch(url=url, extras=extras, tag=tags)
         if new_uuid:
             self.update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': new_uuid, 'skip_when_checksum_same': True}))
             return {'uuid': new_uuid}, 201
@@ -259,13 +271,16 @@ class CreateWatch(Resource):
         """
         list = {}
 
-        tag_limit = request.args.get('tag', None)
-        for k, watch in self.datastore.data['watching'].items():
-            if tag_limit:
-                if not tag_limit.lower() in watch.all_tags:
-                    continue
+        tag_limit = request.args.get('tag', '').lower()
 
-            list[k] = {'url': watch['url'],
+
+        for uuid, watch in self.datastore.data['watching'].items():
+            # Watch tags by name (replace the other calls?)
+            tags = self.datastore.get_all_tags_for_watch(uuid=uuid)
+            if tag_limit and not any(v.get('title').lower() == tag_limit for k, v in tags.items()):
+                continue
+
+            list[uuid] = {'url': watch['url'],
                        'title': watch['title'],
                        'last_checked': watch['last_checked'],
                        'last_changed': watch.last_changed,

changedetectionio/blueprint/check_proxies/__init__.py

@@ -0,0 +1,116 @@
+from concurrent.futures import ThreadPoolExecutor
+
+from functools import wraps
+
+from flask import Blueprint
+from flask_login import login_required
+
+from changedetectionio.processors import text_json_diff
+from changedetectionio.store import ChangeDetectionStore
+
+
+STATUS_CHECKING = 0
+STATUS_FAILED = 1
+STATUS_OK = 2
+THREADPOOL_MAX_WORKERS = 3
+_DEFAULT_POOL = ThreadPoolExecutor(max_workers=THREADPOOL_MAX_WORKERS)
+
+
+# Maybe use fetch-time if its >5 to show some expected load time?
+def threadpool(f, executor=None):
+    @wraps(f)
+    def wrap(*args, **kwargs):
+        return (executor or _DEFAULT_POOL).submit(f, *args, **kwargs)
+
+    return wrap
+
+
+def construct_blueprint(datastore: ChangeDetectionStore):
+    check_proxies_blueprint = Blueprint('check_proxies', __name__)
+    checks_in_progress = {}
+
+    @threadpool
+    def long_task(uuid, preferred_proxy):
+        import time
+        from changedetectionio import content_fetcher
+
+        status = {'status': '', 'length': 0, 'text': ''}
+        from jinja2 import Environment, BaseLoader
+
+        contents = ''
+        now = time.time()
+        try:
+            update_handler = text_json_diff.perform_site_check(datastore=datastore)
+            changed_detected, update_obj, contents = update_handler.run(uuid, preferred_proxy=preferred_proxy, skip_when_checksum_same=False)
+        # title, size is len contents not len xfer
+        except content_fetcher.Non200ErrorCodeReceived as e:
+            if e.status_code == 404:
+                status.update({'status': 'OK', 'length': len(contents), 'text': f"OK but 404 (page not found)"})
+            elif e.status_code == 403 or e.status_code == 401:
+                status.update({'status': 'ERROR', 'length': len(contents), 'text': f"{e.status_code} - Access denied"})
+            else:
+                status.update({'status': 'ERROR', 'length': len(contents), 'text': f"Status code: {e.status_code}"})
+        except text_json_diff.FilterNotFoundInResponse:
+            status.update({'status': 'OK', 'length': len(contents), 'text': f"OK but CSS/xPath filter not found (page changed layout?)"})
+        except content_fetcher.EmptyReply as e:
+            if e.status_code == 403 or e.status_code == 401:
+                status.update({'status': 'ERROR OTHER', 'length': len(contents), 'text': f"Got empty reply with code {e.status_code} - Access denied"})
+            else:
+                status.update({'status': 'ERROR OTHER', 'length': len(contents) if contents else 0, 'text': f"Empty reply with code {e.status_code}, needs chrome?"})
+
+        except Exception as e:
+            status.update({'status': 'ERROR OTHER', 'length': len(contents) if contents else 0, 'text': 'Error: '+str(e)})
+        else:
+            status.update({'status': 'OK', 'length': len(contents), 'text': ''})
+
+        if status.get('text'):
+            status['text'] = Environment(loader=BaseLoader()).from_string('{{text|e}}').render({'text': status['text']})
+
+        status['time'] = "{:.2f}s".format(time.time() - now)
+
+        return status
+
+    def _recalc_check_status(uuid):
+
+        results = {}
+        for k, v in checks_in_progress.get(uuid, {}).items():
+            try:
+                r_1 = v.result(timeout=0.05)
+            except Exception as e:
+                # If timeout error?
+                results[k] = {'status': 'RUNNING'}
+
+            else:
+                results[k] = r_1
+
+        return results
+
+    @login_required
+    @check_proxies_blueprint.route("/<string:uuid>/status", methods=['GET'])
+    def get_recheck_status(uuid):
+        results = _recalc_check_status(uuid=uuid)
+        return results
+
+    @login_required
+    @check_proxies_blueprint.route("/<string:uuid>/start", methods=['GET'])
+    def start_check(uuid):
+
+        if not datastore.proxy_list:
+            return
+
+        if checks_in_progress.get(uuid):
+            state = _recalc_check_status(uuid=uuid)
+            for proxy_key, v in state.items():
+                if v.get('status') == 'RUNNING':
+                    return state
+        else:
+            checks_in_progress[uuid] = {}
+
+        for k, v in datastore.proxy_list.items():
+            if not checks_in_progress[uuid].get(k):
+                checks_in_progress[uuid][k] = long_task(uuid=uuid, preferred_proxy=k)
+
+        results = _recalc_check_status(uuid=uuid)
+        return results
+
+    return check_proxies_blueprint

changedetectionio/blueprint/tags/README.md

@@ -0,0 +1,9 @@
+# Groups tags
+
+## How it works
+
+Watch has a list() of tag UUID's, which relate to a config under application.settings.tags
+
+The 'tag' is actually a watch, because they basically will eventually share 90% of the same config.
+
+So a tag is like an abstract of a watch

changedetectionio/blueprint/tags/__init__.py

@@ -0,0 +1,141 @@
+from flask import Blueprint, request, make_response, render_template, flash, url_for, redirect
+from changedetectionio.store import ChangeDetectionStore
+from changedetectionio import login_optionally_required
+
+
+def construct_blueprint(datastore: ChangeDetectionStore):
+    tags_blueprint = Blueprint('tags', __name__, template_folder="templates")
+
+    @tags_blueprint.route("/list", methods=['GET'])
+    @login_optionally_required
+    def tags_overview_page():
+        from .form import SingleTag
+        add_form = SingleTag(request.form)
+        output = render_template("groups-overview.html",
+                                 form=add_form,
+                                 available_tags=datastore.data['settings']['application'].get('tags', {}),
+                                 )
+
+        return output
+
+    @tags_blueprint.route("/add", methods=['POST'])
+    @login_optionally_required
+    def form_tag_add():
+        from .form import SingleTag
+        add_form = SingleTag(request.form)
+
+        if not add_form.validate():
+            for widget, l in add_form.errors.items():
+                flash(','.join(l), 'error')
+            return redirect(url_for('tags.tags_overview_page'))
+
+        title = request.form.get('name').strip()
+
+        if datastore.tag_exists_by_name(title):
+            flash(f'The tag "{title}" already exists', "error")
+            return redirect(url_for('tags.tags_overview_page'))
+
+        datastore.add_tag(title)
+        flash("Tag added")
+
+
+        return redirect(url_for('tags.tags_overview_page'))
+
+    @tags_blueprint.route("/mute/<string:uuid>", methods=['GET'])
+    @login_optionally_required
+    def mute(uuid):
+        if datastore.data['settings']['application']['tags'].get(uuid):
+            datastore.data['settings']['application']['tags'][uuid]['notification_muted'] = not datastore.data['settings']['application']['tags'][uuid]['notification_muted']
+        return redirect(url_for('tags.tags_overview_page'))
+
+    @tags_blueprint.route("/delete/<string:uuid>", methods=['GET'])
+    @login_optionally_required
+    def delete(uuid):
+        removed = 0
+        # Delete the tag, and any tag reference
+        if datastore.data['settings']['application']['tags'].get(uuid):
+            del datastore.data['settings']['application']['tags'][uuid]
+
+        for watch_uuid, watch in datastore.data['watching'].items():
+            if watch.get('tags') and uuid in watch['tags']:
+                removed += 1
+                watch['tags'].remove(uuid)
+
+        flash(f"Tag deleted and removed from {removed} watches")
+        return redirect(url_for('tags.tags_overview_page'))
+
+    @tags_blueprint.route("/unlink/<string:uuid>", methods=['GET'])
+    @login_optionally_required
+    def unlink(uuid):
+        unlinked = 0
+        for watch_uuid, watch in datastore.data['watching'].items():
+            if watch.get('tags') and uuid in watch['tags']:
+                unlinked += 1
+                watch['tags'].remove(uuid)
+
+        flash(f"Tag unlinked removed from {unlinked} watches")
+        return redirect(url_for('tags.tags_overview_page'))
+
+    @tags_blueprint.route("/delete_all", methods=['GET'])
+    @login_optionally_required
+    def delete_all():
+        for watch_uuid, watch in datastore.data['watching'].items():
+            watch['tags'] = []
+        datastore.data['settings']['application']['tags'] = {}
+
+        flash(f"All tags deleted")
+        return redirect(url_for('tags.tags_overview_page'))
+
+    @tags_blueprint.route("/edit/<string:uuid>", methods=['GET'])
+    @login_optionally_required
+    def form_tag_edit(uuid):
+        from changedetectionio import forms
+
+        if uuid == 'first':
+            uuid = list(datastore.data['settings']['application']['tags'].keys()).pop()
+
+        default = datastore.data['settings']['application']['tags'].get(uuid)
+
+        form = forms.watchForm(formdata=request.form if request.method == 'POST' else None,
+                               data=default,
+                               )
+        form.datastore=datastore # needed?
+
+        output = render_template("edit-tag.html",
+                                 data=default,
+                                 form=form,
+                                 settings_application=datastore.data['settings']['application'],
+                                 )
+
+        return output
+
+
+    @tags_blueprint.route("/edit/<string:uuid>", methods=['POST'])
+    @login_optionally_required
+    def form_tag_edit_submit(uuid):
+        from changedetectionio import forms
+        if uuid == 'first':
+            uuid = list(datastore.data['settings']['application']['tags'].keys()).pop()
+
+        default = datastore.data['settings']['application']['tags'].get(uuid)
+
+        form = forms.watchForm(formdata=request.form if request.method == 'POST' else None,
+                               data=default,
+                               )
+        # @todo subclass form so validation works
+        #if not form.validate():
+#            for widget, l in form.errors.items():
+#                flash(','.join(l), 'error')
+#           return redirect(url_for('tags.form_tag_edit_submit', uuid=uuid))
+
+        datastore.data['settings']['application']['tags'][uuid].update(form.data)
+        datastore.needs_write_urgent = True
+        flash("Updated")
+
+        return redirect(url_for('tags.tags_overview_page'))
+
+
+    @tags_blueprint.route("/delete/<string:uuid>", methods=['GET'])
+    def form_tag_delete(uuid):
+        return redirect(url_for('tags.tags_overview_page'))
+    return tags_blueprint

changedetectionio/blueprint/tags/form.py

@@ -0,0 +1,22 @@
+from wtforms import (
+    BooleanField,
+    Form,
+    IntegerField,
+    RadioField,
+    SelectField,
+    StringField,
+    SubmitField,
+    TextAreaField,
+    validators,
+)
+
+
+
+class SingleTag(Form):
+
+    name = StringField('Tag name', [validators.InputRequired()], render_kw={"placeholder": "Name"})
+    save_button = SubmitField('Save', render_kw={"class": "pure-button pure-button-primary"})
+
+
+
+

changedetectionio/blueprint/tags/templates/edit-tag.html

@@ -0,0 +1,135 @@
+{% extends 'base.html' %}
+{% block content %}
+{% from '_helpers.jinja' import render_field, render_checkbox_field, render_button %}
+{% from '_common_fields.jinja' import render_common_settings_form %}
+<script>
+    const notification_base_url="{{url_for('ajax_callback_send_notification_test')}}";
+</script>
+
+<script src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
+<script>
+
+/*{% if emailprefix %}*/
+    /*const email_notification_prefix=JSON.parse('{{ emailprefix|tojson }}');*/
+/*{% endif %}*/
+
+
+</script>
+
+<script src="{{url_for('static_content', group='js', filename='watch-settings.js')}}" defer></script>
+<!--<script src="{{url_for('static_content', group='js', filename='limit.js')}}" defer></script>-->
+<script src="{{url_for('static_content', group='js', filename='notifications.js')}}" defer></script>
+
+<div class="edit-form monospaced-textarea">
+
+    <div class="tabs collapsable">
+        <ul>
+            <li class="tab" id=""><a href="#general">General</a></li>
+            <li class="tab"><a href="#filters-and-triggers">Filters &amp; Triggers</a></li>
+            <li class="tab"><a href="#notifications">Notifications</a></li>
+        </ul>
+    </div>
+
+    <div class="box-wrap inner">
+        <form class="pure-form pure-form-stacked"
+              action="{{ url_for('tags.form_tag_edit', uuid=data.uuid) }}" method="POST">
+             <input type="hidden" name="csrf_token" value="{{ csrf_token() }}">
+
+            <div class="tab-pane-inner" id="general">
+                <fieldset>
+                    <div class="pure-control-group">
+                        {{ render_field(form.title, placeholder="https://...", required=true, class="m-d") }}
+                    </div>
+                </fieldset>
+            </div>
+
+            <div class="tab-pane-inner" id="filters-and-triggers">
+                    <div class="pure-control-group">
+                        {% set field = render_field(form.include_filters,
+                            rows=5,
+                            placeholder="#example
+xpath://body/div/span[contains(@class, 'example-class')]",
+                            class="m-d")
+                        %}
+                        {{ field }}
+                        {% if '/text()' in  field %}
+                          <span class="pure-form-message-inline"><strong>Note!: //text() function does not work where the &lt;element&gt; contains &lt;![CDATA[]]&gt;</strong></span><br>
+                        {% endif %}
+                        <span class="pure-form-message-inline">One rule per line, <i>any</i> rules that matches will be used.<br>
+
+                    <ul>
+                        <li>CSS - Limit text to this CSS rule, only text matching this CSS rule is included.</li>
+                        <li>JSON - Limit text to this JSON rule, using either <a href="https://pypi.org/project/jsonpath-ng/" target="new">JSONPath</a> or <a href="https://stedolan.github.io/jq/" target="new">jq</a> (if installed).
+                            <ul>
+                                <li>JSONPath: Prefix with <code>json:</code>, use <code>json:$</code> to force re-formatting if required,  <a href="https://jsonpath.com/" target="new">test your JSONPath here</a>.</li>
+                                {% if jq_support %}
+                                <li>jq: Prefix with <code>jq:</code> and <a href="https://jqplay.org/" target="new">test your jq here</a>. Using <a href="https://stedolan.github.io/jq/" target="new">jq</a> allows for complex filtering and processing of JSON data with built-in functions, regex, filtering, and more. See examples and documentation <a href="https://stedolan.github.io/jq/manual/" target="new">here</a>.</li>
+                                {% else %}
+                                <li>jq support not installed</li>
+                                {% endif %}
+                            </ul>
+                        </li>
+                        <li>XPath - Limit text to this XPath rule, simply start with a forward-slash,
+                            <ul>
+                                <li>Example:  <code>//*[contains(@class, 'sametext')]</code> or <code>xpath://*[contains(@class, 'sametext')]</code>, <a
+                                href="http://xpather.com/" target="new">test your XPath here</a></li>
+                                <li>Example: Get all titles from an RSS feed <code>//title/text()</code></li>
+                            </ul>
+                            </li>
+                    </ul>
+                    Please be sure that you thoroughly understand how to write CSS, JSONPath, XPath{% if jq_support %}, or jq selector{%endif%} rules before filing an issue on GitHub! <a
+                                href="https://github.com/dgtlmoon/changedetection.io/wiki/CSS-Selector-help">here for more CSS selector help</a>.<br>
+                </span>
+                    </div>
+                <fieldset class="pure-control-group">
+                    {{ render_field(form.subtractive_selectors, rows=5, placeholder="header
+footer
+nav
+.stockticker") }}
+                    <span class="pure-form-message-inline">
+                        <ul>
+                          <li> Remove HTML element(s) by CSS selector before text conversion. </li>
+                          <li> Add multiple elements or CSS selectors per line to ignore multiple parts of the HTML. </li>
+                        </ul>
+                      </span>
+                </fieldset>
+
+            </div>
+
+            <div class="tab-pane-inner" id="notifications">
+                <fieldset>
+                    <div  class="pure-control-group inline-radio">
+                      {{ render_checkbox_field(form.notification_muted) }}
+                    </div>
+                    {% if is_html_webdriver %}
+                    <div class="pure-control-group inline-radio">
+                      {{ render_checkbox_field(form.notification_screenshot) }}
+                        <span class="pure-form-message-inline">
+                            <strong>Use with caution!</strong> This will easily fill up your email storage quota or flood other storages.
+                        </span>
+                    </div>
+                    {% endif %}
+                    <div class="field-group" id="notification-field-group">
+                        {% if has_default_notification_urls %}
+                        <div class="inline-warning">
+                            <img class="inline-warning-icon" src="{{url_for('static_content', group='images', filename='notice.svg')}}" alt="Look out!" title="Lookout!" >
+                            There are <a href="{{ url_for('settings_page')}}#notifications">system-wide notification URLs enabled</a>, this form will override notification settings for this watch only &dash; an empty Notification URL list here will still send notifications.
+                        </div>
+                        {% endif %}
+                        <a href="#notifications" id="notification-setting-reset-to-default" class="pure-button button-xsmall" style="right: 20px; top: 20px; position: absolute; background-color: #5f42dd; border-radius: 4px; font-size: 70%; color: #fff">Use system defaults</a>
+
+                        {{ render_common_settings_form(form, emailprefix, settings_application) }}
+                    </div>
+                </fieldset>
+            </div>
+
+            <div id="actions">
+                <div class="pure-control-group">
+                    {{ render_button(form.save_button) }}
+                </div>
+            </div>
+        </form>
+    </div>
+</div>
+
+{% endblock %}

changedetectionio/blueprint/tags/templates/groups-overview.html

@@ -0,0 +1,60 @@
+{% extends 'base.html' %}
+{% block content %}
+{% from '_helpers.jinja' import render_simple_field, render_field %}
+<script src="{{url_for('static_content', group='js', filename='jquery-3.6.0.min.js')}}"></script>
+
+<div class="box">
+    <form class="pure-form" action="{{ url_for('tags.form_tag_add') }}" method="POST" id="new-watch-form">
+        <input type="hidden" name="csrf_token" value="{{ csrf_token() }}" >
+        <fieldset>
+            <legend>Add a new organisational tag</legend>
+            <div id="watch-add-wrapper-zone">
+                <div>
+                    {{ render_simple_field(form.name, placeholder="watch label / tag") }}
+                </div>
+                <div>
+                    {{ render_simple_field(form.save_button, title="Save" ) }}
+                </div>
+            </div>
+            <br>
+            <div style="color: #fff;">Groups allows you to manage filters and notifications for multiple watches under a single organisational tag.</div>
+        </fieldset>
+    </form>
+    <!-- @todo maybe some overview matrix, 'tick' with which has notification, filter rules etc -->
+    <div id="watch-table-wrapper">
+
+        <table class="pure-table pure-table-striped watch-table group-overview-table">
+            <thead>
+            <tr>
+                <th></th>
+                <th>Tag / Label name</th>
+                <th></th>
+            </tr>
+            </thead>
+            <tbody>
+            <!--
+            @Todo - connect Last checked, Last Changed, Number of Watches etc
+            --->
+            {% if not available_tags|length %}
+            <tr>
+                <td colspan="3">No website organisational tags/groups configured</td>
+            </tr>
+            {% endif %}
+            {% for uuid, tag in available_tags.items()  %}
+            <tr id="{{ uuid }}" class="{{ loop.cycle('pure-table-odd', 'pure-table-even') }}">
+                <td class="watch-controls">
+                    <a class="link-mute state-{{'on' if tag.notification_muted else 'off'}}" href="{{url_for('tags.mute', uuid=tag.uuid)}}"><img src="{{url_for('static_content', group='images', filename='bell-off.svg')}}" alt="Mute notifications" title="Mute notifications" class="icon icon-mute" ></a>
+                </td>
+                <td class="title-col inline">{{tag.title}}</td>
+                <td>
+                    <a class="pure-button pure-button-primary" href="{{ url_for('tags.form_tag_edit', uuid=uuid) }}">Edit</a>&nbsp;
+                    <a class="pure-button pure-button-primary" href="{{ url_for('tags.delete', uuid=uuid) }}" title="Deletes and removes tag">Delete</a>
+                    <a class="pure-button pure-button-primary" href="{{ url_for('tags.unlink', uuid=uuid) }}" title="Keep the tag but unlink any watches">Unlink</a>
+                </td>
+            </tr>
+            {% endfor %}
+            </tbody>
+        </table>
+    </div>
+</div>
+{% endblock %}

changedetectionio/content_fetcher.py

@@ -147,6 +147,13 @@ class Fetcher():
     def is_ready(self):
         return True
 
+    def get_all_headers(self):
+        """
+        Get all headers but ensure all keys are lowercase
+        :return:
+        """
+        return {k.lower(): v for k, v in self.headers.items()}
+
     def iterate_browser_steps(self):
         from changedetectionio.blueprint.browser_steps.browser_steps import steppable_browser_interface
         from playwright._impl._api_types import TimeoutError
@@ -194,7 +201,8 @@ class Fetcher():
             dest = os.path.join(self.browser_steps_screenshot_path, 'step_*.jpeg')
             files = glob.glob(dest)
             for f in files:
-                os.unlink(f)
+                if os.path.isfile(f):
+                    os.unlink(f)
 
 
 #   Maybe for the future, each fetcher provides its own diff output, could be used for text, image
@@ -384,6 +392,7 @@ class base_html_playwright(Fetcher):
                 self.headers = x.get('headers')
                 self.instock_data = x.get('instock_data')
                 self.screenshot = base64.b64decode(x.get('screenshot'))
+                self.status_code = x.get('status_code')
                 self.xpath_data = x.get('xpath_data')
 
             else:

changedetectionio/diff.py

@@ -54,4 +54,5 @@ def render_diff(previous_version_file_contents, newest_version_file_contents, in
 
     # Recursively join lists
     f = lambda L: line_feed_sep.join([f(x) if type(x) is list else x for x in L])
-    return f(rendered_diff)
+    p= f(rendered_diff)
+    return p

changedetectionio/forms.py

@@ -1,5 +1,6 @@
 import os
 import re
+from distutils.util import strtobool
 
 from wtforms import (
     BooleanField,
@@ -28,6 +29,8 @@ from changedetectionio.notification import (
 
 from wtforms.fields import FormField
 
+dictfilt = lambda x, y: dict([ (i,x[i]) for i in x if i in set(y) ])
+
 valid_method = {
     'GET',
     'POST',
@@ -90,6 +93,29 @@ class SaltyPasswordField(StringField):
         else:
             self.data = False
 
+class StringTagUUID(StringField):
+
+   # process_formdata(self, valuelist) handled manually in POST handler
+
+    # Is what is shown when field <input> is rendered
+    def _value(self):
+        # Tag UUID to name, on submit it will convert it back (in the submit handler of init.py)
+        if self.data and type(self.data) is list:
+            tag_titles = []
+            for i in self.data:
+                tag = self.datastore.data['settings']['application']['tags'].get(i)
+                if tag:
+                    tag_title = tag.get('title')
+                    if tag_title:
+                        tag_titles.append(tag_title)
+
+            return ', '.join(tag_titles)
+
+        if not self.data:
+            return ''
+
+        return 'error'
+
 class TimeBetweenCheckForm(Form):
     weeks = IntegerField('Weeks', validators=[validators.Optional(), validators.NumberRange(min=0, message="Should contain zero or more seconds")])
     days = IntegerField('Days', validators=[validators.Optional(), validators.NumberRange(min=0, message="Should contain zero or more seconds")])
@@ -232,9 +258,10 @@ class validateURL(object):
 
     def __call__(self, form, field):
         import validators
-
+        # If hosts that only contain alphanumerics are allowed ("localhost" for example)
+        allow_simplehost = not strtobool(os.getenv('BLOCK_SIMPLEHOSTS', 'False'))
         try:
-            validators.url(field.data.strip())
+            validators.url(field.data.strip(), simple_host=allow_simplehost)
         except validators.ValidationFailure:
             message = field.gettext('\'%s\' is not a valid URL.' % (field.data.strip()))
             raise ValidationError(message)
@@ -347,7 +374,7 @@ class quickWatchForm(Form):
     from . import processors
 
     url = fields.URLField('URL', validators=[validateURL()])
-    tag = StringField('Group tag', [validators.Optional()])
+    tags = StringTagUUID('Group tag', [validators.Optional()])
     watch_submit_button = SubmitField('Watch', render_kw={"class": "pure-button pure-button-primary"})
     processor = RadioField(u'Processor', choices=processors.available_processors(), default="text_json_diff")
     edit_and_watch_submit_button = SubmitField('Edit > Watch', render_kw={"class": "pure-button pure-button-primary"})
@@ -355,6 +382,7 @@ class quickWatchForm(Form):
 
 # Common to a single watch and the global settings
 class commonSettingsForm(Form):
+
     notification_urls = StringListField('Notification URL List', validators=[validators.Optional(), ValidateAppRiseServers()])
     notification_title = StringField('Notification Title', default='ChangeDetection.io Notification - {{ watch_url }}', validators=[validators.Optional(), ValidateJinja2Template()])
     notification_body = TextAreaField('Notification Body', default='{{ watch_url }} had a change.', validators=[validators.Optional(), ValidateJinja2Template()])
@@ -382,7 +410,7 @@ class SingleBrowserStep(Form):
 class watchForm(commonSettingsForm):
 
     url = fields.URLField('URL', validators=[validateURL()])
-    tag = StringField('Group tag', [validators.Optional()], default='')
+    tags = StringTagUUID('Group tag', [validators.Optional()], default='')
 
     time_between_check = FormField(TimeBetweenCheckForm)
 
@@ -481,6 +509,10 @@ class globalSettingsApplicationForm(commonSettingsForm):
     global_subtractive_selectors = StringListField('Remove elements', [ValidateCSSJSONXPATHInput(allow_xpath=False, allow_json=False)])
     ignore_whitespace = BooleanField('Ignore whitespace')
     password = SaltyPasswordField()
+    pager_size = IntegerField('Pager size',
+                              render_kw={"style": "width: 5em;"},
+                              validators=[validators.NumberRange(min=0,
+                                                                 message="Should be atleast zero (disabled)")])
     removepassword_button = SubmitField('Remove password', render_kw={"class": "pure-button pure-button-primary"})
     render_anchor_tag_content = BooleanField('Render anchor tag content', default=False)
     shared_diff_access = BooleanField('Allow access to view diff page when password is enabled', default=False, validators=[validators.Optional()])

changedetectionio/html_tools.py

@@ -137,12 +137,13 @@ def _get_stripped_text_from_json_match(match):
 def extract_json_as_string(content, json_filter, ensure_is_ldjson_info_type=None):
     stripped_text_from_html = False
 
-    # Try to parse/filter out the JSON, if we get some parser error, then maybe it's embedded <script type=ldjson>
+    # Try to parse/filter out the JSON, if we get some parser error, then maybe it's embedded within HTML tags
     try:
         stripped_text_from_html = _parse_json(json.loads(content), json_filter)
     except json.JSONDecodeError:
 
         # Foreach <script json></script> blob.. just return the first that matches json_filter
+        # As a last resort, try to parse the whole <body>
         s = []
         soup = BeautifulSoup(content, 'html.parser')
 
@@ -150,32 +151,34 @@ def extract_json_as_string(content, json_filter, ensure_is_ldjson_info_type=None
             bs_result = soup.findAll('script', {"type": "application/ld+json"})
         else:
             bs_result = soup.findAll('script')
+        bs_result += soup.findAll('body')
 
-
-        if not bs_result:
-            raise JSONNotFound("No parsable JSON found in this document")
-
+        bs_jsons = []
         for result in bs_result:
             # Skip empty tags, and things that dont even look like JSON
-            if not result.string or not '{' in result.string:
+            if not result.text or '{' not in result.text:
                 continue
-                
             try:
-                json_data = json.loads(result.string)
+                json_data = json.loads(result.text)
+                bs_jsons.append(json_data)
             except json.JSONDecodeError:
-                # Just skip it
+                # Skip objects which cannot be parsed
                 continue
-            else:
-                stripped_text_from_html = _parse_json(json_data, json_filter)
-                if ensure_is_ldjson_info_type:
-                    # Could sometimes be list, string or something else random
-                    if isinstance(json_data, dict):
-                        # If it has LD JSON 'key' @type, and @type is 'product', and something was found for the search
-                        # (Some sites have multiple of the same ld+json @type='product', but some have the review part, some have the 'price' part)
-                        if json_data.get('@type', False) and json_data.get('@type','').lower() == ensure_is_ldjson_info_type.lower() and stripped_text_from_html:
-                            break
-                elif stripped_text_from_html:
-                    break
+
+        if not bs_jsons:
+            raise JSONNotFound("No parsable JSON found in this document")
+        
+        for json_data in bs_jsons:
+            stripped_text_from_html = _parse_json(json_data, json_filter)
+            if ensure_is_ldjson_info_type:
+                # Could sometimes be list, string or something else random
+                if isinstance(json_data, dict):
+                    # If it has LD JSON 'key' @type, and @type is 'product', and something was found for the search
+                    # (Some sites have multiple of the same ld+json @type='product', but some have the review part, some have the 'price' part)
+                    if json_data.get('@type', False) and json_data.get('@type','').lower() == ensure_is_ldjson_info_type.lower() and stripped_text_from_html:
+                        break
+            elif stripped_text_from_html:
+                break
 
     if not stripped_text_from_html:
         # Re 265 - Just return an empty string when filter not found
@@ -188,42 +191,50 @@ def extract_json_as_string(content, json_filter, ensure_is_ldjson_info_type=None
 #
 # wordlist - list of regex's (str) or words (str)
 def strip_ignore_text(content, wordlist, mode="content"):
-    ignore = []
-    ignore_regex = []
-
-    # @todo check this runs case insensitive
-    for k in wordlist:
-
-        # Is it a regex?
-        if k[0] == '/':
-            ignore_regex.append(k.strip(" /"))
-        else:
-            ignore.append(k)
-
     i = 0
     output = []
+    ignore_text = []
+    ignore_regex = []
+
     ignored_line_numbers = []
+
+    for k in wordlist:
+        # Is it a regex?
+        x = re.search('^\/(.*)\/(.*)', k.strip())
+        if x:
+            # Starts with / but doesn't look like a regex
+            p = x.group(1)
+            try:
+                # @Todo python regex options can go before the regex str, but not really many of the options apply on a per-line basis
+                ignore_regex.append(re.compile(rf"{p}", re.IGNORECASE))
+            except Exception as e:
+                # Badly formed regex, treat as text
+                ignore_text.append(k.strip())
+        else:
+            # Had a / but doesn't work as regex
+            ignore_text.append(k.strip())
+
     for line in content.splitlines():
         i += 1
         # Always ignore blank lines in this mode. (when this function gets called)
+        got_match = False
         if len(line.strip()):
-            regex_matches = False
+            for l in ignore_text:
+                if l.lower() in line.lower():
+                    got_match = True
 
-            # if any of these match, skip
-            for regex in ignore_regex:
-                try:
-                    if re.search(regex, line, re.IGNORECASE):
-                        regex_matches = True
-                except Exception as e:
-                    continue
+            if not got_match:
+                for r in ignore_regex:
+                    if r.search(line):
+                        got_match = True
 
-            if not regex_matches and not any(skip_text.lower() in line.lower() for skip_text in ignore):
+            if not got_match:
+                # Not ignored
                 output.append(line.encode('utf8'))
             else:
                 ignored_line_numbers.append(i)
 
 
-
     # Used for finding out what to highlight
     if mode == "line numbers":
         return ignored_line_numbers

changedetectionio/importer.py

@@ -52,7 +52,8 @@ class import_url_list(Importer):
 
             # Flask wtform validators wont work with basic auth, use validators package
             # Up to 5000 per batch so we dont flood the server
-            if len(url) and validators.url(url.replace('source:', '')) and good < 5000:
+            # @todo validators.url failed on local hostnames (such as referring to ourself when using browserless)
+            if len(url) and 'http' in url.lower() and good < 5000:
                 extras = None
                 if processor:
                     extras = {'processor': processor}
@@ -84,7 +85,8 @@ class import_distill_io_json(Importer):
         now = time.time()
         self.new_uuids=[]
 
-
+        # @todo Use JSONSchema like in the API to validate here.
+        
         try:
             data = json.loads(data.strip())
         except json.decoder.JSONDecodeError:
@@ -119,11 +121,8 @@ class import_distill_io_json(Importer):
                 except IndexError:
                     pass
 
-
-                if d.get('tags', False):
-                    extras['tag'] = ", ".join(d['tags'])
-
                 new_uuid = datastore.add_watch(url=d['uri'].strip(),
+                                               tag=",".join(d.get('tags', [])),
                                                extras=extras,
                                                write_to_disk_now=False)
 

changedetectionio/model/App.py

@@ -23,25 +23,27 @@ class model(dict):
                     'workers': int(getenv("DEFAULT_SETTINGS_REQUESTS_WORKERS", "10")),  # Number of threads, lower is better for slow connections
                 },
                 'application': {
+                    # Custom notification content
                     'api_access_token_enabled': True,
-                    'password': False,
                     'base_url' : None,
-                    'extract_title_as_title': False,
                     'empty_pages_are_a_change': False,
+                    'extract_title_as_title': False,
                     'fetch_backend': getenv("DEFAULT_FETCH_BACKEND", "html_requests"),
                     'filter_failure_notification_threshold_attempts': _FILTER_FAILURE_THRESHOLD_ATTEMPTS_DEFAULT,
                     'global_ignore_text': [], # List of text to ignore when calculating the comparison checksum
                     'global_subtractive_selectors': [],
                     'ignore_whitespace': True,
-                    'render_anchor_tag_content': False,
-                    'notification_urls': [], # Apprise URL list
-                    # Custom notification content
-                    'notification_title': default_notification_title,
                     'notification_body': default_notification_body,
                     'notification_format': default_notification_format,
+                    'notification_title': default_notification_title,
+                    'notification_urls': [], # Apprise URL list
+                    'pager_size': 50,
+                    'password': False,
+                    'render_anchor_tag_content': False,
                     'schema_version' : 0,
                     'shared_diff_access': False,
-                    'webdriver_delay': None  # Extra delay in seconds before extracting text
+                    'webdriver_delay': None , # Extra delay in seconds before extracting text
+                    'tags': {} #@todo use Tag.model initialisers
                 }
             }
         }
@@ -49,3 +51,15 @@ class model(dict):
     def __init__(self, *arg, **kw):
         super(model, self).__init__(*arg, **kw)
         self.update(self.base_config)
+
+
+def parse_headers_from_text_file(filepath):
+    headers = {}
+    with open(filepath, 'r') as f:
+        for l in f.readlines():
+            l = l.strip()
+            if not l.startswith('#') and ':' in l:
+                (k, v) = l.split(':')
+                headers[k.strip()] = v.strip()
+
+    return headers

changedetectionio/model/Tag.py

@@ -0,0 +1,19 @@
+from .Watch import base_config
+import uuid
+
+class model(dict):
+
+    def __init__(self, *arg, **kw):
+
+        self.update(base_config)
+
+        self['uuid'] = str(uuid.uuid4())
+
+        if kw.get('default'):
+            self.update(kw['default'])
+            del kw['default']
+
+
+        # Goes at the end so we update the default object with the initialiser
+        super(model, self).__init__(*arg, **kw)
+

changedetectionio/model/Watch.py

@@ -52,7 +52,8 @@ base_config = {
     'previous_md5_before_filters': False,  # Used for skipping changedetection entirely
     'proxy': None,  # Preferred proxy connection
     'subtractive_selectors': [],
-    'tag': None,
+    'tag': '', # Old system of text name for a tag, to be removed
+    'tags': [], # list of UUIDs to App.Tags
     'text_should_not_be_present': [],  # Text that should not present
     # Re #110, so then if this is set to None, we know to use the default value instead
     # Requires setting to None on submit if it's the same as the default
@@ -455,10 +456,6 @@ class model(dict):
 
         return csv_output_filename
 
-    @property
-    # Return list of tags, stripped and lowercase, used for searching
-    def all_tags(self):
-        return [s.strip().lower() for s in self.get('tag','').split(',')]
 
     def has_special_diff_filter_options_set(self):
 

changedetectionio/notification.py

@@ -92,6 +92,12 @@ def process_notification(n_object, datastore):
         n_object.get('notification_format', default_notification_format),
         valid_notification_formats[default_notification_format],
     )
+
+    # If we arrived with 'System default' then look it up
+    if n_format == default_notification_format_for_watch and datastore.data['settings']['application'].get('notification_format') != default_notification_format_for_watch:
+        # Initially text or whatever
+        n_format = datastore.data['settings']['application'].get('notification_format', valid_notification_formats[default_notification_format])
+
     
     # https://github.com/caronc/apprise/wiki/Development_LogCapture
     # Anything higher than or equal to WARNING (which covers things like Connection errors)
@@ -145,9 +151,12 @@ def process_notification(n_object, datastore):
                     # Apprise will default to HTML, so we need to override it
                     # So that whats' generated in n_body is in line with what is going to be sent.
                     # https://github.com/caronc/apprise/issues/633#issuecomment-1191449321
-                    if not 'format=' in url and (n_format == 'text' or n_format == 'markdown'):
+                    if not 'format=' in url and (n_format == 'Text' or n_format == 'Markdown'):
                         prefix = '?' if not '?' in url else '&'
+                        # Apprise format is lowercase text https://github.com/caronc/apprise/issues/633
+                        n_format = n_format.tolower()
                         url = "{}{}format={}".format(url, prefix, n_format)
+                    # If n_format == HTML, then apprise email should default to text/html and we should be sending HTML only
 
                 apobj.add(url)
 
@@ -186,8 +195,13 @@ def create_notification_parameters(n_object, datastore):
     uuid = n_object['uuid'] if 'uuid' in n_object else ''
 
     if uuid != '':
-        watch_title = datastore.data['watching'][uuid]['title']
-        watch_tag = datastore.data['watching'][uuid]['tag']
+        watch_title = datastore.data['watching'][uuid].get('title', '')
+        tag_list = []
+        tags = datastore.get_all_tags_for_watch(uuid)
+        if tags:
+            for tag_uuid, tag in tags.items():
+                tag_list.append(tag.get('title'))
+        watch_tag = ', '.join(tag_list)
     else:
         watch_title = 'Change Detection'
         watch_tag = ''

changedetectionio/processors/__init__.py

@@ -9,7 +9,7 @@ class difference_detection_processor():
         super().__init__(*args, **kwargs)
 
     @abstractmethod
-    def run(self, uuid, skip_when_checksum_same=True):
+    def run(self, uuid, skip_when_checksum_same=True, preferred_proxy=None):
         update_obj = {'last_notification_error': False, 'last_error': False}
         some_data = 'xxxxx'
         update_obj["previous_md5"] = hashlib.md5(some_data.encode('utf-8')).hexdigest()

changedetectionio/processors/restock_diff.py

@@ -12,6 +12,12 @@ urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 name = 'Re-stock detection for single product pages'
 description = 'Detects if the product goes back to in-stock'
 
+class UnableToExtractRestockData(Exception):
+    def __init__(self, status_code):
+        # Set this so we can use it in other parts of the app
+        self.status_code = status_code
+        return
+
 class perform_site_check(difference_detection_processor):
     screenshot = None
     xpath_data = None
@@ -36,11 +42,10 @@ class perform_site_check(difference_detection_processor):
 
         # Unset any existing notification error
         update_obj = {'last_notification_error': False, 'last_error': False}
-        extra_headers = watch.get('headers', [])
 
-        # Tweak the base config with the per-watch ones
-        request_headers = deepcopy(self.datastore.data['settings']['headers'])
-        request_headers.update(extra_headers)
+        request_headers = watch.get('headers', [])
+        request_headers.update(self.datastore.get_all_base_headers())
+        request_headers.update(self.datastore.get_all_headers_in_textfile_for_watch(uuid=uuid))
 
         # https://github.com/psf/requests/issues/4525
         # Requests doesnt yet support brotli encoding, so don't put 'br' here, be totally sure that the user cannot
@@ -105,7 +110,8 @@ class perform_site_check(difference_detection_processor):
             fetched_md5 = hashlib.md5(fetcher.instock_data.encode('utf-8')).hexdigest()
             # 'Possibly in stock' comes from stock-not-in-stock.js when no string found above the fold.
             update_obj["in_stock"] = True if fetcher.instock_data == 'Possibly in stock' else False
-
+        else:
+            raise UnableToExtractRestockData(status_code=fetcher.status_code)
 
         # The main thing that all this at the moment comes down to :)
         changed_detected = False

changedetectionio/processors/text_json_diff.py

@@ -50,14 +50,13 @@ class perform_site_check(difference_detection_processor):
 
         return regex
 
-    def run(self, uuid, skip_when_checksum_same=True):
+    def run(self, uuid, skip_when_checksum_same=True, preferred_proxy=None):
         changed_detected = False
         screenshot = False  # as bytes
         stripped_text_from_html = ""
 
         # DeepCopy so we can be sure we don't accidently change anything by reference
         watch = deepcopy(self.datastore.data['watching'].get(uuid))
-
         if not watch:
             raise Exception("Watch no longer exists.")
 
@@ -70,11 +69,10 @@ class perform_site_check(difference_detection_processor):
         # Unset any existing notification error
         update_obj = {'last_notification_error': False, 'last_error': False}
 
-        extra_headers = watch.get('headers', [])
-
         # Tweak the base config with the per-watch ones
-        request_headers = deepcopy(self.datastore.data['settings']['headers'])
-        request_headers.update(extra_headers)
+        request_headers = watch.get('headers', [])
+        request_headers.update(self.datastore.get_all_base_headers())
+        request_headers.update(self.datastore.get_all_headers_in_textfile_for_watch(uuid=uuid))
 
         # https://github.com/psf/requests/issues/4525
         # Requests doesnt yet support brotli encoding, so don't put 'br' here, be totally sure that the user cannot
@@ -107,7 +105,11 @@ class perform_site_check(difference_detection_processor):
             # If the klass doesnt exist, just use a default
             klass = getattr(content_fetcher, "html_requests")
 
-        proxy_id = self.datastore.get_preferred_proxy_for_watch(uuid=uuid)
+        if preferred_proxy:
+            proxy_id = preferred_proxy
+        else:
+            proxy_id = self.datastore.get_preferred_proxy_for_watch(uuid=uuid)
+
         proxy_url = None
         if proxy_id:
             proxy_url = self.datastore.proxy_list.get(proxy_id).get('url')
@@ -140,7 +142,7 @@ class perform_site_check(difference_detection_processor):
         self.xpath_data = fetcher.xpath_data
 
         # Track the content type
-        update_obj['content_type'] = fetcher.headers.get('Content-Type', '')
+        update_obj['content_type'] = fetcher.get_all_headers().get('content-type', '').lower()
 
         # Watches added automatically in the queue manager will skip if its the same checksum as the previous run
         # Saves a lot of CPU
@@ -160,7 +162,7 @@ class perform_site_check(difference_detection_processor):
         # https://stackoverflow.com/questions/41817578/basic-method-chaining ?
         # return content().textfilter().jsonextract().checksumcompare() ?
 
-        is_json = 'application/json' in fetcher.headers.get('Content-Type', '')
+        is_json = 'application/json' in fetcher.get_all_headers().get('content-type', '').lower()
         is_html = not is_json
 
         # source: support, basically treat it as plaintext
@@ -168,7 +170,7 @@ class perform_site_check(difference_detection_processor):
             is_html = False
             is_json = False
 
-        if watch.is_pdf or 'application/pdf' in fetcher.headers.get('Content-Type', '').lower():
+        if watch.is_pdf or 'application/pdf' in fetcher.get_all_headers().get('content-type', '').lower():
             from shutil import which
             tool = os.getenv("PDF_TO_HTML_TOOL", "pdftohtml")
             if not which(tool):
@@ -192,21 +194,23 @@ class perform_site_check(difference_detection_processor):
 
             fetcher.content = fetcher.content.replace('</body>', metadata + '</body>')
 
+        # Better would be if Watch.model could access the global data also
+        # and then use getattr https://docs.python.org/3/reference/datamodel.html#object.__getitem__
+        # https://realpython.com/inherit-python-dict/ instead of doing it procedurely
+        include_filters_from_tags = self.datastore.get_tag_overrides_for_watch(uuid=uuid, attr='include_filters')
+        include_filters_rule = [*watch.get('include_filters', []), *include_filters_from_tags]
 
-        include_filters_rule = deepcopy(watch.get('include_filters', []))
-        # include_filters_rule = watch['include_filters']
-        subtractive_selectors = watch.get(
-            "subtractive_selectors", []
-        ) + self.datastore.data["settings"]["application"].get(
-            "global_subtractive_selectors", []
-        )
+        subtractive_selectors = [*self.datastore.get_tag_overrides_for_watch(uuid=uuid, attr='subtractive_selectors'),
+                                 *watch.get("subtractive_selectors", []),
+                                 *self.datastore.data["settings"]["application"].get("global_subtractive_selectors", [])
+                                 ]
 
         # Inject a virtual LD+JSON price tracker rule
         if watch.get('track_ldjson_price_data', '') == PRICE_DATA_TRACK_ACCEPT:
             include_filters_rule.append(html_tools.LD_JSON_PRODUCT_OFFER_SELECTOR)
 
-        has_filter_rule = include_filters_rule and len("".join(include_filters_rule).strip())
-        has_subtractive_selectors = subtractive_selectors and len(subtractive_selectors[0].strip())
+        has_filter_rule = len(include_filters_rule) and len(include_filters_rule[0].strip())
+        has_subtractive_selectors = len(subtractive_selectors) and len(subtractive_selectors[0].strip())
 
         if is_json and not has_filter_rule:
             include_filters_rule.append("json:$")
@@ -236,7 +240,7 @@ class perform_site_check(difference_detection_processor):
             html_content = fetcher.content
 
             # If not JSON,  and if it's not text/plain..
-            if 'text/plain' in fetcher.headers.get('Content-Type', '').lower():
+            if 'text/plain' in fetcher.get_all_headers().get('content-type', '').lower():
                 # Don't run get_text or xpath/css filters on plaintext
                 stripped_text_from_html = html_content
             else:

changedetectionio/res/stock-not-in-stock.js

@@ -10,6 +10,7 @@ function isItemInStock() {
     'brak na stanie',
     'brak w magazynie',
     'coming soon',
+    'currently have any tickets for this',
     'currently unavailable',
     'en rupture de stock',
     'item is no longer available',
@@ -20,7 +21,9 @@ function isItemInStock() {
     'nicht zur verfügung',
     'no disponible temporalmente',
     'no longer in stock',
+    'no tickets available',
     'not available',
+    'not currently available',
     'not in stock',
     'notify me when available',
     'não estamos a aceitar encomendas',
@@ -30,6 +33,8 @@ function isItemInStock() {
     'sold out',
     'temporarily out of stock',
     'temporarily unavailable',
+    'tickets unavailable',
+    'unavailable tickets',
     'we do not currently have an estimate of when this product will be back in stock.',
     'zur zeit nicht an lager',
   ];

changedetectionio/static/images/brightdata.svg

@@ -0,0 +1,58 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<svg
+   height="59.553207"
+   viewBox="-0.36 95.21 25.082135 59.553208"
+   width="249.99138"
+   version="1.1"
+   id="svg12"
+   sodipodi:docname="brightdata.svg"
+   inkscape:version="1.1.2 (0a00cf5339, 2022-02-04)"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:svg="http://www.w3.org/2000/svg">
+  <defs
+     id="defs16" />
+  <sodipodi:namedview
+     id="namedview14"
+     pagecolor="#ffffff"
+     bordercolor="#666666"
+     borderopacity="1.0"
+     inkscape:pageshadow="2"
+     inkscape:pageopacity="0.0"
+     inkscape:pagecheckerboard="0"
+     showgrid="false"
+     fit-margin-top="0"
+     fit-margin-left="0"
+     fit-margin-right="0"
+     fit-margin-bottom="0"
+     inkscape:zoom="0.9464"
+     inkscape:cx="22.189349"
+     inkscape:cy="-90.870668"
+     inkscape:window-width="1920"
+     inkscape:window-height="1051"
+     inkscape:window-x="1920"
+     inkscape:window-y="0"
+     inkscape:window-maximized="1"
+     inkscape:current-layer="svg12" />
+  <path
+     d="m -34.416031,129.28 c -3.97,-2.43 -5.1,-6.09 -4.32,-10.35 0.81,-4.4 3.95,-6.75 8.04,-7.75 4.23,-1.04 8.44,-0.86 12.3,1.5 0.63,0.39 0.93,0.03 1.31,-0.29 1.5,-1.26 3.27,-1.72 5.189999,-1.83 0.79,-0.05 1.04,0.24 1.01,1.01 -0.05,1.31 -0.04,2.63 0,3.95 0.02,0.65 -0.19,0.93 -0.87,0.89 -0.889999,-0.04 -1.789999,0.03 -2.669999,-0.02 -0.82,-0.04 -1.08,0.1 -0.88,1.04 0.83,3.9 -0.06,7.37 -3.1,10.06 -2.76,2.44 -6.13,3.15 -9.72,3.04 -0.51,-0.02 -1.03,-0.02 -1.52,-0.13 -1.22,-0.25 -1.96,0.14 -2.19,1.41 -0.28,1.54 0.16,2.62 1.37,3.07 0.84,0.31 1.74,0.35 2.63,0.39 2.97,0.13 5.95,-0.18 8.91,0.21 2.93,0.39 5.69,1.16 6.85,4.25 1.269999,3.38 0.809999,6.62 -1.48,9.47 -2.73,3.39 -6.52,4.78 -10.66,5.33 -3.53,0.48 -7.04,0.27 -10.39,-1.11 -3.89,-1.6 -5.75,-4.95 -4.84,-8.72 0.51,-2.11 1.85,-3.58 3.69,-4.65 0.38,-0.22 0.93,-0.32 0.28,-0.96 -2.91,-2.83 -2.85,-6.16 0.1,-8.95 0.28,-0.26 0.6,-0.53 0.96,-0.86 z m 8.07,21.5 c 0.95,0.04 1.87,-0.13 2.78,-0.33 1.89,-0.42 3.51,-1.3 4.49,-3.06 1.82,-3.25 0.24,-6.2 -3.37,-6.58 -2.88,-0.3 -5.76,0.24 -8.63,-0.13 -0.53,-0.07 -0.75,0.34 -0.95,0.71 -1.16,2.24 -1.08,4.53 0,6.73 1.15,2.34 3.46,2.48 5.68,2.66 z m -5,-30.61 c -0.03,1.67 0.08,3.19 0.74,4.61 0.76,1.62 2.17,2.42 4.03,2.31 1.62,-0.1 2.9,-1.12 3.36,-2.84 0.66,-2.46 0.69,-4.95 0.01,-7.42 -0.49,-1.76 -1.7,-2.64 -3.56,-2.7 -2.08,-0.07 -3.37,0.7 -4.04,2.42 -0.47,1.21 -0.6,2.47 -0.54,3.62 z m 32.9399993,6.56 c 0,2.59 0.05,5.18 -0.02,7.77 -0.03,1.03 0.31,1.46 1.32,1.52 0.65,0.04 1.61,-0.09 1.82,0.57 0.26,0.81 0.11,1.76 0.06,2.65 -0.03,0.48 -0.81,0.39 -0.81,0.39 l -11.47,0.01 c 0,0 -0.95,-0.21 -0.88,-0.88 0.03,-0.29 0.04,-0.6 0,-0.89 -0.19,-1.24 0.21,-1.92 1.58,-1.9 0.99,0.01 1.28,-0.52 1.28,-1.53 -0.05,-8.75 -0.05,-17.49 0,-26.24 0.01,-1.15 -0.36,-1.62 -1.44,-1.67 -0.17,-0.01 -0.34,-0.04 -0.5,-0.07 -1.43,-0.22 -2.12,-1.57 -1.53,-2.91 0.15,-0.35 0.43,-0.36 0.72,-0.4 2.94,-0.41 5.88,-0.81 8.82000002,-1.23 0.81999998,-0.12 0.99999998,0.27 0.98999998,1.01 -0.02,3.35 0,6.71 0.02,10.06 0,0.35 -0.23,0.84 0.18,1.03 0.38,0.17 0.69,-0.25 0.99,-0.45 2.56,-1.74 5.33,-2.73 8.4900007,-2.56 3.51005,0.19 5.65005,1.95 6.35005,5.46 0.42,2.09 0.52,4.21 0.51,6.33 -0.02,3.86 0.05,7.73 -0.04,11.59 -0.02,1.12 0.37,1.5 1.39,1.6 0.61,0.05 1.55,-0.13 1.74,0.47 0.26,0.85 0.12,1.84 0.1,2.77 -0.01,0.41 -0.69,0.37 -0.69,0.37 l -11.4700504,0.01 c 0,0 -0.81,-0.29 -0.8,-0.85 0.01,-0.38 0.04,-0.77 -0.01,-1.15 -0.13,-1.01 0.32,-1.52 1.31,-1.56 1.0600004,-0.05 1.3800004,-0.55 1.3500004,-1.63 -0.14,-4.84 0.16,-9.68 -0.18,-14.51 -0.26,-3.66 -2.1100004,-4.95 -5.6700007,-3.99 -0.25,0.07 -0.49,0.15 -0.73,0.22 -2.57,0.8 -2.79,1.09 -2.79,3.71 0.01,2.3 0.01,4.59 0.01,6.88 z M -109.26603,122.56 c 0,-4.75 -0.02,-9.51 0.02,-14.26 0.01,-0.92 -0.17,-1.47 -1.19,-1.45 -0.16,0 -0.33,-0.07 -0.5,-0.1 -1.56,-0.27 -2.24,-1.47 -1.69,-2.92 0.14,-0.37 0.41,-0.38 0.7,-0.42 2.98,-0.41 5.97,-0.81 8.94,-1.24 0.85,-0.12 0.88,0.33 0.88,0.96 -0.01,3.01 -0.01,6.03 0,9.04 0,0.4 -0.18,0.96 0.27,1.16 0.36,0.16 0.66,-0.3 0.96,-0.52 4.729999,-3.51 12.459999,-2.61 14.889999,4.48 1.89,5.51 1.91,11.06 -0.96,16.28 -2.37,4.31 -6.19,6.49 -11.15,6.59 -3.379999,0.07 -6.679999,-0.3 -9.909999,-1.37 -0.93,-0.31 -1.3,-0.78 -1.28,-1.83 0.05,-4.81 0.02,-9.6 0.02,-14.4 z m 7.15,3.89 c 0,2.76 0.02,5.52 -0.01,8.28 -0.01,0.76 0.18,1.29 0.91,1.64 1.899999,0.9 4.299999,0.5 5.759999,-1.01 0.97,-1 1.56,-2.21 1.96,-3.52 1.03,-3.36 0.97,-6.78 0.61,-10.22 a 9.991,9.991 0 0 0 -0.93,-3.29 c -1.47,-3.06 -4.67,-3.85 -7.439999,-1.86 -0.6,0.43 -0.88,0.93 -0.87,1.7 0.04,2.76 0.01,5.52 0.01,8.28 z"
+     fill="#4280f6"
+     id="path2" />
+  <path
+     d="m 68.644019,137.2 c -1.62,1.46 -3.41,2.56 -5.62,2.96 -4.4,0.8 -8.7,-1.39 -10.49,-5.49 -2.31,-5.31 -2.3,-10.67 -0.1,-15.98 2.31,-5.58 8.29,-8.65 14.24,-7.46 1.71,0.34 1.9,0.18 1.9,-1.55 0,-0.68 -0.05,-1.36 0.01,-2.04 0.09,-1.02 -0.25,-1.54 -1.34,-1.43 -0.64,0.06 -1.26,-0.1 -1.88,-0.21 -1.32,-0.24 -1.6,-0.62 -1.37,-1.97 0.07,-0.41 0.25,-0.57 0.65,-0.62 2.63,-0.33 5.27,-0.66 7.9,-1.02 1.04,-0.14 1.17,0.37 1.17,1.25 -0.02,10.23 -0.02,20.45 -0.01,30.68 v 1.02 c 0.02,0.99 0.35,1.6 1.52,1.47 0.52,-0.06 1.35,-0.27 1.25,0.73 -0.08,0.8 0.58,1.93 -0.94,2.18 -1.29,0.22 -2.51,0.69 -3.86,0.65 -2.04,-0.06 -2.3,-0.23 -2.76,-2.19 -0.09,-0.3 0.06,-0.67 -0.27,-0.98 z m -0.07,-12.46 c 0,-2.8 -0.04,-5.6 0.02,-8.39 0.02,-0.9 -0.28,-1.47 -1.05,-1.81 -3.18,-1.4 -7.54,-0.8 -9.3,2.87 -0.83,1.74 -1.31,3.54 -1.49,5.46 -0.28,2.93 -0.38,5.83 0.61,8.65 0.73,2.09 1.81,3.9 4.11,4.67 2.49,0.83 4.55,-0.04 6.5,-1.48 0.54,-0.4 0.62,-0.95 0.61,-1.57 -0.02,-2.8 -0.01,-5.6 -0.01,-8.4 z m 28.79,2.53 c 0,3.24 0.04,5.83 -0.02,8.41 -0.02,1 0.19,1.49 1.309998,1.41 0.55,-0.04 1.460003,-0.46 1.520003,0.73 0.05,1.02 0.1,1.89 -1.330003,2.08 -1.289998,0.17 -2.559998,0.51 -3.889998,0.48 -1.88,-0.05 -2.15,-0.26 -2.42,-2.15 -0.04,-0.27 0.14,-0.65 -0.22,-0.79 -0.34,-0.13 -0.5,0.24 -0.72,0.42 -3.61,3 -8.15,3.4 -11.64,1.08 -1.61,-1.07 -2.49,-2.63 -2.67,-4.43 -0.51,-5.13 0.77,-7.91 6.3,-10.22 2.44,-1.02 5.07,-1.27 7.68,-1.49 0.77,-0.07 1.03,-0.28 1.02,-1.05 -0.03,-1.48 -0.05,-2.94 -0.64,-4.36 -0.59,-1.42 -1.67,-1.92 -3.08,-2.03 -3.04,-0.24 -5.88,0.5 -8.63,1.71 -0.51,0.23 -1.19,0.75 -1.48,-0.13 -0.26,-0.77 -1.35,-1.61 0.05,-2.47 3.27,-2 6.7,-3.44 10.61,-3.42 1.44,0.01 2.88,0.27 4.21,0.81 2.67,1.08 3.44,3.4 3.8,5.99 0.46,3.37 0.1,6.73 0.24,9.42 z m -5.09,2.9 c 0,-1.23 -0.01,-2.46 0,-3.69 0,-0.52 -0.06,-0.98 -0.75,-0.84 -1.45,0.3 -2.93,0.28 -4.37,0.69 -3.71,1.04 -5.46,4.48 -3.97,8.03 0.51,1.22 1.48,1.98 2.79,2.16 2.01,0.28 3.86,-0.29 5.6,-1.28 0.54,-0.31 0.73,-0.76 0.72,-1.37 -0.05,-1.23 -0.02,-2.47 -0.02,-3.7 z m 43.060001,-2.89 c 0,2.72 0.01,5.43 -0.01,8.15 0,0.66 0.02,1.21 0.91,1.12 0.54,-0.06 0.99,0.12 0.86,0.75 -0.15,0.71 0.56,1.7 -0.58,2.09 -1.55,0.52 -3.16,0.59 -4.77,0.4 -0.99,-0.12 -1.12,-1.01 -1.18,-1.73 -0.08,-1.15 -0.16,-1.45 -1.24,-0.54 -3.41,2.87 -8.05,3.17 -11.43,0.88 -1.75,-1.18 -2.49,-2.91 -2.7,-4.94 -0.64,-6.24 3.16,-8.74 7.83,-10.17 2.04,-0.62 4.14,-0.8 6.24,-0.99 0.81,-0.07 1,-0.36 0.98,-1.09 -0.04,-1.31 0.04,-2.62 -0.42,-3.89 -0.57,-1.57 -1.53,-2.34 -3.18,-2.45 -3.03,-0.21 -5.88,0.46 -8.64,1.66 -0.6,0.26 -1.25,0.81 -1.68,-0.2 -0.34,-0.8 -1.08,-1.61 0.16,-2.36 4.12,-2.5 8.44,-4.16 13.36,-3.07 3.21,0.71 4.89,2.91 5.26,6.34 0.18,1.69 0.22,3.37 0.22,5.07 0.01,1.66 0.01,3.32 0.01,4.97 z m -5.09,2.54 c 0,-1.27 -0.03,-2.54 0.01,-3.81 0.02,-0.74 -0.27,-1.02 -0.98,-0.92 -1.21,0.17 -2.43,0.28 -3.62,0.55 -3.72,0.83 -5.47,3.48 -4.82,7.21 0.29,1.66 1.57,2.94 3.21,3.16 2.02,0.27 3.85,-0.34 5.57,-1.34 0.49,-0.29 0.64,-0.73 0.63,-1.29 -0.02,-1.18 0,-2.37 0,-3.56 z"
+     fill="#c8dbfb"
+     id="path4" />
+  <path
+     d="m 26.314019,125.77 c 0,-2.89 -0.05,-5.77 0.02,-8.66 0.03,-1.04 -0.33,-1.39 -1.31,-1.24 a 0.7,0.7 0 0 1 -0.25,0 c -0.57,-0.18 -1.44,0.48 -1.68,-0.58 -0.35,-1.48 -0.02,-2.3 1.21,-2.7 1.3,-0.43 2.16,-1.26 2.76,-2.46 0.78,-1.56 1.44,-3.17 1.91,-4.84 0.18,-0.63 0.47,-0.86 1.15,-0.88 3.28,-0.09 3.27,-0.11 3.32,3.17 0.01,1.06 0.09,2.12 0.09,3.18 -0.01,0.67 0.27,0.89 0.91,0.88 1.61,-0.02 3.23,0.03 4.84,-0.02 0.77,-0.02 1.01,0.23 1.03,1.01 0.08,3.27 0.1,3.27 -3.09,3.27 -0.93,0 -1.87,0.03 -2.8,-0.01 -0.67,-0.02 -0.89,0.26 -0.88,0.91 0.04,5.43 0.04,10.86 0.12,16.29 0.02,1.7 0.75,2.26 2.46,2.1 1.1,-0.1 2.19,-0.26 3.23,-0.65 0.59,-0.22 0.89,-0.09 1.14,0.53 0.93,2.29 0.92,2.37 -1.32,3.52 -2.54,1.3 -5.22,1.99 -8.1,1.79 -2.27,-0.16 -3.68,-1.27 -4.35,-3.45 -0.3,-0.98 -0.41,-1.99 -0.41,-3.01 z m -97.67005,-8.99 c 0.57,-0.84 1.11,-1.74 1.76,-2.55 1.68,-2.09 3.68,-3.62 6.54,-3.66 1.08,-0.01 1.63,0.28 1.57,1.52 -0.1,2.08 -0.05,4.16 -0.02,6.24 0.01,0.74 -0.17,0.96 -0.96,0.76 -2.36,-0.59 -4.71,-0.42 -7.03,0.28 -0.8,0.24 -1.16,0.62 -1.15,1.52 0.05,4.5 0.04,9 0,13.5 -0.01,0.89 0.29,1.16 1.15,1.2 1.23,0.06 2.44,0.32 3.67,0.39 0.75,0.05 0.91,0.38 0.89,1.04 -0.06,2.86 0.29,2.28 -2.25,2.3 -4.2,0.04 -8.41,-0.02 -12.61,0.03 -0.91,0.01 -1.39,-0.18 -1.22,-1.18 0.02,-0.12 0,-0.25 0,-0.38 0.02,-2.1 -0.24,-1.88 1.77,-2.04 1.33,-0.11 1.6,-0.67 1.58,-1.9 -0.07,-5.35 -0.04,-10.7 -0.02,-16.05 0,-0.78 -0.17,-1.2 -1,-1.46 -2.21,-0.68 -2.7,-1.69 -2.22,-3.99 0.11,-0.52 0.45,-0.56 0.82,-0.62 2.22,-0.34 4.44,-0.7 6.67,-0.99 0.99,-0.13 1.82,0.7 1.84,1.76 0.03,1.4 0.03,2.8 0.04,4.2 -0.01,0.02 0.06,0.04 0.18,0.08 z m 25.24,6.59 c 0,3.69 0.04,7.38 -0.03,11.07 -0.02,1.04 0.31,1.48 1.32,1.49 0.29,0 0.59,0.12 0.88,0.13 0.93,0.01 1.18,0.47 1.16,1.37 -0.05,2.19 0,2.19 -2.24,2.19 -3.48,0 -6.96,-0.04 -10.44,0.03 -1.09,0.02 -1.47,-0.33 -1.3,-1.36 0.02,-0.12 0.02,-0.26 0,-0.38 -0.28,-1.39 0.39,-1.96 1.7,-1.9 1.36,0.06 1.76,-0.51 1.74,-1.88 -0.09,-5.17 -0.08,-10.35 0,-15.53 0.02,-1.22 -0.32,-1.87 -1.52,-2.17 -0.57,-0.14 -1.47,-0.11 -1.57,-0.85 -0.15,-1.04 -0.05,-2.11 0.01,-3.17 0.02,-0.34 0.44,-0.35 0.73,-0.39 2.81,-0.39 5.63,-0.77 8.44,-1.18 0.92,-0.14 1.15,0.2 1.14,1.09 -0.04,3.8 -0.02,7.62 -0.02,11.44 z"
+     fill="#4280f6"
+     id="path6" />
+  <path
+     d="m 101.44402,125.64 c 0,-3.18 -0.03,-6.37 0.02,-9.55 0.02,-0.94 -0.26,-1.36 -1.22,-1.22 -0.21,0.03 -0.430003,0.04 -0.630003,0 -0.51,-0.12 -1.35,0.39 -1.44,-0.55 -0.08,-0.85 -0.429998,-1.87 0.93,-2.24 2.080003,-0.57 2.720003,-2.39 3.350003,-4.17 0.31,-0.88 0.62,-1.76 0.87,-2.66 0.18,-0.64 0.52,-0.85 1.19,-0.84 2.46,0.05 2,-0.15 2.04,2.04 0.02,1.1 0.08,2.21 -0.02,3.31 -0.11,1.16 0.46,1.52 1.46,1.53 1.78,0.01 3.57,0.04 5.35,-0.01 0.82,-0.02 1.12,0.23 1.11,1.08 -0.05,2.86 0.19,2.49 -2.42,2.51 -1.53,0.01 -3.06,0.02 -4.59,-0.01 -0.65,-0.01 -0.9,0.22 -0.9,0.89 0.02,5.52 0,11.04 0.03,16.56 0,0.67 0.14,1.34 0.25,2.01 0.17,1.04 1.17,1.62 2.59,1.42 1.29,-0.19 2.57,-0.49 3.86,-0.69 0.43,-0.07 1.05,-0.47 1.19,0.4 0.12,0.75 1.05,1.61 -0.09,2.24 -2.09,1.16 -4.28,2.07 -6.71,2.16 -1.05,0.04 -2.13,0.2 -3.16,-0.14 -1.92,-0.65 -3.03,-2.28 -3.05,-4.51 -0.02,-3.19 -0.01,-6.37 -0.01,-9.56 z"
+     fill="#c8dbfb"
+     id="path8" />
+  <path
+     d="m -50.816031,95.21 c 0.19,2.160002 1.85,3.240002 2.82,4.740002 0.25,0.379998 0.48,0.109998 0.67,-0.16 0.21,-0.31 0.6,-1.21 1.15,-1.28 -0.35,1.38 -0.04,3.149998 0.16,4.449998 0.49,3.05 -1.22,5.64 -4.07,6.18 -3.38,0.65 -6.22,-2.21 -5.6,-5.62 0.23,-1.24 1.37,-2.5 0.77,-3.699998 -0.85,-1.7 0.54,-0.52 0.79,-0.22 1.04,1.199998 1.21,0.09 1.45,-0.55 0.24,-0.63 0.31,-1.31 0.47,-1.97 0.19,-0.770002 0.55,-1.400002 1.39,-1.870002 z"
+     fill="#4280f6"
+     id="path10" />
+</svg>

changedetectionio/static/js/diff-overview.js

@@ -1,4 +1,13 @@
 $(document).ready(function () {
+    var csrftoken = $('input[name=csrf_token]').val();
+    $.ajaxSetup({
+        beforeSend: function (xhr, settings) {
+            if (!/^(GET|HEAD|OPTIONS|TRACE)$/i.test(settings.type) && !this.crossDomain) {
+                xhr.setRequestHeader("X-CSRFToken", csrftoken)
+            }
+        }
+    })
+
     // Load it when the #screenshot tab is in use, so we dont give a slow experience when waiting for the text diff to load
     window.addEventListener('hashchange', function (e) {
         toggle(location.hash);
@@ -15,11 +24,71 @@ $(document).ready(function () {
             $("#settings").hide();
         } else if (hash_name === '#extract') {
             $("#settings").hide();
-        }
-
-
-        else {
+        } else {
             $("#settings").show();
         }
     }
+
+    const article = $('.highlightable-filter')[0];
+
+    // We could also add the  'touchend' event for touch devices, but since
+    // most iOS/Android browsers already show a dialog when you select
+    // text (often with a Share option) we'll skip that
+    article.addEventListener('mouseup', dragTextHandler, false);
+    article.addEventListener('mousedown', clean, false);
+
+    function clean(event) {
+      $("#highlightSnippet").remove();
+    }
+
+
+    function dragTextHandler(event) {
+        console.log('mouseupped');
+
+        // Check if any text was selected
+        if (window.getSelection().toString().length > 0) {
+
+            // Find out how much (if any) user has scrolled
+            var scrollTop = (window.pageYOffset !== undefined) ? window.pageYOffset : (document.documentElement || document.body.parentNode || document.body).scrollTop;
+
+            // Get cursor position
+            const posX = event.clientX;
+            const posY = event.clientY + 20 + scrollTop;
+
+            // Append HTML to the body, create the "Tweet Selection" dialog
+            document.body.insertAdjacentHTML('beforeend', '<div id="highlightSnippet" style="position: absolute; top: ' + posY + 'px; left: ' + posX + 'px;"><div class="pure-form-message-inline" style="font-size: 70%">Ignore any change on any line which contains the selected text.</div><br><a data-mode="exact" href="javascript:void(0);" class="pure-button button-secondary button-xsmall">Ignore exact text</a>&nbsp;</div>');
+
+            if (/\d/.test(window.getSelection().toString())) {
+                // Offer regex replacement
+                document.getElementById("highlightSnippet").insertAdjacentHTML('beforeend', '<a data-mode="digit-regex"  href="javascript:void(0);" class="pure-button button-secondary button-xsmall">Ignore text including number changes</a>');
+            }
+
+            $('#highlightSnippet a').bind('click', function (e) {
+                if(!window.getSelection().toString().trim().length) {
+                    alert('Oops no text selected!');
+                    return;
+                }
+
+                $.ajax({
+                    type: "POST",
+                    url: highlight_submit_ignore_url,
+                    data: {'mode': $(this).data('mode'), 'selection': window.getSelection().toString()},
+                    statusCode: {
+                        400: function () {
+                            // More than likely the CSRF token was lost when the server restarted
+                            alert("There was a problem processing the request, please reload the page.");
+                        }
+                    }
+                }).done(function (data) {
+                    $("#highlightSnippet").html(data)
+                }).fail(function (data) {
+                    console.log(data);
+                    alert('There was an error communicating with the server.');
+                });
+            });
+
+        }
+    }
+
+
 });

changedetectionio/static/js/recheck-proxy.js

@@ -0,0 +1,87 @@
+$(function () {
+    /* add container before each proxy location to show status */
+
+    var option_li = $('.fetch-backend-proxy li').filter(function() {
+        return $("input",this)[0].value.length >0;
+    });
+
+    //var option_li = $('.fetch-backend-proxy li');
+    var isActive = false;
+    $(option_li).prepend('<div class="proxy-status"></div>');
+    $(option_li).append('<div class="proxy-timing"></div><div class="proxy-check-details"></div>');
+
+    function set_proxy_check_status(proxy_key, state) {
+        // select input by value name
+        const proxy_li = $('input[value="' + proxy_key + '" ]').parent();
+        if (state['status'] === 'RUNNING') {
+            $('.proxy-status', proxy_li).html('<span class="spinner"></span>');
+        }
+        if (state['status'] === 'OK') {
+            $('.proxy-status', proxy_li).html('<span style="color: green; font-weight: bold" >OK</span>');
+            $('.proxy-check-details', proxy_li).html(state['text']);
+        }
+        if (state['status'] === 'ERROR' || state['status'] === 'ERROR OTHER') {
+            $('.proxy-status', proxy_li).html('<span style="color: red; font-weight: bold" >X</span>');
+            $('.proxy-check-details', proxy_li).html(state['text']);
+        }
+        $('.proxy-timing', proxy_li).html(state['time']);
+    }
+
+
+    function pollServer() {
+        if (isActive) {
+            window.setTimeout(function () {
+                $.ajax({
+                    url: proxy_recheck_status_url,
+                    success: function (data) {
+                        var all_done = true;
+                        $.each(data, function (proxy_key, state) {
+                            set_proxy_check_status(proxy_key, state);
+                            if (state['status'] === 'RUNNING') {
+                                all_done = false;
+                            }
+                        });
+
+                        if (all_done) {
+                            console.log("Shutting down poller, all done.")
+                            isActive = false;
+                        } else {
+                            pollServer();
+                        }
+                    },
+                    error: function () {
+                        //ERROR HANDLING
+                        pollServer();
+                    }
+                });
+            }, 2000);
+        }
+    }
+
+    $('#check-all-proxies').click(function (e) {
+        e.preventDefault()
+        $('body').addClass('proxy-check-active');
+        $('.proxy-check-details').html('');
+        $('.proxy-status').html('<span class="spinner"></span>').fadeIn();
+        $('.proxy-timing').html('');
+
+        // Request start, needs CSRF?
+        $.ajax({
+            type: "GET",
+            url: recheck_proxy_start_url,
+        }).done(function (data) {
+            $.each(data, function (proxy_key, state) {
+                set_proxy_check_status(proxy_key, state['status'])
+            });
+            isActive = true;
+            pollServer();
+
+        }).fail(function (data) {
+            console.log(data);
+            alert('There was an error communicating with the server.');
+        });
+
+    });
+
+});
+

changedetectionio/static/js/toggle-theme.js

@@ -3,22 +3,45 @@
  * Toggles theme between light and dark mode.
  */
 $(document).ready(function () {
-  const button = document.getElementsByClassName("toggle-theme")[0];
+  const button = document.getElementById("toggle-light-mode");
 
   button.onclick = () => {
     const htmlElement = document.getElementsByTagName("html");
     const isDarkMode = htmlElement[0].dataset.darkmode === "true";
     htmlElement[0].dataset.darkmode = !isDarkMode;
-    if (isDarkMode) {
-      button.classList.remove("dark");
-      setCookieValue(false);
-    } else {
-      button.classList.add("dark");
-      setCookieValue(true);
-    }
+    setCookieValue(!isDarkMode);
   };
 
   const setCookieValue = (value) => {
     document.cookie = `css_dark_mode=${value};max-age=31536000;path=/`
   }
+
+  // Search input box behaviour
+    const toggle_search = document.getElementById("toggle-search");
+  const search_q = document.getElementById("search-q");
+  window.addEventListener('keydown', function (e) {
+
+    if (e.altKey == true && e.keyCode == 83)
+      search_q.classList.toggle('expanded');
+      search_q.focus();
+  });
+
+
+  search_q.onkeydown = (e) => {
+    var key = e.keyCode || e.which;
+    if (key === 13) {
+      document.searchForm.submit();
+    }
+  };
+  toggle_search.onclick = () => {
+    // Could be that they want to search something once text is in there
+    if (search_q.value.length) {
+      document.searchForm.submit();
+    } else {
+      // If not..
+      search_q.classList.toggle('expanded');
+      search_q.focus();
+    }
+  };
+
 });

changedetectionio/static/js/watch-overview.js

@@ -1,31 +1,45 @@
 $(function () {
-  // Remove unviewed status when normally clicked
-  $('.diff-link').click(function () {
-    $(this).closest('.unviewed').removeClass('unviewed');
-  });
+    // Remove unviewed status when normally clicked
+    $('.diff-link').click(function () {
+        $(this).closest('.unviewed').removeClass('unviewed');
+    });
 
+    $("#checkbox-assign-tag").click(function (e) {
+        $('#op_extradata').val(prompt("Enter a tag name"));
+    });
 
-  $('.with-share-link > *').click(function () {
-      $("#copied-clipboard").remove();
+    $('.with-share-link > *').click(function () {
+        $("#copied-clipboard").remove();
 
-      var range = document.createRange();
-      var n=$("#share-link")[0];
-      range.selectNode(n);
-      window.getSelection().removeAllRanges();
-      window.getSelection().addRange(range);
-      document.execCommand("copy");
-      window.getSelection().removeAllRanges();
+        var range = document.createRange();
+        var n = $("#share-link")[0];
+        range.selectNode(n);
+        window.getSelection().removeAllRanges();
+        window.getSelection().addRange(range);
+        document.execCommand("copy");
+        window.getSelection().removeAllRanges();
 
-      $('.with-share-link').append('<span style="font-size: 80%; color: #fff;" id="copied-clipboard">Copied to clipboard</span>');
-      $("#copied-clipboard").fadeOut(2500, function() {
-       $(this).remove();
-      });
-  });
+        $('.with-share-link').append('<span style="font-size: 80%; color: #fff;" id="copied-clipboard">Copied to clipboard</span>');
+        $("#copied-clipboard").fadeOut(2500, function () {
+            $(this).remove();
+        });
+    });
+
+    $(".watch-table tr").click(function (event) {
+        var tagName = event.target.tagName.toLowerCase();
+        if (tagName === 'tr' || tagName === 'td') {
+            var x = $('input[type=checkbox]', this);
+            if (x) {
+                $(x).click();
+            }
+        }
+    });
 
     // checkboxes - check all
     $("#check-all").click(function (e) {
         $('input[type=checkbox]').not(this).prop('checked', this.checked);
     });
+
     // checkboxes - show/hide buttons
     $("input[type=checkbox]").click(function (e) {
         if ($('input[type=checkbox]:checked').length) {

changedetectionio/static/styles/diff.css

@@ -218,3 +218,10 @@ td#diff-col div {
   text-align: center; }
   .tab-pane-inner#screenshot img {
     max-width: 99%; }
+
+#highlightSnippet {
+  background: var(--color-background);
+  padding: 1em;
+  border-radius: 5px;
+  background: var(--color-background);
+  box-shadow: 1px 1px 4px var(--color-shadow-jump); }

changedetectionio/static/styles/scss/diff.scss

@@ -119,3 +119,11 @@ td#diff-col div {
     max-width: 99%;
   }
 }
+
+#highlightSnippet {
+  background: var(--color-background);
+  padding: 1em;
+  border-radius: 5px;
+  background: var(--color-background);
+  box-shadow: 1px 1px 4px var(--color-shadow-jump);
+}

changedetectionio/static/styles/scss/parts/_darkmode.scss

@@ -0,0 +1,25 @@
+
+#toggle-light-mode {
+  width: 3rem;
+  /* default */
+  .icon-dark {
+    display: none;
+  }
+
+}
+
+html[data-darkmode="true"] {
+  #toggle-light-mode {
+    .icon-light {
+      display: none;
+    }
+
+    .icon-dark {
+      display: block;
+    }
+  }
+}
+
+
+
+

changedetectionio/static/styles/scss/parts/_extra_proxies.scss

@@ -7,6 +7,7 @@ ul#requests-extra_proxies {
     }
 
   }
+
   /* each proxy entry is a `table` */
   table {
     tr {
@@ -15,3 +16,47 @@ ul#requests-extra_proxies {
   }
 }
 
+#request {
+  /* Auto proxy scan/checker */
+  label[for=proxy] {
+    display: inline-block;
+  }
+}
+
+body.proxy-check-active {
+  #request {
+    .proxy-status {
+      width: 2em;
+    }
+
+    .proxy-check-details {
+      font-size: 80%;
+      color: #555;
+      display: block;
+      padding-left: 4em;
+    }
+
+    .proxy-timing {
+      font-size: 80%;
+      padding-left: 1rem;
+      color: var(--color-link);
+    }
+  }
+}
+
+
+#recommended-proxy {
+  display: grid;
+  gap: 2rem;
+    @media  (min-width: 991px) {
+      grid-template-columns: repeat(2, 1fr);
+    }
+
+  > div {
+    border: 1px #aaa solid;
+    border-radius: 4px;
+    padding: 1em;
+  }
+
+  padding-bottom: 1em;
+}

changedetectionio/static/styles/scss/styles.scss

@@ -8,6 +8,7 @@
 @import "parts/_pagination";
 @import "parts/_spinners";
 @import "parts/_variables";
+@import "parts/_darkmode";
 
 body {
   color: var(--color-text);
@@ -54,8 +55,31 @@ a.github-link {
   }
 }
 
-button.toggle-theme {
-  width: 4rem;
+
+#toggle-search {
+  width: 2rem;
+}
+
+#search-q {
+  opacity: 0;
+  -webkit-transition: all .9s ease;
+  -moz-transition: all .9s ease;
+  transition: all .9s ease;
+  width: 0;
+  display: none;
+  &.expanded {
+    width: auto;
+    display: inline-block;
+
+    opacity: 1;
+  }
+}
+#search-result-info {
+  color: #fff;
+}
+
+button.toggle-button {
+  vertical-align: middle;
   background: transparent;
   border: none;
   cursor: pointer;
@@ -74,19 +98,7 @@ button.toggle-theme {
     display: block;
   }
 
-  .icon-dark {
-    display: none;
-  }
 
-  &.dark {
-    .icon-light {
-      display: none;
-    }
-
-    .icon-dark {
-      display: block;
-    }
-  }
 }
 
 .pure-menu-horizontal {

changedetectionio/static/styles/styles.css

@@ -95,6 +95,37 @@ ul#requests-extra_proxies {
   ul#requests-extra_proxies table tr {
     display: inline; }
 
+#request {
+  /* Auto proxy scan/checker */ }
+  #request label[for=proxy] {
+    display: inline-block; }
+
+body.proxy-check-active #request .proxy-status {
+  width: 2em; }
+
+body.proxy-check-active #request .proxy-check-details {
+  font-size: 80%;
+  color: #555;
+  display: block;
+  padding-left: 4em; }
+
+body.proxy-check-active #request .proxy-timing {
+  font-size: 80%;
+  padding-left: 1rem;
+  color: var(--color-link); }
+
+#recommended-proxy {
+  display: grid;
+  gap: 2rem;
+  padding-bottom: 1em; }
+  @media (min-width: 991px) {
+    #recommended-proxy {
+      grid-template-columns: repeat(2, 1fr); } }
+  #recommended-proxy > div {
+    border: 1px #aaa solid;
+    border-radius: 4px;
+    padding: 1em; }
+
 .pagination-page-info {
   color: #fff;
   font-size: 0.85rem;
@@ -297,6 +328,18 @@ html[data-darkmode="true"] {
     html[data-darkmode="true"] .watch-table .unviewed.error {
       color: var(--color-watch-table-error); }
 
+#toggle-light-mode {
+  width: 3rem;
+  /* default */ }
+  #toggle-light-mode .icon-dark {
+    display: none; }
+
+html[data-darkmode="true"] #toggle-light-mode .icon-light {
+  display: none; }
+
+html[data-darkmode="true"] #toggle-light-mode .icon-dark {
+  display: block; }
+
 body {
   color: var(--color-text);
   background: var(--color-background-page); }
@@ -331,23 +374,35 @@ a.github-link {
   a.github-link:hover {
     color: var(--color-icon-github-hover); }
 
-button.toggle-theme {
-  width: 4rem;
+#toggle-search {
+  width: 2rem; }
+
+#search-q {
+  opacity: 0;
+  -webkit-transition: all .9s ease;
+  -moz-transition: all .9s ease;
+  transition: all .9s ease;
+  width: 0;
+  display: none; }
+  #search-q.expanded {
+    width: auto;
+    display: inline-block;
+    opacity: 1; }
+
+#search-result-info {
+  color: #fff; }
+
+button.toggle-button {
+  vertical-align: middle;
   background: transparent;
   border: none;
   cursor: pointer;
   color: var(--color-icon-github); }
-  button.toggle-theme:hover {
+  button.toggle-button:hover {
     color: var(--color-icon-github-hover); }
-  button.toggle-theme svg {
+  button.toggle-button svg {
     fill: currentColor; }
-  button.toggle-theme .icon-light {
-    display: block; }
-  button.toggle-theme .icon-dark {
-    display: none; }
-  button.toggle-theme.dark .icon-light {
-    display: none; }
-  button.toggle-theme.dark .icon-dark {
+  button.toggle-button .icon-light {
     display: block; }
 
 .pure-menu-horizontal {

changedetectionio/store.py

@@ -3,7 +3,7 @@ from flask import (
 )
 
 from . model import App, Watch
-from copy import deepcopy
+from copy import deepcopy, copy
 from os import path, unlink
 from threading import Lock
 import json
@@ -16,6 +16,8 @@ import threading
 import time
 import uuid as uuid_builder
 
+dictfilt = lambda x, y: dict([ (i,x[i]) for i in x if i in set(y) ])
+
 # Is there an existing library to ensure some data store (JSON etc) is in sync with CRUD methods?
 # Open a github issue if you know something :)
 # https://stackoverflow.com/questions/6190468/how-to-trigger-function-on-value-change
@@ -178,20 +180,6 @@ class ChangeDetectionStore:
 
         return self.__data
 
-    def get_all_tags(self):
-        tags = []
-        for uuid, watch in self.data['watching'].items():
-            if watch['tag'] is None:
-                continue
-            # Support for comma separated list of tags.
-            for tag in watch['tag'].split(','):
-                tag = tag.strip()
-                if tag not in tags:
-                    tags.append(tag)
-
-        tags.sort()
-        return tags
-
     # Delete a single watch by UUID
     def delete(self, uuid):
         import pathlib
@@ -204,22 +192,22 @@ class ChangeDetectionStore:
                 # GitHub #30 also delete history records
                 for uuid in self.data['watching']:
                     path = pathlib.Path(os.path.join(self.datastore_path, uuid))
-                    shutil.rmtree(path)
-                    self.needs_write_urgent = True
+                    if os.path.exists(path):
+                        shutil.rmtree(path)
 
             else:
                 path = pathlib.Path(os.path.join(self.datastore_path, uuid))
-                shutil.rmtree(path)
+                if os.path.exists(path):
+                    shutil.rmtree(path)
                 del self.data['watching'][uuid]
 
-            self.needs_write_urgent = True
+        self.needs_write_urgent = True
 
     # Clone a watch by UUID
     def clone(self, uuid):
-        url = self.data['watching'][uuid]['url']
-        tag = self.data['watching'][uuid]['tag']
+        url = self.data['watching'][uuid].get('url')
         extras = self.data['watching'][uuid]
-        new_uuid = self.add_watch(url=url, tag=tag, extras=extras)
+        new_uuid = self.add_watch(url=url, extras=extras)
         return new_uuid
 
     def url_exists(self, url):
@@ -254,16 +242,14 @@ class ChangeDetectionStore:
 
         self.needs_write_urgent = True
 
-    def add_watch(self, url, tag="", extras=None, write_to_disk_now=True):
+    def add_watch(self, url, tag='', extras=None, tag_uuids=None, write_to_disk_now=True):
 
         if extras is None:
             extras = {}
-        # should always be str
-        if tag is None or not tag:
-            tag = ''
 
         # Incase these are copied across, assume it's a reference and deepcopy()
         apply_extras = deepcopy(extras)
+        apply_extras['tags'] = [] if not apply_extras.get('tags') else apply_extras.get('tags')
 
         # Was it a share link? try to fetch the data
         if (url.startswith("https://changedetection.io/share/")):
@@ -290,6 +276,7 @@ class ChangeDetectionStore:
                     'processor',
                     'subtractive_selectors',
                     'tag',
+                    'tags',
                     'text_should_not_be_present',
                     'title',
                     'trigger_text',
@@ -312,25 +299,36 @@ class ChangeDetectionStore:
             flash('Watch protocol is not permitted by SAFE_PROTOCOL_REGEX', 'error')
             return None
 
-        with self.lock:
-            # #Re 569
-            new_watch = Watch.model(datastore_path=self.datastore_path, default={
-                'url': url,
-                'tag': tag,
-                'date_created': int(time.time())
-            })
+        if tag and type(tag) == str:
+            # Then it's probably a string of the actual tag by name, split and add it
+            for t in tag.split(','):
+                # for each stripped tag, add tag as UUID
+                for a_t in t.split(','):
+                    tag_uuid = self.add_tag(a_t)
+                    apply_extras['tags'].append(tag_uuid)
 
-            new_uuid = new_watch['uuid']
-            logging.debug("Added URL {} - {}".format(url, new_uuid))
+        # Or if UUIDs given directly
+        if tag_uuids:
+            apply_extras['tags'] = list(set(apply_extras['tags'] + tag_uuids))
 
-            for k in ['uuid', 'history', 'last_checked', 'last_changed', 'newest_history_key', 'previous_md5', 'viewed']:
-                if k in apply_extras:
-                    del apply_extras[k]
+        # Make any uuids unique
+        if apply_extras.get('tags'):
+            apply_extras['tags'] = list(set(apply_extras.get('tags')))
 
-            new_watch.update(apply_extras)
-            self.__data['watching'][new_uuid] = new_watch
+        new_watch = Watch.model(datastore_path=self.datastore_path, url=url)
+
+        new_uuid = new_watch.get('uuid')
+
+        logging.debug("Added URL {} - {}".format(url, new_uuid))
+
+        for k in ['uuid', 'history', 'last_checked', 'last_changed', 'newest_history_key', 'previous_md5', 'viewed']:
+            if k in apply_extras:
+                del apply_extras[k]
+
+        new_watch.update(apply_extras)
+        new_watch.ensure_data_dir_exists()
+        self.__data['watching'][new_uuid] = new_watch
 
-        self.__data['watching'][new_uuid].ensure_data_dir_exists()
 
         if write_to_disk_now:
             self.sync_to_json()
@@ -366,19 +364,21 @@ class ChangeDetectionStore:
     def save_error_text(self, watch_uuid, contents):
         if not self.data['watching'].get(watch_uuid):
             return
-        target_path = os.path.join(self.datastore_path, watch_uuid, "last-error.txt")
 
+        self.data['watching'][watch_uuid].ensure_data_dir_exists()
+        target_path = os.path.join(self.datastore_path, watch_uuid, "last-error.txt")
         with open(target_path, 'w') as f:
             f.write(contents)
 
     def save_xpath_data(self, watch_uuid, data, as_error=False):
+
         if not self.data['watching'].get(watch_uuid):
             return
         if as_error:
             target_path = os.path.join(self.datastore_path, watch_uuid, "elements-error.json")
         else:
             target_path = os.path.join(self.datastore_path, watch_uuid, "elements.json")
-
+        self.data['watching'][watch_uuid].ensure_data_dir_exists()
         with open(target_path, 'w') as f:
             f.write(json.dumps(data))
             f.close()
@@ -472,8 +472,6 @@ class ChangeDetectionStore:
         return proxy_list if len(proxy_list) else None
 
 
-
-
     def get_preferred_proxy_for_watch(self, uuid):
         """
         Returns the preferred proxy by ID key
@@ -505,6 +503,105 @@ class ChangeDetectionStore:
 
         return None
 
+    @property
+    def has_extra_headers_file(self):
+        filepath = os.path.join(self.datastore_path, 'headers.txt')
+        return os.path.isfile(filepath)
+
+    def get_all_base_headers(self):
+        from .model.App import parse_headers_from_text_file
+        headers = {}
+        # Global app settings
+        headers.update(self.data['settings'].get('headers', {}))
+
+        return headers
+
+    def get_all_headers_in_textfile_for_watch(self, uuid):
+        from .model.App import parse_headers_from_text_file
+        headers = {}
+
+        # Global in /datastore/headers.txt
+        filepath = os.path.join(self.datastore_path, 'headers.txt')
+        try:
+            if os.path.isfile(filepath):
+                headers.update(parse_headers_from_text_file(filepath))
+        except Exception as e:
+            print(f"ERROR reading headers.txt at {filepath}", str(e))
+
+        watch = self.data['watching'].get(uuid)
+        if watch:
+
+            # In /datastore/xyz-xyz/headers.txt
+            filepath = os.path.join(watch.watch_data_dir, 'headers.txt')
+            try:
+                if os.path.isfile(filepath):
+                    headers.update(parse_headers_from_text_file(filepath))
+            except Exception as e:
+                print(f"ERROR reading headers.txt at {filepath}", str(e))
+
+            # In /datastore/tag-name.txt
+            tags = self.get_all_tags_for_watch(uuid=uuid)
+            for tag_uuid, tag in tags.items():
+                fname = "headers-"+re.sub(r'[\W_]', '', tag.get('title')).lower().strip() + ".txt"
+                filepath = os.path.join(self.datastore_path, fname)
+                try:
+                    if os.path.isfile(filepath):
+                        headers.update(parse_headers_from_text_file(filepath))
+                except Exception as e:
+                    print(f"ERROR reading headers.txt at {filepath}", str(e))
+
+        return headers
+
+    def get_tag_overrides_for_watch(self, uuid, attr):
+        tags = self.get_all_tags_for_watch(uuid=uuid)
+        ret = []
+
+        if tags:
+            for tag_uuid, tag in tags.items():
+                if attr in tag and tag[attr]:
+                    ret=[*ret, *tag[attr]]
+
+        return ret
+
+    def add_tag(self, name):
+        # If name exists, return that
+        n = name.strip().lower()
+        print (f">>> Adding new tag - '{n}'")
+        if not n:
+            return False
+
+        for uuid, tag in self.__data['settings']['application'].get('tags', {}).items():
+            if n == tag.get('title', '').lower().strip():
+                print (f">>> Tag {name} already exists")
+                return uuid
+
+        # Eventually almost everything todo with a watch will apply as a Tag
+        # So we use the same model as a Watch
+        with self.lock:
+            new_tag = Watch.model(datastore_path=self.datastore_path, default={
+                'title': name.strip(),
+                'date_created': int(time.time())
+            })
+
+            new_uuid = new_tag.get('uuid')
+
+            self.__data['settings']['application']['tags'][new_uuid] = new_tag
+
+        return new_uuid
+
+    def get_all_tags_for_watch(self, uuid):
+        """This should be in Watch model but Watch doesn't have access to datastore, not sure how to solve that yet"""
+        watch = self.data['watching'].get(uuid)
+
+        # Should return a dict of full tag info linked by UUID
+        if watch:
+            return dictfilt(self.__data['settings']['application']['tags'], watch.get('tags', []))
+
+        return {}
+
+    def tag_exists_by_name(self, tag_name):
+        return any(v.get('title', '').lower() == tag_name.lower() for k, v in self.__data['settings']['application']['tags'].items())
+
     # Run all updates
     # IMPORTANT - Each update could be run even when they have a new install and the schema is correct
     #             So therefor - each `update_n` should be very careful about checking if it needs to actually run
@@ -690,3 +787,16 @@ class ChangeDetectionStore:
             i+=1
         return
 
+    # Create tag objects and their references from existing tag text
+    def update_12(self):
+        i = 0
+        for uuid, watch in self.data['watching'].items():
+            # Split out and convert old tag string
+            tag = watch.get('tag')
+            if tag:
+                tag_uuids = []
+                for t in tag.split(','):
+                    tag_uuids.append(self.add_tag(name=t))
+
+                self.data['watching'][uuid]['tags'] = tag_uuids
+

changedetectionio/templates/_common_fields.jinja

@@ -115,7 +115,7 @@
                                     URLs generated by changedetection.io (such as <code>{{ '{{diff_url}}' }}</code>) require the <code>BASE_URL</code> environment variable set.<br>
                                     Your <code>BASE_URL</code> var is currently "{{settings_application['current_base_url']}}"
 									<br>
-									Warning: Contents of <code>{{ '{{diff}}' }}</code>, <code>{{ '{{diff_removed}}' }}</code>, and <code>{{ '{{diff_added}}' }}</code> depend on how the difference algorithm perceives the change. For example, an addition or removal could be perceived as a change in some cases. <a target="_new" href="https://github.com/dgtlmoon/changedetection.io/wiki/Using-the-%7B%7Bdiff%7D%7D,-%7B%7Bdiff_added%7D%7D,-and-%7B%7Bdiff_removal%7D%7D-notification-tokens">More Here</a> <br>
+									Warning: Contents of <code>{{ '{{diff}}' }}</code>, <code>{{ '{{diff_removed}}' }}</code>, and <code>{{ '{{diff_added}}' }}</code> depend on how the difference algorithm perceives the change. For example, an addition or removal could be perceived as a change in some cases. <a target="_new" href="https://github.com/dgtlmoon/changedetection.io/wiki/Using-the-%7B%7Bdiff%7D%7D,-%7B%7Bdiff_added%7D%7D,-and-%7B%7Bdiff_removed%7D%7D-notification-tokens">More Here</a> <br>
                                 </div>
                             </div>
                         </div>

changedetectionio/templates/_helpers.jinja

@@ -1,7 +1,6 @@
 {% macro render_field(field) %}
-  <div {% if field.errors %} class="error" {% endif %}>{{ field(**kwargs)|safe }}
   <div {% if field.errors %} class="error" {% endif %}>{{ field.label }}</div>
-
+  <div {% if field.errors %} class="error" {% endif %}>{{ field(**kwargs)|safe }}
   {% if field.errors %}
     <ul class=errors>
     {% for error in field.errors %}
@@ -25,18 +24,6 @@
   </div>
 {% endmacro %}
 
-{% macro render_field(field) %}
-  <div {% if field.errors %} class="error" {% endif %}>{{ field.label }}</div>
-  <div {% if field.errors %} class="error" {% endif %}>{{ field(**kwargs)|safe }}
-  {% if field.errors %}
-    <ul class=errors>
-    {% for error in field.errors %}
-      <li>{{ error }}</li>
-    {% endfor %}
-    </ul>
-  {% endif %}
-  </div>
-{% endmacro %}
 
 {% macro render_simple_field(field) %}
   <span class="label {% if field.errors %}error{% endif %}">{{ field.label }}</span>

changedetectionio/templates/base.html

@@ -37,7 +37,7 @@
     <div class="header">
       <div class="home-menu pure-menu pure-menu-horizontal pure-menu-fixed" id="nav-menu">
         {% if has_password and not current_user.is_authenticated %}
-          <a class="pure-menu-heading" href="https://github.com/dgtlmoon/changedetection.io" rel="noopener">
+          <a class="pure-menu-heading" href="https://changedetection.io" rel="noopener">
             <strong>Change</strong>Detection.io</a>
         {% else %}
           <a class="pure-menu-heading" href="{{url_for('index')}}">
@@ -49,7 +49,7 @@
         {% else %}
           {% if new_version_available and not(has_password and not current_user.is_authenticated) %}
             <span id="new-version-text" class="pure-menu-heading">
-              <a href="https://github.com/dgtlmoon/changedetection.io">A new version is available</a>
+              <a href="https://changedetection.io">A new version is available</a>
             </span>
           {% endif %}
         {% endif %}
@@ -58,6 +58,9 @@
           {% if current_user.is_authenticated or not has_password %}
             {% if not
             current_diff_url %}
+              <li class="pure-menu-item">
+                <a href="{{ url_for('tags.tags_overview_page')}}" class="pure-menu-link">GROUPS</a>
+              </li>
               <li class="pure-menu-item">
                 <a href="{{ url_for('settings_page')}}" class="pure-menu-link">SETTINGS</a>
               </li>
@@ -74,7 +77,7 @@
             {% endif %}
           {% else %}
             <li class="pure-menu-item">
-              <a class="pure-menu-link" href="https://github.com/dgtlmoon/changedetection.io">Website Change Detection and Notification.</a>
+              <a class="pure-menu-link" href="https://changedetection.io">Website Change Detection and Notification.</a>
             </li>
           {% endif %}
           {% if current_user.is_authenticated %}
@@ -82,11 +85,18 @@
               <a href="{{url_for('logout')}}" class="pure-menu-link">LOG OUT</a>
             </li>
           {% endif %}
+          <li class="pure-menu-item pure-form" id="search-menu-item">
+            <!-- We use GET here so it offers people a chance to set bookmarks etc -->
+            <form name="searchForm" action="" method="GET">
+              <input id="search-q" class="" name="q" placeholder="URL or Title {% if active_tag %}in '{{ active_tag }}'{% endif %}" required="" type="text" value="">
+              <input name="tags" type="hidden" value="{% if active_tag %}{{active_tag}}{% endif %}">
+              <button class="toggle-button " id="toggle-search" type="button" title="Search, or Use Alt+S Key" >
+                {% include "svgs/search-icon.svg" %}
+              </button>
+            </form>
+          </li>
           <li class="pure-menu-item">
-            {% if dark_mode %}
-            {% set darkClass = 'dark' %}
-            {% endif %}
-            <button class="toggle-theme {{darkClass}}" type="button" title="Toggle Light/Dark Mode">
+            <button class="toggle-button" id ="toggle-light-mode" type="button" title="Toggle Light/Dark Mode">
               <span class="visually-hidden">Toggle light/dark mode</span>
               <span class="icon-light">
                 {% include "svgs/light-mode-toggle-icon.svg" %}

changedetectionio/templates/diff.html

@@ -6,6 +6,9 @@
     {% if last_error_screenshot %}
     const error_screenshot_url="{{url_for('static_content', group='screenshot', filename=uuid, error_screenshot=1) }}";
     {% endif %}
+
+    const highlight_submit_ignore_url="{{url_for('highlight_submit_ignore_url', uuid=uuid)}}";
+
 </script>
 <script src="{{url_for('static_content', group='js', filename='diff-overview.js')}}" defer></script>
 
@@ -76,7 +79,7 @@
     </div>
 
      <div class="tab-pane-inner" id="text">
-         <div class="tip">Pro-tip: Use <strong>show current snapshot</strong> tab to visualise what will be ignored.</div>
+         <div class="tip">Pro-tip: Use <strong>show current snapshot</strong> tab to visualise what will be ignored, highlight text to add to ignore filters</div>
 
          {% if password_enabled_and_share_is_off %}
            <div class="tip">Pro-tip: You can enable <strong>"share access when password is enabled"</strong> from settings</div>
@@ -91,7 +94,7 @@
                  <td id="a" style="display: none;">{{previous}}</td>
                  <td id="b" style="display: none;">{{newest}}</td>
                  <td id="diff-col">
-                     <span id="result"></span>
+                     <span id="result" class="highlightable-filter"></span>
                  </td>
              </tr>
              </tbody>

changedetectionio/templates/edit.html

@@ -4,18 +4,19 @@
 {% from '_common_fields.jinja' import render_common_settings_form %}
 <script src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
 <script>
-    const notification_base_url="{{url_for('ajax_callback_send_notification_test')}}";
-    const watch_visual_selector_data_url="{{url_for('static_content', group='visual_selector_data', filename=uuid)}}";
-    const screenshot_url="{{url_for('static_content', group='screenshot', filename=uuid)}}";
-    const playwright_enabled={% if playwright_enabled %} true {% else %} false {% endif %};
-
-{% if emailprefix %}
-    const email_notification_prefix=JSON.parse('{{ emailprefix|tojson }}');
-{% endif %}
 
     const browser_steps_config=JSON.parse('{{ browser_steps_config|tojson }}');
     const browser_steps_start_url="{{url_for('browser_steps.browsersteps_start_session', uuid=uuid)}}";
     const browser_steps_sync_url="{{url_for('browser_steps.browsersteps_ui_update', uuid=uuid)}}";
+{% if emailprefix %}
+    const email_notification_prefix=JSON.parse('{{ emailprefix|tojson }}');
+{% endif %}
+    const notification_base_url="{{url_for('ajax_callback_send_notification_test')}}";
+    const playwright_enabled={% if playwright_enabled %} true {% else %} false {% endif %};
+    const recheck_proxy_start_url="{{url_for('check_proxies.start_check', uuid=uuid)}}";
+    const proxy_recheck_status_url="{{url_for('check_proxies.get_recheck_status', uuid=uuid)}}";
+    const screenshot_url="{{url_for('static_content', group='screenshot', filename=uuid)}}";
+    const watch_visual_selector_data_url="{{url_for('static_content', group='visual_selector_data', filename=uuid)}}";
 
 </script>
 
@@ -27,6 +28,8 @@
 <script src="{{url_for('static_content', group='js', filename='browser-steps.js')}}" defer></script>
 {% endif %}
 
+<script src="{{url_for('static_content', group='js', filename='recheck-proxy.js')}}" defer></script>
+
 <div class="edit-form monospaced-textarea">
 
     <div class="tabs collapsable">
@@ -75,7 +78,7 @@
                         {{ render_field(form.title, class="m-d") }}
                     </div>
                     <div class="pure-control-group">
-                        {{ render_field(form.tag) }}
+                        {{ render_field(form.tags) }}
                         <span class="pure-form-message-inline">Organisational tag/group name used in the main listing page</span>
                     </div>
                     <div class="pure-control-group">
@@ -111,7 +114,8 @@
                     </div>
                 {% if form.proxy %}
                     <div class="pure-control-group inline-radio">
-                        {{ render_field(form.proxy, class="fetch-backend-proxy") }}
+                          <div>{{ form.proxy.label }} <a href="" id="check-all-proxies" class="pure-button button-secondary button-xsmall" >Check/Scan all</a></div>
+                          <div>{{ form.proxy(class="fetch-backend-proxy") }}</div>
                         <span class="pure-form-message-inline">
                         Choose a proxy for this watch
                         </span>
@@ -152,6 +156,17 @@
 {{ render_field(form.headers, rows=5, placeholder="Example
 Cookie: foobar
 User-Agent: wonderbra 1.0") }}
+
+                        <div class="pure-form-message-inline">
+                            {% if has_extra_headers_file %}
+                                <strong>Alert! Extra headers file found and will be added to this watch!</strong>
+                            {% else %}
+                                Headers can be also read from a file in your data-directory <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Adding-headers-from-an-external-file">Read more here</a>
+                            {% endif %}
+                            <br>
+                            (Not supported by Selenium browser)
+                        </div>
+
                     </div>
                     <div class="pure-control-group" id="request-body">
                                         {{ render_field(form.body, rows=5, placeholder="Example

changedetectionio/templates/preview.html

@@ -6,6 +6,7 @@
     {% if last_error_screenshot %}
     const error_screenshot_url="{{url_for('static_content', group='screenshot', filename=uuid, error_screenshot=1) }}";
     {% endif %}
+    const highlight_submit_ignore_url="{{url_for('highlight_submit_ignore_url', uuid=uuid)}}";
 </script>
 <script src="{{url_for('static_content', group='js', filename='diff-overview.js')}}" defer></script>
 
@@ -20,7 +21,7 @@
         {% endif %}
     </ul>
 </div>
-
+<form><input type="hidden" name="csrf_token" value="{{ csrf_token() }}"></form>
 <div id="diff-ui">
     <div class="tab-pane-inner" id="error-text">
         <div class="snapshot-age error">{{watch.error_text_ctime|format_seconds_ago}} seconds ago</div>
@@ -36,11 +37,12 @@
 
     <div class="tab-pane-inner" id="text">
         <div class="snapshot-age">{{watch.snapshot_text_ctime|format_timestamp_timeago}}</div>
-        <span class="ignored">Grey lines are ignored</span> <span class="triggered">Blue lines are triggers</span>
+        <span class="ignored">Grey lines are ignored</span> <span class="triggered">Blue lines are triggers</span> <span class="tip"><strong>Pro-tip</strong>: Highlight text to add to ignore filters</span>
+
         <table>
             <tbody>
             <tr>
-                <td id="diff-col">
+                <td id="diff-col" class="highlightable-filter">
                     {% for row in content %}
                     <div class="{{row.classes}}">{{row.line}}</div>
                     {% endfor %}

changedetectionio/templates/settings.html

@@ -70,6 +70,10 @@
                             <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Configurable-BASE_URL-setting">read more here</a>.
                         </span>
                     </div>
+                    <div class="pure-control-group">
+                        {{ render_field(form.application.form.pager_size) }}
+                        <span class="pure-form-message-inline">Number of items per page in the watch overview list, 0 to disable.</span>
+                    </div>
 
                     <div class="pure-control-group">
                         {{ render_checkbox_field(form.application.form.extract_title_as_title) }}
@@ -177,21 +181,56 @@ nav
                 </div>
             </div>
             <div class="tab-pane-inner" id="proxies">
+                <div id="recommended-proxy">
+                    <div>
+                        <img style="height: 2em;" src="{{url_for('static_content', group='images', filename='brightdata.svg')}}" alt="BrightData Proxy Provider">
+                        <p>BrightData offer world-class proxy services, "Data Center" proxies are a very affordable way to proxy your requests, whilst <strong><a href="https://brightdata.grsm.io/n0r16zf7eivq">WebUnlocker</a></strong> can help solve most CAPTCHAs.</p>
+                        <p>
+                            BrightData offer many <a href="https://brightdata.com/proxy-types" target="new">many different types of proxies</a>, it is worth reading about what is best for your use-case.
+                        </p>
 
-                <p><strong>Tip</strong>: You can connect to websites using <a href="https://brightdata.grsm.io/n0r16zf7eivq">BrightData</a> proxies, their service <strong>WebUnlocker</strong> will solve most CAPTCHAs, whilst their <strong>Residential Proxies</strong> may help to avoid CAPTCHA altogether. </p>
-                <p>It may be easier to try <strong>WebUnlocker</strong> first, WebUnlocker also supports country selection.</p>
+                        <p>
+                            When you have <a href="https://brightdata.grsm.io/n0r16zf7eivq">registered</a>, enabled the required services, visit the <A href="https://brightdata.com/cp/api_example?">API example page</A>, then select <strong>Python</strong>, set the country you wish to use, then copy+paste the access Proxy URL into the "Extra Proxies" boxes below.<br>
+                        </p>
+                        <p>
+                            The Proxy URL with BrightData should start with <code>http://brd-customer...</code>
+                        </p>
+                        <p>When you sign up using <a href="https://brightdata.grsm.io/n0r16zf7eivq">https://brightdata.grsm.io/n0r16zf7eivq</a> BrightData will match any first deposit up to $150</p>
+                    </div>
+                    <div>
+                        <img style="height: 2em;"
+                             src="{{url_for('static_content', group='images', filename='oxylabs.svg')}}"
+                             alt="Oxylabs Proxy Provider">
+                        <p>
+                            Collect public data at scale with industry-leading web scraping solutions and the world’s
+                            largest ethical proxy network.
+                        </p>
+                        <p>
+                            Oxylabs also provide a <a href="https://oxylabs.io/products/web-unblocker"><strong>WebUnlocker</strong></a>
+                            proxy that bypasses sophisticated anti-bot systems, so you don’t have to.<br>
+                        </p>
+                        <p>
+                            Serve over <a href="https://oxylabs.io/location-proxy">195 countries</a>, providing <a
+                                href="https://oxylabs.io/products/residential-proxy-pool">Residential</a>, <a
+                                href="https://oxylabs.io/products/mobile-proxies">Mobile</a> and <a
+                                href="https://oxylabs.io/products/rotating-isp-proxies">ISP proxies</a> and much more.
+                        </p>
+                        <p>
+                            Use the promo code <strong>boost35</strong> with this link <a href="https://oxylabs.go2cloud.org/SH2d">https://oxylabs.go2cloud.org/SH2d</a> for 35% off Residential, Mobile proxies, Web Unblocker, and Scraper APIs. Built-in proxies enable you to access data from all around the world and help overcome anti-bot solutions.
+
+                        </p>
+
+                        
+                    </div>
+                </div>
                 <p>
-                    When you have <a href="https://brightdata.grsm.io/n0r16zf7eivq">registered</a>, enabled the required services, visit the <A href="https://brightdata.com/cp/api_example?">API example page</A>, then select <strong>Python</strong>, set the country you wish to use, then copy+paste the example URL below<br>
-                    The Proxy URL with BrightData should start with <code>http://brd-customer...</code>
+                    Your proxy provider may need to whitelist our IP of <code>204.15.192.195</code>
                 </p>
-
-                <p>When you sign up using <a href="https://brightdata.grsm.io/n0r16zf7eivq">https://brightdata.grsm.io/n0r16zf7eivq</a> BrightData will match any first deposit up to $150</p>
-
+               <p><strong>Tip</strong>: "Residential" and "Mobile" proxy type can be more successfull than "Data Center" for blocked websites.
 
                 <div class="pure-control-group">
-                    {{ render_field(form.requests.form.extra_proxies) }}
-                    <span class="pure-form-message-inline">"Name" will be used for selecting the proxy in the Watch Edit settings</span>
-                </div>
+                {{ render_field(form.requests.form.extra_proxies) }}
+                <span class="pure-form-message-inline">"Name" will be used for selecting the proxy in the Watch Edit settings</span>
             </div>
             <div id="actions">
                 <div class="pure-control-group">

changedetectionio/templates/svgs/search-icon.svg

@@ -0,0 +1 @@
+<?xml version="1.0" encoding="utf-8"?><svg version="1.1" id="Layer_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 0 122.879 119.799" enable-background="new 0 0 122.879 119.799" xml:space="preserve"><g><path d="M49.988,0h0.016v0.007C63.803,0.011,76.298,5.608,85.34,14.652c9.027,9.031,14.619,21.515,14.628,35.303h0.007v0.033v0.04 h-0.007c-0.005,5.557-0.917,10.905-2.594,15.892c-0.281,0.837-0.575,1.641-0.877,2.409v0.007c-1.446,3.66-3.315,7.12-5.547,10.307 l29.082,26.139l0.018,0.016l0.157,0.146l0.011,0.011c1.642,1.563,2.536,3.656,2.649,5.78c0.11,2.1-0.543,4.248-1.979,5.971 l-0.011,0.016l-0.175,0.203l-0.035,0.035l-0.146,0.16l-0.016,0.021c-1.565,1.642-3.654,2.534-5.78,2.646 c-2.097,0.111-4.247-0.54-5.971-1.978l-0.015-0.011l-0.204-0.175l-0.029-0.024L78.761,90.865c-0.88,0.62-1.778,1.209-2.687,1.765 c-1.233,0.755-2.51,1.466-3.813,2.115c-6.699,3.342-14.269,5.222-22.272,5.222v0.007h-0.016v-0.007 c-13.799-0.004-26.296-5.601-35.338-14.645C5.605,76.291,0.016,63.805,0.007,50.021H0v-0.033v-0.016h0.007 c0.004-13.799,5.601-26.296,14.645-35.338C23.683,5.608,36.167,0.016,49.955,0.007V0H49.988L49.988,0z M50.004,11.21v0.007h-0.016 h-0.033V11.21c-10.686,0.007-20.372,4.35-27.384,11.359C15.56,29.578,11.213,39.274,11.21,49.973h0.007v0.016v0.033H11.21 c0.007,10.686,4.347,20.367,11.359,27.381c7.009,7.012,16.705,11.359,27.403,11.361v-0.007h0.016h0.033v0.007 c10.686-0.007,20.368-4.348,27.382-11.359c7.011-7.009,11.358-16.702,11.36-27.4h-0.006v-0.016v-0.033h0.006 c-0.006-10.686-4.35-20.372-11.358-27.384C70.396,15.56,60.703,11.213,50.004,11.21L50.004,11.21z"/></g></svg>

changedetectionio/templates/watch-overview.html

@@ -13,7 +13,7 @@
             <div id="watch-add-wrapper-zone">
                 <div>
                     {{ render_simple_field(form.url, placeholder="https://...", required=true) }}
-                    {{ render_simple_field(form.tag, value=active_tag if active_tag else '', placeholder="watch label / tag") }}
+                    {{ render_simple_field(form.tags, value=tags[active_tag].title if active_tag else '', placeholder="watch label / tag") }}
                 </div>
                 <div>
                     {{ render_simple_field(form.watch_submit_button, title="Watch this URL!" ) }}
@@ -30,12 +30,14 @@
 
     <form class="pure-form" action="{{ url_for('form_watch_list_checkbox_operations') }}" method="POST" id="watch-list-form">
     <input type="hidden" name="csrf_token" value="{{ csrf_token() }}" >
+    <input type="hidden" id="op_extradata" name="op_extradata" value="" >
     <div id="checkbox-operations">
         <button class="pure-button button-secondary button-xsmall"  name="op" value="pause">Pause</button>
         <button class="pure-button button-secondary button-xsmall"  name="op" value="unpause">UnPause</button>
         <button class="pure-button button-secondary button-xsmall"  name="op" value="mute">Mute</button>
         <button class="pure-button button-secondary button-xsmall"  name="op" value="unmute">UnMute</button>
         <button class="pure-button button-secondary button-xsmall" name="op" value="recheck">Recheck</button>
+        <button class="pure-button button-secondary button-xsmall" name="op" value="assign-tag" id="checkbox-assign-tag">Tag</button>
         <button class="pure-button button-secondary button-xsmall" name="op" value="mark-viewed">Mark viewed</button>
         <button class="pure-button button-secondary button-xsmall" name="op" value="notification-default">Use default notification</button>
         <button class="pure-button button-secondary button-xsmall" style="background: #dd4242;" name="op" value="clear-history">Clear/reset history</button>
@@ -44,11 +46,12 @@
     {% if watches|length >= pagination.per_page %}
         {{ pagination.info }}
     {% endif %}
+    {% if search_q %}<div id="search-result-info">Searching "<strong><i>{{search_q}}</i></strong>"</div>{% endif %}
     <div>
         <a href="{{url_for('index')}}" class="pure-button button-tag {{'active' if not active_tag }}">All</a>
-        {% for tag in tags %}
+        {% for uuid, tag in tags.items() %}
             {% if tag != "" %}
-                <a href="{{url_for('index', tag=tag) }}" class="pure-button button-tag {{'active' if active_tag == tag }}">{{ tag }}</a>
+                <a href="{{url_for('index', tag=uuid) }}" class="pure-button button-tag {{'active' if active_tag == uuid }}">{{ tag.title }}</a>
             {% endif %}
         {% endfor %}
     </div>
@@ -73,8 +76,12 @@
             </tr>
             </thead>
             <tbody>
-
-            {% for watch in (watches|sort(attribute=sort_attribute, reverse=sort_order == 'asc'))[pagination.skip:pagination.skip+pagination.per_page] %}
+            {% if not watches|length %}
+            <tr>
+                <td colspan="6">No website watches configured, please add a URL in the box above, or <a href="{{ url_for('import_page')}}" >import a list</a>.</td>
+            </tr>
+            {% endif %}
+            {% for watch in (watches|sort(attribute=sort_attribute, reverse=sort_order == 'asc'))|pagination_slice(skip=pagination.skip) %}
             <tr id="{{ watch.uuid }}"
                 class="{{ loop.cycle('pure-table-odd', 'pure-table-even') }} processor-{{ watch['processor'] }}
                 {% if watch.last_error is defined and watch.last_error != False %}error{% endif %}
@@ -138,9 +145,11 @@
                     </span>
                     {% endif %}
 
-                    {% if not active_tag %}
-                    <span class="watch-tag-list">{{ watch.tag}}</span>
-                    {% endif %}
+
+                    {% for watch_tag_uuid, watch_tag in datastore.get_all_tags_for_watch(watch['uuid']).items() %}
+                      <span class="watch-tag-list">{{ watch_tag.title }}</span>
+                    {% endfor %}
+
                 </td>
                 <td class="last-checked">{{watch|format_last_checked_time|safe}}</td>
                 <td class="last-changed">{% if watch.history_n >=2 and watch.last_changed >0 %}
@@ -173,7 +182,7 @@
             {% endif %}
             <li>
                <a href="{{ url_for('form_watch_checknow', tag=active_tag) }}" class="pure-button button-tag ">Recheck
-                all {% if active_tag%}in "{{active_tag}}"{%endif%}</a>
+                all {% if active_tag%} in "{{tags[active_tag].title}}"{%endif%}</a>
             </li>
             <li>
                 <a href="{{ url_for('rss', tag=active_tag , token=app_rss_token)}}"><img alt="RSS Feed" id="feed-icon" src="{{url_for('static_content', group='images', filename='Generic_Feed-icon.svg')}}" height="15"></a>

changedetectionio/tests/conftest.py

@@ -14,13 +14,16 @@ global app
 
 def cleanup(datastore_path):
     # Unlink test output files
-    files = ['output.txt',
-             'url-watches.json',
-             'secret.txt',
-             'notification.txt',
-             'count.txt',
-             'endpoint-content.txt'
-                 ]
+    files = [
+        'count.txt',
+        'endpoint-content.txt'
+        'headers.txt',
+        'headers-testtag.txt',
+        'notification.txt',
+        'secret.txt',
+        'url-watches.json',
+        'output.txt',
+    ]
     for file in files:
         try:
             os.unlink("{}/{}".format(datastore_path, file))

changedetectionio/tests/proxy_list/test_multiple_proxy.py

@@ -28,7 +28,7 @@ def test_preferred_proxy(client, live_server):
                 "fetch_backend": "html_requests",
                 "headers": "",
                 "proxy": "proxy-two",
-                "tag": "",
+                "tags": "",
                 "url": url,
               },
         follow_redirects=True

changedetectionio/tests/restock/test_restock.py

@@ -77,7 +77,7 @@ def test_restock_detection(client, live_server):
 
     client.post(
         url_for("form_quick_watch_add"),
-        data={"url": test_url, "tag": '', 'processor': 'restock_diff'},
+        data={"url": test_url, "tags": '', 'processor': 'restock_diff'},
         follow_redirects=True
     )
 

changedetectionio/tests/smtp/smtp-test-server.py

@@ -0,0 +1,42 @@
+#!/usr/bin/python3
+import smtpd
+import asyncore
+
+# Accept a SMTP message and offer a way to retrieve the last message via TCP Socket
+
+last_received_message = b"Nothing"
+
+
+class CustomSMTPServer(smtpd.SMTPServer):
+
+    def process_message(self, peer, mailfrom, rcpttos, data, **kwargs):
+        global last_received_message
+        last_received_message = data
+        print('Receiving message from:', peer)
+        print('Message addressed from:', mailfrom)
+        print('Message addressed to  :', rcpttos)
+        print('Message length        :', len(data))
+        print(data.decode('utf8'))
+        return
+
+
+# Just print out the last message received on plain TCP socket server
+class EchoServer(asyncore.dispatcher):
+
+    def __init__(self, host, port):
+        asyncore.dispatcher.__init__(self)
+        self.create_socket()
+        self.set_reuse_addr()
+        self.bind((host, port))
+        self.listen(5)
+
+    def handle_accepted(self, sock, addr):
+        global last_received_message
+        print('Incoming connection from %s' % repr(addr))
+        sock.send(last_received_message)
+        last_received_message = b''
+
+
+server = CustomSMTPServer(('0.0.0.0', 11025), None)  # SMTP mail goes here
+server2 = EchoServer('0.0.0.0', 11080)  # Echo back last message received
+asyncore.loop()

changedetectionio/tests/smtp/test_notification_smtp.py

@@ -0,0 +1,165 @@
+import json
+import os
+import time
+import re
+from flask import url_for
+from changedetectionio.tests.util import set_original_response, set_modified_response, set_more_modified_response, live_server_setup, \
+    wait_for_all_checks, \
+    set_longer_modified_response
+from changedetectionio.tests.util import extract_UUID_from_client
+import logging
+import base64
+
+# NOTE - RELIES ON mailserver as hostname running, see github build recipes
+smtp_test_server = 'mailserver'
+
+from changedetectionio.notification import (
+    default_notification_body,
+    default_notification_format,
+    default_notification_title,
+    valid_notification_formats,
+)
+
+def test_setup(live_server):
+    live_server_setup(live_server)
+
+def get_last_message_from_smtp_server():
+    import socket
+    global smtp_test_server
+    port = 11080  # socket server port number
+
+    client_socket = socket.socket()  # instantiate
+    client_socket.connect((smtp_test_server, port))  # connect to the server
+
+    data = client_socket.recv(50024).decode()  # receive response
+    client_socket.close()  # close the connection
+    return data
+
+
+# Requires running the test SMTP server
+
+def test_check_notification_email_formats_default_HTML(client, live_server):
+    # live_server_setup(live_server)
+    set_original_response()
+
+    global smtp_test_server
+    notification_url = f'mailto://changedetection@{smtp_test_server}:11025/?to=fff@home.com'
+
+    #####################
+    # Set this up for when we remove the notification from the watch, it should fallback with these details
+    res = client.post(
+        url_for("settings_page"),
+        data={"application-notification_urls": notification_url,
+              "application-notification_title": "fallback-title " + default_notification_title,
+              "application-notification_body": "fallback-body<br> " + default_notification_body,
+              "application-notification_format": 'HTML',
+              "requests-time_between_check-minutes": 180,
+              'application-fetch_backend': "html_requests"},
+        follow_redirects=True
+    )
+    assert b"Settings updated." in res.data
+
+    # Add a watch and trigger a HTTP POST
+    test_url = url_for('test_endpoint', _external=True)
+    res = client.post(
+        url_for("form_quick_watch_add"),
+        data={"url": test_url, "tags": 'nice one'},
+        follow_redirects=True
+    )
+
+    assert b"Watch added" in res.data
+
+    wait_for_all_checks(client)
+    set_longer_modified_response()
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    wait_for_all_checks(client)
+
+    time.sleep(3)
+
+    msg = get_last_message_from_smtp_server()
+    assert len(msg) >= 1
+
+    # The email should have two bodies, and the text/html part should be <br>
+    assert 'Content-Type: text/plain' in msg
+    assert '(added) So let\'s see what happens.\n' in msg  # The plaintext part with \n
+    assert 'Content-Type: text/html' in msg
+    assert '(added) So let\'s see what happens.<br>' in msg  # the html part
+    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    assert b'Deleted' in res.data
+
+
+def test_check_notification_email_formats_default_Text_override_HTML(client, live_server):
+    # live_server_setup(live_server)
+
+    # HTML problems? see this
+    # https://github.com/caronc/apprise/issues/633
+
+    set_original_response()
+    global smtp_test_server
+    notification_url = f'mailto://changedetection@{smtp_test_server}:11025/?to=fff@home.com'
+
+    #####################
+    # Set this up for when we remove the notification from the watch, it should fallback with these details
+    res = client.post(
+        url_for("settings_page"),
+        data={"application-notification_urls": notification_url,
+              "application-notification_title": "fallback-title " + default_notification_title,
+              "application-notification_body": default_notification_body,
+              "application-notification_format": 'Text',
+              "requests-time_between_check-minutes": 180,
+              'application-fetch_backend': "html_requests"},
+        follow_redirects=True
+    )
+    assert b"Settings updated." in res.data
+
+    # Add a watch and trigger a HTTP POST
+    test_url = url_for('test_endpoint', _external=True)
+    res = client.post(
+        url_for("form_quick_watch_add"),
+        data={"url": test_url, "tags": 'nice one'},
+        follow_redirects=True
+    )
+
+    assert b"Watch added" in res.data
+
+    wait_for_all_checks(client)
+    set_longer_modified_response()
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    wait_for_all_checks(client)
+
+    time.sleep(3)
+    msg = get_last_message_from_smtp_server()
+    assert len(msg) >= 1
+    #    with open('/tmp/m.txt', 'w') as f:
+    #        f.write(msg)
+
+    # The email should not have two bodies, should be TEXT only
+
+    assert 'Content-Type: text/plain' in msg
+    assert '(added) So let\'s see what happens.\n' in msg  # The plaintext part with \n
+
+    set_original_response()
+    # Now override as HTML format
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={
+            "url": test_url,
+            "notification_format": 'HTML',
+            'fetch_backend': "html_requests"},
+        follow_redirects=True
+    )
+    assert b"Updated watch." in res.data
+    wait_for_all_checks(client)
+
+    time.sleep(3)
+    msg = get_last_message_from_smtp_server()
+    assert len(msg) >= 1
+
+    # The email should have two bodies, and the text/html part should be <br>
+    assert 'Content-Type: text/plain' in msg
+    assert '(removed) So let\'s see what happens.\n' in msg  # The plaintext part with \n
+    assert 'Content-Type: text/html' in msg
+    assert '(removed) So let\'s see what happens.<br>' in msg  # the html part
+
+    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    assert b'Deleted' in res.data

changedetectionio/tests/test_access_control.py

@@ -45,6 +45,15 @@ def test_check_access_control(app, client, live_server):
         res = client.get(url_for("diff_history_page", uuid="first"))
         assert b'Random content' in res.data
 
+        # Check wrong password does not let us in
+        res = c.post(
+            url_for("login"),
+            data={"password": "WRONG PASSWORD"},
+            follow_redirects=True
+        )
+
+        assert b"LOG OUT" not in res.data
+        assert b"Incorrect password" in res.data
 
 
         # Menu should not be available yet

changedetectionio/tests/test_add_replace_remove_filter.py

@@ -2,7 +2,7 @@
 
 import time
 from flask import url_for
-from .util import live_server_setup
+from .util import live_server_setup, wait_for_all_checks
 from changedetectionio import html_tools
 
 
@@ -39,7 +39,6 @@ def test_setup(client, live_server):
     live_server_setup(live_server)
 
 def test_check_removed_line_contains_trigger(client, live_server):
-    sleep_time_for_fetch_thread = 3
 
     # Give the endpoint time to spin up
     time.sleep(1)
@@ -54,7 +53,7 @@ def test_check_removed_line_contains_trigger(client, live_server):
     assert b"1 Imported" in res.data
 
     # Give the thread time to pick it up
-    time.sleep(sleep_time_for_fetch_thread)
+    wait_for_all_checks(client)
 
     # Goto the edit page, add our ignore text
     # Add our URL to the import page
@@ -67,20 +66,20 @@ def test_check_removed_line_contains_trigger(client, live_server):
         follow_redirects=True
     )
     assert b"Updated watch." in res.data
-    time.sleep(sleep_time_for_fetch_thread)
+    wait_for_all_checks(client)
     set_original(excluding='Something irrelevant')
 
     # A line thats not the trigger should not trigger anything
     res = client.get(url_for("form_watch_checknow"), follow_redirects=True)
     assert b'1 watches queued for rechecking.' in res.data
-    time.sleep(sleep_time_for_fetch_thread)
+    wait_for_all_checks(client)
     res = client.get(url_for("index"))
     assert b'unviewed' not in res.data
 
     # The trigger line is REMOVED,  this should trigger
     set_original(excluding='The golden line')
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    time.sleep(sleep_time_for_fetch_thread)
+    wait_for_all_checks(client)
     res = client.get(url_for("index"))
     assert b'unviewed' in res.data
 
@@ -89,14 +88,14 @@ def test_check_removed_line_contains_trigger(client, live_server):
     client.get(url_for("mark_all_viewed"), follow_redirects=True)
     set_original(excluding=None)
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    time.sleep(sleep_time_for_fetch_thread)
+    wait_for_all_checks(client)
     res = client.get(url_for("index"))
     assert b'unviewed' not in res.data
 
     # Remove it again, and we should get a trigger
     set_original(excluding='The golden line')
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    time.sleep(sleep_time_for_fetch_thread)
+    wait_for_all_checks(client)
     res = client.get(url_for("index"))
     assert b'unviewed' in res.data
 
@@ -105,8 +104,7 @@ def test_check_removed_line_contains_trigger(client, live_server):
 
 
 def test_check_add_line_contains_trigger(client, live_server):
-
-    sleep_time_for_fetch_thread = 3
+    #live_server_setup(live_server)
 
     # Give the endpoint time to spin up
     time.sleep(1)
@@ -136,8 +134,7 @@ def test_check_add_line_contains_trigger(client, live_server):
     assert b"1 Imported" in res.data
 
     # Give the thread time to pick it up
-    time.sleep(sleep_time_for_fetch_thread)
-
+    wait_for_all_checks(client)
     # Goto the edit page, add our ignore text
     # Add our URL to the import page
     res = client.post(
@@ -150,23 +147,25 @@ def test_check_add_line_contains_trigger(client, live_server):
         follow_redirects=True
     )
     assert b"Updated watch." in res.data
-    time.sleep(sleep_time_for_fetch_thread)
+    wait_for_all_checks(client)
     set_original(excluding='Something irrelevant')
 
     # A line thats not the trigger should not trigger anything
     res = client.get(url_for("form_watch_checknow"), follow_redirects=True)
     assert b'1 watches queued for rechecking.' in res.data
-    time.sleep(sleep_time_for_fetch_thread)
+    wait_for_all_checks(client)
     res = client.get(url_for("index"))
     assert b'unviewed' not in res.data
 
     # The trigger line is ADDED,  this should trigger
     set_original(add_line='<p>Oh yes please</p>')
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    time.sleep(sleep_time_for_fetch_thread)
+    wait_for_all_checks(client)
     res = client.get(url_for("index"))
     assert b'unviewed' in res.data
 
+    # Takes a moment for apprise to fire
+    time.sleep(3)
     with open("test-datastore/notification.txt", 'r') as f:
         response= f.read()
         assert '-Oh yes please-' in response

changedetectionio/tests/test_api.py

@@ -2,7 +2,7 @@
 
 import time
 from flask import url_for
-from .util import live_server_setup, extract_api_key_from_UI
+from .util import live_server_setup, extract_api_key_from_UI, wait_for_all_checks
 
 import json
 import uuid
@@ -57,6 +57,7 @@ def test_setup(client, live_server):
     live_server_setup(live_server)
 
 def test_api_simple(client, live_server):
+    #live_server_setup(live_server)
 
     api_key = extract_api_key_from_UI(client)
 
@@ -86,7 +87,7 @@ def test_api_simple(client, live_server):
     watch_uuid = res.json.get('uuid')
     assert res.status_code == 201
 
-    time.sleep(3)
+    wait_for_all_checks(client)
 
     # Verify its in the list and that recheck worked
     res = client.get(
@@ -107,7 +108,7 @@ def test_api_simple(client, live_server):
     )
     assert len(res.json) == 0
 
-    time.sleep(2)
+    wait_for_all_checks(client)
 
     set_modified_response()
     # Trigger recheck of all ?recheck_all=1
@@ -115,7 +116,7 @@ def test_api_simple(client, live_server):
         url_for("createwatch", recheck_all='1'),
         headers={'x-api-key': api_key},
     )
-    time.sleep(3)
+    wait_for_all_checks(client)
 
     # Did the recheck fire?
     res = client.get(
@@ -266,7 +267,7 @@ def test_api_watch_PUT_update(client, live_server):
 
     #live_server_setup(live_server)
     api_key = extract_api_key_from_UI(client)
-    time.sleep(1)
+
     # Create a watch
     set_original_response()
     test_url = url_for('test_endpoint', _external=True,
@@ -282,7 +283,6 @@ def test_api_watch_PUT_update(client, live_server):
 
     assert res.status_code == 201
 
-    time.sleep(1)
 
     # Get a listing, it will be the first one
     res = client.get(
@@ -297,6 +297,8 @@ def test_api_watch_PUT_update(client, live_server):
         url_for("edit_page", uuid=watch_uuid),
     )
     assert b"cookie: yum" in res.data, "'cookie: yum' found in 'headers' section"
+    assert b"One" in res.data, "Tag 'One' was found"
+    assert b"Two" in res.data, "Tag 'Two' was found"
 
     # HTTP PUT ( UPDATE an existing watch )
     res = client.put(
@@ -319,7 +321,8 @@ def test_api_watch_PUT_update(client, live_server):
     )
     assert b"new title" in res.data, "new title found in edit page"
     assert b"552" in res.data, "552 minutes found in edit page"
-    assert b"One, Two" in res.data, "Tag 'One, Two' was found"
+    assert b"One" in res.data, "Tag 'One' was found"
+    assert b"Two" in res.data, "Tag 'Two' was found"
     assert b"cookie: all eaten" in res.data, "'cookie: all eaten' found in 'headers' section"
 
     ######################################################

changedetectionio/tests/test_auth.py

@@ -24,7 +24,7 @@ def test_basic_auth(client, live_server):
     # Check form validation
     res = client.post(
         url_for("edit_page", uuid="first"),
-        data={"include_filters": "", "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
+        data={"include_filters": "", "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests"},
         follow_redirects=True
     )
     assert b"Updated watch." in res.data

changedetectionio/tests/test_block_while_text_present.py

@@ -2,7 +2,7 @@
 
 import time
 from flask import url_for
-from . util import live_server_setup
+from .util import live_server_setup, wait_for_all_checks
 from changedetectionio import html_tools
 
 def set_original_ignore_response():
@@ -61,7 +61,7 @@ def set_modified_response_minus_block_text():
 
 
 def test_check_block_changedetection_text_NOT_present(client, live_server):
-    sleep_time_for_fetch_thread = 3
+
     live_server_setup(live_server)
     # Use a mix of case in ZzZ to prove it works case-insensitive.
     ignore_text = "out of stoCk\r\nfoobar"
@@ -81,7 +81,7 @@ def test_check_block_changedetection_text_NOT_present(client, live_server):
     assert b"1 Imported" in res.data
 
     # Give the thread time to pick it up
-    time.sleep(sleep_time_for_fetch_thread)
+    wait_for_all_checks(client)
 
     # Goto the edit page, add our ignore text
     # Add our URL to the import page
@@ -96,7 +96,7 @@ def test_check_block_changedetection_text_NOT_present(client, live_server):
     assert b"Updated watch." in res.data
 
     # Give the thread time to pick it up
-    time.sleep(sleep_time_for_fetch_thread)
+    wait_for_all_checks(client)
     # Check it saved
     res = client.get(
         url_for("edit_page", uuid="first"),
@@ -107,7 +107,7 @@ def test_check_block_changedetection_text_NOT_present(client, live_server):
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
-    time.sleep(sleep_time_for_fetch_thread)
+    wait_for_all_checks(client)
 
     # It should report nothing found (no new 'unviewed' class)
     res = client.get(url_for("index"))
@@ -120,7 +120,7 @@ def test_check_block_changedetection_text_NOT_present(client, live_server):
     # Trigger a check
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
     # Give the thread time to pick it up
-    time.sleep(sleep_time_for_fetch_thread)
+    wait_for_all_checks(client)
 
     # It should report nothing found (no new 'unviewed' class)
     res = client.get(url_for("index"))
@@ -131,7 +131,7 @@ def test_check_block_changedetection_text_NOT_present(client, live_server):
     # Now we set a change where the text is gone, it should now trigger
     set_modified_response_minus_block_text()
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    time.sleep(sleep_time_for_fetch_thread)
+    wait_for_all_checks(client)
     res = client.get(url_for("index"))
     assert b'unviewed' in res.data
 

changedetectionio/tests/test_css_selector.py

@@ -96,7 +96,7 @@ def test_check_markup_include_filters_restriction(client, live_server):
     # Add our URL to the import page
     res = client.post(
         url_for("edit_page", uuid="first"),
-        data={"include_filters": include_filters, "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
+        data={"include_filters": include_filters, "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests"},
         follow_redirects=True
     )
     assert b"Updated watch." in res.data
@@ -157,7 +157,7 @@ def test_check_multiple_filters(client, live_server):
         url_for("edit_page", uuid="first"),
         data={"include_filters": include_filters,
               "url": test_url,
-              "tag": "",
+              "tags": "",
               "headers": "",
               'fetch_backend': "html_requests"},
         follow_redirects=True

changedetectionio/tests/test_element_removal.py

@@ -129,7 +129,7 @@ def test_element_removal_full(client, live_server):
         data={
             "subtractive_selectors": subtractive_selectors_data,
             "url": test_url,
-            "tag": "",
+            "tags": "",
             "headers": "",
             "fetch_backend": "html_requests",
         },

changedetectionio/tests/test_extract_regex.py

@@ -91,7 +91,7 @@ def test_check_filter_multiline(client, live_server):
         data={"include_filters": '',
               'extract_text': '/something.+?6 billion.+?lines/si',
               "url": test_url,
-              "tag": "",
+              "tags": "",
               "headers": "",
               'fetch_backend': "html_requests"
               },
@@ -146,7 +146,7 @@ def test_check_filter_and_regex_extract(client, live_server):
         data={"include_filters": include_filters,
               'extract_text': '\d+ online\r\n\d+ guests\r\n/somecase insensitive \d+/i\r\n/somecase insensitive (345\d)/i',
               "url": test_url,
-              "tag": "",
+              "tags": "",
               "headers": "",
               'fetch_backend': "html_requests"
               },

changedetectionio/tests/test_filter_exist_changes.py

@@ -56,7 +56,7 @@ def test_filter_doesnt_exist_then_exists_should_get_notification(client, live_se
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
         url_for("form_quick_watch_add"),
-        data={"url": test_url, "tag": 'cinema'},
+        data={"url": test_url, "tags": 'cinema'},
         follow_redirects=True
     )
     assert b"Watch added" in res.data
@@ -89,7 +89,7 @@ def test_filter_doesnt_exist_then_exists_should_get_notification(client, live_se
 
     notification_form_data.update({
         "url": test_url,
-        "tag": "my tag",
+        "tags": "my tag",
         "title": "my title",
         "headers": "",
         "include_filters": '.ticket-available',

changedetectionio/tests/test_filter_failure_notification.py

@@ -1,7 +1,7 @@
 import os
 import time
 from flask import url_for
-from .util import set_original_response, live_server_setup, extract_UUID_from_client
+from .util import set_original_response, live_server_setup, extract_UUID_from_client, wait_for_all_checks
 from changedetectionio.model import App
 
 
@@ -37,14 +37,14 @@ def run_filter_test(client, content_filter):
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
         url_for("form_quick_watch_add"),
-        data={"url": test_url, "tag": ''},
+        data={"url": test_url, "tags": ''},
         follow_redirects=True
     )
 
     assert b"Watch added" in res.data
 
     # Give the thread time to pick up the first version
-    time.sleep(3)
+    wait_for_all_checks(client)
 
     # Goto the edit page, add our ignore text
     # Add our URL to the import page
@@ -71,8 +71,8 @@ def run_filter_test(client, content_filter):
 
     notification_form_data.update({
         "url": test_url,
-        "tag": "my tag",
-        "title": "my title",
+        "tags": "my tag",
+        "title": "my title 123",
         "headers": "",
         "filter_failure_notification_send": 'y',
         "include_filters": content_filter,
@@ -85,43 +85,55 @@ def run_filter_test(client, content_filter):
     )
 
     assert b"Updated watch." in res.data
-    time.sleep(3)
+    wait_for_all_checks(client)
 
     # Now the notification should not exist, because we didnt reach the threshold
     assert not os.path.isfile("test-datastore/notification.txt")
 
-    for i in range(0, App._FILTER_FAILURE_THRESHOLD_ATTEMPTS_DEFAULT):
+    # -2 because we would have checked twice above (on adding and on edit)
+    for i in range(0, App._FILTER_FAILURE_THRESHOLD_ATTEMPTS_DEFAULT-2):
         res = client.get(url_for("form_watch_checknow"), follow_redirects=True)
-        time.sleep(3)
+        wait_for_all_checks(client)
+        assert not os.path.isfile("test-datastore/notification.txt"), f"test-datastore/notification.txt should not exist - Attempt {i}"
 
     # We should see something in the frontend
     assert b'Warning, no filters were found' in res.data
 
+    # One more check should trigger it (see -2 above)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    wait_for_all_checks(client)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    wait_for_all_checks(client)
     # Now it should exist and contain our "filter not found" alert
     assert os.path.isfile("test-datastore/notification.txt")
-    notification = False
+
     with open("test-datastore/notification.txt", 'r') as f:
         notification = f.read()
+
     assert 'CSS/xPath filter was not present in the page' in notification
     assert content_filter.replace('"', '\\"') in notification
 
-    # Remove it and prove that it doesnt trigger when not expected
+    # Remove it and prove that it doesn't trigger when not expected
+    # It should register a change, but no 'filter not found'
     os.unlink("test-datastore/notification.txt")
     set_response_with_filter()
 
+    # Try several times, it should NOT have 'filter not found'
     for i in range(0, App._FILTER_FAILURE_THRESHOLD_ATTEMPTS_DEFAULT):
         client.get(url_for("form_watch_checknow"), follow_redirects=True)
-        time.sleep(3)
+        wait_for_all_checks(client)
 
     # It should have sent a notification, but..
     assert os.path.isfile("test-datastore/notification.txt")
-    # but it should not contain the info about the failed filter
+    # but it should not contain the info about a failed filter (because there was none in this case)
     with open("test-datastore/notification.txt", 'r') as f:
         notification = f.read()
     assert not 'CSS/xPath filter was not present in the page' in notification
 
-    # Re #1247 - All tokens got replaced
+    # Re #1247 - All tokens got replaced correctly in the notification
+    res = client.get(url_for("index"))
     uuid = extract_UUID_from_client(client)
+    # UUID is correct, but notification contains tag uuid as UUIID wtf
     assert uuid in notification
 
     # cleanup for the next
@@ -137,7 +149,7 @@ def test_setup(live_server):
 
 def test_check_include_filters_failure_notification(client, live_server):
     set_original_response()
-    time.sleep(1)
+    wait_for_all_checks(client)
     run_filter_test(client, '#nope-doesnt-exist')
 
 def test_check_xpath_filter_failure_notification(client, live_server):

changedetectionio/tests/test_group.py

@@ -0,0 +1,322 @@
+#!/usr/bin/python3
+
+import time
+from flask import url_for
+from .util import live_server_setup, wait_for_all_checks, extract_rss_token_from_UI, get_UUID_for_tag_name, extract_UUID_from_client
+import os
+
+
+def test_setup(client, live_server):
+    live_server_setup(live_server)
+
+def set_original_response():
+    test_return_data = """<html>
+       <body>
+     Some initial text<br>
+     <p id="only-this">Should be only this</p>
+     <br>
+     <p id="not-this">And never this</p>     
+     </body>
+     </html>
+    """
+
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write(test_return_data)
+    return None
+
+def set_modified_response():
+    test_return_data = """<html>
+       <body>
+     Some initial text<br>
+     <p id="only-this">Should be REALLY only this</p>
+     <br>
+     <p id="not-this">And never this</p>     
+     </body>
+     </html>
+    """
+
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write(test_return_data)
+    return None
+
+def test_setup_group_tag(client, live_server):
+    #live_server_setup(live_server)
+    set_original_response()
+
+    # Add a tag with some config, import a tag and it should roughly work
+    res = client.post(
+        url_for("tags.form_tag_add"),
+        data={"name": "test-tag"},
+        follow_redirects=True
+    )
+    assert b"Tag added" in res.data
+    assert b"test-tag" in res.data
+
+    res = client.post(
+        url_for("tags.form_tag_edit_submit", uuid="first"),
+        data={"name": "test-tag",
+              "include_filters": '#only-this',
+              "subtractive_selectors": '#not-this'},
+        follow_redirects=True
+    )
+    assert b"Updated" in res.data
+    tag_uuid = get_UUID_for_tag_name(client, name="test-tag")
+    res = client.get(
+        url_for("tags.form_tag_edit", uuid="first")
+    )
+    assert b"#only-this" in res.data
+    assert b"#not-this" in res.data
+
+    # Tag should be setup and ready, now add a watch
+
+    test_url = url_for('test_endpoint', _external=True)
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url + "?first-imported=1 test-tag, extra-import-tag"},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+    res = client.get(url_for("index"))
+    assert b'import-tag' in res.data
+    assert b'extra-import-tag' in res.data
+
+    res = client.get(
+        url_for("tags.tags_overview_page"),
+        follow_redirects=True
+    )
+    assert b'import-tag' in res.data
+    assert b'extra-import-tag' in res.data
+
+    wait_for_all_checks(client)
+
+    res = client.get(url_for("index"))
+    assert b'Warning, no filters were found' not in res.data
+
+    res = client.get(
+        url_for("preview_page", uuid="first"),
+        follow_redirects=True
+    )
+    assert b'Should be only this' in res.data
+    assert b'And never this' not in res.data
+
+
+    # RSS Group tag filter
+    # An extra one that should be excluded
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url + "?should-be-excluded=1 some-tag"},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+    wait_for_all_checks(client)
+    set_modified_response()
+    res = client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    wait_for_all_checks(client)
+    rss_token = extract_rss_token_from_UI(client)
+    res = client.get(
+        url_for("rss", token=rss_token, tag="extra-import-tag", _external=True),
+        follow_redirects=True
+    )
+    assert b"should-be-excluded" not in res.data
+    assert res.status_code == 200
+    assert b"first-imported=1" in res.data
+    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    assert b'Deleted' in res.data
+
+def test_tag_import_singular(client, live_server):
+    #live_server_setup(live_server)
+
+    test_url = url_for('test_endpoint', _external=True)
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url + " test-tag, test-tag\r\n"+ test_url + "?x=1 test-tag, test-tag\r\n"},
+        follow_redirects=True
+    )
+    assert b"2 Imported" in res.data
+
+    res = client.get(
+        url_for("tags.tags_overview_page"),
+        follow_redirects=True
+    )
+    # Should be only 1 tag because they both had the same
+    assert res.data.count(b'test-tag') == 1
+    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    assert b'Deleted' in res.data
+
+def test_tag_add_in_ui(client, live_server):
+    #live_server_setup(live_server)
+#
+    res = client.post(
+        url_for("tags.form_tag_add"),
+        data={"name": "new-test-tag"},
+        follow_redirects=True
+    )
+    assert b"Tag added" in res.data
+    assert b"new-test-tag" in res.data
+
+    res = client.get(url_for("tags.delete_all"), follow_redirects=True)
+    assert b'All tags deleted' in res.data
+
+    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    assert b'Deleted' in res.data
+
+def test_group_tag_notification(client, live_server):
+    #live_server_setup(live_server)
+    set_original_response()
+
+    test_url = url_for('test_endpoint', _external=True)
+    res = client.post(
+        url_for("form_quick_watch_add"),
+        data={"url": test_url, "tags": 'test-tag, other-tag'},
+        follow_redirects=True
+    )
+
+    assert b"Watch added" in res.data
+
+    notification_url = url_for('test_notification_endpoint', _external=True).replace('http', 'json')
+    notification_form_data = {"notification_urls": notification_url,
+                              "notification_title": "New GROUP TAG ChangeDetection.io Notification - {{watch_url}}",
+                              "notification_body": "BASE URL: {{base_url}}\n"
+                                                   "Watch URL: {{watch_url}}\n"
+                                                   "Watch UUID: {{watch_uuid}}\n"
+                                                   "Watch title: {{watch_title}}\n"
+                                                   "Watch tag: {{watch_tag}}\n"
+                                                   "Preview: {{preview_url}}\n"
+                                                   "Diff URL: {{diff_url}}\n"
+                                                   "Snapshot: {{current_snapshot}}\n"
+                                                   "Diff: {{diff}}\n"
+                                                   "Diff Added: {{diff_added}}\n"
+                                                   "Diff Removed: {{diff_removed}}\n"
+                                                   "Diff Full: {{diff_full}}\n"
+                                                   ":-)",
+                              "notification_screenshot": True,
+                              "notification_format": "Text",
+                              "title": "test-tag"}
+
+    res = client.post(
+        url_for("tags.form_tag_edit_submit", uuid=get_UUID_for_tag_name(client, name="test-tag")),
+        data=notification_form_data,
+        follow_redirects=True
+    )
+    assert b"Updated" in res.data
+
+    wait_for_all_checks(client)
+
+    set_modified_response()
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    time.sleep(3)
+
+    assert os.path.isfile("test-datastore/notification.txt")
+
+    # Verify what was sent as a notification, this file should exist
+    with open("test-datastore/notification.txt", "r") as f:
+        notification_submission = f.read()
+    os.unlink("test-datastore/notification.txt")
+
+    # Did we see the URL that had a change, in the notification?
+    # Diff was correctly executed
+    assert test_url in notification_submission
+    assert ':-)' in notification_submission
+    assert "Diff Full: Some initial text" in notification_submission
+    assert "New GROUP TAG ChangeDetection.io" in notification_submission
+    assert "test-tag" in notification_submission
+    assert "other-tag" in notification_submission
+
+    #@todo Test that multiple notifications fired
+    #@todo Test that each of multiple notifications with different settings
+    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    assert b'Deleted' in res.data
+
+def test_limit_tag_ui(client, live_server):
+    #live_server_setup(live_server)
+
+    test_url = url_for('test_endpoint', _external=True)
+    urls=[]
+
+    for i in range(20):
+        urls.append(test_url+"?x="+str(i)+" test-tag")
+
+    for i in range(20):
+        urls.append(test_url+"?non-grouped="+str(i))
+
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": "\r\n".join(urls)},
+        follow_redirects=True
+    )
+
+    assert b"40 Imported" in res.data
+
+    res = client.get(url_for("index"))
+    assert b'test-tag' in res.data
+
+    # All should be here
+    assert res.data.count(b'processor-text_json_diff') == 40
+
+    tag_uuid = get_UUID_for_tag_name(client, name="test-tag")
+
+    res = client.get(url_for("index", tag=tag_uuid))
+
+    # Just a subset should be here
+    assert b'test-tag' in res.data
+    assert res.data.count(b'processor-text_json_diff') == 20
+    assert b"object at" not in res.data
+    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    assert b'Deleted' in res.data
+    res = client.get(url_for("tags.delete_all"), follow_redirects=True)
+    assert b'All tags deleted' in res.data
+def test_clone_tag_on_import(client, live_server):
+    #live_server_setup(live_server)
+    test_url = url_for('test_endpoint', _external=True)
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url + " test-tag, another-tag\r\n"},
+        follow_redirects=True
+    )
+
+    assert b"1 Imported" in res.data
+
+    res = client.get(url_for("index"))
+    assert b'test-tag' in res.data
+    assert b'another-tag' in res.data
+
+    watch_uuid = extract_UUID_from_client(client)
+    res = client.get(url_for("form_clone", uuid=watch_uuid), follow_redirects=True)
+
+    assert b'Cloned' in res.data
+    # 2 times plus the top link to tag
+    assert res.data.count(b'test-tag') == 3
+    assert res.data.count(b'another-tag') == 3
+    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    assert b'Deleted' in res.data
+
+def test_clone_tag_on_quickwatchform_add(client, live_server):
+    #live_server_setup(live_server)
+
+    test_url = url_for('test_endpoint', _external=True)
+
+    res = client.post(
+        url_for("form_quick_watch_add"),
+        data={"url": test_url, "tags": ' test-tag, another-tag      '},
+        follow_redirects=True
+    )
+
+    assert b"Watch added" in res.data
+
+    res = client.get(url_for("index"))
+    assert b'test-tag' in res.data
+    assert b'another-tag' in res.data
+
+    watch_uuid = extract_UUID_from_client(client)
+    res = client.get(url_for("form_clone", uuid=watch_uuid), follow_redirects=True)
+
+    assert b'Cloned' in res.data
+    # 2 times plus the top link to tag
+    assert res.data.count(b'test-tag') == 3
+    assert res.data.count(b'another-tag') == 3
+    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    assert b'Deleted' in res.data
+
+    res = client.get(url_for("tags.delete_all"), follow_redirects=True)
+    assert b'All tags deleted' in res.data

changedetectionio/tests/test_ignore_regex_text.py

@@ -15,11 +15,24 @@ def test_strip_regex_text_func():
     but sometimes we want to remove the lines.
     
     but 1 lines
+    skip 5 lines
+    really? yes man
+#/not this tries weirdly formed regex or just strings starting with /
+/not this
     but including 1234 lines
     igNORe-cAse text we dont want to keep    
     but not always."""
 
-    ignore_lines = ["sometimes", "/\s\d{2,3}\s/", "/ignore-case text/"]
+
+    ignore_lines = [
+        "sometimes",
+        "/\s\d{2,3}\s/",
+        "/ignore-case text/",
+        "really?",
+        "/skip \d lines/i",
+        "/not"
+    ]
+
 
     fetcher = fetch_site_status.perform_site_check(datastore=False)
     stripped_content = html_tools.strip_ignore_text(test_content, ignore_lines)
@@ -27,4 +40,10 @@ def test_strip_regex_text_func():
     assert b"but 1 lines" in stripped_content
     assert b"igNORe-cAse text" not in stripped_content
     assert b"but 1234 lines" not in stripped_content
+    assert b"really" not in stripped_content
+    assert b"not this" not in stripped_content
+
+    # Check line number reporting
+    stripped_content = html_tools.strip_ignore_text(test_content, ignore_lines, mode="line numbers")
+    assert stripped_content == [2, 5, 6, 7, 8, 10]
 

changedetectionio/tests/test_ignore_text.py

@@ -2,7 +2,7 @@
 
 import time
 from flask import url_for
-from . util import live_server_setup
+from .util import live_server_setup, wait_for_all_checks
 from changedetectionio import html_tools
 
 def test_setup(live_server):
@@ -84,7 +84,6 @@ def set_modified_ignore_response():
 
 
 def test_check_ignore_text_functionality(client, live_server):
-    sleep_time_for_fetch_thread = 3
 
     # Use a mix of case in ZzZ to prove it works case-insensitive.
     ignore_text = "XXXXX\r\nYYYYY\r\nzZzZZ\r\nnew ignore stuff"
@@ -103,7 +102,7 @@ def test_check_ignore_text_functionality(client, live_server):
     assert b"1 Imported" in res.data
 
     # Give the thread time to pick it up
-    time.sleep(sleep_time_for_fetch_thread)
+    wait_for_all_checks(client)
 
     # Goto the edit page, add our ignore text
     # Add our URL to the import page
@@ -124,7 +123,7 @@ def test_check_ignore_text_functionality(client, live_server):
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
-    time.sleep(sleep_time_for_fetch_thread)
+    wait_for_all_checks(client)
 
     # It should report nothing found (no new 'unviewed' class)
     res = client.get(url_for("index"))
@@ -137,7 +136,7 @@ def test_check_ignore_text_functionality(client, live_server):
     # Trigger a check
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
     # Give the thread time to pick it up
-    time.sleep(sleep_time_for_fetch_thread)
+    wait_for_all_checks(client)
 
     # It should report nothing found (no new 'unviewed' class)
     res = client.get(url_for("index"))
@@ -151,7 +150,7 @@ def test_check_ignore_text_functionality(client, live_server):
     # Just to be sure.. set a regular modified change..
     set_modified_original_ignore_response()
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    time.sleep(sleep_time_for_fetch_thread)
+    wait_for_all_checks(client)
 
     res = client.get(url_for("index"))
     assert b'unviewed' in res.data
@@ -167,7 +166,6 @@ def test_check_ignore_text_functionality(client, live_server):
     assert b'Deleted' in res.data
 
 def test_check_global_ignore_text_functionality(client, live_server):
-    sleep_time_for_fetch_thread = 3
 
     # Give the endpoint time to spin up
     time.sleep(1)
@@ -198,7 +196,7 @@ def test_check_global_ignore_text_functionality(client, live_server):
     assert b"1 Imported" in res.data
 
     # Give the thread time to pick it up
-    time.sleep(sleep_time_for_fetch_thread)
+    wait_for_all_checks(client)
 
 
     # Goto the edit page of the item, add our ignore text
@@ -220,7 +218,7 @@ def test_check_global_ignore_text_functionality(client, live_server):
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
-    time.sleep(sleep_time_for_fetch_thread)
+    wait_for_all_checks(client)
 
     # so that we are sure everything is viewed and in a known 'nothing changed' state
     res = client.get(url_for("diff_history_page", uuid="first"))
@@ -237,7 +235,7 @@ def test_check_global_ignore_text_functionality(client, live_server):
     # Trigger a check
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
     # Give the thread time to pick it up
-    time.sleep(sleep_time_for_fetch_thread)
+    wait_for_all_checks(client)
 
     # It should report nothing found (no new 'unviewed' class)
     res = client.get(url_for("index"))
@@ -247,7 +245,7 @@ def test_check_global_ignore_text_functionality(client, live_server):
     # Just to be sure.. set a regular modified change that will trigger it
     set_modified_original_ignore_response()
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    time.sleep(sleep_time_for_fetch_thread)
+    wait_for_all_checks(client)
     res = client.get(url_for("index"))
     assert b'unviewed' in res.data
 

changedetectionio/tests/test_ignorehighlighter.py

@@ -0,0 +1,57 @@
+#!/usr/bin/python3
+
+import time
+from flask import url_for
+from .util import live_server_setup, wait_for_all_checks
+from changedetectionio import html_tools
+from . util import  extract_UUID_from_client
+
+def set_original_ignore_response():
+    test_return_data = """<html>
+       <body>
+     Some initial text<br>
+     <p>Which is across multiple lines</p>
+     <br>
+     So let's see what happens.  <br>
+     <p>oh yeah 456</p>
+     </body>
+     </html>
+
+    """
+
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write(test_return_data)
+
+
+def test_highlight_ignore(client, live_server):
+    live_server_setup(live_server)
+    set_original_ignore_response()
+    test_url = url_for('test_endpoint', _external=True)
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+    # Give the thread time to pick it up
+    wait_for_all_checks(client)
+    uuid = extract_UUID_from_client(client)
+    # use the highlighter endpoint
+    res = client.post(
+        url_for("highlight_submit_ignore_url", uuid=uuid),
+        data={"mode": 'digit-regex', 'selection': 'oh yeah 123'},
+        follow_redirects=True
+    )
+
+    res = client.get(url_for("edit_page", uuid=uuid))
+
+    # should be a regex now
+    assert b'/oh\ yeah\ \d+/' in res.data
+
+    # Should return a link
+    assert b'href' in res.data
+
+    # And it should register in the preview page
+    res = client.get(url_for("preview_page", uuid=uuid))
+    assert b'<div class="ignored">oh yeah 456' in res.data

changedetectionio/tests/test_ignorestatuscode.py

@@ -2,7 +2,7 @@
 
 import time
 from flask import url_for
-from . util import live_server_setup
+from .util import live_server_setup, wait_for_all_checks
 
 
 def test_setup(live_server):
@@ -40,7 +40,7 @@ def set_some_changed_response():
 
 
 def test_normal_page_check_works_with_ignore_status_code(client, live_server):
-    sleep_time_for_fetch_thread = 3
+
 
     # Give the endpoint time to spin up
     time.sleep(1)
@@ -68,15 +68,15 @@ def test_normal_page_check_works_with_ignore_status_code(client, live_server):
     )
     assert b"1 Imported" in res.data
 
-    time.sleep(sleep_time_for_fetch_thread)
+    wait_for_all_checks(client)
 
     set_some_changed_response()
-    time.sleep(sleep_time_for_fetch_thread)
+    wait_for_all_checks(client)
     # Trigger a check
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
-    time.sleep(sleep_time_for_fetch_thread)
+    wait_for_all_checks(client)
 
     # It should report nothing found (no new 'unviewed' class)
     res = client.get(url_for("index"))
@@ -109,13 +109,13 @@ def test_403_page_check_works_with_ignore_status_code(client, live_server):
     # Add our URL to the import page
     res = client.post(
         url_for("edit_page", uuid="first"),
-        data={"ignore_status_codes": "y", "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
+        data={"ignore_status_codes": "y", "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests"},
         follow_redirects=True
     )
     assert b"Updated watch." in res.data
 
     # Give the thread time to pick it up
-    time.sleep(sleep_time_for_fetch_thread)
+    wait_for_all_checks(client)
 
     #  Make a change
     set_some_changed_response()
@@ -123,7 +123,7 @@ def test_403_page_check_works_with_ignore_status_code(client, live_server):
     # Trigger a check
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
     # Give the thread time to pick it up
-    time.sleep(sleep_time_for_fetch_thread)
+    wait_for_all_checks(client)
 
     # It should have 'unviewed' still
     # Because it should be looking at only that 'sametext' id

changedetectionio/tests/test_import.py

@@ -112,6 +112,7 @@ def test_import_distillio(client, live_server):
     # did the tags work?
     res = client.get( url_for("index"))
 
+    # check tags
     assert b"nice stuff" in res.data
     assert b"nerd-news" in res.data
 

changedetectionio/tests/test_jinja2.py

@@ -20,7 +20,7 @@ def test_jinja2_in_url_query(client, live_server):
                               "date={% now 'Europe/Berlin', '%Y' %}.{% now 'Europe/Berlin', '%m' %}.{% now 'Europe/Berlin', '%d' %}", )
     res = client.post(
         url_for("form_quick_watch_add"),
-        data={"url": full_url, "tag": "test"},
+        data={"url": full_url, "tags": "test"},
         follow_redirects=True
     )
     assert b"Watch added" in res.data

changedetectionio/tests/test_jsonpath_jq_selector.py

@@ -3,7 +3,7 @@
 
 import time
 from flask import url_for, escape
-from . util import live_server_setup
+from . util import live_server_setup, wait_for_all_checks
 import pytest
 jq_support = True
 
@@ -64,6 +64,24 @@ and it can also be repeated
         with pytest.raises(html_tools.JSONNotFound) as e_info:
             html_tools.extract_json_as_string('COMPLETE GIBBERISH, NO JSON!', "jq:.id")
 
+
+def test_unittest_inline_extract_body():
+    content = """
+    <html>
+        <head></head>
+        <body>
+            <pre style="word-wrap: break-word; white-space: pre-wrap;">
+                {"testKey": 42}
+            </pre>
+        </body>
+    </html>
+    """
+    from .. import html_tools
+
+    # See that we can find the second <script> one, which is not broken, and matches our filter
+    text = html_tools.extract_json_as_string(content, "json:$.testKey")
+    assert text == '42'
+
 def set_original_ext_response():
     data = """
         [
@@ -190,7 +208,7 @@ def test_check_json_without_filter(client, live_server):
     )
 
     # Give the thread time to pick it up
-    time.sleep(3)
+    wait_for_all_checks(client)
 
     res = client.get(
         url_for("preview_page", uuid="first"),
@@ -220,7 +238,7 @@ def check_json_filter(json_filter, client, live_server):
     assert b"1 Imported" in res.data
 
     # Give the thread time to pick it up
-    time.sleep(3)
+    wait_for_all_checks(client)
 
     # Goto the edit page, add our ignore text
     # Add our URL to the import page
@@ -228,7 +246,7 @@ def check_json_filter(json_filter, client, live_server):
         url_for("edit_page", uuid="first"),
         data={"include_filters": json_filter,
               "url": test_url,
-              "tag": "",
+              "tags": "",
               "headers": "",
               "fetch_backend": "html_requests"
               },
@@ -243,14 +261,14 @@ def check_json_filter(json_filter, client, live_server):
     assert bytes(escape(json_filter).encode('utf-8')) in res.data
 
     # Give the thread time to pick it up
-    time.sleep(3)
+    wait_for_all_checks(client)
     #  Make a change
     set_modified_response()
 
     # Trigger a check
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
     # Give the thread time to pick it up
-    time.sleep(4)
+    wait_for_all_checks(client)
 
     # It should have 'unviewed' still
     res = client.get(url_for("index"))
@@ -288,14 +306,14 @@ def check_json_filter_bool_val(json_filter, client, live_server):
     )
     assert b"1 Imported" in res.data
 
-    time.sleep(3)
+    wait_for_all_checks(client)
     # Goto the edit page, add our ignore text
     # Add our URL to the import page
     res = client.post(
         url_for("edit_page", uuid="first"),
         data={"include_filters": json_filter,
               "url": test_url,
-              "tag": "",
+              "tags": "",
               "headers": "",
               "fetch_backend": "html_requests"
               },
@@ -304,14 +322,14 @@ def check_json_filter_bool_val(json_filter, client, live_server):
     assert b"Updated watch." in res.data
 
     # Give the thread time to pick it up
-    time.sleep(3)
+    wait_for_all_checks(client)
     #  Make a change
     set_modified_response()
 
     # Trigger a check
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
     # Give the thread time to pick it up
-    time.sleep(3)
+    wait_for_all_checks(client)
 
     res = client.get(url_for("diff_history_page", uuid="first"))
     # But the change should be there, tho its hard to test the change was detected because it will show old and new versions
@@ -348,7 +366,7 @@ def check_json_ext_filter(json_filter, client, live_server):
     assert b"1 Imported" in res.data
 
     # Give the thread time to pick it up
-    time.sleep(3)
+    wait_for_all_checks(client)
 
     # Goto the edit page, add our ignore text
     # Add our URL to the import page
@@ -356,7 +374,7 @@ def check_json_ext_filter(json_filter, client, live_server):
         url_for("edit_page", uuid="first"),
         data={"include_filters": json_filter,
               "url": test_url,
-              "tag": "",
+              "tags": "",
               "headers": "",
               "fetch_backend": "html_requests"
               },
@@ -371,14 +389,14 @@ def check_json_ext_filter(json_filter, client, live_server):
     assert bytes(escape(json_filter).encode('utf-8')) in res.data
 
     # Give the thread time to pick it up
-    time.sleep(3)
+    wait_for_all_checks(client)
     #  Make a change
     set_modified_ext_response()
 
     # Trigger a check
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
     # Give the thread time to pick it up
-    time.sleep(4)
+    wait_for_all_checks(client)
 
     # It should have 'unviewed'
     res = client.get(url_for("index"))
@@ -410,14 +428,14 @@ def test_ignore_json_order(client, live_server):
     )
     assert b"1 Imported" in res.data
 
-    time.sleep(2)
+    wait_for_all_checks(client)
 
     with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write('{"world" : 123, "hello": 123}')
 
     # Trigger a check
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    time.sleep(2)
+    wait_for_all_checks(client)
 
     res = client.get(url_for("index"))
     assert b'unviewed' not in res.data
@@ -428,7 +446,7 @@ def test_ignore_json_order(client, live_server):
 
     # Trigger a check
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    time.sleep(2)
+    wait_for_all_checks(client)
 
     res = client.get(url_for("index"))
     assert b'unviewed' in res.data
@@ -436,6 +454,37 @@ def test_ignore_json_order(client, live_server):
     res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
 
+def test_correct_header_detect(client, live_server):
+    # Like in https://github.com/dgtlmoon/changedetection.io/pull/1593
+    # Specify extra html that JSON is sometimes wrapped in - when using Browserless/Puppeteer etc
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write('<html><body>{"hello" : 123, "world": 123}')
+
+    # Add our URL to the import page
+    # Check weird casing is cleaned up and detected also
+    test_url = url_for('test_endpoint', content_type="aPPlication/JSon", uppercase_headers=True, _external=True)
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+    wait_for_all_checks(client)
+    res = client.get(url_for("index"))
+
+    # Fixed in #1593
+    assert b'No parsable JSON found in this document' not in res.data
+
+    res = client.get(
+        url_for("preview_page", uuid="first"),
+        follow_redirects=True
+    )
+    assert b'&#34;world&#34;:' in res.data
+    assert res.data.count(b'{') >= 2
+
+    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    assert b'Deleted' in res.data
+
 def test_check_jsonpath_ext_filter(client, live_server):
     check_json_ext_filter('json:$[?(@.status==Sold)]', client, live_server)
 

changedetectionio/tests/test_notification.py

@@ -3,7 +3,8 @@ import os
 import time
 import re
 from flask import url_for
-from . util import set_original_response, set_modified_response, set_more_modified_response, live_server_setup
+from .util import set_original_response, set_modified_response, set_more_modified_response, live_server_setup, wait_for_all_checks, \
+    set_longer_modified_response
 from . util import  extract_UUID_from_client
 import logging
 import base64
@@ -21,11 +22,9 @@ def test_setup(live_server):
 # Hard to just add more live server URLs when one test is already running (I think)
 # So we add our test here (was in a different file)
 def test_check_notification(client, live_server):
+    #live_server_setup(live_server)
     set_original_response()
 
-    # Give the endpoint time to spin up
-    time.sleep(3)
-
     # Re 360 - new install should have defaults set
     res = client.get(url_for("settings_page"))
     notification_url = url_for('test_notification_endpoint', _external=True).replace('http', 'json')
@@ -62,13 +61,13 @@ def test_check_notification(client, live_server):
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
         url_for("form_quick_watch_add"),
-        data={"url": test_url, "tag": ''},
+        data={"url": test_url, "tags": ''},
         follow_redirects=True
     )
     assert b"Watch added" in res.data
 
     # Give the thread time to pick up the first version
-    time.sleep(3)
+    wait_for_all_checks(client)
 
     # We write the PNG to disk, but a JPEG should appear in the notification
     # Write the last screenshot png
@@ -105,7 +104,7 @@ def test_check_notification(client, live_server):
 
     notification_form_data.update({
         "url": test_url,
-        "tag": "my tag",
+        "tags": "my tag, my second tag",
         "title": "my title",
         "headers": "",
         "fetch_backend": "html_requests"})
@@ -128,7 +127,7 @@ def test_check_notification(client, live_server):
 
 
     ## Now recheck, and it should have sent the notification
-    time.sleep(3)
+    wait_for_all_checks(client)
     set_modified_response()
 
     # Trigger a check
@@ -141,8 +140,7 @@ def test_check_notification(client, live_server):
 
     # Did we see the URL that had a change, in the notification?
     # Diff was correctly executed
-    assert test_url in notification_submission
-    assert ':-)' in notification_submission
+
     assert "Diff Full: Some initial text" in notification_submission
     assert "Diff: (changed) Which is across multiple lines" in notification_submission
     assert "(into) which has this one new line" in notification_submission
@@ -150,12 +148,13 @@ def test_check_notification(client, live_server):
     assert "b'" not in notification_submission
     assert re.search('Watch UUID: [0-9a-f]{8}(-[0-9a-f]{4}){3}-[0-9a-f]{12}', notification_submission, re.IGNORECASE)
     assert "Watch title: my title" in notification_submission
-    assert "Watch tag: my tag" in notification_submission
+    assert "Watch tag: my tag, my second tag" in notification_submission
     assert "diff/" in notification_submission
     assert "preview/" in notification_submission
     assert ":-)" in notification_submission
     assert "New ChangeDetection.io Notification - {}".format(test_url) in notification_submission
-
+    assert test_url in notification_submission
+    assert ':-)' in notification_submission
     # Check the attachment was added, and that it is a JPEG from the original PNG
     notification_submission_object = json.loads(notification_submission)
     # We keep PNG screenshots for now
@@ -193,11 +192,11 @@ def test_check_notification(client, live_server):
 
     # Trigger a check
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    time.sleep(1)
+    wait_for_all_checks(client)
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    time.sleep(1)
+    wait_for_all_checks(client)
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    time.sleep(1)
+    wait_for_all_checks(client)
     assert os.path.exists("test-datastore/notification.txt") == False
 
     res = client.get(url_for("notification_logs"))
@@ -209,7 +208,7 @@ def test_check_notification(client, live_server):
         url_for("edit_page", uuid="first"),
         data={
         "url": test_url,
-        "tag": "my tag",
+        "tags": "my tag",
         "title": "my title",
         "notification_urls": '',
         "notification_title": '',
@@ -243,7 +242,7 @@ def test_notification_validation(client, live_server):
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
         url_for("form_quick_watch_add"),
-        data={"url": test_url, "tag": 'nice one'},
+        data={"url": test_url, "tags": 'nice one'},
         follow_redirects=True
     )
 
@@ -274,7 +273,7 @@ def test_notification_validation(client, live_server):
 
 
 def test_notification_custom_endpoint_and_jinja2(client, live_server):
-    time.sleep(1)
+    #live_server_setup(live_server)
 
     # test_endpoint - that sends the contents of a file
     # test_notification_endpoint - that takes a POST and writes it to file (test-datastore/notification.txt)
@@ -285,12 +284,14 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server):
 
     res = client.post(
         url_for("settings_page"),
-        data={"application-notification_title": "New ChangeDetection.io Notification - {{ watch_url }}",
-              "application-notification_body": '{ "url" : "{{ watch_url }}", "secret": 444 }',
-              # https://github.com/caronc/apprise/wiki/Notify_Custom_JSON#get-parameter-manipulation
-              "application-notification_urls": test_notification_url,
+        data={
+              "application-fetch_backend": "html_requests",
               "application-minutes_between_check": 180,
-              "application-fetch_backend": "html_requests"
+              "application-notification_body": '{ "url" : "{{ watch_url }}", "secret": 444 }',
+              "application-notification_format": default_notification_format,
+              "application-notification_urls": test_notification_url,
+              # https://github.com/caronc/apprise/wiki/Notify_Custom_JSON#get-parameter-manipulation
+              "application-notification_title": "New ChangeDetection.io Notification - {{ watch_url }}",
               },
         follow_redirects=True
     )
@@ -303,21 +304,20 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server):
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
         url_for("form_quick_watch_add"),
-        data={"url": test_url, "tag": 'nice one'},
+        data={"url": test_url, "tags": 'nice one'},
         follow_redirects=True
     )
 
     assert b"Watch added" in res.data
 
-    time.sleep(2)
+    wait_for_all_checks(client)
     set_modified_response()
 
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
     time.sleep(2)
 
-
     with open("test-datastore/notification.txt", 'r') as f:
-        x=f.read()
+        x = f.read()
         j = json.loads(x)
         assert j['url'].startswith('http://localhost')
         assert j['secret'] == 444
@@ -328,5 +328,9 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server):
         notification_url = f.read()
         assert 'xxx=http' in notification_url
 
-    os.unlink("test-datastore/notification-url.txt")
+    # Should always be automatically detected as JSON content type even when we set it as 'Text' (default)
+    assert os.path.isfile("test-datastore/notification-content-type.txt")
+    with open("test-datastore/notification-content-type.txt", 'r') as f:
+        assert 'application/json' in f.read()
 
+    os.unlink("test-datastore/notification-url.txt")

changedetectionio/tests/test_notification_errors.py

@@ -17,7 +17,7 @@ def test_check_notification_error_handling(client, live_server):
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
         url_for("form_quick_watch_add"),
-        data={"url": test_url, "tag": ''},
+        data={"url": test_url, "tags": ''},
         follow_redirects=True
     )
     assert b"Watch added" in res.data
@@ -32,7 +32,7 @@ def test_check_notification_error_handling(client, live_server):
               "notification_body": "xxxxx",
               "notification_format": "Text",
               "url": test_url,
-              "tag": "",
+              "tags": "",
               "title": "",
               "headers": "",
               "time_between_check-minutes": "180",

changedetectionio/tests/test_request.py

@@ -1,7 +1,8 @@
 import json
+import os
 import time
 from flask import url_for
-from . util import set_original_response, set_modified_response, live_server_setup
+from . util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks, extract_UUID_from_client
 
 def test_setup(live_server):
     live_server_setup(live_server)
@@ -9,8 +10,12 @@ def test_setup(live_server):
 # Hard to just add more live server URLs when one test is already running (I think)
 # So we add our test here (was in a different file)
 def test_headers_in_request(client, live_server):
+    #live_server_setup(live_server)
     # Add our URL to the import page
     test_url = url_for('test_headers', _external=True)
+    if os.getenv('PLAYWRIGHT_DRIVER_URL'):
+        # Because its no longer calling back to localhost but from browserless, set in test-only.yml
+        test_url = test_url.replace('localhost', 'changedet')
 
     # Add the test URL twice, we will check
     res = client.post(
@@ -20,7 +25,7 @@ def test_headers_in_request(client, live_server):
     )
     assert b"1 Imported" in res.data
 
-    time.sleep(1)
+    wait_for_all_checks(client)
 
     res = client.post(
         url_for("import_page"),
@@ -29,7 +34,7 @@ def test_headers_in_request(client, live_server):
     )
     assert b"1 Imported" in res.data
 
-    time.sleep(3)
+    wait_for_all_checks(client)
     cookie_header = '_ga=GA1.2.1022228332; cookie-preferences=analytics:accepted;'
 
 
@@ -38,8 +43,8 @@ def test_headers_in_request(client, live_server):
         url_for("edit_page", uuid="first"),
         data={
               "url": test_url,
-              "tag": "",
-              "fetch_backend": "html_requests",
+              "tags": "",
+              "fetch_backend": 'html_webdriver' if os.getenv('PLAYWRIGHT_DRIVER_URL') else 'html_requests',
               "headers": "xxx:ooo\ncool:yeah\r\ncookie:"+cookie_header},
         follow_redirects=True
     )
@@ -47,7 +52,7 @@ def test_headers_in_request(client, live_server):
 
 
     # Give the thread time to pick up the first version
-    time.sleep(5)
+    wait_for_all_checks(client)
 
     # The service should echo back the request headers
     res = client.get(
@@ -63,7 +68,7 @@ def test_headers_in_request(client, live_server):
     from html import escape
     assert escape(cookie_header).encode('utf-8') in res.data
 
-    time.sleep(5)
+    wait_for_all_checks(client)
 
     # Re #137 -  Examine the JSON index file, it should have only one set of headers entered
     watches_with_headers = 0
@@ -79,6 +84,9 @@ def test_headers_in_request(client, live_server):
 def test_body_in_request(client, live_server):
     # Add our URL to the import page
     test_url = url_for('test_body', _external=True)
+    if os.getenv('PLAYWRIGHT_DRIVER_URL'):
+        # Because its no longer calling back to localhost but from browserless, set in test-only.yml
+        test_url = test_url.replace('localhost', 'cdio')
 
     res = client.post(
         url_for("import_page"),
@@ -87,14 +95,14 @@ def test_body_in_request(client, live_server):
     )
     assert b"1 Imported" in res.data
 
-    time.sleep(3)
+    wait_for_all_checks(client)
 
     # add the first 'version'
     res = client.post(
         url_for("edit_page", uuid="first"),
         data={
               "url": test_url,
-              "tag": "",
+              "tags": "",
               "method": "POST",
               "fetch_backend": "html_requests",
               "body": "something something"},
@@ -102,7 +110,7 @@ def test_body_in_request(client, live_server):
     )
     assert b"Updated watch." in res.data
 
-    time.sleep(3)
+    wait_for_all_checks(client)
 
     # Now the change which should trigger a change
     body_value = 'Test Body Value'
@@ -110,7 +118,7 @@ def test_body_in_request(client, live_server):
         url_for("edit_page", uuid="first"),
         data={
               "url": test_url,
-              "tag": "",
+              "tags": "",
               "method": "POST",
               "fetch_backend": "html_requests",
               "body": body_value},
@@ -118,7 +126,7 @@ def test_body_in_request(client, live_server):
     )
     assert b"Updated watch." in res.data
 
-    time.sleep(3)
+    wait_for_all_checks(client)
 
     # The service should echo back the body
     res = client.get(
@@ -155,7 +163,7 @@ def test_body_in_request(client, live_server):
         url_for("edit_page", uuid="first"),
         data={
               "url": test_url,
-              "tag": "",
+              "tags": "",
               "method": "GET",
               "fetch_backend": "html_requests",
               "body": "invalid"},
@@ -167,6 +175,9 @@ def test_body_in_request(client, live_server):
 def test_method_in_request(client, live_server):
     # Add our URL to the import page
     test_url = url_for('test_method', _external=True)
+    if os.getenv('PLAYWRIGHT_DRIVER_URL'):
+        # Because its no longer calling back to localhost but from browserless, set in test-only.yml
+        test_url = test_url.replace('localhost', 'cdio')
 
     # Add the test URL twice, we will check
     res = client.post(
@@ -176,7 +187,7 @@ def test_method_in_request(client, live_server):
     )
     assert b"1 Imported" in res.data
 
-    time.sleep(2)
+    wait_for_all_checks(client)
     res = client.post(
         url_for("import_page"),
         data={"urls": test_url},
@@ -184,14 +195,14 @@ def test_method_in_request(client, live_server):
     )
     assert b"1 Imported" in res.data
 
-    time.sleep(2)
+    wait_for_all_checks(client)
 
     # Attempt to add a method which is not valid
     res = client.post(
         url_for("edit_page", uuid="first"),
         data={
             "url": test_url,
-            "tag": "",
+            "tags": "",
             "fetch_backend": "html_requests",
             "method": "invalid"},
         follow_redirects=True
@@ -203,7 +214,7 @@ def test_method_in_request(client, live_server):
         url_for("edit_page", uuid="first"),
         data={
             "url": test_url,
-            "tag": "",
+            "tags": "",
             "fetch_backend": "html_requests",
             "method": "PATCH"},
         follow_redirects=True
@@ -211,7 +222,7 @@ def test_method_in_request(client, live_server):
     assert b"Updated watch." in res.data
 
     # Give the thread time to pick up the first version
-    time.sleep(2)
+    wait_for_all_checks(client)
 
     # The service should echo back the request verb
     res = client.get(
@@ -222,7 +233,7 @@ def test_method_in_request(client, live_server):
     # The test call service will return the verb as the body
     assert b"PATCH" in res.data
 
-    time.sleep(2)
+    wait_for_all_checks(client)
 
     watches_with_method = 0
     with open('test-datastore/url-watches.json') as f:
@@ -234,3 +245,76 @@ def test_method_in_request(client, live_server):
     # Should be only one with method set to PATCH
     assert watches_with_method == 1
 
+    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    assert b'Deleted' in res.data
+
+def test_headers_textfile_in_request(client, live_server):
+    #live_server_setup(live_server)
+    # Add our URL to the import page
+    test_url = url_for('test_headers', _external=True)
+    if os.getenv('PLAYWRIGHT_DRIVER_URL'):
+        # Because its no longer calling back to localhost but from browserless, set in test-only.yml
+        test_url = test_url.replace('localhost', 'cdio')
+
+    print ("TEST URL IS ",test_url)
+    # Add the test URL twice, we will check
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+    wait_for_all_checks(client)
+
+
+    # Add some headers to a request
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={
+              "url": test_url,
+              "tags": "testtag",
+              "fetch_backend": 'html_webdriver' if os.getenv('PLAYWRIGHT_DRIVER_URL') else 'html_requests',
+              "headers": "xxx:ooo\ncool:yeah\r\n"},
+        follow_redirects=True
+    )
+    assert b"Updated watch." in res.data
+    wait_for_all_checks(client)
+
+    with open('test-datastore/headers-testtag.txt', 'w') as f:
+        f.write("tag-header: test")
+
+    with open('test-datastore/headers.txt', 'w') as f:
+        f.write("global-header: nice\r\nnext-global-header: nice")
+
+    with open('test-datastore/'+extract_UUID_from_client(client)+'/headers.txt', 'w') as f:
+        f.write("watch-header: nice")
+
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    wait_for_all_checks(client)
+
+    res = client.get(url_for("edit_page", uuid="first"))
+    assert b"Extra headers file found and will be added to this watch" in res.data
+
+    # Not needed anymore
+    os.unlink('test-datastore/headers.txt')
+    os.unlink('test-datastore/headers-testtag.txt')
+    os.unlink('test-datastore/'+extract_UUID_from_client(client)+'/headers.txt')
+    # The service should echo back the request verb
+    res = client.get(
+        url_for("preview_page", uuid="first"),
+        follow_redirects=True
+    )
+
+    assert b"Global-Header:nice" in res.data
+    assert b"Next-Global-Header:nice" in res.data
+    assert b"Xxx:ooo" in res.data
+    assert b"Watch-Header:nice" in res.data
+    assert b"Tag-Header:test" in res.data
+
+
+    #unlink headers.txt on start/stop
+    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    assert b'Deleted' in res.data

changedetectionio/tests/test_search.py

@@ -0,0 +1,74 @@
+from flask import url_for
+from .util import set_original_response, set_modified_response, live_server_setup
+import time
+
+def test_setup(live_server):
+    live_server_setup(live_server)
+
+def test_basic_search(client, live_server):
+    #live_server_setup(live_server)
+
+    urls = ['https://localhost:12300?first-result=1',
+            'https://localhost:5000?second-result=1'
+            ]
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": "\r\n".join(urls)},
+        follow_redirects=True
+    )
+
+    assert b"2 Imported" in res.data
+
+    # By URL
+    res = client.get(url_for("index") + "?q=first-res")
+    assert urls[0].encode('utf-8') in res.data
+    assert urls[1].encode('utf-8') not in res.data
+
+    # By Title
+
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={"title": "xxx-title", "url": urls[0], "tags": "", "headers": "", 'fetch_backend': "html_requests"},
+        follow_redirects=True
+    )
+    assert b"Updated watch." in res.data
+
+    res = client.get(url_for("index") + "?q=xxx-title")
+    assert urls[0].encode('utf-8') in res.data
+    assert urls[1].encode('utf-8') not in res.data
+
+
+def test_search_in_tag_limit(client, live_server):
+    #live_server_setup(live_server)
+
+    urls = ['https://localhost:12300?first-result=1 tag-one',
+            'https://localhost:5000?second-result=1 tag-two'
+            ]
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": "\r\n".join(urls)},
+        follow_redirects=True
+    )
+
+    assert b"2 Imported" in res.data
+
+    # By URL
+
+    res = client.get(url_for("index") + "?q=first-res")
+    # Split because of the import tag separation
+    assert urls[0].split(' ')[0].encode('utf-8') in res.data, urls[0].encode('utf-8')
+    assert urls[1].split(' ')[0].encode('utf-8') not in res.data, urls[0].encode('utf-8')
+
+    # By Title
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={"title": "xxx-title", "url": urls[0].split(' ')[0], "tags": urls[0].split(' ')[1], "headers": "",
+              'fetch_backend': "html_requests"},
+        follow_redirects=True
+    )
+    assert b"Updated watch." in res.data
+
+    res = client.get(url_for("index") + "?q=xxx-title")
+    assert urls[0].split(' ')[0].encode('utf-8') in res.data, urls[0].encode('utf-8')
+    assert urls[1].split(' ')[0].encode('utf-8') not in res.data, urls[0].encode('utf-8')
+

changedetectionio/tests/test_security.py

@@ -18,7 +18,7 @@ def test_bad_access(client, live_server):
         url_for("edit_page", uuid="first"),
         data={
               "url": 'javascript:alert(document.domain)',
-              "tag": "",
+              "tags": "",
               "method": "GET",
               "fetch_backend": "html_requests",
               "body": ""},
@@ -29,7 +29,7 @@ def test_bad_access(client, live_server):
 
     res = client.post(
         url_for("form_quick_watch_add"),
-        data={"url": '            javascript:alert(123)', "tag": ''},
+        data={"url": '            javascript:alert(123)', "tags": ''},
         follow_redirects=True
     )
 
@@ -37,7 +37,7 @@ def test_bad_access(client, live_server):
 
     res = client.post(
         url_for("form_quick_watch_add"),
-        data={"url": '%20%20%20javascript:alert(123)%20%20', "tag": ''},
+        data={"url": '%20%20%20javascript:alert(123)%20%20', "tags": ''},
         follow_redirects=True
     )
 
@@ -46,7 +46,7 @@ def test_bad_access(client, live_server):
 
     res = client.post(
         url_for("form_quick_watch_add"),
-        data={"url": ' source:javascript:alert(document.domain)', "tag": ''},
+        data={"url": ' source:javascript:alert(document.domain)', "tags": ''},
         follow_redirects=True
     )
 
@@ -56,7 +56,7 @@ def test_bad_access(client, live_server):
 
     client.post(
         url_for("form_quick_watch_add"),
-        data={"url": 'file:///tasty/disk/drive', "tag": ''},
+        data={"url": 'file:///tasty/disk/drive', "tags": ''},
         follow_redirects=True
     )
     time.sleep(1)

changedetectionio/tests/test_share_watch.py

@@ -29,7 +29,7 @@ def test_share_watch(client, live_server):
     # Add our URL to the import page
     res = client.post(
         url_for("edit_page", uuid="first"),
-        data={"include_filters": include_filters, "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
+        data={"include_filters": include_filters, "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests"},
         follow_redirects=True
     )
     assert b"Updated watch." in res.data

changedetectionio/tests/test_source.py

@@ -3,7 +3,7 @@
 import time
 from flask import url_for
 from urllib.request import urlopen
-from .util import set_original_response, set_modified_response, live_server_setup
+from .util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks
 
 sleep_time_for_fetch_thread = 3
 
@@ -42,7 +42,7 @@ def test_check_basic_change_detection_functionality_source(client, live_server):
     res = client.get(url_for("form_watch_checknow"), follow_redirects=True)
     assert b'1 watches queued for rechecking.' in res.data
 
-    time.sleep(5)
+    wait_for_all_checks(client)
 
     # Now something should be ready, indicated by having a 'unviewed' class
     res = client.get(url_for("index"))
@@ -60,7 +60,7 @@ def test_check_basic_change_detection_functionality_source(client, live_server):
 # `subtractive_selectors` should still work in `source:` type requests
 def test_check_ignore_elements(client, live_server):
     set_original_response()
-    time.sleep(2)
+    time.sleep(1)
     test_url = 'source:'+url_for('test_endpoint', _external=True)
     # Add our URL to the import page
     res = client.post(
@@ -71,14 +71,14 @@ def test_check_ignore_elements(client, live_server):
 
     assert b"1 Imported" in res.data
 
-    time.sleep(sleep_time_for_fetch_thread)
+    wait_for_all_checks(client)
 
     #####################
     # We want <span> and <p> ONLY, but ignore span with .foobar-detection
 
     client.post(
         url_for("edit_page", uuid="first"),
-        data={"include_filters": 'span,p', "url": test_url, "tag": "", "subtractive_selectors": ".foobar-detection", 'fetch_backend': "html_requests"},
+        data={"include_filters": 'span,p', "url": test_url, "tags": "", "subtractive_selectors": ".foobar-detection", 'fetch_backend': "html_requests"},
         follow_redirects=True
     )
 

changedetectionio/tests/test_watch_fields_storage.py

@@ -26,7 +26,7 @@ def test_check_watch_field_storage(client, live_server):
                "title" : "My title",
                "ignore_text" : "ignore this",
                "url": test_url,
-               "tag": "woohoo",
+               "tags": "woohoo",
                "headers": "curl:foo",
                'fetch_backend': "html_requests"
                },

changedetectionio/tests/test_xpath_selector.py

@@ -89,7 +89,7 @@ def test_check_xpath_filter_utf8(client, live_server):
     time.sleep(1)
     res = client.post(
         url_for("edit_page", uuid="first"),
-        data={"include_filters": filter, "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
+        data={"include_filters": filter, "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests"},
         follow_redirects=True
     )
     assert b"Updated watch." in res.data
@@ -143,7 +143,7 @@ def test_check_xpath_text_function_utf8(client, live_server):
     time.sleep(1)
     res = client.post(
         url_for("edit_page", uuid="first"),
-        data={"include_filters": filter, "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
+        data={"include_filters": filter, "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests"},
         follow_redirects=True
     )
     assert b"Updated watch." in res.data
@@ -189,7 +189,7 @@ def test_check_markup_xpath_filter_restriction(client, live_server):
     # Add our URL to the import page
     res = client.post(
         url_for("edit_page", uuid="first"),
-        data={"include_filters": xpath_filter, "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
+        data={"include_filters": xpath_filter, "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests"},
         follow_redirects=True
     )
     assert b"Updated watch." in res.data
@@ -231,7 +231,7 @@ def test_xpath_validation(client, live_server):
 
     res = client.post(
         url_for("edit_page", uuid="first"),
-        data={"include_filters": "/something horrible", "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
+        data={"include_filters": "/something horrible", "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests"},
         follow_redirects=True
     )
     assert b"is not a valid XPath expression" in res.data
@@ -261,7 +261,7 @@ def test_check_with_prefix_include_filters(client, live_server):
 
     res = client.post(
         url_for("edit_page", uuid="first"),
-        data={"include_filters":  "xpath://*[contains(@class, 'sametext')]", "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
+        data={"include_filters":  "xpath://*[contains(@class, 'sametext')]", "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests"},
         follow_redirects=True
     )
 

changedetectionio/tests/util.py

@@ -38,7 +38,25 @@ def set_modified_response():
         f.write(test_return_data)
 
     return None
+def set_longer_modified_response():
+    test_return_data = """<html>
+    <head><title>modified head title</title></head>
+    <body>
+     Some initial text<br>
+     <p>which has this one new line</p>
+     <br>
+     So let's see what happens.  <br>
+     So let's see what happens.  <br>
+      So let's see what happens.  <br>
+     So let's see what happens.  <br>           
+     </body>
+     </html>
+    """
 
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write(test_return_data)
+
+    return None
 def set_more_modified_response():
     test_return_data = """<html>
     <head><title>modified head title</title></head>
@@ -70,6 +88,16 @@ def extract_api_key_from_UI(client):
     api_key = m.group(1)
     return api_key.strip()
 
+
+# kinda funky, but works for now
+def get_UUID_for_tag_name(client, name):
+    app_config = client.application.config.get('DATASTORE').data
+    for uuid, tag in app_config['settings']['application'].get('tags', {}).items():
+        if name == tag.get('title', '').lower().strip():
+            return uuid
+    return None
+
+
 # kinda funky, but works for now
 def extract_rss_token_from_UI(client):
     import re
@@ -119,16 +147,26 @@ def live_server_setup(live_server):
         status_code = request.args.get('status_code')
         content = request.args.get('content') or None
 
+        # Used to just try to break the header detection
+        uppercase_headers = request.args.get('uppercase_headers')
+
         try:
             if content is not None:
                 resp = make_response(content, status_code)
-                resp.headers['Content-Type'] = ctype if ctype else 'text/html'
+                if uppercase_headers:
+                    ctype=ctype.upper()
+                    resp.headers['CONTENT-TYPE'] = ctype if ctype else 'text/html'
+                else:
+                    resp.headers['Content-Type'] = ctype if ctype else 'text/html'
                 return resp
 
             # Tried using a global var here but didn't seem to work, so reading from a file instead.
             with open("test-datastore/endpoint-content.txt", "r") as f:
                 resp = make_response(f.read(), status_code)
-                resp.headers['Content-Type'] = ctype if ctype else 'text/html'
+                if uppercase_headers:
+                    resp.headers['CONTENT-TYPE'] = ctype if ctype else 'text/html'
+                else:
+                    resp.headers['Content-Type'] = ctype if ctype else 'text/html'
                 return resp
         except FileNotFoundError:
             return make_response('', status_code)
@@ -167,6 +205,10 @@ def live_server_setup(live_server):
         with open("test-datastore/notification-url.txt", "w") as f:
             f.write(request.url)
 
+        if request.content_type:
+            with open("test-datastore/notification-content-type.txt", "w") as f:
+                f.write(request.content_type)
+
         print("\n>> Test notification endpoint was hit.\n", data)
         return "Text was set"
 

changedetectionio/tests/visualselector/test_fetch_data.py

@@ -19,7 +19,7 @@ def test_visual_selector_content_ready(client, live_server):
 
     res = client.post(
         url_for("form_quick_watch_add"),
-        data={"url": test_url, "tag": '', 'edit_and_watch_submit_button': 'Edit > Watch'},
+        data={"url": test_url, "tags": '', 'edit_and_watch_submit_button': 'Edit > Watch'},
         follow_redirects=True
     )
     assert b"Watch added in Paused state, saving will unpause" in res.data
@@ -28,7 +28,7 @@ def test_visual_selector_content_ready(client, live_server):
         url_for("edit_page", uuid="first", unpause_on_save=1),
         data={
               "url": test_url,
-              "tag": "",
+              "tags": "",
               "headers": "",
               'fetch_backend': "html_webdriver",
               'webdriver_js_execute_code': 'document.querySelector("button[name=test-button]").click();'

changedetectionio/update_worker.py

@@ -5,7 +5,7 @@ import time
 
 from changedetectionio import content_fetcher
 from .processors.text_json_diff import FilterNotFoundInResponse
-
+from .processors.restock_diff import UnableToExtractRestockData
 
 # A single update worker
 #
@@ -26,16 +26,89 @@ class update_worker(threading.Thread):
         self.datastore = datastore
         super().__init__(*args, **kwargs)
 
-    def send_content_changed_notification(self, t, watch_uuid):
+    def queue_notification_for_watch(self, n_object, watch):
 
         from changedetectionio import diff
 
+        watch_history = watch.history
+        dates = list(watch_history.keys())
+        # Add text that was triggered
+        snapshot_contents = watch.get_history_snapshot(dates[-1])
+
+        # HTML needs linebreak, but MarkDown and Text can use a linefeed
+        if n_object['notification_format'] == 'HTML':
+            line_feed_sep = "<br>"
+            # Snapshot will be plaintext on the disk, convert to some kind of HTML
+            snapshot_contents = snapshot_contents.replace('\n', line_feed_sep)
+        else:
+            line_feed_sep = "\n"
+
+        trigger_text = watch.get('trigger_text', [])
+        triggered_text = ''
+
+        if len(trigger_text):
+            from . import html_tools
+            triggered_text = html_tools.get_triggered_text(content=snapshot_contents, trigger_text=trigger_text)
+            if triggered_text:
+                triggered_text = line_feed_sep.join(triggered_text)
+
+
+        n_object.update({
+            'current_snapshot': snapshot_contents,
+            'diff': diff.render_diff(watch.get_history_snapshot(dates[-2]), watch.get_history_snapshot(dates[-1]), line_feed_sep=line_feed_sep),
+            'diff_added': diff.render_diff(watch.get_history_snapshot(dates[-2]), watch.get_history_snapshot(dates[-1]), include_removed=False, line_feed_sep=line_feed_sep),
+            'diff_full': diff.render_diff(watch.get_history_snapshot(dates[-2]), watch.get_history_snapshot(dates[-1]), include_equal=True, line_feed_sep=line_feed_sep),
+            'diff_removed': diff.render_diff(watch.get_history_snapshot(dates[-2]), watch.get_history_snapshot(dates[-1]), include_added=False, line_feed_sep=line_feed_sep),
+            'screenshot': watch.get_screenshot() if watch.get('notification_screenshot') else None,
+            'triggered_text': triggered_text,
+            'uuid': watch.get('uuid'),
+            'watch_url': watch.get('url'),
+        })
+        logging.info (">> SENDING NOTIFICATION")
+        self.notification_q.put(n_object)
+
+    # Prefer - Individual watch settings > Tag settings >  Global settings (in that order)
+    def _check_cascading_vars(self, var_name, watch):
+
         from changedetectionio.notification import (
-            default_notification_format_for_watch
+            default_notification_format_for_watch,
+            default_notification_body,
+            default_notification_title
         )
 
+
+        # Would be better if this was some kind of Object where Watch can reference the parent datastore etc
+        v = watch.get(var_name)
+        if v and not watch.get('notification_muted'):
+            if var_name == 'notification_format' and v == default_notification_format_for_watch:
+                return self.datastore.data['settings']['application'].get('notification_format')
+
+            return v
+
+        tags = self.datastore.get_all_tags_for_watch(uuid=watch.get('uuid'))
+        if tags:
+            for tag_uuid, tag in tags.items():
+                v = tag.get(var_name)
+                if v and not tag.get('notification_muted'):
+                    return v
+
+        if self.datastore.data['settings']['application'].get(var_name):
+            return self.datastore.data['settings']['application'].get(var_name)
+
+        # Otherwise could be defaults
+        if var_name == 'notification_format':
+            return default_notification_format_for_watch
+        if var_name == 'notification_body':
+            return default_notification_body
+        if var_name == 'notification_title':
+            return default_notification_title
+
+        return None
+
+    def send_content_changed_notification(self, watch_uuid):
+
         n_object = {}
-        watch = self.datastore.data['watching'].get(watch_uuid, False)
+        watch = self.datastore.data['watching'].get(watch_uuid)
         if not watch:
             return
 
@@ -48,59 +121,27 @@ class update_worker(threading.Thread):
                 "History index had 2 or more, but only 1 date loaded, timestamps were not unique? maybe two of the same timestamps got written, needs more delay?"
             )
 
-        n_object['notification_urls'] = watch['notification_urls'] if len(watch['notification_urls']) else \
-            self.datastore.data['settings']['application']['notification_urls']
+        # Should be a better parent getter in the model object
 
-        n_object['notification_title'] = watch['notification_title'] if watch['notification_title'] else \
-            self.datastore.data['settings']['application']['notification_title']
+        # Prefer - Individual watch settings > Tag settings >  Global settings (in that order)
+        n_object['notification_urls'] = self._check_cascading_vars('notification_urls', watch)
+        n_object['notification_title'] = self._check_cascading_vars('notification_title', watch)
+        n_object['notification_body'] = self._check_cascading_vars('notification_body', watch)
+        n_object['notification_format'] = self._check_cascading_vars('notification_format', watch)
 
-        n_object['notification_body'] = watch['notification_body'] if watch['notification_body'] else \
-            self.datastore.data['settings']['application']['notification_body']
+        # (Individual watch) Only prepare to notify if the rules above matched
+        queued = False
+        if n_object and n_object.get('notification_urls'):
+            queued = True
+            self.queue_notification_for_watch(n_object, watch)
 
-        n_object['notification_format'] = watch['notification_format'] if watch['notification_format'] != default_notification_format_for_watch else \
-            self.datastore.data['settings']['application']['notification_format']
+        return queued
 
 
-        # Only prepare to notify if the rules above matched
-        if 'notification_urls' in n_object and n_object['notification_urls']:
-            # HTML needs linebreak, but MarkDown and Text can use a linefeed
-            if n_object['notification_format'] == 'HTML':
-                line_feed_sep = "<br>"
-            else:
-                line_feed_sep = "\n"
-
-            # Add text that was triggered
-            snapshot_contents = watch.get_history_snapshot(dates[-1])
-            trigger_text = watch.get('trigger_text', [])
-            triggered_text = ''
-
-            if len(trigger_text):
-                from . import html_tools
-                triggered_text = html_tools.get_triggered_text(content=snapshot_contents, trigger_text=trigger_text)
-                if triggered_text:
-                    triggered_text = line_feed_sep.join(triggered_text)
-
-
-            n_object.update({
-                'current_snapshot': snapshot_contents,
-                'diff': diff.render_diff(watch.get_history_snapshot(dates[-2]), watch.get_history_snapshot(dates[-1]), line_feed_sep=line_feed_sep),
-                'diff_added': diff.render_diff(watch.get_history_snapshot(dates[-2]), watch.get_history_snapshot(dates[-1]), include_removed=False, line_feed_sep=line_feed_sep),
-                'diff_full': diff.render_diff(watch.get_history_snapshot(dates[-2]), watch.get_history_snapshot(dates[-1]), include_equal=True, line_feed_sep=line_feed_sep),
-                'diff_removed': diff.render_diff(watch.get_history_snapshot(dates[-2]), watch.get_history_snapshot(dates[-1]), include_added=False, line_feed_sep=line_feed_sep),
-                'screenshot': watch.get_screenshot() if watch.get('notification_screenshot') else None,
-                'triggered_text': triggered_text,
-                'uuid': watch_uuid,
-                'watch_url': watch['url'],
-            })
-            logging.info (">> SENDING NOTIFICATION")
-            self.notification_q.put(n_object)
-        else:
-            logging.info (">> NO Notification sent, notification_url was empty in both watch and system")
-
     def send_filter_failure_notification(self, watch_uuid):
 
         threshold = self.datastore.data['settings']['application'].get('filter_failure_notification_threshold_attempts')
-        watch = self.datastore.data['watching'].get(watch_uuid, False)
+        watch = self.datastore.data['watching'].get(watch_uuid)
         if not watch:
             return
 
@@ -177,7 +218,7 @@ class update_worker(threading.Thread):
                 uuid = queued_item_data.item.get('uuid')
                 self.current_uuid = uuid
 
-                if uuid in list(self.datastore.data['watching'].keys()):
+                if uuid in list(self.datastore.data['watching'].keys()) and self.datastore.data['watching'][uuid].get('url'):
                     changed_detected = False
                     contents = b''
                     process_changedetection_results = True
@@ -238,7 +279,7 @@ class update_worker(threading.Thread):
                         if not self.datastore.data['watching'].get(uuid):
                             continue
 
-                        err_text = "Warning, no filters were found, no change detection ran."
+                        err_text = "Warning, no filters were found, no change detection ran - Did the page change layout? update your Visual Filter if necessary."
                         self.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text})
 
                         # Only when enabled, send the notification
@@ -262,6 +303,7 @@ class update_worker(threading.Thread):
                         # Yes fine, so nothing todo, don't continue to process.
                         process_changedetection_results = False
                         changed_detected = False
+                        self.datastore.update_watch(uuid=uuid, update_obj={'last_error': False})
 
                     except content_fetcher.BrowserStepsStepTimout as e:
 
@@ -318,6 +360,11 @@ class update_worker(threading.Thread):
                                                                            'last_check_status': e.status_code,
                                                                            'has_ldjson_price_data': None})
                         process_changedetection_results = False
+                    except UnableToExtractRestockData as e:
+                        # Usually when fetcher.instock_data returns empty
+                        self.app.logger.error("Exception reached processing watch UUID: %s - %s", uuid, str(e))
+                        self.datastore.update_watch(uuid=uuid, update_obj={'last_error': f"Unable to extract restock data for this page unfortunately. (Got code {e.status_code} from server)"})
+                        process_changedetection_results = False
                     except Exception as e:
                         self.app.logger.error("Exception reached processing watch UUID: %s - %s", uuid, str(e))
                         self.datastore.update_watch(uuid=uuid, update_obj={'last_error': str(e)})
@@ -332,6 +379,9 @@ class update_worker(threading.Thread):
                         if not self.datastore.data['watching'][uuid].get('ignore_status_codes'):
                             update_obj['consecutive_filter_failures'] = 0
 
+                        # Everything ran OK, clean off any previous error
+                        update_obj['last_error'] = False
+
                         self.cleanup_error_artifacts(uuid)
 
                     #
@@ -354,7 +404,7 @@ class update_worker(threading.Thread):
                                 # Notifications should only trigger on the second time (first time, we gather the initial snapshot)
                                 if watch.history_n >= 2:
                                     if not self.datastore.data['watching'][uuid].get('notification_muted'):
-                                        self.send_content_changed_notification(self, watch_uuid=uuid)
+                                        self.send_content_changed_notification(watch_uuid=uuid)
 
 
                         except Exception as e:

docker-compose.yml

@@ -47,6 +47,9 @@ services:
   #
   #        Hides the `Referer` header so that monitored websites can't see the changedetection.io hostname.
   #      - HIDE_REFERER=true
+  #        
+  #        Default number of parallel/concurrent fetchers
+  #      - FETCH_WORKERS=10
 
       # Comment out ports: when using behind a reverse proxy , enable networks: etc.
       ports:

requirements.txt

@@ -10,7 +10,8 @@ flask~=2.0
 inscriptis~=2.2
 pytz
 timeago~=1.0
-validators
+validators~=0.21
+
 
 # Set these versions together to avoid a RequestsDependencyWarning
 # >= 2.26 also adds Brotli support if brotli is installed
@@ -32,7 +33,7 @@ dnspython<2.3.0
 # jq not available on Windows so must be installed manually
 
 # Notification library
-apprise~=1.3.0
+apprise~=1.4.5
 
 # apprise mqtt https://github.com/dgtlmoon/changedetection.io/issues/315
 paho-mqtt
@@ -71,3 +72,6 @@ pillow
 # Include pytest, so if theres a support issue we can ask them to run these tests on their setup
 pytest ~=7.2
 pytest-flask ~=1.2
+
+# Pin jsonschema version to prevent build errors on armv6 while rpds-py wheels aren't available (1708)
+jsonschema==4.17.3