Bug fix - Delete watch from UI when watch is manually deleted from datadir

.dockerignore

@@ -1,18 +1,2 @@
 .git
 .github
-changedetectionio/processors/__pycache__
-changedetectionio/api/__pycache__
-changedetectionio/model/__pycache__
-changedetectionio/blueprint/price_data_follower/__pycache__
-changedetectionio/blueprint/tags/__pycache__
-changedetectionio/blueprint/__pycache__
-changedetectionio/blueprint/browser_steps/__pycache__
-changedetectionio/fetchers/__pycache__
-changedetectionio/tests/visualselector/__pycache__
-changedetectionio/tests/restock/__pycache__
-changedetectionio/tests/__pycache__
-changedetectionio/tests/fetchers/__pycache__
-changedetectionio/tests/unit/__pycache__
-changedetectionio/tests/proxy_list/__pycache__
-changedetectionio/__pycache__
-

.github/workflows/containers.yml

@@ -95,7 +95,7 @@ jobs:
           push: true
           tags: |
             ${{ secrets.DOCKER_HUB_USERNAME }}/changedetection.io:dev,ghcr.io/${{ github.repository }}:dev
-          platforms: linux/amd64,linux/arm64,linux/arm/v6,linux/arm/v7,linux/arm/v8
+          platforms: linux/amd64,linux/arm64,linux/arm/v6,linux/arm/v7
           cache-from: type=local,src=/tmp/.buildx-cache
           cache-to: type=local,dest=/tmp/.buildx-cache
 # Looks like this was disabled
@@ -115,7 +115,7 @@ jobs:
             ghcr.io/dgtlmoon/changedetection.io:${{ github.event.release.tag_name }}
             ${{ secrets.DOCKER_HUB_USERNAME }}/changedetection.io:latest
             ghcr.io/dgtlmoon/changedetection.io:latest
-          platforms: linux/amd64,linux/arm64,linux/arm/v6,linux/arm/v7,linux/arm/v8
+          platforms: linux/amd64,linux/arm64,linux/arm/v6,linux/arm/v7
           cache-from: type=local,src=/tmp/.buildx-cache
           cache-to: type=local,dest=/tmp/.buildx-cache
 # Looks like this was disabled

.github/workflows/test-container-build.yml

@@ -62,7 +62,7 @@ jobs:
           with:
             context: ./
             file: ./Dockerfile
-            platforms: linux/amd64,linux/arm64,linux/arm/v6,linux/arm/v7,linux/arm/v8
+            platforms: linux/arm/v7,linux/arm/v6,linux/amd64,linux/arm64,
             cache-from: type=local,src=/tmp/.buildx-cache
             cache-to: type=local,dest=/tmp/.buildx-cache
 

.github/workflows/test-only.yml

@@ -36,16 +36,10 @@ jobs:
         run: |         
           # Build a changedetection.io container and start testing inside
           docker build . -t test-changedetectionio
-          # Debug info
-          docker run test-changedetectionio  bash -c 'pip list'
-
-      - name: Spin up ancillary SMTP+Echo message test server
-        run: |
-          # Debug SMTP server/echo message back server
-          docker run --network changedet-network -d -p 11025:11025 -p 11080:11080  --hostname mailserver test-changedetectionio  bash -c 'python changedetectionio/tests/smtp/smtp-test-server.py' 
 
       - name: Test built container with pytest
         run: |
+          
           # Unit tests
           docker run test-changedetectionio  bash -c 'python3 -m unittest changedetectionio.tests.unit.test_notification_diff'
           
@@ -64,16 +58,11 @@ jobs:
           # Settings headers playwright tests - Call back in from Browserless, check headers
           docker run --name "changedet" --hostname changedet --rm -e "FLASK_SERVER_NAME=changedet" -e "PLAYWRIGHT_DRIVER_URL=ws://browserless:3000?dumpio=true" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0  --live-server-port=5004 tests/test_request.py'
           docker run --name "changedet" --hostname changedet --rm -e "FLASK_SERVER_NAME=changedet" -e "WEBDRIVER_URL=http://selenium:4444/wd/hub" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0  --live-server-port=5004 tests/test_request.py'
-          docker run --name "changedet" --hostname changedet --rm -e "FLASK_SERVER_NAME=changedet" -e "USE_EXPERIMENTAL_PUPPETEER_FETCH=yes" -e "PLAYWRIGHT_DRIVER_URL=ws://browserless:3000?dumpio=true" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0  --live-server-port=5004 tests/test_request.py'          
+          docker run --name "changedet" --hostname changedet --rm -e "FLASK_SERVER_NAME=changedet" -e "USE_EXPERIMENTAL_PUPPETEER_FETCH=yes" -e "PLAYWRIGHT_DRIVER_URL=ws://browserless:3000?dumpio=true" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0  --live-server-port=5004 tests/test_request.py'
           
           # restock detection via playwright - added name=changedet here so that playwright/browserless can connect to it
           docker run --rm --name "changedet" -e "FLASK_SERVER_NAME=changedet" -e "PLAYWRIGHT_DRIVER_URL=ws://browserless:3000" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-port=5004 --live-server-host=0.0.0.0 tests/restock/test_restock.py'
 
-      - name: Test SMTP notification mime types
-        run: |
-          # SMTP content types - needs the 'Debug SMTP server/echo message back server' container from above
-          docker run --rm  --network changedet-network test-changedetectionio bash -c 'cd changedetectionio;pytest tests/smtp/test_notification_smtp.py'
-
       - name: Test with puppeteer fetcher and disk cache
         run: |
           docker run --rm -e "PUPPETEER_DISK_CACHE=/tmp/data/" -e "USE_EXPERIMENTAL_PUPPETEER_FETCH=yes" -e "PLAYWRIGHT_DRIVER_URL=ws://browserless:3000" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio;pytest tests/fetchers/test_content.py && pytest tests/test_errorhandling.py && pytest tests/visualselector/test_fetch_data.py'

Dockerfile

@@ -1,5 +1,5 @@
 # pip dependencies install stage
-FROM python:3.10-slim-bullseye as builder
+FROM python:3.10-slim as builder
 
 # See `cryptography` pin comment in requirements.txt
 ARG CRYPTOGRAPHY_DONT_BUILD_RUST=1
@@ -29,7 +29,7 @@ RUN pip install --target=/dependencies playwright~=1.27.1 \
     || echo "WARN: Failed to install Playwright. The application can still run, but the Playwright option will be disabled."
 
 # Final image stage
-FROM python:3.10-slim-bullseye
+FROM python:3.10-slim
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
     libssl1.1 \

MANIFEST.in

@@ -13,6 +13,3 @@ include changedetection.py
 global-exclude *.pyc
 global-exclude node_modules
 global-exclude venv
-
-global-exclude test-datastore
-global-exclude changedetection.io*dist-info

README-pip.md

@@ -54,5 +54,5 @@ $ changedetection.io -d /path/to/empty/data/dir -p 5000
 
 Then visit http://127.0.0.1:5000 , You should now be able to access the UI.
 
-See https://changedetection.io for more information.
+See https://github.com/dgtlmoon/changedetection.io for more information.
 

README.md

@@ -67,14 +67,13 @@ Requires Playwright to be enabled.
 - Get alerts when new job positions are open on Bamboo HR and other job platforms
 - Website defacement monitoring
 - Pokémon Card Restock Tracker / Pokémon TCG Tracker
-- RegTech - stay ahead of regulatory changes, regulatory compliance
 
 _Need an actual Chrome runner with Javascript support? We support fetching via WebDriver and Playwright!</a>_
 
 #### Key Features
 
 - Lots of trigger filters, such as "Trigger on text", "Remove text by selector", "Ignore text", "Extract text", also using regular-expressions!
-- Target elements with xPath(1.0) and CSS Selectors, Easily monitor complex JSON with JSONPath or jq
+- Target elements with xPath and CSS Selectors, Easily monitor complex JSON with JSONPath or jq
 - Switch between fast non-JS and Chrome JS based "fetchers"
 - Track changes in PDF files (Monitor text changed in the PDF, Also monitor PDF filesize and checksums)
 - Easily specify how often a site should be checked
@@ -86,8 +85,6 @@ _Need an actual Chrome runner with Javascript support? We support fetching via W
 
 We [recommend and use Bright Data](https://brightdata.grsm.io/n0r16zf7eivq) global proxy services, Bright Data will match any first deposit up to $100 using our signup link.
 
-[Oxylabs](https://oxylabs.go2cloud.org/SH2d) is also an excellent proxy provider and well worth using, they offer Residental, ISP, Rotating and many other proxy types to suit your project. 
-
 Please :star: star :star: this project and help it grow! https://github.com/dgtlmoon/changedetection.io/
 
 ## Installation
@@ -147,7 +144,7 @@ See the wiki for more information https://github.com/dgtlmoon/changedetection.io
 
 ## Filters
 
-XPath(1.0), JSONPath, jq, and CSS support comes baked in! You can be as specific as you need, use XPath exported from various XPath element query creation tools. 
+XPath, JSONPath, jq, and CSS support comes baked in! You can be as specific as you need, use XPath exported from various XPath element query creation tools. 
 (We support LXML `re:test`, `re:match` and `re:replace`.)
 
 ## Notifications
@@ -186,7 +183,7 @@ This will re-parse the JSON and apply formatting to the text, making it super ea
 
 ### JSONPath or jq?
 
-For more complex parsing, filtering, and modifying of JSON data, jq is recommended due to the built-in operators and functions. Refer to the [documentation](https://stedolan.github.io/jq/manual/) for more specific information on jq.
+For more complex parsing, filtering, and modifying of JSON data, jq is recommended due to the built-in operators and functions. Refer to the [documentation](https://stedolan.github.io/jq/manual/) for more specifc information on jq.
 
 One big advantage of `jq` is that you can use logic in your JSON filter, such as filters to only show items that have a value greater than/less than etc.
 
@@ -226,7 +223,7 @@ The application also supports notifying you that it can follow this information
 
 ## Proxy Configuration
 
-See the wiki https://github.com/dgtlmoon/changedetection.io/wiki/Proxy-configuration , we also support using [Bright Data proxy services where possible]( https://github.com/dgtlmoon/changedetection.io/wiki/Proxy-configuration#brightdata-proxy-support)
+See the wiki https://github.com/dgtlmoon/changedetection.io/wiki/Proxy-configuration , we also support using [BrightData proxy services where possible]( https://github.com/dgtlmoon/changedetection.io/wiki/Proxy-configuration#brightdata-proxy-support)
 
 ## Raspberry Pi support?
 

changedetectionio/__init__.py

@@ -38,7 +38,7 @@ from flask_paginate import Pagination, get_page_parameter
 from changedetectionio import html_tools
 from changedetectionio.api import api_v1
 
-__version__ = '0.45.1'
+__version__ = '0.42.2'
 
 datastore = None
 
@@ -317,21 +317,25 @@ def changedetection_app(config=None, datastore_o=None):
             return "Access denied, bad token", 403
 
         from . import diff
-        limit_tag = request.args.get('tag', '').lower().strip()
-        # Be sure limit_tag is a uuid
-        for uuid, tag in datastore.data['settings']['application'].get('tags', {}).items():
-            if limit_tag == tag.get('title', '').lower().strip():
-                limit_tag = uuid
+        limit_tag = request.args.get('tag')
 
         # Sort by last_changed and add the uuid which is usually the key..
         sorted_watches = []
 
         # @todo needs a .itemsWithTag() or something - then we can use that in Jinaj2 and throw this away
         for uuid, watch in datastore.data['watching'].items():
-            if limit_tag and not limit_tag in watch['tags']:
-                    continue
-            watch['uuid'] = uuid
-            sorted_watches.append(watch)
+
+            if limit_tag != None:
+                # Support for comma separated list of tags.
+                for tag_in_watch in watch['tag'].split(','):
+                    tag_in_watch = tag_in_watch.strip()
+                    if tag_in_watch == limit_tag:
+                        watch['uuid'] = uuid
+                        sorted_watches.append(watch)
+
+            else:
+                watch['uuid'] = uuid
+                sorted_watches.append(watch)
 
         sorted_watches.sort(key=lambda x: x.last_changed, reverse=False)
 
@@ -355,13 +359,11 @@ def changedetection_app(config=None, datastore_o=None):
 
                 # Include a link to the diff page, they will have to login here to see if password protection is enabled.
                 # Description is the page you watch, link takes you to the diff JS UI page
-                # Dict val base_url will get overriden with the env var if it is set.
-                ext_base_url = datastore.data['settings']['application'].get('base_url')
-                if ext_base_url:
-                    # Go with overriden value
-                    diff_link = {'href': "{}{}".format(ext_base_url, url_for('diff_history_page', uuid=watch['uuid'], _external=False))}
-                else:
-                    diff_link = {'href': url_for('diff_history_page', uuid=watch['uuid'], _external=True)}
+                base_url = datastore.data['settings']['application']['base_url']
+                if base_url == '':
+                    base_url = "<base-url-env-var-not-set>"
+
+                diff_link = {'href': "{}{}".format(base_url, url_for('diff_history_page', uuid=watch['uuid']))}
 
                 fe.link(link=diff_link)
 
@@ -390,17 +392,9 @@ def changedetection_app(config=None, datastore_o=None):
     @app.route("/", methods=['GET'])
     @login_optionally_required
     def index():
-        global datastore
         from changedetectionio import forms
 
-        limit_tag = request.args.get('tag', '').lower().strip()
-
-        # Be sure limit_tag is a uuid
-        for uuid, tag in datastore.data['settings']['application'].get('tags', {}).items():
-            if limit_tag == tag.get('title', '').lower().strip():
-                limit_tag = uuid
-
-
+        limit_tag = request.args.get('tag')
         # Redirect for the old rss path which used the /?rss=true
         if request.args.get('rss'):
             return redirect(url_for('rss', tag=limit_tag))
@@ -420,15 +414,30 @@ def changedetection_app(config=None, datastore_o=None):
         sorted_watches = []
         search_q = request.args.get('q').strip().lower() if request.args.get('q') else False
         for uuid, watch in datastore.data['watching'].items():
-            if limit_tag and not limit_tag in watch['tags']:
+
+            if limit_tag:
+                # Support for comma separated list of tags.
+                if not watch.get('tag'):
                     continue
+                for tag_in_watch in watch.get('tag', '').split(','):
+                    tag_in_watch = tag_in_watch.strip()
+                    if tag_in_watch == limit_tag:
+                        watch['uuid'] = uuid
+                        if search_q:
+                            if (watch.get('title') and search_q in watch.get('title').lower()) or search_q in watch.get('url', '').lower():
+                                sorted_watches.append(watch)
+                        else:
+                            sorted_watches.append(watch)
 
-            if search_q:
-                if (watch.get('title') and search_q in watch.get('title').lower()) or search_q in watch.get('url', '').lower():
-                    sorted_watches.append(watch)
             else:
-                sorted_watches.append(watch)
+                #watch['uuid'] = uuid
+                if search_q:
+                    if (watch.get('title') and search_q in watch.get('title').lower()) or search_q in watch.get('url', '').lower():
+                        sorted_watches.append(watch)
+                else:
+                    sorted_watches.append(watch)
 
+        existing_tags = datastore.get_all_tags()
         form = forms.quickWatchForm(request.form)
         page = request.args.get(get_page_parameter(), type=int, default=1)
         total_count = len(sorted_watches)
@@ -443,7 +452,6 @@ def changedetection_app(config=None, datastore_o=None):
                                  # Don't link to hosting when we're on the hosting environment
                                  active_tag=limit_tag,
                                  app_rss_token=datastore.data['settings']['application']['rss_access_token'],
-                                 datastore=datastore,
                                  form=form,
                                  guid=datastore.data['app_guid'],
                                  has_proxies=datastore.proxy_list,
@@ -455,7 +463,7 @@ def changedetection_app(config=None, datastore_o=None):
                                  sort_attribute=request.args.get('sort') if request.args.get('sort') else request.cookies.get('sort'),
                                  sort_order=request.args.get('order') if request.args.get('order') else request.cookies.get('order'),
                                  system_default_fetcher=datastore.data['settings']['application'].get('fetch_backend'),
-                                 tags=datastore.data['settings']['application'].get('tags'),
+                                 tags=existing_tags,
                                  watches=sorted_watches
                                  )
 
@@ -598,13 +606,9 @@ def changedetection_app(config=None, datastore_o=None):
 
         # proxy_override set to the json/text list of the items
         form = forms.watchForm(formdata=request.form if request.method == 'POST' else None,
-                               data=default
+                               data=default,
                                )
 
-        # For the form widget tag uuid lookup
-        form.tags.datastore = datastore # in _value
-
-
         form.fetch_backend.choices.append(("system", 'System settings default'))
 
         # form.browser_steps[0] can be assumed that we 'goto url' first
@@ -655,16 +659,6 @@ def changedetection_app(config=None, datastore_o=None):
                 extra_update_obj['filter_text_replaced'] = True
                 extra_update_obj['filter_text_removed'] = True
 
-            # Because wtforms doesn't support accessing other data in process_ , but we convert the CSV list of tags back to a list of UUIDs
-            tag_uuids = []
-            if form.data.get('tags'):
-                # Sometimes in testing this can be list, dont know why
-                if type(form.data.get('tags')) == list:
-                    extra_update_obj['tags'] = form.data.get('tags')
-                else:
-                    for t in form.data.get('tags').split(','):
-                        tag_uuids.append(datastore.add_tag(name=t))
-                    extra_update_obj['tags'] = tag_uuids
 
             datastore.data['watching'][uuid].update(form.data)
             datastore.data['watching'][uuid].update(extra_update_obj)
@@ -719,7 +713,7 @@ def changedetection_app(config=None, datastore_o=None):
                                      form=form,
                                      has_default_notification_urls=True if len(datastore.data['settings']['application']['notification_urls']) else False,
                                      has_empty_checktime=using_default_check_time,
-                                     has_extra_headers_file=len(datastore.get_all_headers_in_textfile_for_watch(uuid=uuid)) > 0,
+                                     has_extra_headers_file=watch.has_extra_headers_file or datastore.has_extra_headers_file,
                                      is_html_webdriver=is_html_webdriver,
                                      jq_support=jq_support,
                                      playwright_enabled=os.getenv('PLAYWRIGHT_DRIVER_URL', False),
@@ -1116,8 +1110,8 @@ def changedetection_app(config=None, datastore_o=None):
                 os.path.join(datastore_o.datastore_path, list_with_tags_file), "w"
             ) as f:
                 for uuid in datastore.data["watching"]:
-                    url = datastore.data["watching"][uuid].get('url')
-                    tag = datastore.data["watching"][uuid].get('tags', {})
+                    url = datastore.data["watching"][uuid]["url"]
+                    tag = datastore.data["watching"][uuid]["tag"]
                     f.write("{} {}\r\n".format(url, tag))
 
             # Add it to the Zip
@@ -1205,7 +1199,7 @@ def changedetection_app(config=None, datastore_o=None):
 
         add_paused = request.form.get('edit_and_watch_submit_button') != None
         processor = request.form.get('processor', 'text_json_diff')
-        new_uuid = datastore.add_watch(url=url, tag=request.form.get('tags').strip(), extras={'paused': add_paused, 'processor': processor})
+        new_uuid = datastore.add_watch(url=url, tag=request.form.get('tag').strip(), extras={'paused': add_paused, 'processor': processor})
 
         if new_uuid:
             if add_paused:
@@ -1270,14 +1264,12 @@ def changedetection_app(config=None, datastore_o=None):
                 update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid, 'skip_when_checksum_same': False}))
             i = 1
 
-        elif tag:
+        elif tag != None:
             # Items that have this current tag
             for watch_uuid, watch in datastore.data['watching'].items():
-                if tag in watch.get('tags', {}):
+                if (tag != None and tag in watch['tag']):
                     if watch_uuid not in running_uuids and not datastore.data['watching'][watch_uuid]['paused']:
-                        update_q.put(
-                            queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': watch_uuid, 'skip_when_checksum_same': False})
-                        )
+                        update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': watch_uuid, 'skip_when_checksum_same': False}))
                         i += 1
 
         else:
@@ -1365,18 +1357,6 @@ def changedetection_app(config=None, datastore_o=None):
                     datastore.data['watching'][uuid.strip()]['notification_format'] = default_notification_format_for_watch
             flash("{} watches set to use default notification settings".format(len(uuids)))
 
-        elif (op == 'assign-tag'):
-            op_extradata = request.form.get('op_extradata', '').strip()
-            if op_extradata:
-                tag_uuid = datastore.add_tag(name=op_extradata)
-                if op_extradata and tag_uuid:
-                    for uuid in uuids:
-                        uuid = uuid.strip()
-                        if datastore.data['watching'].get(uuid):
-                            datastore.data['watching'][uuid]['tags'].append(tag_uuid)
-
-            flash("{} watches assigned tag".format(len(uuids)))
-
         return redirect(url_for('index'))
 
     @app.route("/api/share-url", methods=['GET'])
@@ -1386,6 +1366,7 @@ def changedetection_app(config=None, datastore_o=None):
            the share-link can be imported/added"""
         import requests
         import json
+        tag = request.args.get('tag')
         uuid = request.args.get('uuid')
 
         # more for testing
@@ -1432,39 +1413,12 @@ def changedetection_app(config=None, datastore_o=None):
         # paste in etc
         return redirect(url_for('index'))
 
-    @app.route("/highlight_submit_ignore_url", methods=['POST'])
-    def highlight_submit_ignore_url():
-        import re
-        mode = request.form.get('mode')
-        selection = request.form.get('selection')
-
-        uuid = request.args.get('uuid','')
-        if datastore.data["watching"].get(uuid):
-            if mode == 'exact':
-                for l in selection.splitlines():
-                    datastore.data["watching"][uuid]['ignore_text'].append(l.strip())
-            elif mode == 'digit-regex':
-                for l in selection.splitlines():
-                    # Replace any series of numbers with a regex
-                    s = re.escape(l.strip())
-                    s = re.sub(r'[0-9]+', r'\\d+', s)
-                    datastore.data["watching"][uuid]['ignore_text'].append('/' + s + '/')
-
-        return f"<a href={url_for('preview_page', uuid=uuid)}>Click to preview</a>"
-
-
     import changedetectionio.blueprint.browser_steps as browser_steps
     app.register_blueprint(browser_steps.construct_blueprint(datastore), url_prefix='/browser-steps')
 
     import changedetectionio.blueprint.price_data_follower as price_data_follower
     app.register_blueprint(price_data_follower.construct_blueprint(datastore, update_q), url_prefix='/price_data_follower')
 
-    import changedetectionio.blueprint.tags as tags
-    app.register_blueprint(tags.construct_blueprint(datastore), url_prefix='/tags')
-
-    import changedetectionio.blueprint.check_proxies as check_proxies
-    app.register_blueprint(check_proxies.construct_blueprint(datastore=datastore), url_prefix='/check_proxy')
-
 
     # @todo handle ctrl break
     ticker_thread = threading.Thread(target=ticker_thread_check_time_launch_checks).start()

changedetectionio/api/api_v1.py

@@ -1,6 +1,3 @@
-import os
-from distutils.util import strtobool
-
 from flask_expects_json import expects_json
 from changedetectionio import queuedWatchMetaData
 from flask_restful import abort, Resource
@@ -36,7 +33,7 @@ class Watch(Resource):
     @auth.check_token
     def get(self, uuid):
         """
-        @api {get} /api/v1/watch/:uuid Single watch - get data, recheck, pause, mute.
+        @api {get} /api/v1/watch/:uuid Get a single watch data
         @apiDescription Retrieve watch information and set muted/paused status
         @apiExample {curl} Example usage:
             curl http://localhost:4000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091  -H"x-api-key:813031b16330fe25e3780cf0325daa45"
@@ -212,9 +209,7 @@ class CreateWatch(Resource):
         json_data = request.get_json()
         url = json_data['url'].strip()
 
-        # If hosts that only contain alphanumerics are allowed ("localhost" for example)
-        allow_simplehost = not strtobool(os.getenv('BLOCK_SIMPLEHOSTS', 'False'))
-        if not validators.url(url, simple_host=allow_simplehost):
+        if not validators.url(json_data['url'].strip()):
             return "Invalid or unsupported URL", 400
 
         if json_data.get('proxy'):
@@ -223,16 +218,9 @@ class CreateWatch(Resource):
                 return "Invalid proxy choice, currently supported proxies are '{}'".format(', '.join(plist)), 400
 
         extras = copy.deepcopy(json_data)
-
-        # Because we renamed 'tag' to 'tags' but don't want to change the API (can do this in v2 of the API)
-        tags = None
-        if extras.get('tag'):
-            tags = extras.get('tag')
-            del extras['tag']
-
         del extras['url']
 
-        new_uuid = self.datastore.add_watch(url=url, extras=extras, tag=tags)
+        new_uuid = self.datastore.add_watch(url=url, extras=extras)
         if new_uuid:
             self.update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': new_uuid, 'skip_when_checksum_same': True}))
             return {'uuid': new_uuid}, 201
@@ -271,16 +259,13 @@ class CreateWatch(Resource):
         """
         list = {}
 
-        tag_limit = request.args.get('tag', '').lower()
+        tag_limit = request.args.get('tag', None)
+        for k, watch in self.datastore.data['watching'].items():
+            if tag_limit:
+                if not tag_limit.lower() in watch.all_tags:
+                    continue
 
-
-        for uuid, watch in self.datastore.data['watching'].items():
-            # Watch tags by name (replace the other calls?)
-            tags = self.datastore.get_all_tags_for_watch(uuid=uuid)
-            if tag_limit and not any(v.get('title').lower() == tag_limit for k, v in tags.items()):
-                continue
-
-            list[uuid] = {'url': watch['url'],
+            list[k] = {'url': watch['url'],
                        'title': watch['title'],
                        'last_checked': watch['last_checked'],
                        'last_changed': watch.last_changed,

changedetectionio/blueprint/check_proxies/__init__.py

@@ -1,116 +0,0 @@
-from concurrent.futures import ThreadPoolExecutor
-
-from functools import wraps
-
-from flask import Blueprint
-from flask_login import login_required
-
-from changedetectionio.processors import text_json_diff
-from changedetectionio.store import ChangeDetectionStore
-
-
-STATUS_CHECKING = 0
-STATUS_FAILED = 1
-STATUS_OK = 2
-THREADPOOL_MAX_WORKERS = 3
-_DEFAULT_POOL = ThreadPoolExecutor(max_workers=THREADPOOL_MAX_WORKERS)
-
-
-# Maybe use fetch-time if its >5 to show some expected load time?
-def threadpool(f, executor=None):
-    @wraps(f)
-    def wrap(*args, **kwargs):
-        return (executor or _DEFAULT_POOL).submit(f, *args, **kwargs)
-
-    return wrap
-
-
-def construct_blueprint(datastore: ChangeDetectionStore):
-    check_proxies_blueprint = Blueprint('check_proxies', __name__)
-    checks_in_progress = {}
-
-    @threadpool
-    def long_task(uuid, preferred_proxy):
-        import time
-        from changedetectionio import content_fetcher
-
-        status = {'status': '', 'length': 0, 'text': ''}
-        from jinja2 import Environment, BaseLoader
-
-        contents = ''
-        now = time.time()
-        try:
-            update_handler = text_json_diff.perform_site_check(datastore=datastore)
-            changed_detected, update_obj, contents = update_handler.run(uuid, preferred_proxy=preferred_proxy, skip_when_checksum_same=False)
-        # title, size is len contents not len xfer
-        except content_fetcher.Non200ErrorCodeReceived as e:
-            if e.status_code == 404:
-                status.update({'status': 'OK', 'length': len(contents), 'text': f"OK but 404 (page not found)"})
-            elif e.status_code == 403 or e.status_code == 401:
-                status.update({'status': 'ERROR', 'length': len(contents), 'text': f"{e.status_code} - Access denied"})
-            else:
-                status.update({'status': 'ERROR', 'length': len(contents), 'text': f"Status code: {e.status_code}"})
-        except text_json_diff.FilterNotFoundInResponse:
-            status.update({'status': 'OK', 'length': len(contents), 'text': f"OK but CSS/xPath filter not found (page changed layout?)"})
-        except content_fetcher.EmptyReply as e:
-            if e.status_code == 403 or e.status_code == 401:
-                status.update({'status': 'ERROR OTHER', 'length': len(contents), 'text': f"Got empty reply with code {e.status_code} - Access denied"})
-            else:
-                status.update({'status': 'ERROR OTHER', 'length': len(contents) if contents else 0, 'text': f"Empty reply with code {e.status_code}, needs chrome?"})
-
-        except Exception as e:
-            status.update({'status': 'ERROR OTHER', 'length': len(contents) if contents else 0, 'text': 'Error: '+str(e)})
-        else:
-            status.update({'status': 'OK', 'length': len(contents), 'text': ''})
-
-        if status.get('text'):
-            status['text'] = Environment(loader=BaseLoader()).from_string('{{text|e}}').render({'text': status['text']})
-
-        status['time'] = "{:.2f}s".format(time.time() - now)
-
-        return status
-
-    def _recalc_check_status(uuid):
-
-        results = {}
-        for k, v in checks_in_progress.get(uuid, {}).items():
-            try:
-                r_1 = v.result(timeout=0.05)
-            except Exception as e:
-                # If timeout error?
-                results[k] = {'status': 'RUNNING'}
-
-            else:
-                results[k] = r_1
-
-        return results
-
-    @login_required
-    @check_proxies_blueprint.route("/<string:uuid>/status", methods=['GET'])
-    def get_recheck_status(uuid):
-        results = _recalc_check_status(uuid=uuid)
-        return results
-
-    @login_required
-    @check_proxies_blueprint.route("/<string:uuid>/start", methods=['GET'])
-    def start_check(uuid):
-
-        if not datastore.proxy_list:
-            return
-
-        if checks_in_progress.get(uuid):
-            state = _recalc_check_status(uuid=uuid)
-            for proxy_key, v in state.items():
-                if v.get('status') == 'RUNNING':
-                    return state
-        else:
-            checks_in_progress[uuid] = {}
-
-        for k, v in datastore.proxy_list.items():
-            if not checks_in_progress[uuid].get(k):
-                checks_in_progress[uuid][k] = long_task(uuid=uuid, preferred_proxy=k)
-
-        results = _recalc_check_status(uuid=uuid)
-        return results
-
-    return check_proxies_blueprint

changedetectionio/blueprint/tags/README.md

@@ -1,9 +0,0 @@
-# Groups tags
-
-## How it works
-
-Watch has a list() of tag UUID's, which relate to a config under application.settings.tags
-
-The 'tag' is actually a watch, because they basically will eventually share 90% of the same config.
-
-So a tag is like an abstract of a watch

changedetectionio/blueprint/tags/__init__.py

@@ -1,141 +0,0 @@
-from flask import Blueprint, request, make_response, render_template, flash, url_for, redirect
-from changedetectionio.store import ChangeDetectionStore
-from changedetectionio import login_optionally_required
-
-
-def construct_blueprint(datastore: ChangeDetectionStore):
-    tags_blueprint = Blueprint('tags', __name__, template_folder="templates")
-
-    @tags_blueprint.route("/list", methods=['GET'])
-    @login_optionally_required
-    def tags_overview_page():
-        from .form import SingleTag
-        add_form = SingleTag(request.form)
-        output = render_template("groups-overview.html",
-                                 form=add_form,
-                                 available_tags=datastore.data['settings']['application'].get('tags', {}),
-                                 )
-
-        return output
-
-    @tags_blueprint.route("/add", methods=['POST'])
-    @login_optionally_required
-    def form_tag_add():
-        from .form import SingleTag
-        add_form = SingleTag(request.form)
-
-        if not add_form.validate():
-            for widget, l in add_form.errors.items():
-                flash(','.join(l), 'error')
-            return redirect(url_for('tags.tags_overview_page'))
-
-        title = request.form.get('name').strip()
-
-        if datastore.tag_exists_by_name(title):
-            flash(f'The tag "{title}" already exists', "error")
-            return redirect(url_for('tags.tags_overview_page'))
-
-        datastore.add_tag(title)
-        flash("Tag added")
-
-
-        return redirect(url_for('tags.tags_overview_page'))
-
-    @tags_blueprint.route("/mute/<string:uuid>", methods=['GET'])
-    @login_optionally_required
-    def mute(uuid):
-        if datastore.data['settings']['application']['tags'].get(uuid):
-            datastore.data['settings']['application']['tags'][uuid]['notification_muted'] = not datastore.data['settings']['application']['tags'][uuid]['notification_muted']
-        return redirect(url_for('tags.tags_overview_page'))
-
-    @tags_blueprint.route("/delete/<string:uuid>", methods=['GET'])
-    @login_optionally_required
-    def delete(uuid):
-        removed = 0
-        # Delete the tag, and any tag reference
-        if datastore.data['settings']['application']['tags'].get(uuid):
-            del datastore.data['settings']['application']['tags'][uuid]
-
-        for watch_uuid, watch in datastore.data['watching'].items():
-            if watch.get('tags') and uuid in watch['tags']:
-                removed += 1
-                watch['tags'].remove(uuid)
-
-        flash(f"Tag deleted and removed from {removed} watches")
-        return redirect(url_for('tags.tags_overview_page'))
-
-    @tags_blueprint.route("/unlink/<string:uuid>", methods=['GET'])
-    @login_optionally_required
-    def unlink(uuid):
-        unlinked = 0
-        for watch_uuid, watch in datastore.data['watching'].items():
-            if watch.get('tags') and uuid in watch['tags']:
-                unlinked += 1
-                watch['tags'].remove(uuid)
-
-        flash(f"Tag unlinked removed from {unlinked} watches")
-        return redirect(url_for('tags.tags_overview_page'))
-
-    @tags_blueprint.route("/delete_all", methods=['GET'])
-    @login_optionally_required
-    def delete_all():
-        for watch_uuid, watch in datastore.data['watching'].items():
-            watch['tags'] = []
-        datastore.data['settings']['application']['tags'] = {}
-
-        flash(f"All tags deleted")
-        return redirect(url_for('tags.tags_overview_page'))
-
-    @tags_blueprint.route("/edit/<string:uuid>", methods=['GET'])
-    @login_optionally_required
-    def form_tag_edit(uuid):
-        from changedetectionio import forms
-
-        if uuid == 'first':
-            uuid = list(datastore.data['settings']['application']['tags'].keys()).pop()
-
-        default = datastore.data['settings']['application']['tags'].get(uuid)
-
-        form = forms.watchForm(formdata=request.form if request.method == 'POST' else None,
-                               data=default,
-                               )
-        form.datastore=datastore # needed?
-
-        output = render_template("edit-tag.html",
-                                 data=default,
-                                 form=form,
-                                 settings_application=datastore.data['settings']['application'],
-                                 )
-
-        return output
-
-
-    @tags_blueprint.route("/edit/<string:uuid>", methods=['POST'])
-    @login_optionally_required
-    def form_tag_edit_submit(uuid):
-        from changedetectionio import forms
-        if uuid == 'first':
-            uuid = list(datastore.data['settings']['application']['tags'].keys()).pop()
-
-        default = datastore.data['settings']['application']['tags'].get(uuid)
-
-        form = forms.watchForm(formdata=request.form if request.method == 'POST' else None,
-                               data=default,
-                               )
-        # @todo subclass form so validation works
-        #if not form.validate():
-#            for widget, l in form.errors.items():
-#                flash(','.join(l), 'error')
-#           return redirect(url_for('tags.form_tag_edit_submit', uuid=uuid))
-
-        datastore.data['settings']['application']['tags'][uuid].update(form.data)
-        datastore.needs_write_urgent = True
-        flash("Updated")
-
-        return redirect(url_for('tags.tags_overview_page'))
-
-
-    @tags_blueprint.route("/delete/<string:uuid>", methods=['GET'])
-    def form_tag_delete(uuid):
-        return redirect(url_for('tags.tags_overview_page'))
-    return tags_blueprint

changedetectionio/blueprint/tags/form.py

@@ -1,22 +0,0 @@
-from wtforms import (
-    BooleanField,
-    Form,
-    IntegerField,
-    RadioField,
-    SelectField,
-    StringField,
-    SubmitField,
-    TextAreaField,
-    validators,
-)
-
-
-
-class SingleTag(Form):
-
-    name = StringField('Tag name', [validators.InputRequired()], render_kw={"placeholder": "Name"})
-    save_button = SubmitField('Save', render_kw={"class": "pure-button pure-button-primary"})
-
-
-
-

changedetectionio/blueprint/tags/templates/edit-tag.html

@@ -1,135 +0,0 @@
-{% extends 'base.html' %}
-{% block content %}
-{% from '_helpers.jinja' import render_field, render_checkbox_field, render_button %}
-{% from '_common_fields.jinja' import render_common_settings_form %}
-<script>
-    const notification_base_url="{{url_for('ajax_callback_send_notification_test')}}";
-</script>
-
-<script src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
-<script>
-
-/*{% if emailprefix %}*/
-    /*const email_notification_prefix=JSON.parse('{{ emailprefix|tojson }}');*/
-/*{% endif %}*/
-
-
-</script>
-
-<script src="{{url_for('static_content', group='js', filename='watch-settings.js')}}" defer></script>
-<!--<script src="{{url_for('static_content', group='js', filename='limit.js')}}" defer></script>-->
-<script src="{{url_for('static_content', group='js', filename='notifications.js')}}" defer></script>
-
-<div class="edit-form monospaced-textarea">
-
-    <div class="tabs collapsable">
-        <ul>
-            <li class="tab" id=""><a href="#general">General</a></li>
-            <li class="tab"><a href="#filters-and-triggers">Filters &amp; Triggers</a></li>
-            <li class="tab"><a href="#notifications">Notifications</a></li>
-        </ul>
-    </div>
-
-    <div class="box-wrap inner">
-        <form class="pure-form pure-form-stacked"
-              action="{{ url_for('tags.form_tag_edit', uuid=data.uuid) }}" method="POST">
-             <input type="hidden" name="csrf_token" value="{{ csrf_token() }}">
-
-            <div class="tab-pane-inner" id="general">
-                <fieldset>
-                    <div class="pure-control-group">
-                        {{ render_field(form.title, placeholder="https://...", required=true, class="m-d") }}
-                    </div>
-                </fieldset>
-            </div>
-
-            <div class="tab-pane-inner" id="filters-and-triggers">
-                    <div class="pure-control-group">
-                        {% set field = render_field(form.include_filters,
-                            rows=5,
-                            placeholder="#example
-xpath://body/div/span[contains(@class, 'example-class')]",
-                            class="m-d")
-                        %}
-                        {{ field }}
-                        {% if '/text()' in  field %}
-                          <span class="pure-form-message-inline"><strong>Note!: //text() function does not work where the &lt;element&gt; contains &lt;![CDATA[]]&gt;</strong></span><br>
-                        {% endif %}
-                        <span class="pure-form-message-inline">One rule per line, <i>any</i> rules that matches will be used.<br>
-
-                    <ul>
-                        <li>CSS - Limit text to this CSS rule, only text matching this CSS rule is included.</li>
-                        <li>JSON - Limit text to this JSON rule, using either <a href="https://pypi.org/project/jsonpath-ng/" target="new">JSONPath</a> or <a href="https://stedolan.github.io/jq/" target="new">jq</a> (if installed).
-                            <ul>
-                                <li>JSONPath: Prefix with <code>json:</code>, use <code>json:$</code> to force re-formatting if required,  <a href="https://jsonpath.com/" target="new">test your JSONPath here</a>.</li>
-                                {% if jq_support %}
-                                <li>jq: Prefix with <code>jq:</code> and <a href="https://jqplay.org/" target="new">test your jq here</a>. Using <a href="https://stedolan.github.io/jq/" target="new">jq</a> allows for complex filtering and processing of JSON data with built-in functions, regex, filtering, and more. See examples and documentation <a href="https://stedolan.github.io/jq/manual/" target="new">here</a>.</li>
-                                {% else %}
-                                <li>jq support not installed</li>
-                                {% endif %}
-                            </ul>
-                        </li>
-                        <li>XPath - Limit text to this XPath rule, simply start with a forward-slash,
-                            <ul>
-                                <li>Example:  <code>//*[contains(@class, 'sametext')]</code> or <code>xpath://*[contains(@class, 'sametext')]</code>, <a
-                                href="http://xpather.com/" target="new">test your XPath here</a></li>
-                                <li>Example: Get all titles from an RSS feed <code>//title/text()</code></li>
-                            </ul>
-                            </li>
-                    </ul>
-                    Please be sure that you thoroughly understand how to write CSS, JSONPath, XPath{% if jq_support %}, or jq selector{%endif%} rules before filing an issue on GitHub! <a
-                                href="https://github.com/dgtlmoon/changedetection.io/wiki/CSS-Selector-help">here for more CSS selector help</a>.<br>
-                </span>
-                    </div>
-                <fieldset class="pure-control-group">
-                    {{ render_field(form.subtractive_selectors, rows=5, placeholder="header
-footer
-nav
-.stockticker") }}
-                    <span class="pure-form-message-inline">
-                        <ul>
-                          <li> Remove HTML element(s) by CSS selector before text conversion. </li>
-                          <li> Add multiple elements or CSS selectors per line to ignore multiple parts of the HTML. </li>
-                        </ul>
-                      </span>
-                </fieldset>
-
-            </div>
-
-            <div class="tab-pane-inner" id="notifications">
-                <fieldset>
-                    <div  class="pure-control-group inline-radio">
-                      {{ render_checkbox_field(form.notification_muted) }}
-                    </div>
-                    {% if is_html_webdriver %}
-                    <div class="pure-control-group inline-radio">
-                      {{ render_checkbox_field(form.notification_screenshot) }}
-                        <span class="pure-form-message-inline">
-                            <strong>Use with caution!</strong> This will easily fill up your email storage quota or flood other storages.
-                        </span>
-                    </div>
-                    {% endif %}
-                    <div class="field-group" id="notification-field-group">
-                        {% if has_default_notification_urls %}
-                        <div class="inline-warning">
-                            <img class="inline-warning-icon" src="{{url_for('static_content', group='images', filename='notice.svg')}}" alt="Look out!" title="Lookout!" >
-                            There are <a href="{{ url_for('settings_page')}}#notifications">system-wide notification URLs enabled</a>, this form will override notification settings for this watch only &dash; an empty Notification URL list here will still send notifications.
-                        </div>
-                        {% endif %}
-                        <a href="#notifications" id="notification-setting-reset-to-default" class="pure-button button-xsmall" style="right: 20px; top: 20px; position: absolute; background-color: #5f42dd; border-radius: 4px; font-size: 70%; color: #fff">Use system defaults</a>
-
-                        {{ render_common_settings_form(form, emailprefix, settings_application) }}
-                    </div>
-                </fieldset>
-            </div>
-
-            <div id="actions">
-                <div class="pure-control-group">
-                    {{ render_button(form.save_button) }}
-                </div>
-            </div>
-        </form>
-    </div>
-</div>
-
-{% endblock %}

changedetectionio/blueprint/tags/templates/groups-overview.html

@@ -1,60 +0,0 @@
-{% extends 'base.html' %}
-{% block content %}
-{% from '_helpers.jinja' import render_simple_field, render_field %}
-<script src="{{url_for('static_content', group='js', filename='jquery-3.6.0.min.js')}}"></script>
-
-<div class="box">
-    <form class="pure-form" action="{{ url_for('tags.form_tag_add') }}" method="POST" id="new-watch-form">
-        <input type="hidden" name="csrf_token" value="{{ csrf_token() }}" >
-        <fieldset>
-            <legend>Add a new organisational tag</legend>
-            <div id="watch-add-wrapper-zone">
-                <div>
-                    {{ render_simple_field(form.name, placeholder="watch label / tag") }}
-                </div>
-                <div>
-                    {{ render_simple_field(form.save_button, title="Save" ) }}
-                </div>
-            </div>
-            <br>
-            <div style="color: #fff;">Groups allows you to manage filters and notifications for multiple watches under a single organisational tag.</div>
-        </fieldset>
-    </form>
-    <!-- @todo maybe some overview matrix, 'tick' with which has notification, filter rules etc -->
-    <div id="watch-table-wrapper">
-
-        <table class="pure-table pure-table-striped watch-table group-overview-table">
-            <thead>
-            <tr>
-                <th></th>
-                <th>Tag / Label name</th>
-                <th></th>
-            </tr>
-            </thead>
-            <tbody>
-            <!--
-            @Todo - connect Last checked, Last Changed, Number of Watches etc
-            --->
-            {% if not available_tags|length %}
-            <tr>
-                <td colspan="3">No website organisational tags/groups configured</td>
-            </tr>
-            {% endif %}
-            {% for uuid, tag in available_tags.items()  %}
-            <tr id="{{ uuid }}" class="{{ loop.cycle('pure-table-odd', 'pure-table-even') }}">
-                <td class="watch-controls">
-                    <a class="link-mute state-{{'on' if tag.notification_muted else 'off'}}" href="{{url_for('tags.mute', uuid=tag.uuid)}}"><img src="{{url_for('static_content', group='images', filename='bell-off.svg')}}" alt="Mute notifications" title="Mute notifications" class="icon icon-mute" ></a>
-                </td>
-                <td class="title-col inline">{{tag.title}}</td>
-                <td>
-                    <a class="pure-button pure-button-primary" href="{{ url_for('tags.form_tag_edit', uuid=uuid) }}">Edit</a>&nbsp;
-                    <a class="pure-button pure-button-primary" href="{{ url_for('tags.delete', uuid=uuid) }}" title="Deletes and removes tag">Delete</a>
-                    <a class="pure-button pure-button-primary" href="{{ url_for('tags.unlink', uuid=uuid) }}" title="Keep the tag but unlink any watches">Unlink</a>
-                </td>
-            </tr>
-            {% endfor %}
-            </tbody>
-        </table>
-    </div>
-</div>
-{% endblock %}

changedetectionio/content_fetcher.py

@@ -201,8 +201,7 @@ class Fetcher():
             dest = os.path.join(self.browser_steps_screenshot_path, 'step_*.jpeg')
             files = glob.glob(dest)
             for f in files:
-                if os.path.isfile(f):
-                    os.unlink(f)
+                os.unlink(f)
 
 
 #   Maybe for the future, each fetcher provides its own diff output, could be used for text, image

changedetectionio/diff.py

@@ -35,19 +35,15 @@ def customSequenceMatcher(before, after, include_equal=False, include_removed=Tr
 
 # only_differences - only return info about the differences, no context
 # line_feed_sep could be "<br>" or "<li>" or "\n" etc
-def render_diff(previous_version_file_contents, newest_version_file_contents, include_equal=False, include_removed=True, include_added=True, include_replaced=True, line_feed_sep="\n", include_change_type_prefix=True, patch_format=False):
+def render_diff(previous_version_file_contents, newest_version_file_contents, include_equal=False, include_removed=True, include_added=True, include_replaced=True, line_feed_sep="\n", include_change_type_prefix=True):
 
     newest_version_file_contents = [line.rstrip() for line in newest_version_file_contents.splitlines()]
 
     if previous_version_file_contents:
-        previous_version_file_contents = [line.rstrip() for line in previous_version_file_contents.splitlines()]
+            previous_version_file_contents = [line.rstrip() for line in previous_version_file_contents.splitlines()]
     else:
         previous_version_file_contents = ""
 
-    if patch_format:
-        patch = difflib.unified_diff(previous_version_file_contents, newest_version_file_contents)
-        return line_feed_sep.join(patch)
-
     rendered_diff = customSequenceMatcher(before=previous_version_file_contents,
                                           after=newest_version_file_contents,
                                           include_equal=include_equal,
@@ -58,5 +54,4 @@ def render_diff(previous_version_file_contents, newest_version_file_contents, in
 
     # Recursively join lists
     f = lambda L: line_feed_sep.join([f(x) if type(x) is list else x for x in L])
-    p= f(rendered_diff)
-    return p
+    return f(rendered_diff)

changedetectionio/forms.py

@@ -1,6 +1,5 @@
 import os
 import re
-from distutils.util import strtobool
 
 from wtforms import (
     BooleanField,
@@ -29,8 +28,6 @@ from changedetectionio.notification import (
 
 from wtforms.fields import FormField
 
-dictfilt = lambda x, y: dict([ (i,x[i]) for i in x if i in set(y) ])
-
 valid_method = {
     'GET',
     'POST',
@@ -93,29 +90,6 @@ class SaltyPasswordField(StringField):
         else:
             self.data = False
 
-class StringTagUUID(StringField):
-
-   # process_formdata(self, valuelist) handled manually in POST handler
-
-    # Is what is shown when field <input> is rendered
-    def _value(self):
-        # Tag UUID to name, on submit it will convert it back (in the submit handler of init.py)
-        if self.data and type(self.data) is list:
-            tag_titles = []
-            for i in self.data:
-                tag = self.datastore.data['settings']['application']['tags'].get(i)
-                if tag:
-                    tag_title = tag.get('title')
-                    if tag_title:
-                        tag_titles.append(tag_title)
-
-            return ', '.join(tag_titles)
-
-        if not self.data:
-            return ''
-
-        return 'error'
-
 class TimeBetweenCheckForm(Form):
     weeks = IntegerField('Weeks', validators=[validators.Optional(), validators.NumberRange(min=0, message="Should contain zero or more seconds")])
     days = IntegerField('Days', validators=[validators.Optional(), validators.NumberRange(min=0, message="Should contain zero or more seconds")])
@@ -258,10 +232,9 @@ class validateURL(object):
 
     def __call__(self, form, field):
         import validators
-        # If hosts that only contain alphanumerics are allowed ("localhost" for example)
-        allow_simplehost = not strtobool(os.getenv('BLOCK_SIMPLEHOSTS', 'False'))
+
         try:
-            validators.url(field.data.strip(), simple_host=allow_simplehost)
+            validators.url(field.data.strip())
         except validators.ValidationFailure:
             message = field.gettext('\'%s\' is not a valid URL.' % (field.data.strip()))
             raise ValidationError(message)
@@ -374,7 +347,7 @@ class quickWatchForm(Form):
     from . import processors
 
     url = fields.URLField('URL', validators=[validateURL()])
-    tags = StringTagUUID('Group tag', [validators.Optional()])
+    tag = StringField('Group tag', [validators.Optional()])
     watch_submit_button = SubmitField('Watch', render_kw={"class": "pure-button pure-button-primary"})
     processor = RadioField(u'Processor', choices=processors.available_processors(), default="text_json_diff")
     edit_and_watch_submit_button = SubmitField('Edit > Watch', render_kw={"class": "pure-button pure-button-primary"})
@@ -382,7 +355,6 @@ class quickWatchForm(Form):
 
 # Common to a single watch and the global settings
 class commonSettingsForm(Form):
-
     notification_urls = StringListField('Notification URL List', validators=[validators.Optional(), ValidateAppRiseServers()])
     notification_title = StringField('Notification Title', default='ChangeDetection.io Notification - {{ watch_url }}', validators=[validators.Optional(), ValidateJinja2Template()])
     notification_body = TextAreaField('Notification Body', default='{{ watch_url }} had a change.', validators=[validators.Optional(), ValidateJinja2Template()])
@@ -410,7 +382,7 @@ class SingleBrowserStep(Form):
 class watchForm(commonSettingsForm):
 
     url = fields.URLField('URL', validators=[validateURL()])
-    tags = StringTagUUID('Group tag', [validators.Optional()], default='')
+    tag = StringField('Group tag', [validators.Optional()], default='')
 
     time_between_check = FormField(TimeBetweenCheckForm)
 

changedetectionio/html_tools.py

@@ -191,50 +191,42 @@ def extract_json_as_string(content, json_filter, ensure_is_ldjson_info_type=None
 #
 # wordlist - list of regex's (str) or words (str)
 def strip_ignore_text(content, wordlist, mode="content"):
-    i = 0
-    output = []
-    ignore_text = []
+    ignore = []
     ignore_regex = []
 
-    ignored_line_numbers = []
-
+    # @todo check this runs case insensitive
     for k in wordlist:
-        # Is it a regex?
-        x = re.search('^\/(.*)\/(.*)', k.strip())
-        if x:
-            # Starts with / but doesn't look like a regex
-            p = x.group(1)
-            try:
-                # @Todo python regex options can go before the regex str, but not really many of the options apply on a per-line basis
-                ignore_regex.append(re.compile(rf"{p}", re.IGNORECASE))
-            except Exception as e:
-                # Badly formed regex, treat as text
-                ignore_text.append(k.strip())
-        else:
-            # Had a / but doesn't work as regex
-            ignore_text.append(k.strip())
 
+        # Is it a regex?
+        if k[0] == '/':
+            ignore_regex.append(k.strip(" /"))
+        else:
+            ignore.append(k)
+
+    i = 0
+    output = []
+    ignored_line_numbers = []
     for line in content.splitlines():
         i += 1
         # Always ignore blank lines in this mode. (when this function gets called)
-        got_match = False
         if len(line.strip()):
-            for l in ignore_text:
-                if l.lower() in line.lower():
-                    got_match = True
+            regex_matches = False
 
-            if not got_match:
-                for r in ignore_regex:
-                    if r.search(line):
-                        got_match = True
+            # if any of these match, skip
+            for regex in ignore_regex:
+                try:
+                    if re.search(regex, line, re.IGNORECASE):
+                        regex_matches = True
+                except Exception as e:
+                    continue
 
-            if not got_match:
-                # Not ignored
+            if not regex_matches and not any(skip_text.lower() in line.lower() for skip_text in ignore):
                 output.append(line.encode('utf8'))
             else:
                 ignored_line_numbers.append(i)
 
 
+
     # Used for finding out what to highlight
     if mode == "line numbers":
         return ignored_line_numbers

changedetectionio/importer.py

@@ -85,8 +85,7 @@ class import_distill_io_json(Importer):
         now = time.time()
         self.new_uuids=[]
 
-        # @todo Use JSONSchema like in the API to validate here.
-        
+
         try:
             data = json.loads(data.strip())
         except json.decoder.JSONDecodeError:
@@ -121,8 +120,11 @@ class import_distill_io_json(Importer):
                 except IndexError:
                     pass
 
+
+                if d.get('tags', False):
+                    extras['tag'] = ", ".join(d['tags'])
+
                 new_uuid = datastore.add_watch(url=d['uri'].strip(),
-                                               tag=",".join(d.get('tags', [])),
                                                extras=extras,
                                                write_to_disk_now=False)
 

changedetectionio/model/App.py

@@ -43,7 +43,6 @@ class model(dict):
                     'schema_version' : 0,
                     'shared_diff_access': False,
                     'webdriver_delay': None , # Extra delay in seconds before extracting text
-                    'tags': {} #@todo use Tag.model initialisers
                 }
             }
         }

changedetectionio/model/Tag.py

@@ -1,19 +0,0 @@
-from .Watch import base_config
-import uuid
-
-class model(dict):
-
-    def __init__(self, *arg, **kw):
-
-        self.update(base_config)
-
-        self['uuid'] = str(uuid.uuid4())
-
-        if kw.get('default'):
-            self.update(kw['default'])
-            del kw['default']
-
-
-        # Goes at the end so we update the default object with the initialiser
-        super(model, self).__init__(*arg, **kw)
-

changedetectionio/model/Watch.py

@@ -52,8 +52,7 @@ base_config = {
     'previous_md5_before_filters': False,  # Used for skipping changedetection entirely
     'proxy': None,  # Preferred proxy connection
     'subtractive_selectors': [],
-    'tag': '', # Old system of text name for a tag, to be removed
-    'tags': [], # list of UUIDs to App.Tags
+    'tag': None,
     'text_should_not_be_present': [],  # Text that should not present
     # Re #110, so then if this is set to None, we know to use the default value instead
     # Requires setting to None on submit if it's the same as the default
@@ -456,6 +455,10 @@ class model(dict):
 
         return csv_output_filename
 
+    @property
+    # Return list of tags, stripped and lowercase, used for searching
+    def all_tags(self):
+        return [s.strip().lower() for s in self.get('tag','').split(',')]
 
     def has_special_diff_filter_options_set(self):
 
@@ -470,6 +473,40 @@ class model(dict):
         # None is set
         return False
 
+    @property
+    def has_extra_headers_file(self):
+        if os.path.isfile(os.path.join(self.watch_data_dir, 'headers.txt')):
+            return True
+
+        for f in self.all_tags:
+            fname = "headers-"+re.sub(r'[\W_]', '', f).lower().strip() + ".txt"
+            filepath = os.path.join(self.__datastore_path, fname)
+            if os.path.isfile(filepath):
+                return True
+
+        return False
+
+    def get_all_headers(self):
+        from .App import parse_headers_from_text_file
+        headers = self.get('headers', {}).copy()
+        # Available headers on the disk could 'headers.txt' in the watch data dir
+        filepath = os.path.join(self.watch_data_dir, 'headers.txt')
+        try:
+            if os.path.isfile(filepath):
+                headers.update(parse_headers_from_text_file(filepath))
+        except Exception as e:
+            print(f"ERROR reading headers.txt at {filepath}", str(e))
+
+        # Or each by tag, as tagname.txt in the main datadir
+        for f in self.all_tags:
+            fname = "headers-"+re.sub(r'[\W_]', '', f).lower().strip() + ".txt"
+            filepath = os.path.join(self.__datastore_path, fname)
+            try:
+                if os.path.isfile(filepath):
+                    headers.update(parse_headers_from_text_file(filepath))
+            except Exception as e:
+                print(f"ERROR reading headers.txt at {filepath}", str(e))
+        return headers
 
     def get_last_fetched_before_filters(self):
         import brotli

changedetectionio/notification.py

@@ -9,7 +9,6 @@ valid_tokens = {
     'diff': '',
     'diff_added': '',
     'diff_full': '',
-    'diff_patch': '',
     'diff_removed': '',
     'diff_url': '',
     'preview_url': '',
@@ -93,13 +92,7 @@ def process_notification(n_object, datastore):
         n_object.get('notification_format', default_notification_format),
         valid_notification_formats[default_notification_format],
     )
-
-    # If we arrived with 'System default' then look it up
-    if n_format == default_notification_format_for_watch and datastore.data['settings']['application'].get('notification_format') != default_notification_format_for_watch:
-        # Initially text or whatever
-        n_format = datastore.data['settings']['application'].get('notification_format', valid_notification_formats[default_notification_format])
-
-
+    
     # https://github.com/caronc/apprise/wiki/Development_LogCapture
     # Anything higher than or equal to WARNING (which covers things like Connection errors)
     # raise it as an exception
@@ -152,12 +145,9 @@ def process_notification(n_object, datastore):
                     # Apprise will default to HTML, so we need to override it
                     # So that whats' generated in n_body is in line with what is going to be sent.
                     # https://github.com/caronc/apprise/issues/633#issuecomment-1191449321
-                    if not 'format=' in url and (n_format == 'Text' or n_format == 'Markdown'):
+                    if not 'format=' in url and (n_format == 'text' or n_format == 'markdown'):
                         prefix = '?' if not '?' in url else '&'
-                        # Apprise format is lowercase text https://github.com/caronc/apprise/issues/633
-                        n_format = n_format.tolower()
                         url = "{}{}format={}".format(url, prefix, n_format)
-                    # If n_format == HTML, then apprise email should default to text/html and we should be sending HTML only
 
                 apobj.add(url)
 
@@ -178,7 +168,7 @@ def process_notification(n_object, datastore):
                 log_value = logs.getvalue()
                 if log_value and 'WARNING' in log_value or 'ERROR' in log_value:
                     raise Exception(log_value)
-
+                
                 sent_objs.append({'title': n_title,
                                   'body': n_body,
                                   'url' : url,
@@ -196,13 +186,8 @@ def create_notification_parameters(n_object, datastore):
     uuid = n_object['uuid'] if 'uuid' in n_object else ''
 
     if uuid != '':
-        watch_title = datastore.data['watching'][uuid].get('title', '')
-        tag_list = []
-        tags = datastore.get_all_tags_for_watch(uuid)
-        if tags:
-            for tag_uuid, tag in tags.items():
-                tag_list.append(tag.get('title'))
-        watch_tag = ', '.join(tag_list)
+        watch_title = datastore.data['watching'][uuid]['title']
+        watch_tag = datastore.data['watching'][uuid]['tag']
     else:
         watch_title = 'Change Detection'
         watch_tag = ''
@@ -231,7 +216,6 @@ def create_notification_parameters(n_object, datastore):
             'diff': n_object.get('diff', ''),  # Null default in the case we use a test
             'diff_added': n_object.get('diff_added', ''),  # Null default in the case we use a test
             'diff_full': n_object.get('diff_full', ''),  # Null default in the case we use a test
-            'diff_patch': n_object.get('diff_patch', ''),  # Null default in the case we use a test
             'diff_removed': n_object.get('diff_removed', ''),  # Null default in the case we use a test
             'diff_url': diff_url,
             'preview_url': preview_url,

changedetectionio/processors/__init__.py

@@ -9,7 +9,7 @@ class difference_detection_processor():
         super().__init__(*args, **kwargs)
 
     @abstractmethod
-    def run(self, uuid, skip_when_checksum_same=True, preferred_proxy=None):
+    def run(self, uuid, skip_when_checksum_same=True):
         update_obj = {'last_notification_error': False, 'last_error': False}
         some_data = 'xxxxx'
         update_obj["previous_md5"] = hashlib.md5(some_data.encode('utf-8')).hexdigest()

changedetectionio/processors/restock_diff.py

@@ -42,10 +42,11 @@ class perform_site_check(difference_detection_processor):
 
         # Unset any existing notification error
         update_obj = {'last_notification_error': False, 'last_error': False}
+        extra_headers = watch.get('headers', [])
 
-        request_headers = watch.get('headers', [])
-        request_headers.update(self.datastore.get_all_base_headers())
-        request_headers.update(self.datastore.get_all_headers_in_textfile_for_watch(uuid=uuid))
+        # Tweak the base config with the per-watch ones
+        request_headers = deepcopy(self.datastore.data['settings']['headers'])
+        request_headers.update(extra_headers)
 
         # https://github.com/psf/requests/issues/4525
         # Requests doesnt yet support brotli encoding, so don't put 'br' here, be totally sure that the user cannot

changedetectionio/processors/text_json_diff.py

@@ -50,13 +50,14 @@ class perform_site_check(difference_detection_processor):
 
         return regex
 
-    def run(self, uuid, skip_when_checksum_same=True, preferred_proxy=None):
+    def run(self, uuid, skip_when_checksum_same=True):
         changed_detected = False
         screenshot = False  # as bytes
         stripped_text_from_html = ""
 
         # DeepCopy so we can be sure we don't accidently change anything by reference
         watch = deepcopy(self.datastore.data['watching'].get(uuid))
+
         if not watch:
             raise Exception("Watch no longer exists.")
 
@@ -70,9 +71,9 @@ class perform_site_check(difference_detection_processor):
         update_obj = {'last_notification_error': False, 'last_error': False}
 
         # Tweak the base config with the per-watch ones
-        request_headers = watch.get('headers', [])
-        request_headers.update(self.datastore.get_all_base_headers())
-        request_headers.update(self.datastore.get_all_headers_in_textfile_for_watch(uuid=uuid))
+        extra_headers = watch.get_all_headers()
+        request_headers = self.datastore.get_all_headers()
+        request_headers.update(extra_headers)
 
         # https://github.com/psf/requests/issues/4525
         # Requests doesnt yet support brotli encoding, so don't put 'br' here, be totally sure that the user cannot
@@ -105,11 +106,7 @@ class perform_site_check(difference_detection_processor):
             # If the klass doesnt exist, just use a default
             klass = getattr(content_fetcher, "html_requests")
 
-        if preferred_proxy:
-            proxy_id = preferred_proxy
-        else:
-            proxy_id = self.datastore.get_preferred_proxy_for_watch(uuid=uuid)
-
+        proxy_id = self.datastore.get_preferred_proxy_for_watch(uuid=uuid)
         proxy_url = None
         if proxy_id:
             proxy_url = self.datastore.proxy_list.get(proxy_id).get('url')
@@ -194,23 +191,21 @@ class perform_site_check(difference_detection_processor):
 
             fetcher.content = fetcher.content.replace('</body>', metadata + '</body>')
 
-        # Better would be if Watch.model could access the global data also
-        # and then use getattr https://docs.python.org/3/reference/datamodel.html#object.__getitem__
-        # https://realpython.com/inherit-python-dict/ instead of doing it procedurely
-        include_filters_from_tags = self.datastore.get_tag_overrides_for_watch(uuid=uuid, attr='include_filters')
-        include_filters_rule = [*watch.get('include_filters', []), *include_filters_from_tags]
 
-        subtractive_selectors = [*self.datastore.get_tag_overrides_for_watch(uuid=uuid, attr='subtractive_selectors'),
-                                 *watch.get("subtractive_selectors", []),
-                                 *self.datastore.data["settings"]["application"].get("global_subtractive_selectors", [])
-                                 ]
+        include_filters_rule = deepcopy(watch.get('include_filters', []))
+        # include_filters_rule = watch['include_filters']
+        subtractive_selectors = watch.get(
+            "subtractive_selectors", []
+        ) + self.datastore.data["settings"]["application"].get(
+            "global_subtractive_selectors", []
+        )
 
         # Inject a virtual LD+JSON price tracker rule
         if watch.get('track_ldjson_price_data', '') == PRICE_DATA_TRACK_ACCEPT:
             include_filters_rule.append(html_tools.LD_JSON_PRODUCT_OFFER_SELECTOR)
 
-        has_filter_rule = len(include_filters_rule) and len(include_filters_rule[0].strip())
-        has_subtractive_selectors = len(subtractive_selectors) and len(subtractive_selectors[0].strip())
+        has_filter_rule = include_filters_rule and len("".join(include_filters_rule).strip())
+        has_subtractive_selectors = subtractive_selectors and len(subtractive_selectors[0].strip())
 
         if is_json and not has_filter_rule:
             include_filters_rule.append("json:$")

changedetectionio/run_proxy_tests.sh

@@ -2,8 +2,6 @@
 
 # exit when any command fails
 set -e
-# enable debug
-set -x
 
 # Test proxy list handling, starting two squids on different ports
 # Each squid adds a different header to the response, which is the main thing we test for.
@@ -21,6 +19,7 @@ docker run --network changedet-network -d \
 
 
 ## 2nd test actually choose the preferred proxy from proxies.json
+
 docker run --network changedet-network \
   -v `pwd`/tests/proxy_list/proxies.json-example:/app/changedetectionio/test-datastore/proxies.json \
   test-changedetectionio \
@@ -45,6 +44,7 @@ fi
 
 
 # Test the UI configurable proxies
+
 docker run --network changedet-network \
   test-changedetectionio \
   bash -c 'cd changedetectionio && pytest tests/proxy_list/test_select_custom_proxy.py'
@@ -58,25 +58,4 @@ then
   exit 1
 fi
 
-# Test "no-proxy" option
-docker run --network changedet-network \
-  test-changedetectionio \
-  bash -c 'cd changedetectionio && pytest tests/proxy_list/test_noproxy.py'
-
-# We need to handle grep returning 1
-set +e
-# Check request was never seen in any container
-for c in $(echo "squid-one squid-two squid-custom"); do
-  echo Checking $c
-  docker logs $c &> $c.txt
-  grep noproxy $c.txt
-  if [ $? -ne 1 ]
-  then
-    echo "Saw request for noproxy in $c container"
-    cat $c.txt
-    exit 1
-  fi
-done
-
-
 docker kill squid-one squid-two squid-custom

changedetectionio/static/images/brightdata.svg

@@ -1,58 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<svg
-   height="59.553207"
-   viewBox="-0.36 95.21 25.082135 59.553208"
-   width="249.99138"
-   version="1.1"
-   id="svg12"
-   sodipodi:docname="brightdata.svg"
-   inkscape:version="1.1.2 (0a00cf5339, 2022-02-04)"
-   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
-   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
-   xmlns="http://www.w3.org/2000/svg"
-   xmlns:svg="http://www.w3.org/2000/svg">
-  <defs
-     id="defs16" />
-  <sodipodi:namedview
-     id="namedview14"
-     pagecolor="#ffffff"
-     bordercolor="#666666"
-     borderopacity="1.0"
-     inkscape:pageshadow="2"
-     inkscape:pageopacity="0.0"
-     inkscape:pagecheckerboard="0"
-     showgrid="false"
-     fit-margin-top="0"
-     fit-margin-left="0"
-     fit-margin-right="0"
-     fit-margin-bottom="0"
-     inkscape:zoom="0.9464"
-     inkscape:cx="22.189349"
-     inkscape:cy="-90.870668"
-     inkscape:window-width="1920"
-     inkscape:window-height="1051"
-     inkscape:window-x="1920"
-     inkscape:window-y="0"
-     inkscape:window-maximized="1"
-     inkscape:current-layer="svg12" />
-  <path
-     d="m -34.416031,129.28 c -3.97,-2.43 -5.1,-6.09 -4.32,-10.35 0.81,-4.4 3.95,-6.75 8.04,-7.75 4.23,-1.04 8.44,-0.86 12.3,1.5 0.63,0.39 0.93,0.03 1.31,-0.29 1.5,-1.26 3.27,-1.72 5.189999,-1.83 0.79,-0.05 1.04,0.24 1.01,1.01 -0.05,1.31 -0.04,2.63 0,3.95 0.02,0.65 -0.19,0.93 -0.87,0.89 -0.889999,-0.04 -1.789999,0.03 -2.669999,-0.02 -0.82,-0.04 -1.08,0.1 -0.88,1.04 0.83,3.9 -0.06,7.37 -3.1,10.06 -2.76,2.44 -6.13,3.15 -9.72,3.04 -0.51,-0.02 -1.03,-0.02 -1.52,-0.13 -1.22,-0.25 -1.96,0.14 -2.19,1.41 -0.28,1.54 0.16,2.62 1.37,3.07 0.84,0.31 1.74,0.35 2.63,0.39 2.97,0.13 5.95,-0.18 8.91,0.21 2.93,0.39 5.69,1.16 6.85,4.25 1.269999,3.38 0.809999,6.62 -1.48,9.47 -2.73,3.39 -6.52,4.78 -10.66,5.33 -3.53,0.48 -7.04,0.27 -10.39,-1.11 -3.89,-1.6 -5.75,-4.95 -4.84,-8.72 0.51,-2.11 1.85,-3.58 3.69,-4.65 0.38,-0.22 0.93,-0.32 0.28,-0.96 -2.91,-2.83 -2.85,-6.16 0.1,-8.95 0.28,-0.26 0.6,-0.53 0.96,-0.86 z m 8.07,21.5 c 0.95,0.04 1.87,-0.13 2.78,-0.33 1.89,-0.42 3.51,-1.3 4.49,-3.06 1.82,-3.25 0.24,-6.2 -3.37,-6.58 -2.88,-0.3 -5.76,0.24 -8.63,-0.13 -0.53,-0.07 -0.75,0.34 -0.95,0.71 -1.16,2.24 -1.08,4.53 0,6.73 1.15,2.34 3.46,2.48 5.68,2.66 z m -5,-30.61 c -0.03,1.67 0.08,3.19 0.74,4.61 0.76,1.62 2.17,2.42 4.03,2.31 1.62,-0.1 2.9,-1.12 3.36,-2.84 0.66,-2.46 0.69,-4.95 0.01,-7.42 -0.49,-1.76 -1.7,-2.64 -3.56,-2.7 -2.08,-0.07 -3.37,0.7 -4.04,2.42 -0.47,1.21 -0.6,2.47 -0.54,3.62 z m 32.9399993,6.56 c 0,2.59 0.05,5.18 -0.02,7.77 -0.03,1.03 0.31,1.46 1.32,1.52 0.65,0.04 1.61,-0.09 1.82,0.57 0.26,0.81 0.11,1.76 0.06,2.65 -0.03,0.48 -0.81,0.39 -0.81,0.39 l -11.47,0.01 c 0,0 -0.95,-0.21 -0.88,-0.88 0.03,-0.29 0.04,-0.6 0,-0.89 -0.19,-1.24 0.21,-1.92 1.58,-1.9 0.99,0.01 1.28,-0.52 1.28,-1.53 -0.05,-8.75 -0.05,-17.49 0,-26.24 0.01,-1.15 -0.36,-1.62 -1.44,-1.67 -0.17,-0.01 -0.34,-0.04 -0.5,-0.07 -1.43,-0.22 -2.12,-1.57 -1.53,-2.91 0.15,-0.35 0.43,-0.36 0.72,-0.4 2.94,-0.41 5.88,-0.81 8.82000002,-1.23 0.81999998,-0.12 0.99999998,0.27 0.98999998,1.01 -0.02,3.35 0,6.71 0.02,10.06 0,0.35 -0.23,0.84 0.18,1.03 0.38,0.17 0.69,-0.25 0.99,-0.45 2.56,-1.74 5.33,-2.73 8.4900007,-2.56 3.51005,0.19 5.65005,1.95 6.35005,5.46 0.42,2.09 0.52,4.21 0.51,6.33 -0.02,3.86 0.05,7.73 -0.04,11.59 -0.02,1.12 0.37,1.5 1.39,1.6 0.61,0.05 1.55,-0.13 1.74,0.47 0.26,0.85 0.12,1.84 0.1,2.77 -0.01,0.41 -0.69,0.37 -0.69,0.37 l -11.4700504,0.01 c 0,0 -0.81,-0.29 -0.8,-0.85 0.01,-0.38 0.04,-0.77 -0.01,-1.15 -0.13,-1.01 0.32,-1.52 1.31,-1.56 1.0600004,-0.05 1.3800004,-0.55 1.3500004,-1.63 -0.14,-4.84 0.16,-9.68 -0.18,-14.51 -0.26,-3.66 -2.1100004,-4.95 -5.6700007,-3.99 -0.25,0.07 -0.49,0.15 -0.73,0.22 -2.57,0.8 -2.79,1.09 -2.79,3.71 0.01,2.3 0.01,4.59 0.01,6.88 z M -109.26603,122.56 c 0,-4.75 -0.02,-9.51 0.02,-14.26 0.01,-0.92 -0.17,-1.47 -1.19,-1.45 -0.16,0 -0.33,-0.07 -0.5,-0.1 -1.56,-0.27 -2.24,-1.47 -1.69,-2.92 0.14,-0.37 0.41,-0.38 0.7,-0.42 2.98,-0.41 5.97,-0.81 8.94,-1.24 0.85,-0.12 0.88,0.33 0.88,0.96 -0.01,3.01 -0.01,6.03 0,9.04 0,0.4 -0.18,0.96 0.27,1.16 0.36,0.16 0.66,-0.3 0.96,-0.52 4.729999,-3.51 12.459999,-2.61 14.889999,4.48 1.89,5.51 1.91,11.06 -0.96,16.28 -2.37,4.31 -6.19,6.49 -11.15,6.59 -3.379999,0.07 -6.679999,-0.3 -9.909999,-1.37 -0.93,-0.31 -1.3,-0.78 -1.28,-1.83 0.05,-4.81 0.02,-9.6 0.02,-14.4 z m 7.15,3.89 c 0,2.76 0.02,5.52 -0.01,8.28 -0.01,0.76 0.18,1.29 0.91,1.64 1.899999,0.9 4.299999,0.5 5.759999,-1.01 0.97,-1 1.56,-2.21 1.96,-3.52 1.03,-3.36 0.97,-6.78 0.61,-10.22 a 9.991,9.991 0 0 0 -0.93,-3.29 c -1.47,-3.06 -4.67,-3.85 -7.439999,-1.86 -0.6,0.43 -0.88,0.93 -0.87,1.7 0.04,2.76 0.01,5.52 0.01,8.28 z"
-     fill="#4280f6"
-     id="path2" />
-  <path
-     d="m 68.644019,137.2 c -1.62,1.46 -3.41,2.56 -5.62,2.96 -4.4,0.8 -8.7,-1.39 -10.49,-5.49 -2.31,-5.31 -2.3,-10.67 -0.1,-15.98 2.31,-5.58 8.29,-8.65 14.24,-7.46 1.71,0.34 1.9,0.18 1.9,-1.55 0,-0.68 -0.05,-1.36 0.01,-2.04 0.09,-1.02 -0.25,-1.54 -1.34,-1.43 -0.64,0.06 -1.26,-0.1 -1.88,-0.21 -1.32,-0.24 -1.6,-0.62 -1.37,-1.97 0.07,-0.41 0.25,-0.57 0.65,-0.62 2.63,-0.33 5.27,-0.66 7.9,-1.02 1.04,-0.14 1.17,0.37 1.17,1.25 -0.02,10.23 -0.02,20.45 -0.01,30.68 v 1.02 c 0.02,0.99 0.35,1.6 1.52,1.47 0.52,-0.06 1.35,-0.27 1.25,0.73 -0.08,0.8 0.58,1.93 -0.94,2.18 -1.29,0.22 -2.51,0.69 -3.86,0.65 -2.04,-0.06 -2.3,-0.23 -2.76,-2.19 -0.09,-0.3 0.06,-0.67 -0.27,-0.98 z m -0.07,-12.46 c 0,-2.8 -0.04,-5.6 0.02,-8.39 0.02,-0.9 -0.28,-1.47 -1.05,-1.81 -3.18,-1.4 -7.54,-0.8 -9.3,2.87 -0.83,1.74 -1.31,3.54 -1.49,5.46 -0.28,2.93 -0.38,5.83 0.61,8.65 0.73,2.09 1.81,3.9 4.11,4.67 2.49,0.83 4.55,-0.04 6.5,-1.48 0.54,-0.4 0.62,-0.95 0.61,-1.57 -0.02,-2.8 -0.01,-5.6 -0.01,-8.4 z m 28.79,2.53 c 0,3.24 0.04,5.83 -0.02,8.41 -0.02,1 0.19,1.49 1.309998,1.41 0.55,-0.04 1.460003,-0.46 1.520003,0.73 0.05,1.02 0.1,1.89 -1.330003,2.08 -1.289998,0.17 -2.559998,0.51 -3.889998,0.48 -1.88,-0.05 -2.15,-0.26 -2.42,-2.15 -0.04,-0.27 0.14,-0.65 -0.22,-0.79 -0.34,-0.13 -0.5,0.24 -0.72,0.42 -3.61,3 -8.15,3.4 -11.64,1.08 -1.61,-1.07 -2.49,-2.63 -2.67,-4.43 -0.51,-5.13 0.77,-7.91 6.3,-10.22 2.44,-1.02 5.07,-1.27 7.68,-1.49 0.77,-0.07 1.03,-0.28 1.02,-1.05 -0.03,-1.48 -0.05,-2.94 -0.64,-4.36 -0.59,-1.42 -1.67,-1.92 -3.08,-2.03 -3.04,-0.24 -5.88,0.5 -8.63,1.71 -0.51,0.23 -1.19,0.75 -1.48,-0.13 -0.26,-0.77 -1.35,-1.61 0.05,-2.47 3.27,-2 6.7,-3.44 10.61,-3.42 1.44,0.01 2.88,0.27 4.21,0.81 2.67,1.08 3.44,3.4 3.8,5.99 0.46,3.37 0.1,6.73 0.24,9.42 z m -5.09,2.9 c 0,-1.23 -0.01,-2.46 0,-3.69 0,-0.52 -0.06,-0.98 -0.75,-0.84 -1.45,0.3 -2.93,0.28 -4.37,0.69 -3.71,1.04 -5.46,4.48 -3.97,8.03 0.51,1.22 1.48,1.98 2.79,2.16 2.01,0.28 3.86,-0.29 5.6,-1.28 0.54,-0.31 0.73,-0.76 0.72,-1.37 -0.05,-1.23 -0.02,-2.47 -0.02,-3.7 z m 43.060001,-2.89 c 0,2.72 0.01,5.43 -0.01,8.15 0,0.66 0.02,1.21 0.91,1.12 0.54,-0.06 0.99,0.12 0.86,0.75 -0.15,0.71 0.56,1.7 -0.58,2.09 -1.55,0.52 -3.16,0.59 -4.77,0.4 -0.99,-0.12 -1.12,-1.01 -1.18,-1.73 -0.08,-1.15 -0.16,-1.45 -1.24,-0.54 -3.41,2.87 -8.05,3.17 -11.43,0.88 -1.75,-1.18 -2.49,-2.91 -2.7,-4.94 -0.64,-6.24 3.16,-8.74 7.83,-10.17 2.04,-0.62 4.14,-0.8 6.24,-0.99 0.81,-0.07 1,-0.36 0.98,-1.09 -0.04,-1.31 0.04,-2.62 -0.42,-3.89 -0.57,-1.57 -1.53,-2.34 -3.18,-2.45 -3.03,-0.21 -5.88,0.46 -8.64,1.66 -0.6,0.26 -1.25,0.81 -1.68,-0.2 -0.34,-0.8 -1.08,-1.61 0.16,-2.36 4.12,-2.5 8.44,-4.16 13.36,-3.07 3.21,0.71 4.89,2.91 5.26,6.34 0.18,1.69 0.22,3.37 0.22,5.07 0.01,1.66 0.01,3.32 0.01,4.97 z m -5.09,2.54 c 0,-1.27 -0.03,-2.54 0.01,-3.81 0.02,-0.74 -0.27,-1.02 -0.98,-0.92 -1.21,0.17 -2.43,0.28 -3.62,0.55 -3.72,0.83 -5.47,3.48 -4.82,7.21 0.29,1.66 1.57,2.94 3.21,3.16 2.02,0.27 3.85,-0.34 5.57,-1.34 0.49,-0.29 0.64,-0.73 0.63,-1.29 -0.02,-1.18 0,-2.37 0,-3.56 z"
-     fill="#c8dbfb"
-     id="path4" />
-  <path
-     d="m 26.314019,125.77 c 0,-2.89 -0.05,-5.77 0.02,-8.66 0.03,-1.04 -0.33,-1.39 -1.31,-1.24 a 0.7,0.7 0 0 1 -0.25,0 c -0.57,-0.18 -1.44,0.48 -1.68,-0.58 -0.35,-1.48 -0.02,-2.3 1.21,-2.7 1.3,-0.43 2.16,-1.26 2.76,-2.46 0.78,-1.56 1.44,-3.17 1.91,-4.84 0.18,-0.63 0.47,-0.86 1.15,-0.88 3.28,-0.09 3.27,-0.11 3.32,3.17 0.01,1.06 0.09,2.12 0.09,3.18 -0.01,0.67 0.27,0.89 0.91,0.88 1.61,-0.02 3.23,0.03 4.84,-0.02 0.77,-0.02 1.01,0.23 1.03,1.01 0.08,3.27 0.1,3.27 -3.09,3.27 -0.93,0 -1.87,0.03 -2.8,-0.01 -0.67,-0.02 -0.89,0.26 -0.88,0.91 0.04,5.43 0.04,10.86 0.12,16.29 0.02,1.7 0.75,2.26 2.46,2.1 1.1,-0.1 2.19,-0.26 3.23,-0.65 0.59,-0.22 0.89,-0.09 1.14,0.53 0.93,2.29 0.92,2.37 -1.32,3.52 -2.54,1.3 -5.22,1.99 -8.1,1.79 -2.27,-0.16 -3.68,-1.27 -4.35,-3.45 -0.3,-0.98 -0.41,-1.99 -0.41,-3.01 z m -97.67005,-8.99 c 0.57,-0.84 1.11,-1.74 1.76,-2.55 1.68,-2.09 3.68,-3.62 6.54,-3.66 1.08,-0.01 1.63,0.28 1.57,1.52 -0.1,2.08 -0.05,4.16 -0.02,6.24 0.01,0.74 -0.17,0.96 -0.96,0.76 -2.36,-0.59 -4.71,-0.42 -7.03,0.28 -0.8,0.24 -1.16,0.62 -1.15,1.52 0.05,4.5 0.04,9 0,13.5 -0.01,0.89 0.29,1.16 1.15,1.2 1.23,0.06 2.44,0.32 3.67,0.39 0.75,0.05 0.91,0.38 0.89,1.04 -0.06,2.86 0.29,2.28 -2.25,2.3 -4.2,0.04 -8.41,-0.02 -12.61,0.03 -0.91,0.01 -1.39,-0.18 -1.22,-1.18 0.02,-0.12 0,-0.25 0,-0.38 0.02,-2.1 -0.24,-1.88 1.77,-2.04 1.33,-0.11 1.6,-0.67 1.58,-1.9 -0.07,-5.35 -0.04,-10.7 -0.02,-16.05 0,-0.78 -0.17,-1.2 -1,-1.46 -2.21,-0.68 -2.7,-1.69 -2.22,-3.99 0.11,-0.52 0.45,-0.56 0.82,-0.62 2.22,-0.34 4.44,-0.7 6.67,-0.99 0.99,-0.13 1.82,0.7 1.84,1.76 0.03,1.4 0.03,2.8 0.04,4.2 -0.01,0.02 0.06,0.04 0.18,0.08 z m 25.24,6.59 c 0,3.69 0.04,7.38 -0.03,11.07 -0.02,1.04 0.31,1.48 1.32,1.49 0.29,0 0.59,0.12 0.88,0.13 0.93,0.01 1.18,0.47 1.16,1.37 -0.05,2.19 0,2.19 -2.24,2.19 -3.48,0 -6.96,-0.04 -10.44,0.03 -1.09,0.02 -1.47,-0.33 -1.3,-1.36 0.02,-0.12 0.02,-0.26 0,-0.38 -0.28,-1.39 0.39,-1.96 1.7,-1.9 1.36,0.06 1.76,-0.51 1.74,-1.88 -0.09,-5.17 -0.08,-10.35 0,-15.53 0.02,-1.22 -0.32,-1.87 -1.52,-2.17 -0.57,-0.14 -1.47,-0.11 -1.57,-0.85 -0.15,-1.04 -0.05,-2.11 0.01,-3.17 0.02,-0.34 0.44,-0.35 0.73,-0.39 2.81,-0.39 5.63,-0.77 8.44,-1.18 0.92,-0.14 1.15,0.2 1.14,1.09 -0.04,3.8 -0.02,7.62 -0.02,11.44 z"
-     fill="#4280f6"
-     id="path6" />
-  <path
-     d="m 101.44402,125.64 c 0,-3.18 -0.03,-6.37 0.02,-9.55 0.02,-0.94 -0.26,-1.36 -1.22,-1.22 -0.21,0.03 -0.430003,0.04 -0.630003,0 -0.51,-0.12 -1.35,0.39 -1.44,-0.55 -0.08,-0.85 -0.429998,-1.87 0.93,-2.24 2.080003,-0.57 2.720003,-2.39 3.350003,-4.17 0.31,-0.88 0.62,-1.76 0.87,-2.66 0.18,-0.64 0.52,-0.85 1.19,-0.84 2.46,0.05 2,-0.15 2.04,2.04 0.02,1.1 0.08,2.21 -0.02,3.31 -0.11,1.16 0.46,1.52 1.46,1.53 1.78,0.01 3.57,0.04 5.35,-0.01 0.82,-0.02 1.12,0.23 1.11,1.08 -0.05,2.86 0.19,2.49 -2.42,2.51 -1.53,0.01 -3.06,0.02 -4.59,-0.01 -0.65,-0.01 -0.9,0.22 -0.9,0.89 0.02,5.52 0,11.04 0.03,16.56 0,0.67 0.14,1.34 0.25,2.01 0.17,1.04 1.17,1.62 2.59,1.42 1.29,-0.19 2.57,-0.49 3.86,-0.69 0.43,-0.07 1.05,-0.47 1.19,0.4 0.12,0.75 1.05,1.61 -0.09,2.24 -2.09,1.16 -4.28,2.07 -6.71,2.16 -1.05,0.04 -2.13,0.2 -3.16,-0.14 -1.92,-0.65 -3.03,-2.28 -3.05,-4.51 -0.02,-3.19 -0.01,-6.37 -0.01,-9.56 z"
-     fill="#c8dbfb"
-     id="path8" />
-  <path
-     d="m -50.816031,95.21 c 0.19,2.160002 1.85,3.240002 2.82,4.740002 0.25,0.379998 0.48,0.109998 0.67,-0.16 0.21,-0.31 0.6,-1.21 1.15,-1.28 -0.35,1.38 -0.04,3.149998 0.16,4.449998 0.49,3.05 -1.22,5.64 -4.07,6.18 -3.38,0.65 -6.22,-2.21 -5.6,-5.62 0.23,-1.24 1.37,-2.5 0.77,-3.699998 -0.85,-1.7 0.54,-0.52 0.79,-0.22 1.04,1.199998 1.21,0.09 1.45,-0.55 0.24,-0.63 0.31,-1.31 0.47,-1.97 0.19,-0.770002 0.55,-1.400002 1.39,-1.870002 z"
-     fill="#4280f6"
-     id="path10" />
-</svg>

changedetectionio/static/js/diff-overview.js

@@ -1,13 +1,4 @@
 $(document).ready(function () {
-    var csrftoken = $('input[name=csrf_token]').val();
-    $.ajaxSetup({
-        beforeSend: function (xhr, settings) {
-            if (!/^(GET|HEAD|OPTIONS|TRACE)$/i.test(settings.type) && !this.crossDomain) {
-                xhr.setRequestHeader("X-CSRFToken", csrftoken)
-            }
-        }
-    })
-
     // Load it when the #screenshot tab is in use, so we dont give a slow experience when waiting for the text diff to load
     window.addEventListener('hashchange', function (e) {
         toggle(location.hash);
@@ -24,71 +15,11 @@ $(document).ready(function () {
             $("#settings").hide();
         } else if (hash_name === '#extract') {
             $("#settings").hide();
-        } else {
+        }
+
+
+        else {
             $("#settings").show();
         }
     }
-
-    const article = $('.highlightable-filter')[0];
-
-    // We could also add the  'touchend' event for touch devices, but since
-    // most iOS/Android browsers already show a dialog when you select
-    // text (often with a Share option) we'll skip that
-    article.addEventListener('mouseup', dragTextHandler, false);
-    article.addEventListener('mousedown', clean, false);
-
-    function clean(event) {
-      $("#highlightSnippet").remove();
-    }
-
-
-    function dragTextHandler(event) {
-        console.log('mouseupped');
-
-        // Check if any text was selected
-        if (window.getSelection().toString().length > 0) {
-
-            // Find out how much (if any) user has scrolled
-            var scrollTop = (window.pageYOffset !== undefined) ? window.pageYOffset : (document.documentElement || document.body.parentNode || document.body).scrollTop;
-
-            // Get cursor position
-            const posX = event.clientX;
-            const posY = event.clientY + 20 + scrollTop;
-
-            // Append HTML to the body, create the "Tweet Selection" dialog
-            document.body.insertAdjacentHTML('beforeend', '<div id="highlightSnippet" style="position: absolute; top: ' + posY + 'px; left: ' + posX + 'px;"><div class="pure-form-message-inline" style="font-size: 70%">Ignore any change on any line which contains the selected text.</div><br><a data-mode="exact" href="javascript:void(0);" class="pure-button button-secondary button-xsmall">Ignore exact text</a>&nbsp;</div>');
-
-            if (/\d/.test(window.getSelection().toString())) {
-                // Offer regex replacement
-                document.getElementById("highlightSnippet").insertAdjacentHTML('beforeend', '<a data-mode="digit-regex"  href="javascript:void(0);" class="pure-button button-secondary button-xsmall">Ignore text including number changes</a>');
-            }
-
-            $('#highlightSnippet a').bind('click', function (e) {
-                if(!window.getSelection().toString().trim().length) {
-                    alert('Oops no text selected!');
-                    return;
-                }
-
-                $.ajax({
-                    type: "POST",
-                    url: highlight_submit_ignore_url,
-                    data: {'mode': $(this).data('mode'), 'selection': window.getSelection().toString()},
-                    statusCode: {
-                        400: function () {
-                            // More than likely the CSRF token was lost when the server restarted
-                            alert("There was a problem processing the request, please reload the page.");
-                        }
-                    }
-                }).done(function (data) {
-                    $("#highlightSnippet").html(data)
-                }).fail(function (data) {
-                    console.log(data);
-                    alert('There was an error communicating with the server.');
-                });
-            });
-
-        }
-    }
-
-
 });

changedetectionio/static/js/recheck-proxy.js

@@ -1,87 +0,0 @@
-$(function () {
-    /* add container before each proxy location to show status */
-
-    var option_li = $('.fetch-backend-proxy li').filter(function() {
-        return $("input",this)[0].value.length >0;
-    });
-
-    //var option_li = $('.fetch-backend-proxy li');
-    var isActive = false;
-    $(option_li).prepend('<div class="proxy-status"></div>');
-    $(option_li).append('<div class="proxy-timing"></div><div class="proxy-check-details"></div>');
-
-    function set_proxy_check_status(proxy_key, state) {
-        // select input by value name
-        const proxy_li = $('input[value="' + proxy_key + '" ]').parent();
-        if (state['status'] === 'RUNNING') {
-            $('.proxy-status', proxy_li).html('<span class="spinner"></span>');
-        }
-        if (state['status'] === 'OK') {
-            $('.proxy-status', proxy_li).html('<span style="color: green; font-weight: bold" >OK</span>');
-            $('.proxy-check-details', proxy_li).html(state['text']);
-        }
-        if (state['status'] === 'ERROR' || state['status'] === 'ERROR OTHER') {
-            $('.proxy-status', proxy_li).html('<span style="color: red; font-weight: bold" >X</span>');
-            $('.proxy-check-details', proxy_li).html(state['text']);
-        }
-        $('.proxy-timing', proxy_li).html(state['time']);
-    }
-
-
-    function pollServer() {
-        if (isActive) {
-            window.setTimeout(function () {
-                $.ajax({
-                    url: proxy_recheck_status_url,
-                    success: function (data) {
-                        var all_done = true;
-                        $.each(data, function (proxy_key, state) {
-                            set_proxy_check_status(proxy_key, state);
-                            if (state['status'] === 'RUNNING') {
-                                all_done = false;
-                            }
-                        });
-
-                        if (all_done) {
-                            console.log("Shutting down poller, all done.")
-                            isActive = false;
-                        } else {
-                            pollServer();
-                        }
-                    },
-                    error: function () {
-                        //ERROR HANDLING
-                        pollServer();
-                    }
-                });
-            }, 2000);
-        }
-    }
-
-    $('#check-all-proxies').click(function (e) {
-        e.preventDefault()
-        $('body').addClass('proxy-check-active');
-        $('.proxy-check-details').html('');
-        $('.proxy-status').html('<span class="spinner"></span>').fadeIn();
-        $('.proxy-timing').html('');
-
-        // Request start, needs CSRF?
-        $.ajax({
-            type: "GET",
-            url: recheck_proxy_start_url,
-        }).done(function (data) {
-            $.each(data, function (proxy_key, state) {
-                set_proxy_check_status(proxy_key, state['status'])
-            });
-            isActive = true;
-            pollServer();
-
-        }).fail(function (data) {
-            console.log(data);
-            alert('There was an error communicating with the server.');
-        });
-
-    });
-
-});
-

changedetectionio/static/js/toggle-theme.js

@@ -9,7 +9,13 @@ $(document).ready(function () {
     const htmlElement = document.getElementsByTagName("html");
     const isDarkMode = htmlElement[0].dataset.darkmode === "true";
     htmlElement[0].dataset.darkmode = !isDarkMode;
-    setCookieValue(!isDarkMode);
+    if (isDarkMode) {
+      button.classList.remove("dark");
+      setCookieValue(false);
+    } else {
+      button.classList.add("dark");
+      setCookieValue(true);
+    }
   };
 
   const setCookieValue = (value) => {

changedetectionio/static/js/watch-overview.js

@@ -1,45 +1,31 @@
 $(function () {
-    // Remove unviewed status when normally clicked
-    $('.diff-link').click(function () {
-        $(this).closest('.unviewed').removeClass('unviewed');
-    });
+  // Remove unviewed status when normally clicked
+  $('.diff-link').click(function () {
+    $(this).closest('.unviewed').removeClass('unviewed');
+  });
 
-    $("#checkbox-assign-tag").click(function (e) {
-        $('#op_extradata').val(prompt("Enter a tag name"));
-    });
 
-    $('.with-share-link > *').click(function () {
-        $("#copied-clipboard").remove();
+  $('.with-share-link > *').click(function () {
+      $("#copied-clipboard").remove();
 
-        var range = document.createRange();
-        var n = $("#share-link")[0];
-        range.selectNode(n);
-        window.getSelection().removeAllRanges();
-        window.getSelection().addRange(range);
-        document.execCommand("copy");
-        window.getSelection().removeAllRanges();
+      var range = document.createRange();
+      var n=$("#share-link")[0];
+      range.selectNode(n);
+      window.getSelection().removeAllRanges();
+      window.getSelection().addRange(range);
+      document.execCommand("copy");
+      window.getSelection().removeAllRanges();
 
-        $('.with-share-link').append('<span style="font-size: 80%; color: #fff;" id="copied-clipboard">Copied to clipboard</span>');
-        $("#copied-clipboard").fadeOut(2500, function () {
-            $(this).remove();
-        });
-    });
-
-    $(".watch-table tr").click(function (event) {
-        var tagName = event.target.tagName.toLowerCase();
-        if (tagName === 'tr' || tagName === 'td') {
-            var x = $('input[type=checkbox]', this);
-            if (x) {
-                $(x).click();
-            }
-        }
-    });
+      $('.with-share-link').append('<span style="font-size: 80%; color: #fff;" id="copied-clipboard">Copied to clipboard</span>');
+      $("#copied-clipboard").fadeOut(2500, function() {
+       $(this).remove();
+      });
+  });
 
     // checkboxes - check all
     $("#check-all").click(function (e) {
         $('input[type=checkbox]').not(this).prop('checked', this.checked);
     });
-
     // checkboxes - show/hide buttons
     $("input[type=checkbox]").click(function (e) {
         if ($('input[type=checkbox]:checked').length) {

changedetectionio/static/styles/diff.css

@@ -218,10 +218,3 @@ td#diff-col div {
   text-align: center; }
   .tab-pane-inner#screenshot img {
     max-width: 99%; }
-
-#highlightSnippet {
-  background: var(--color-background);
-  padding: 1em;
-  border-radius: 5px;
-  background: var(--color-background);
-  box-shadow: 1px 1px 4px var(--color-shadow-jump); }

changedetectionio/static/styles/scss/diff.scss

@@ -119,11 +119,3 @@ td#diff-col div {
     max-width: 99%;
   }
 }
-
-#highlightSnippet {
-  background: var(--color-background);
-  padding: 1em;
-  border-radius: 5px;
-  background: var(--color-background);
-  box-shadow: 1px 1px 4px var(--color-shadow-jump);
-}

changedetectionio/static/styles/scss/parts/_darkmode.scss

@@ -1,25 +0,0 @@
-
-#toggle-light-mode {
-  width: 3rem;
-  /* default */
-  .icon-dark {
-    display: none;
-  }
-
-}
-
-html[data-darkmode="true"] {
-  #toggle-light-mode {
-    .icon-light {
-      display: none;
-    }
-
-    .icon-dark {
-      display: block;
-    }
-  }
-}
-
-
-
-

changedetectionio/static/styles/scss/parts/_extra_proxies.scss

@@ -7,7 +7,6 @@ ul#requests-extra_proxies {
     }
 
   }
-
   /* each proxy entry is a `table` */
   table {
     tr {
@@ -16,47 +15,3 @@ ul#requests-extra_proxies {
   }
 }
 
-#request {
-  /* Auto proxy scan/checker */
-  label[for=proxy] {
-    display: inline-block;
-  }
-}
-
-body.proxy-check-active {
-  #request {
-    .proxy-status {
-      width: 2em;
-    }
-
-    .proxy-check-details {
-      font-size: 80%;
-      color: #555;
-      display: block;
-      padding-left: 4em;
-    }
-
-    .proxy-timing {
-      font-size: 80%;
-      padding-left: 1rem;
-      color: var(--color-link);
-    }
-  }
-}
-
-
-#recommended-proxy {
-  display: grid;
-  gap: 2rem;
-    @media  (min-width: 991px) {
-      grid-template-columns: repeat(2, 1fr);
-    }
-
-  > div {
-    border: 1px #aaa solid;
-    border-radius: 4px;
-    padding: 1em;
-  }
-
-  padding-bottom: 1em;
-}

changedetectionio/static/styles/scss/styles.scss

@@ -8,7 +8,6 @@
 @import "parts/_pagination";
 @import "parts/_spinners";
 @import "parts/_variables";
-@import "parts/_darkmode";
 
 body {
   color: var(--color-text);
@@ -55,6 +54,22 @@ a.github-link {
   }
 }
 
+#toggle-light-mode {
+  width: 3rem;
+  .icon-dark {
+    display: none;
+  }
+
+  &.dark {
+    .icon-light {
+      display: none;
+    }
+
+    .icon-dark {
+      display: block;
+    }
+  }
+}
 
 #toggle-search {
   width: 2rem;

changedetectionio/static/styles/styles.css

@@ -95,37 +95,6 @@ ul#requests-extra_proxies {
   ul#requests-extra_proxies table tr {
     display: inline; }
 
-#request {
-  /* Auto proxy scan/checker */ }
-  #request label[for=proxy] {
-    display: inline-block; }
-
-body.proxy-check-active #request .proxy-status {
-  width: 2em; }
-
-body.proxy-check-active #request .proxy-check-details {
-  font-size: 80%;
-  color: #555;
-  display: block;
-  padding-left: 4em; }
-
-body.proxy-check-active #request .proxy-timing {
-  font-size: 80%;
-  padding-left: 1rem;
-  color: var(--color-link); }
-
-#recommended-proxy {
-  display: grid;
-  gap: 2rem;
-  padding-bottom: 1em; }
-  @media (min-width: 991px) {
-    #recommended-proxy {
-      grid-template-columns: repeat(2, 1fr); } }
-  #recommended-proxy > div {
-    border: 1px #aaa solid;
-    border-radius: 4px;
-    padding: 1em; }
-
 .pagination-page-info {
   color: #fff;
   font-size: 0.85rem;
@@ -328,18 +297,6 @@ html[data-darkmode="true"] {
     html[data-darkmode="true"] .watch-table .unviewed.error {
       color: var(--color-watch-table-error); }
 
-#toggle-light-mode {
-  width: 3rem;
-  /* default */ }
-  #toggle-light-mode .icon-dark {
-    display: none; }
-
-html[data-darkmode="true"] #toggle-light-mode .icon-light {
-  display: none; }
-
-html[data-darkmode="true"] #toggle-light-mode .icon-dark {
-  display: block; }
-
 body {
   color: var(--color-text);
   background: var(--color-background-page); }
@@ -374,6 +331,15 @@ a.github-link {
   a.github-link:hover {
     color: var(--color-icon-github-hover); }
 
+#toggle-light-mode {
+  width: 3rem; }
+  #toggle-light-mode .icon-dark {
+    display: none; }
+  #toggle-light-mode.dark .icon-light {
+    display: none; }
+  #toggle-light-mode.dark .icon-dark {
+    display: block; }
+
 #toggle-search {
   width: 2rem; }
 

changedetectionio/store.py

@@ -1,5 +1,3 @@
-from distutils.util import strtobool
-
 from flask import (
     flash
 )
@@ -18,8 +16,6 @@ import threading
 import time
 import uuid as uuid_builder
 
-dictfilt = lambda x, y: dict([ (i,x[i]) for i in x if i in set(y) ])
-
 # Is there an existing library to ensure some data store (JSON etc) is in sync with CRUD methods?
 # Open a github issue if you know something :)
 # https://stackoverflow.com/questions/6190468/how-to-trigger-function-on-value-change
@@ -182,6 +178,20 @@ class ChangeDetectionStore:
 
         return self.__data
 
+    def get_all_tags(self):
+        tags = []
+        for uuid, watch in self.data['watching'].items():
+            if watch['tag'] is None:
+                continue
+            # Support for comma separated list of tags.
+            for tag in watch['tag'].split(','):
+                tag = tag.strip()
+                if tag not in tags:
+                    tags.append(tag)
+
+        tags.sort()
+        return tags
+
     # Delete a single watch by UUID
     def delete(self, uuid):
         import pathlib
@@ -207,9 +217,10 @@ class ChangeDetectionStore:
 
     # Clone a watch by UUID
     def clone(self, uuid):
-        url = self.data['watching'][uuid].get('url')
+        url = self.data['watching'][uuid]['url']
+        tag = self.data['watching'][uuid]['tag']
         extras = self.data['watching'][uuid]
-        new_uuid = self.add_watch(url=url, extras=extras)
+        new_uuid = self.add_watch(url=url, tag=tag, extras=extras)
         return new_uuid
 
     def url_exists(self, url):
@@ -244,14 +255,16 @@ class ChangeDetectionStore:
 
         self.needs_write_urgent = True
 
-    def add_watch(self, url, tag='', extras=None, tag_uuids=None, write_to_disk_now=True):
+    def add_watch(self, url, tag="", extras=None, write_to_disk_now=True):
 
         if extras is None:
             extras = {}
+        # should always be str
+        if tag is None or not tag:
+            tag = ''
 
         # Incase these are copied across, assume it's a reference and deepcopy()
         apply_extras = deepcopy(extras)
-        apply_extras['tags'] = [] if not apply_extras.get('tags') else apply_extras.get('tags')
 
         # Was it a share link? try to fetch the data
         if (url.startswith("https://changedetection.io/share/")):
@@ -278,7 +291,6 @@ class ChangeDetectionStore:
                     'processor',
                     'subtractive_selectors',
                     'tag',
-                    'tags',
                     'text_should_not_be_present',
                     'title',
                     'trigger_text',
@@ -301,39 +313,25 @@ class ChangeDetectionStore:
             flash('Watch protocol is not permitted by SAFE_PROTOCOL_REGEX', 'error')
             return None
 
-        if tag and type(tag) == str:
-            # Then it's probably a string of the actual tag by name, split and add it
-            for t in tag.split(','):
-                # for each stripped tag, add tag as UUID
-                for a_t in t.split(','):
-                    tag_uuid = self.add_tag(a_t)
-                    apply_extras['tags'].append(tag_uuid)
+        with self.lock:
+            # #Re 569
+            new_watch = Watch.model(datastore_path=self.datastore_path, default={
+                'url': url,
+                'tag': tag,
+                'date_created': int(time.time())
+            })
 
-        # Or if UUIDs given directly
-        if tag_uuids:
-            apply_extras['tags'] = list(set(apply_extras['tags'] + tag_uuids))
+            new_uuid = new_watch['uuid']
+            logging.debug("Added URL {} - {}".format(url, new_uuid))
 
-        # Make any uuids unique
-        if apply_extras.get('tags'):
-            apply_extras['tags'] = list(set(apply_extras.get('tags')))
+            for k in ['uuid', 'history', 'last_checked', 'last_changed', 'newest_history_key', 'previous_md5', 'viewed']:
+                if k in apply_extras:
+                    del apply_extras[k]
 
-        new_watch = Watch.model(datastore_path=self.datastore_path, url=url)
-
-        new_uuid = new_watch.get('uuid')
-
-        logging.debug("Added URL {} - {}".format(url, new_uuid))
-
-        for k in ['uuid', 'history', 'last_checked', 'last_changed', 'newest_history_key', 'previous_md5', 'viewed']:
-            if k in apply_extras:
-                del apply_extras[k]
-
-        if not apply_extras.get('date_created'):
-            apply_extras['date_created'] = int(time.time())
-
-        new_watch.update(apply_extras)
-        new_watch.ensure_data_dir_exists()
-        self.__data['watching'][new_uuid] = new_watch
+            new_watch.update(apply_extras)
+            self.__data['watching'][new_uuid] = new_watch
 
+        self.__data['watching'][new_uuid].ensure_data_dir_exists()
 
         if write_to_disk_now:
             self.sync_to_json()
@@ -473,8 +471,6 @@ class ChangeDetectionStore:
                     k = "ui-" + str(i) + proxy.get('proxy_name')
                     proxy_list[k] = {'label': proxy.get('proxy_name'), 'url': proxy.get('proxy_url')}
 
-        if proxy_list and strtobool(os.getenv('ENABLE_NO_PROXY_OPTION', 'True')):
-            proxy_list["no-proxy"] = {'label': "No proxy", 'url': ''}
 
         return proxy_list if len(proxy_list) else None
 
@@ -492,9 +488,6 @@ class ChangeDetectionStore:
         # If it's a valid one
         watch = self.data['watching'].get(uuid)
 
-        if strtobool(os.getenv('ENABLE_NO_PROXY_OPTION', 'True')) and watch.get('proxy') == "no-proxy":
-            return None
-
         if watch.get('proxy') and watch.get('proxy') in list(self.proxy_list.keys()):
             return watch.get('proxy')
 
@@ -518,19 +511,10 @@ class ChangeDetectionStore:
         filepath = os.path.join(self.datastore_path, 'headers.txt')
         return os.path.isfile(filepath)
 
-    def get_all_base_headers(self):
+    def get_all_headers(self):
         from .model.App import parse_headers_from_text_file
-        headers = {}
-        # Global app settings
-        headers.update(self.data['settings'].get('headers', {}))
+        headers = copy(self.data['settings'].get('headers', {}))
 
-        return headers
-
-    def get_all_headers_in_textfile_for_watch(self, uuid):
-        from .model.App import parse_headers_from_text_file
-        headers = {}
-
-        # Global in /datastore/headers.txt
         filepath = os.path.join(self.datastore_path, 'headers.txt')
         try:
             if os.path.isfile(filepath):
@@ -538,79 +522,8 @@ class ChangeDetectionStore:
         except Exception as e:
             print(f"ERROR reading headers.txt at {filepath}", str(e))
 
-        watch = self.data['watching'].get(uuid)
-        if watch:
-
-            # In /datastore/xyz-xyz/headers.txt
-            filepath = os.path.join(watch.watch_data_dir, 'headers.txt')
-            try:
-                if os.path.isfile(filepath):
-                    headers.update(parse_headers_from_text_file(filepath))
-            except Exception as e:
-                print(f"ERROR reading headers.txt at {filepath}", str(e))
-
-            # In /datastore/tag-name.txt
-            tags = self.get_all_tags_for_watch(uuid=uuid)
-            for tag_uuid, tag in tags.items():
-                fname = "headers-"+re.sub(r'[\W_]', '', tag.get('title')).lower().strip() + ".txt"
-                filepath = os.path.join(self.datastore_path, fname)
-                try:
-                    if os.path.isfile(filepath):
-                        headers.update(parse_headers_from_text_file(filepath))
-                except Exception as e:
-                    print(f"ERROR reading headers.txt at {filepath}", str(e))
-
         return headers
 
-    def get_tag_overrides_for_watch(self, uuid, attr):
-        tags = self.get_all_tags_for_watch(uuid=uuid)
-        ret = []
-
-        if tags:
-            for tag_uuid, tag in tags.items():
-                if attr in tag and tag[attr]:
-                    ret=[*ret, *tag[attr]]
-
-        return ret
-
-    def add_tag(self, name):
-        # If name exists, return that
-        n = name.strip().lower()
-        print (f">>> Adding new tag - '{n}'")
-        if not n:
-            return False
-
-        for uuid, tag in self.__data['settings']['application'].get('tags', {}).items():
-            if n == tag.get('title', '').lower().strip():
-                print (f">>> Tag {name} already exists")
-                return uuid
-
-        # Eventually almost everything todo with a watch will apply as a Tag
-        # So we use the same model as a Watch
-        with self.lock:
-            new_tag = Watch.model(datastore_path=self.datastore_path, default={
-                'title': name.strip(),
-                'date_created': int(time.time())
-            })
-
-            new_uuid = new_tag.get('uuid')
-
-            self.__data['settings']['application']['tags'][new_uuid] = new_tag
-
-        return new_uuid
-
-    def get_all_tags_for_watch(self, uuid):
-        """This should be in Watch model but Watch doesn't have access to datastore, not sure how to solve that yet"""
-        watch = self.data['watching'].get(uuid)
-
-        # Should return a dict of full tag info linked by UUID
-        if watch:
-            return dictfilt(self.__data['settings']['application']['tags'], watch.get('tags', []))
-
-        return {}
-
-    def tag_exists_by_name(self, tag_name):
-        return any(v.get('title', '').lower() == tag_name.lower() for k, v in self.__data['settings']['application']['tags'].items())
 
     # Run all updates
     # IMPORTANT - Each update could be run even when they have a new install and the schema is correct
@@ -788,24 +701,12 @@ class ChangeDetectionStore:
                 continue
         return
 
-    # Create tag objects and their references from existing tag text
-    def update_12(self):
-        i = 0
-        for uuid, watch in self.data['watching'].items():
-            # Split out and convert old tag string
-            tag = watch.get('tag')
-            if tag:
-                tag_uuids = []
-                for t in tag.split(','):
-                    tag_uuids.append(self.add_tag(name=t))
-
-                self.data['watching'][uuid]['tags'] = tag_uuids
-
-    # #1775 - Update 11 did not update the records correctly when adding 'date_created' values for sorting
-    def update_13(self):
+    # We don't know when the date_created was in the past until now, so just add an index number for now.
+    def update_11(self):
         i = 0
         for uuid, watch in self.data['watching'].items():
             if not watch.get('date_created'):
-                self.data['watching'][uuid]['date_created'] = i
+                watch['date_created'] = i
             i+=1
-        return
+        return
+

changedetectionio/templates/_common_fields.jinja

@@ -13,9 +13,9 @@
                             <div class="pure-form-message-inline">
                               <ul>
                                 <li>Use <a target=_new href="https://github.com/caronc/apprise">AppRise URLs</a> for notification to just about any service! <i><a target=_new href="https://github.com/dgtlmoon/changedetection.io/wiki/Notification-configuration-notes">Please read the notification services wiki here for important configuration notes</a></i>.</li>
-                                <li><code><a target=_new href="https://github.com/caronc/apprise/wiki/Notify_discord">discord://</a></code> only supports a maximum <strong>2,000 characters</strong> of notification text, including the title.</li>
-                                <li><code><a target=_new href="https://github.com/caronc/apprise/wiki/Notify_telegram">tgram://</a></code> bots can't send messages to other bots, so you should specify chat ID of non-bot user.</li>
-                                <li><code><a target=_new href="https://github.com/caronc/apprise/wiki/Notify_telegram">tgram://</a></code> only supports very limited HTML and can fail when extra tags are sent, <a href="https://core.telegram.org/bots/api#html-style">read more here</a> (or use plaintext/markdown format)</li>
+                                <li><code>discord://</code> only supports a maximum <strong>2,000 characters</strong> of notification text, including the title.</li>
+                                <li><code>tgram://</code> bots cant send messages to other bots, so you should specify chat ID of non-bot user.</li>
+                                <li><code>tgram://</code> only supports very limited HTML and can fail when extra tags are sent, <a href="https://core.telegram.org/bots/api#html-style">read more here</a> (or use plaintext/markdown format)</li>
                                 <li><code>gets://</code>, <code>posts://</code>, <code>puts://</code>, <code>deletes://</code> for direct API calls (or omit the "<code>s</code>" for non-SSL ie <code>get://</code>)</li>
                                   <li>Accepts the <code>{{ '{{token}}' }}</code> placeholders listed below</li>
                               </ul>
@@ -99,10 +99,6 @@
                                         <td><code>{{ '{{diff_full}}' }}</code></td>
                                         <td>The diff output - full difference output</td>
                                     </tr>
-                                    <tr>
-                                        <td><code>{{ '{{diff_patch}}' }}</code></td>
-                                        <td>The diff output - patch in unified format</td>
-                                    </tr>
                                     <tr>
                                         <td><code>{{ '{{current_snapshot}}' }}</code></td>
                                         <td>The current snapshot value, useful when combined with JSON or CSS filters

changedetectionio/templates/_helpers.jinja

@@ -1,6 +1,7 @@
 {% macro render_field(field) %}
-  <div {% if field.errors %} class="error" {% endif %}>{{ field.label }}</div>
   <div {% if field.errors %} class="error" {% endif %}>{{ field(**kwargs)|safe }}
+  <div {% if field.errors %} class="error" {% endif %}>{{ field.label }}</div>
+
   {% if field.errors %}
     <ul class=errors>
     {% for error in field.errors %}
@@ -24,6 +25,18 @@
   </div>
 {% endmacro %}
 
+{% macro render_field(field) %}
+  <div {% if field.errors %} class="error" {% endif %}>{{ field.label }}</div>
+  <div {% if field.errors %} class="error" {% endif %}>{{ field(**kwargs)|safe }}
+  {% if field.errors %}
+    <ul class=errors>
+    {% for error in field.errors %}
+      <li>{{ error }}</li>
+    {% endfor %}
+    </ul>
+  {% endif %}
+  </div>
+{% endmacro %}
 
 {% macro render_simple_field(field) %}
   <span class="label {% if field.errors %}error{% endif %}">{{ field.label }}</span>

changedetectionio/templates/base.html

@@ -37,7 +37,7 @@
     <div class="header">
       <div class="home-menu pure-menu pure-menu-horizontal pure-menu-fixed" id="nav-menu">
         {% if has_password and not current_user.is_authenticated %}
-          <a class="pure-menu-heading" href="https://changedetection.io" rel="noopener">
+          <a class="pure-menu-heading" href="https://github.com/dgtlmoon/changedetection.io" rel="noopener">
             <strong>Change</strong>Detection.io</a>
         {% else %}
           <a class="pure-menu-heading" href="{{url_for('index')}}">
@@ -49,7 +49,7 @@
         {% else %}
           {% if new_version_available and not(has_password and not current_user.is_authenticated) %}
             <span id="new-version-text" class="pure-menu-heading">
-              <a href="https://changedetection.io">A new version is available</a>
+              <a href="https://github.com/dgtlmoon/changedetection.io">A new version is available</a>
             </span>
           {% endif %}
         {% endif %}
@@ -58,9 +58,6 @@
           {% if current_user.is_authenticated or not has_password %}
             {% if not
             current_diff_url %}
-              <li class="pure-menu-item">
-                <a href="{{ url_for('tags.tags_overview_page')}}" class="pure-menu-link">GROUPS</a>
-              </li>
               <li class="pure-menu-item">
                 <a href="{{ url_for('settings_page')}}" class="pure-menu-link">SETTINGS</a>
               </li>
@@ -77,7 +74,7 @@
             {% endif %}
           {% else %}
             <li class="pure-menu-item">
-              <a class="pure-menu-link" href="https://changedetection.io">Website Change Detection and Notification.</a>
+              <a class="pure-menu-link" href="https://github.com/dgtlmoon/changedetection.io">Website Change Detection and Notification.</a>
             </li>
           {% endif %}
           {% if current_user.is_authenticated %}
@@ -89,14 +86,17 @@
             <!-- We use GET here so it offers people a chance to set bookmarks etc -->
             <form name="searchForm" action="" method="GET">
               <input id="search-q" class="" name="q" placeholder="URL or Title {% if active_tag %}in '{{ active_tag }}'{% endif %}" required="" type="text" value="">
-              <input name="tags" type="hidden" value="{% if active_tag %}{{active_tag}}{% endif %}">
+              <input name="tag" type="hidden" value="{% if active_tag %}{{active_tag}}{% endif %}">
               <button class="toggle-button " id="toggle-search" type="button" title="Search, or Use Alt+S Key" >
                 {% include "svgs/search-icon.svg" %}
               </button>
             </form>
           </li>
           <li class="pure-menu-item">
-            <button class="toggle-button" id ="toggle-light-mode" type="button" title="Toggle Light/Dark Mode">
+            {% if dark_mode %}
+            {% set darkClass = 'dark' %}
+            {% endif %}
+            <button class="toggle-button {{darkClass}}"  id ="toggle-light-mode" type="button" title="Toggle Light/Dark Mode">
               <span class="visually-hidden">Toggle light/dark mode</span>
               <span class="icon-light">
                 {% include "svgs/light-mode-toggle-icon.svg" %}

changedetectionio/templates/diff.html

@@ -6,9 +6,6 @@
     {% if last_error_screenshot %}
     const error_screenshot_url="{{url_for('static_content', group='screenshot', filename=uuid, error_screenshot=1) }}";
     {% endif %}
-
-    const highlight_submit_ignore_url="{{url_for('highlight_submit_ignore_url', uuid=uuid)}}";
-
 </script>
 <script src="{{url_for('static_content', group='js', filename='diff-overview.js')}}" defer></script>
 
@@ -79,7 +76,7 @@
     </div>
 
      <div class="tab-pane-inner" id="text">
-         <div class="tip">Pro-tip: Use <strong>show current snapshot</strong> tab to visualise what will be ignored, highlight text to add to ignore filters</div>
+         <div class="tip">Pro-tip: Use <strong>show current snapshot</strong> tab to visualise what will be ignored.</div>
 
          {% if password_enabled_and_share_is_off %}
            <div class="tip">Pro-tip: You can enable <strong>"share access when password is enabled"</strong> from settings</div>
@@ -94,7 +91,7 @@
                  <td id="a" style="display: none;">{{previous}}</td>
                  <td id="b" style="display: none;">{{newest}}</td>
                  <td id="diff-col">
-                     <span id="result" class="highlightable-filter"></span>
+                     <span id="result"></span>
                  </td>
              </tr>
              </tbody>

changedetectionio/templates/edit.html

@@ -4,19 +4,18 @@
 {% from '_common_fields.jinja' import render_common_settings_form %}
 <script src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
 <script>
+    const notification_base_url="{{url_for('ajax_callback_send_notification_test')}}";
+    const watch_visual_selector_data_url="{{url_for('static_content', group='visual_selector_data', filename=uuid)}}";
+    const screenshot_url="{{url_for('static_content', group='screenshot', filename=uuid)}}";
+    const playwright_enabled={% if playwright_enabled %} true {% else %} false {% endif %};
+
+{% if emailprefix %}
+    const email_notification_prefix=JSON.parse('{{ emailprefix|tojson }}');
+{% endif %}
 
     const browser_steps_config=JSON.parse('{{ browser_steps_config|tojson }}');
     const browser_steps_start_url="{{url_for('browser_steps.browsersteps_start_session', uuid=uuid)}}";
     const browser_steps_sync_url="{{url_for('browser_steps.browsersteps_ui_update', uuid=uuid)}}";
-{% if emailprefix %}
-    const email_notification_prefix=JSON.parse('{{ emailprefix|tojson }}');
-{% endif %}
-    const notification_base_url="{{url_for('ajax_callback_send_notification_test')}}";
-    const playwright_enabled={% if playwright_enabled %} true {% else %} false {% endif %};
-    const recheck_proxy_start_url="{{url_for('check_proxies.start_check', uuid=uuid)}}";
-    const proxy_recheck_status_url="{{url_for('check_proxies.get_recheck_status', uuid=uuid)}}";
-    const screenshot_url="{{url_for('static_content', group='screenshot', filename=uuid)}}";
-    const watch_visual_selector_data_url="{{url_for('static_content', group='visual_selector_data', filename=uuid)}}";
 
 </script>
 
@@ -28,8 +27,6 @@
 <script src="{{url_for('static_content', group='js', filename='browser-steps.js')}}" defer></script>
 {% endif %}
 
-<script src="{{url_for('static_content', group='js', filename='recheck-proxy.js')}}" defer></script>
-
 <div class="edit-form monospaced-textarea">
 
     <div class="tabs collapsable">
@@ -78,7 +75,7 @@
                         {{ render_field(form.title, class="m-d") }}
                     </div>
                     <div class="pure-control-group">
-                        {{ render_field(form.tags) }}
+                        {{ render_field(form.tag) }}
                         <span class="pure-form-message-inline">Organisational tag/group name used in the main listing page</span>
                     </div>
                     <div class="pure-control-group">
@@ -114,8 +111,7 @@
                     </div>
                 {% if form.proxy %}
                     <div class="pure-control-group inline-radio">
-                          <div>{{ form.proxy.label }} <a href="" id="check-all-proxies" class="pure-button button-secondary button-xsmall" >Check/Scan all</a></div>
-                          <div>{{ form.proxy(class="fetch-backend-proxy") }}</div>
+                        {{ render_field(form.proxy, class="fetch-backend-proxy") }}
                         <span class="pure-form-message-inline">
                         Choose a proxy for this watch
                         </span>

changedetectionio/templates/preview.html

@@ -6,7 +6,6 @@
     {% if last_error_screenshot %}
     const error_screenshot_url="{{url_for('static_content', group='screenshot', filename=uuid, error_screenshot=1) }}";
     {% endif %}
-    const highlight_submit_ignore_url="{{url_for('highlight_submit_ignore_url', uuid=uuid)}}";
 </script>
 <script src="{{url_for('static_content', group='js', filename='diff-overview.js')}}" defer></script>
 
@@ -21,7 +20,7 @@
         {% endif %}
     </ul>
 </div>
-<form><input type="hidden" name="csrf_token" value="{{ csrf_token() }}"></form>
+
 <div id="diff-ui">
     <div class="tab-pane-inner" id="error-text">
         <div class="snapshot-age error">{{watch.error_text_ctime|format_seconds_ago}} seconds ago</div>
@@ -37,12 +36,11 @@
 
     <div class="tab-pane-inner" id="text">
         <div class="snapshot-age">{{watch.snapshot_text_ctime|format_timestamp_timeago}}</div>
-        <span class="ignored">Grey lines are ignored</span> <span class="triggered">Blue lines are triggers</span> <span class="tip"><strong>Pro-tip</strong>: Highlight text to add to ignore filters</span>
-
+        <span class="ignored">Grey lines are ignored</span> <span class="triggered">Blue lines are triggers</span>
         <table>
             <tbody>
             <tr>
-                <td id="diff-col" class="highlightable-filter">
+                <td id="diff-col">
                     {% for row in content %}
                     <div class="{{row.classes}}">{{row.line}}</div>
                     {% endfor %}

changedetectionio/templates/settings.html

@@ -181,56 +181,20 @@ nav
                 </div>
             </div>
             <div class="tab-pane-inner" id="proxies">
-                <div id="recommended-proxy">
-                    <div>
-                        <img style="height: 2em;" src="{{url_for('static_content', group='images', filename='brightdata.svg')}}" alt="BrightData Proxy Provider">
-                        <p>BrightData offer world-class proxy services, "Data Center" proxies are a very affordable way to proxy your requests, whilst <strong><a href="https://brightdata.grsm.io/n0r16zf7eivq">WebUnlocker</a></strong> can help solve most CAPTCHAs.</p>
-                        <p>
-                            BrightData offer many <a href="https://brightdata.com/proxy-types" target="new">many different types of proxies</a>, it is worth reading about what is best for your use-case.
-                        </p>
 
-                        <p>
-                            When you have <a href="https://brightdata.grsm.io/n0r16zf7eivq">registered</a>, enabled the required services, visit the <A href="https://brightdata.com/cp/api_example?">API example page</A>, then select <strong>Python</strong>, set the country you wish to use, then copy+paste the access Proxy URL into the "Extra Proxies" boxes below.<br>
-                        </p>
-                        <p>
-                            The Proxy URL with BrightData should start with <code>http://brd-customer...</code>
-                        </p>
-                        <p>When you sign up using <a href="https://brightdata.grsm.io/n0r16zf7eivq">https://brightdata.grsm.io/n0r16zf7eivq</a> BrightData will match any first deposit up to $150</p>
-                    </div>
-                    <div>
-                        <img style="height: 2em;"
-                             src="{{url_for('static_content', group='images', filename='oxylabs.svg')}}"
-                             alt="Oxylabs Proxy Provider">
-                        <p>
-                            Collect public data at scale with industry-leading web scraping solutions and the world’s
-                            largest ethical proxy network.
-                        </p>
-                        <p>
-                            Oxylabs also provide a <a href="https://oxylabs.io/products/web-unblocker"><strong>WebUnlocker</strong></a>
-                            proxy that bypasses sophisticated anti-bot systems, so you don’t have to.<br>
-                        </p>
-                        <p>
-                            Serve over <a href="https://oxylabs.io/location-proxy">195 countries</a>, providing <a
-                                href="https://oxylabs.io/products/residential-proxy-pool">Residential</a>, <a
-                                href="https://oxylabs.io/products/mobile-proxies">Mobile</a> and <a
-                                href="https://oxylabs.io/products/rotating-isp-proxies">ISP proxies</a> and much more.
-                        </p>
-                        <p>
-                            Use the promo code <strong>boost35</strong> with this link <a href="https://oxylabs.go2cloud.org/SH2d">https://oxylabs.go2cloud.org/SH2d</a> for 35% off Residential, Mobile proxies, Web Unblocker, and Scraper APIs. Built-in proxies enable you to access data from all around the world and help overcome anti-bot solutions.
-
-                        </p>
-
-                        
-                    </div>
-                </div>
+                <p><strong>Tip</strong>: You can connect to websites using <a href="https://brightdata.grsm.io/n0r16zf7eivq">BrightData</a> proxies, their service <strong>WebUnlocker</strong> will solve most CAPTCHAs, whilst their <strong>Residential Proxies</strong> may help to avoid CAPTCHA altogether. </p>
+                <p>It may be easier to try <strong>WebUnlocker</strong> first, WebUnlocker also supports country selection.</p>
                 <p>
-                    Your proxy provider may need to whitelist our IP of <code>204.15.192.195</code>
+                    When you have <a href="https://brightdata.grsm.io/n0r16zf7eivq">registered</a>, enabled the required services, visit the <A href="https://brightdata.com/cp/api_example?">API example page</A>, then select <strong>Python</strong>, set the country you wish to use, then copy+paste the example URL below<br>
+                    The Proxy URL with BrightData should start with <code>http://brd-customer...</code>
                 </p>
-               <p><strong>Tip</strong>: "Residential" and "Mobile" proxy type can be more successfull than "Data Center" for blocked websites.
+
+                <p>When you sign up using <a href="https://brightdata.grsm.io/n0r16zf7eivq">https://brightdata.grsm.io/n0r16zf7eivq</a> BrightData will match any first deposit up to $150</p>
+
 
                 <div class="pure-control-group">
-                {{ render_field(form.requests.form.extra_proxies) }}
-                <span class="pure-form-message-inline">"Name" will be used for selecting the proxy in the Watch Edit settings</span>
+                    {{ render_field(form.requests.form.extra_proxies) }}
+                    <span class="pure-form-message-inline">"Name" will be used for selecting the proxy in the Watch Edit settings</span>
                 </div>
             </div>
             <div id="actions">

changedetectionio/templates/watch-overview.html

@@ -13,7 +13,7 @@
             <div id="watch-add-wrapper-zone">
                 <div>
                     {{ render_simple_field(form.url, placeholder="https://...", required=true) }}
-                    {{ render_simple_field(form.tags, value=tags[active_tag].title if active_tag else '', placeholder="watch label / tag") }}
+                    {{ render_simple_field(form.tag, value=active_tag if active_tag else '', placeholder="watch label / tag") }}
                 </div>
                 <div>
                     {{ render_simple_field(form.watch_submit_button, title="Watch this URL!" ) }}
@@ -30,14 +30,12 @@
 
     <form class="pure-form" action="{{ url_for('form_watch_list_checkbox_operations') }}" method="POST" id="watch-list-form">
     <input type="hidden" name="csrf_token" value="{{ csrf_token() }}" >
-    <input type="hidden" id="op_extradata" name="op_extradata" value="" >
     <div id="checkbox-operations">
         <button class="pure-button button-secondary button-xsmall"  name="op" value="pause">Pause</button>
         <button class="pure-button button-secondary button-xsmall"  name="op" value="unpause">UnPause</button>
         <button class="pure-button button-secondary button-xsmall"  name="op" value="mute">Mute</button>
         <button class="pure-button button-secondary button-xsmall"  name="op" value="unmute">UnMute</button>
         <button class="pure-button button-secondary button-xsmall" name="op" value="recheck">Recheck</button>
-        <button class="pure-button button-secondary button-xsmall" name="op" value="assign-tag" id="checkbox-assign-tag">Tag</button>
         <button class="pure-button button-secondary button-xsmall" name="op" value="mark-viewed">Mark viewed</button>
         <button class="pure-button button-secondary button-xsmall" name="op" value="notification-default">Use default notification</button>
         <button class="pure-button button-secondary button-xsmall" style="background: #dd4242;" name="op" value="clear-history">Clear/reset history</button>
@@ -49,9 +47,9 @@
     {% if search_q %}<div id="search-result-info">Searching "<strong><i>{{search_q}}</i></strong>"</div>{% endif %}
     <div>
         <a href="{{url_for('index')}}" class="pure-button button-tag {{'active' if not active_tag }}">All</a>
-        {% for uuid, tag in tags.items() %}
+        {% for tag in tags %}
             {% if tag != "" %}
-                <a href="{{url_for('index', tag=uuid) }}" class="pure-button button-tag {{'active' if active_tag == uuid }}">{{ tag.title }}</a>
+                <a href="{{url_for('index', tag=tag) }}" class="pure-button button-tag {{'active' if active_tag == tag }}">{{ tag }}</a>
             {% endif %}
         {% endfor %}
     </div>
@@ -145,11 +143,9 @@
                     </span>
                     {% endif %}
 
-
-                    {% for watch_tag_uuid, watch_tag in datastore.get_all_tags_for_watch(watch['uuid']).items() %}
-                      <span class="watch-tag-list">{{ watch_tag.title }}</span>
-                    {% endfor %}
-
+                    {% if not active_tag %}
+                    <span class="watch-tag-list">{{ watch.tag}}</span>
+                    {% endif %}
                 </td>
                 <td class="last-checked">{{watch|format_last_checked_time|safe}}</td>
                 <td class="last-changed">{% if watch.history_n >=2 and watch.last_changed >0 %}
@@ -182,7 +178,7 @@
             {% endif %}
             <li>
                <a href="{{ url_for('form_watch_checknow', tag=active_tag) }}" class="pure-button button-tag ">Recheck
-                all {% if active_tag%} in "{{tags[active_tag].title}}"{%endif%}</a>
+                all {% if active_tag%}in "{{active_tag}}"{%endif%}</a>
             </li>
             <li>
                 <a href="{{ url_for('rss', tag=active_tag , token=app_rss_token)}}"><img alt="RSS Feed" id="feed-icon" src="{{url_for('static_content', group='images', filename='Generic_Feed-icon.svg')}}" height="15"></a>

changedetectionio/tests/proxy_list/test_multiple_proxy.py

@@ -28,7 +28,7 @@ def test_preferred_proxy(client, live_server):
                 "fetch_backend": "html_requests",
                 "headers": "",
                 "proxy": "proxy-two",
-                "tags": "",
+                "tag": "",
                 "url": url,
               },
         follow_redirects=True

changedetectionio/tests/proxy_list/test_noproxy.py

@@ -1,77 +0,0 @@
-#!/usr/bin/python3
-
-import time
-from flask import url_for
-from ..util import live_server_setup, wait_for_all_checks, extract_UUID_from_client
-
-
-def test_noproxy_option(client, live_server):
-    live_server_setup(live_server)
-    # Run by run_proxy_tests.sh
-    # Call this URL then scan the containers that it never went through them
-    url = "http://noproxy.changedetection.io"
-
-    # Should only be available when a proxy is setup
-    res = client.get(
-        url_for("edit_page", uuid="first", unpause_on_save=1))
-    assert b'No proxy' not in res.data
-
-    # Setup a proxy
-    res = client.post(
-        url_for("settings_page"),
-        data={
-            "requests-time_between_check-minutes": 180,
-            "application-ignore_whitespace": "y",
-            "application-fetch_backend": "html_requests",
-            "requests-extra_proxies-0-proxy_name": "custom-one-proxy",
-            "requests-extra_proxies-0-proxy_url": "http://test:awesome@squid-one:3128",
-            "requests-extra_proxies-1-proxy_name": "custom-two-proxy",
-            "requests-extra_proxies-1-proxy_url": "http://test:awesome@squid-two:3128",
-            "requests-extra_proxies-2-proxy_name": "custom-proxy",
-            "requests-extra_proxies-2-proxy_url": "http://test:awesome@squid-custom:3128",
-        },
-        follow_redirects=True
-    )
-
-    assert b"Settings updated." in res.data
-
-    # Should be available as an option
-    res = client.get(
-        url_for("settings_page", unpause_on_save=1))
-    assert b'No proxy' in res.data
-
-
-    # This will add it paused
-    res = client.post(
-        url_for("form_quick_watch_add"),
-        data={"url": url, "tags": '', 'edit_and_watch_submit_button': 'Edit > Watch'},
-        follow_redirects=True
-    )
-    assert b"Watch added in Paused state, saving will unpause" in res.data
-    uuid = extract_UUID_from_client(client)
-    res = client.get(
-        url_for("edit_page", uuid=uuid, unpause_on_save=1))
-    assert b'No proxy' in res.data
-
-    res = client.post(
-        url_for("edit_page", uuid=uuid, unpause_on_save=1),
-        data={
-                "include_filters": "",
-                "fetch_backend": "html_requests",
-                "headers": "",
-                "proxy": "no-proxy",
-                "tags": "",
-                "url": url,
-              },
-        follow_redirects=True
-    )
-    assert b"unpaused" in res.data
-    wait_for_all_checks(client)
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    wait_for_all_checks(client)
-    # Now the request should NOT appear in the second-squid logs (handled by the run_test_proxies.sh script)
-
-    # Prove that it actually checked
-
-    assert live_server.app.config['DATASTORE'].data['watching'][uuid]['last_checked'] != 0
-

changedetectionio/tests/restock/test_restock.py

@@ -77,7 +77,7 @@ def test_restock_detection(client, live_server):
 
     client.post(
         url_for("form_quick_watch_add"),
-        data={"url": test_url, "tags": '', 'processor': 'restock_diff'},
+        data={"url": test_url, "tag": '', 'processor': 'restock_diff'},
         follow_redirects=True
     )
 

changedetectionio/tests/smtp/smtp-test-server.py

@@ -1,42 +0,0 @@
-#!/usr/bin/python3
-import smtpd
-import asyncore
-
-# Accept a SMTP message and offer a way to retrieve the last message via TCP Socket
-
-last_received_message = b"Nothing"
-
-
-class CustomSMTPServer(smtpd.SMTPServer):
-
-    def process_message(self, peer, mailfrom, rcpttos, data, **kwargs):
-        global last_received_message
-        last_received_message = data
-        print('Receiving message from:', peer)
-        print('Message addressed from:', mailfrom)
-        print('Message addressed to  :', rcpttos)
-        print('Message length        :', len(data))
-        print(data.decode('utf8'))
-        return
-
-
-# Just print out the last message received on plain TCP socket server
-class EchoServer(asyncore.dispatcher):
-
-    def __init__(self, host, port):
-        asyncore.dispatcher.__init__(self)
-        self.create_socket()
-        self.set_reuse_addr()
-        self.bind((host, port))
-        self.listen(5)
-
-    def handle_accepted(self, sock, addr):
-        global last_received_message
-        print('Incoming connection from %s' % repr(addr))
-        sock.send(last_received_message)
-        last_received_message = b''
-
-
-server = CustomSMTPServer(('0.0.0.0', 11025), None)  # SMTP mail goes here
-server2 = EchoServer('0.0.0.0', 11080)  # Echo back last message received
-asyncore.loop()

changedetectionio/tests/smtp/test_notification_smtp.py

@@ -1,165 +0,0 @@
-import json
-import os
-import time
-import re
-from flask import url_for
-from changedetectionio.tests.util import set_original_response, set_modified_response, set_more_modified_response, live_server_setup, \
-    wait_for_all_checks, \
-    set_longer_modified_response
-from changedetectionio.tests.util import extract_UUID_from_client
-import logging
-import base64
-
-# NOTE - RELIES ON mailserver as hostname running, see github build recipes
-smtp_test_server = 'mailserver'
-
-from changedetectionio.notification import (
-    default_notification_body,
-    default_notification_format,
-    default_notification_title,
-    valid_notification_formats,
-)
-
-def test_setup(live_server):
-    live_server_setup(live_server)
-
-def get_last_message_from_smtp_server():
-    import socket
-    global smtp_test_server
-    port = 11080  # socket server port number
-
-    client_socket = socket.socket()  # instantiate
-    client_socket.connect((smtp_test_server, port))  # connect to the server
-
-    data = client_socket.recv(50024).decode()  # receive response
-    client_socket.close()  # close the connection
-    return data
-
-
-# Requires running the test SMTP server
-
-def test_check_notification_email_formats_default_HTML(client, live_server):
-    # live_server_setup(live_server)
-    set_original_response()
-
-    global smtp_test_server
-    notification_url = f'mailto://changedetection@{smtp_test_server}:11025/?to=fff@home.com'
-
-    #####################
-    # Set this up for when we remove the notification from the watch, it should fallback with these details
-    res = client.post(
-        url_for("settings_page"),
-        data={"application-notification_urls": notification_url,
-              "application-notification_title": "fallback-title " + default_notification_title,
-              "application-notification_body": "fallback-body<br> " + default_notification_body,
-              "application-notification_format": 'HTML',
-              "requests-time_between_check-minutes": 180,
-              'application-fetch_backend': "html_requests"},
-        follow_redirects=True
-    )
-    assert b"Settings updated." in res.data
-
-    # Add a watch and trigger a HTTP POST
-    test_url = url_for('test_endpoint', _external=True)
-    res = client.post(
-        url_for("form_quick_watch_add"),
-        data={"url": test_url, "tags": 'nice one'},
-        follow_redirects=True
-    )
-
-    assert b"Watch added" in res.data
-
-    wait_for_all_checks(client)
-    set_longer_modified_response()
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    wait_for_all_checks(client)
-
-    time.sleep(3)
-
-    msg = get_last_message_from_smtp_server()
-    assert len(msg) >= 1
-
-    # The email should have two bodies, and the text/html part should be <br>
-    assert 'Content-Type: text/plain' in msg
-    assert '(added) So let\'s see what happens.\n' in msg  # The plaintext part with \n
-    assert 'Content-Type: text/html' in msg
-    assert '(added) So let\'s see what happens.<br>' in msg  # the html part
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
-    assert b'Deleted' in res.data
-
-
-def test_check_notification_email_formats_default_Text_override_HTML(client, live_server):
-    # live_server_setup(live_server)
-
-    # HTML problems? see this
-    # https://github.com/caronc/apprise/issues/633
-
-    set_original_response()
-    global smtp_test_server
-    notification_url = f'mailto://changedetection@{smtp_test_server}:11025/?to=fff@home.com'
-
-    #####################
-    # Set this up for when we remove the notification from the watch, it should fallback with these details
-    res = client.post(
-        url_for("settings_page"),
-        data={"application-notification_urls": notification_url,
-              "application-notification_title": "fallback-title " + default_notification_title,
-              "application-notification_body": default_notification_body,
-              "application-notification_format": 'Text',
-              "requests-time_between_check-minutes": 180,
-              'application-fetch_backend': "html_requests"},
-        follow_redirects=True
-    )
-    assert b"Settings updated." in res.data
-
-    # Add a watch and trigger a HTTP POST
-    test_url = url_for('test_endpoint', _external=True)
-    res = client.post(
-        url_for("form_quick_watch_add"),
-        data={"url": test_url, "tags": 'nice one'},
-        follow_redirects=True
-    )
-
-    assert b"Watch added" in res.data
-
-    wait_for_all_checks(client)
-    set_longer_modified_response()
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    wait_for_all_checks(client)
-
-    time.sleep(3)
-    msg = get_last_message_from_smtp_server()
-    assert len(msg) >= 1
-    #    with open('/tmp/m.txt', 'w') as f:
-    #        f.write(msg)
-
-    # The email should not have two bodies, should be TEXT only
-
-    assert 'Content-Type: text/plain' in msg
-    assert '(added) So let\'s see what happens.\n' in msg  # The plaintext part with \n
-
-    set_original_response()
-    # Now override as HTML format
-    res = client.post(
-        url_for("edit_page", uuid="first"),
-        data={
-            "url": test_url,
-            "notification_format": 'HTML',
-            'fetch_backend': "html_requests"},
-        follow_redirects=True
-    )
-    assert b"Updated watch." in res.data
-    wait_for_all_checks(client)
-
-    time.sleep(3)
-    msg = get_last_message_from_smtp_server()
-    assert len(msg) >= 1
-
-    # The email should have two bodies, and the text/html part should be <br>
-    assert 'Content-Type: text/plain' in msg
-    assert '(removed) So let\'s see what happens.\n' in msg  # The plaintext part with \n
-    assert 'Content-Type: text/html' in msg
-    assert '(removed) So let\'s see what happens.<br>' in msg  # the html part
-
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
-    assert b'Deleted' in res.data

changedetectionio/tests/test_access_control.py

@@ -45,15 +45,6 @@ def test_check_access_control(app, client, live_server):
         res = client.get(url_for("diff_history_page", uuid="first"))
         assert b'Random content' in res.data
 
-        # Check wrong password does not let us in
-        res = c.post(
-            url_for("login"),
-            data={"password": "WRONG PASSWORD"},
-            follow_redirects=True
-        )
-
-        assert b"LOG OUT" not in res.data
-        assert b"Incorrect password" in res.data
 
 
         # Menu should not be available yet

changedetectionio/tests/test_add_replace_remove_filter.py

@@ -2,7 +2,7 @@
 
 import time
 from flask import url_for
-from .util import live_server_setup, wait_for_all_checks
+from .util import live_server_setup
 from changedetectionio import html_tools
 
 
@@ -39,6 +39,7 @@ def test_setup(client, live_server):
     live_server_setup(live_server)
 
 def test_check_removed_line_contains_trigger(client, live_server):
+    sleep_time_for_fetch_thread = 3
 
     # Give the endpoint time to spin up
     time.sleep(1)
@@ -53,7 +54,7 @@ def test_check_removed_line_contains_trigger(client, live_server):
     assert b"1 Imported" in res.data
 
     # Give the thread time to pick it up
-    wait_for_all_checks(client)
+    time.sleep(sleep_time_for_fetch_thread)
 
     # Goto the edit page, add our ignore text
     # Add our URL to the import page
@@ -66,20 +67,20 @@ def test_check_removed_line_contains_trigger(client, live_server):
         follow_redirects=True
     )
     assert b"Updated watch." in res.data
-    wait_for_all_checks(client)
+    time.sleep(sleep_time_for_fetch_thread)
     set_original(excluding='Something irrelevant')
 
     # A line thats not the trigger should not trigger anything
     res = client.get(url_for("form_watch_checknow"), follow_redirects=True)
     assert b'1 watches queued for rechecking.' in res.data
-    wait_for_all_checks(client)
+    time.sleep(sleep_time_for_fetch_thread)
     res = client.get(url_for("index"))
     assert b'unviewed' not in res.data
 
     # The trigger line is REMOVED,  this should trigger
     set_original(excluding='The golden line')
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    wait_for_all_checks(client)
+    time.sleep(sleep_time_for_fetch_thread)
     res = client.get(url_for("index"))
     assert b'unviewed' in res.data
 
@@ -88,14 +89,14 @@ def test_check_removed_line_contains_trigger(client, live_server):
     client.get(url_for("mark_all_viewed"), follow_redirects=True)
     set_original(excluding=None)
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    wait_for_all_checks(client)
+    time.sleep(sleep_time_for_fetch_thread)
     res = client.get(url_for("index"))
     assert b'unviewed' not in res.data
 
     # Remove it again, and we should get a trigger
     set_original(excluding='The golden line')
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    wait_for_all_checks(client)
+    time.sleep(sleep_time_for_fetch_thread)
     res = client.get(url_for("index"))
     assert b'unviewed' in res.data
 
@@ -104,7 +105,8 @@ def test_check_removed_line_contains_trigger(client, live_server):
 
 
 def test_check_add_line_contains_trigger(client, live_server):
-    #live_server_setup(live_server)
+
+    sleep_time_for_fetch_thread = 3
 
     # Give the endpoint time to spin up
     time.sleep(1)
@@ -134,7 +136,8 @@ def test_check_add_line_contains_trigger(client, live_server):
     assert b"1 Imported" in res.data
 
     # Give the thread time to pick it up
-    wait_for_all_checks(client)
+    time.sleep(sleep_time_for_fetch_thread)
+
     # Goto the edit page, add our ignore text
     # Add our URL to the import page
     res = client.post(
@@ -147,25 +150,23 @@ def test_check_add_line_contains_trigger(client, live_server):
         follow_redirects=True
     )
     assert b"Updated watch." in res.data
-    wait_for_all_checks(client)
+    time.sleep(sleep_time_for_fetch_thread)
     set_original(excluding='Something irrelevant')
 
     # A line thats not the trigger should not trigger anything
     res = client.get(url_for("form_watch_checknow"), follow_redirects=True)
     assert b'1 watches queued for rechecking.' in res.data
-    wait_for_all_checks(client)
+    time.sleep(sleep_time_for_fetch_thread)
     res = client.get(url_for("index"))
     assert b'unviewed' not in res.data
 
     # The trigger line is ADDED,  this should trigger
     set_original(add_line='<p>Oh yes please</p>')
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    wait_for_all_checks(client)
+    time.sleep(sleep_time_for_fetch_thread)
     res = client.get(url_for("index"))
     assert b'unviewed' in res.data
 
-    # Takes a moment for apprise to fire
-    time.sleep(3)
     with open("test-datastore/notification.txt", 'r') as f:
         response= f.read()
         assert '-Oh yes please-' in response

changedetectionio/tests/test_api.py

@@ -2,7 +2,7 @@
 
 import time
 from flask import url_for
-from .util import live_server_setup, extract_api_key_from_UI, wait_for_all_checks
+from .util import live_server_setup, extract_api_key_from_UI
 
 import json
 import uuid
@@ -57,7 +57,6 @@ def test_setup(client, live_server):
     live_server_setup(live_server)
 
 def test_api_simple(client, live_server):
-    #live_server_setup(live_server)
 
     api_key = extract_api_key_from_UI(client)
 
@@ -87,7 +86,7 @@ def test_api_simple(client, live_server):
     watch_uuid = res.json.get('uuid')
     assert res.status_code == 201
 
-    wait_for_all_checks(client)
+    time.sleep(3)
 
     # Verify its in the list and that recheck worked
     res = client.get(
@@ -108,7 +107,7 @@ def test_api_simple(client, live_server):
     )
     assert len(res.json) == 0
 
-    wait_for_all_checks(client)
+    time.sleep(2)
 
     set_modified_response()
     # Trigger recheck of all ?recheck_all=1
@@ -116,7 +115,7 @@ def test_api_simple(client, live_server):
         url_for("createwatch", recheck_all='1'),
         headers={'x-api-key': api_key},
     )
-    wait_for_all_checks(client)
+    time.sleep(3)
 
     # Did the recheck fire?
     res = client.get(
@@ -267,7 +266,7 @@ def test_api_watch_PUT_update(client, live_server):
 
     #live_server_setup(live_server)
     api_key = extract_api_key_from_UI(client)
-
+    time.sleep(1)
     # Create a watch
     set_original_response()
     test_url = url_for('test_endpoint', _external=True,
@@ -283,6 +282,7 @@ def test_api_watch_PUT_update(client, live_server):
 
     assert res.status_code == 201
 
+    time.sleep(1)
 
     # Get a listing, it will be the first one
     res = client.get(
@@ -297,8 +297,6 @@ def test_api_watch_PUT_update(client, live_server):
         url_for("edit_page", uuid=watch_uuid),
     )
     assert b"cookie: yum" in res.data, "'cookie: yum' found in 'headers' section"
-    assert b"One" in res.data, "Tag 'One' was found"
-    assert b"Two" in res.data, "Tag 'Two' was found"
 
     # HTTP PUT ( UPDATE an existing watch )
     res = client.put(
@@ -321,8 +319,7 @@ def test_api_watch_PUT_update(client, live_server):
     )
     assert b"new title" in res.data, "new title found in edit page"
     assert b"552" in res.data, "552 minutes found in edit page"
-    assert b"One" in res.data, "Tag 'One' was found"
-    assert b"Two" in res.data, "Tag 'Two' was found"
+    assert b"One, Two" in res.data, "Tag 'One, Two' was found"
     assert b"cookie: all eaten" in res.data, "'cookie: all eaten' found in 'headers' section"
 
     ######################################################

changedetectionio/tests/test_auth.py

@@ -24,7 +24,7 @@ def test_basic_auth(client, live_server):
     # Check form validation
     res = client.post(
         url_for("edit_page", uuid="first"),
-        data={"include_filters": "", "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests"},
+        data={"include_filters": "", "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
         follow_redirects=True
     )
     assert b"Updated watch." in res.data

changedetectionio/tests/test_block_while_text_present.py

@@ -2,7 +2,7 @@
 
 import time
 from flask import url_for
-from .util import live_server_setup, wait_for_all_checks
+from . util import live_server_setup
 from changedetectionio import html_tools
 
 def set_original_ignore_response():
@@ -61,7 +61,7 @@ def set_modified_response_minus_block_text():
 
 
 def test_check_block_changedetection_text_NOT_present(client, live_server):
-
+    sleep_time_for_fetch_thread = 3
     live_server_setup(live_server)
     # Use a mix of case in ZzZ to prove it works case-insensitive.
     ignore_text = "out of stoCk\r\nfoobar"
@@ -81,7 +81,7 @@ def test_check_block_changedetection_text_NOT_present(client, live_server):
     assert b"1 Imported" in res.data
 
     # Give the thread time to pick it up
-    wait_for_all_checks(client)
+    time.sleep(sleep_time_for_fetch_thread)
 
     # Goto the edit page, add our ignore text
     # Add our URL to the import page
@@ -96,7 +96,7 @@ def test_check_block_changedetection_text_NOT_present(client, live_server):
     assert b"Updated watch." in res.data
 
     # Give the thread time to pick it up
-    wait_for_all_checks(client)
+    time.sleep(sleep_time_for_fetch_thread)
     # Check it saved
     res = client.get(
         url_for("edit_page", uuid="first"),
@@ -107,7 +107,7 @@ def test_check_block_changedetection_text_NOT_present(client, live_server):
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
-    wait_for_all_checks(client)
+    time.sleep(sleep_time_for_fetch_thread)
 
     # It should report nothing found (no new 'unviewed' class)
     res = client.get(url_for("index"))
@@ -120,7 +120,7 @@ def test_check_block_changedetection_text_NOT_present(client, live_server):
     # Trigger a check
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
     # Give the thread time to pick it up
-    wait_for_all_checks(client)
+    time.sleep(sleep_time_for_fetch_thread)
 
     # It should report nothing found (no new 'unviewed' class)
     res = client.get(url_for("index"))
@@ -131,7 +131,7 @@ def test_check_block_changedetection_text_NOT_present(client, live_server):
     # Now we set a change where the text is gone, it should now trigger
     set_modified_response_minus_block_text()
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    wait_for_all_checks(client)
+    time.sleep(sleep_time_for_fetch_thread)
     res = client.get(url_for("index"))
     assert b'unviewed' in res.data
 

changedetectionio/tests/test_css_selector.py

@@ -96,7 +96,7 @@ def test_check_markup_include_filters_restriction(client, live_server):
     # Add our URL to the import page
     res = client.post(
         url_for("edit_page", uuid="first"),
-        data={"include_filters": include_filters, "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests"},
+        data={"include_filters": include_filters, "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
         follow_redirects=True
     )
     assert b"Updated watch." in res.data
@@ -157,7 +157,7 @@ def test_check_multiple_filters(client, live_server):
         url_for("edit_page", uuid="first"),
         data={"include_filters": include_filters,
               "url": test_url,
-              "tags": "",
+              "tag": "",
               "headers": "",
               'fetch_backend': "html_requests"},
         follow_redirects=True

changedetectionio/tests/test_element_removal.py

@@ -129,7 +129,7 @@ def test_element_removal_full(client, live_server):
         data={
             "subtractive_selectors": subtractive_selectors_data,
             "url": test_url,
-            "tags": "",
+            "tag": "",
             "headers": "",
             "fetch_backend": "html_requests",
         },

changedetectionio/tests/test_extract_regex.py

@@ -91,7 +91,7 @@ def test_check_filter_multiline(client, live_server):
         data={"include_filters": '',
               'extract_text': '/something.+?6 billion.+?lines/si',
               "url": test_url,
-              "tags": "",
+              "tag": "",
               "headers": "",
               'fetch_backend': "html_requests"
               },
@@ -146,7 +146,7 @@ def test_check_filter_and_regex_extract(client, live_server):
         data={"include_filters": include_filters,
               'extract_text': '\d+ online\r\n\d+ guests\r\n/somecase insensitive \d+/i\r\n/somecase insensitive (345\d)/i',
               "url": test_url,
-              "tags": "",
+              "tag": "",
               "headers": "",
               'fetch_backend': "html_requests"
               },

changedetectionio/tests/test_filter_exist_changes.py

@@ -15,7 +15,7 @@ def set_response_without_filter():
      <p>Which is across multiple lines</p>
      <br>
      So let's see what happens.  <br>
-     <div id="nope-doesnt-exist">Some text thats the same</div>
+     <div id="nope-doesnt-exist">Some text thats the same</div>     
      </body>
      </html>
     """
@@ -32,7 +32,7 @@ def set_response_with_filter():
      <p>Which is across multiple lines</p>
      <br>
      So let's see what happens.  <br>
-     <div class="ticket-available">Ticket now on sale!</div>
+     <div class="ticket-available">Ticket now on sale!</div>     
      </body>
      </html>
     """
@@ -56,7 +56,7 @@ def test_filter_doesnt_exist_then_exists_should_get_notification(client, live_se
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
         url_for("form_quick_watch_add"),
-        data={"url": test_url, "tags": 'cinema'},
+        data={"url": test_url, "tag": 'cinema'},
         follow_redirects=True
     )
     assert b"Watch added" in res.data
@@ -84,13 +84,12 @@ def test_filter_doesnt_exist_then_exists_should_get_notification(client, live_se
                                                    "Snapshot: {{current_snapshot}}\n"
                                                    "Diff: {{diff}}\n"
                                                    "Diff Full: {{diff_full}}\n"
-                                                   "Diff as Patch: {{diff_patch}}\n"
                                                    ":-)",
                               "notification_format": "Text"}
 
     notification_form_data.update({
         "url": test_url,
-        "tags": "my tag",
+        "tag": "my tag",
         "title": "my title",
         "headers": "",
         "include_filters": '.ticket-available',

changedetectionio/tests/test_filter_failure_notification.py

@@ -1,7 +1,7 @@
 import os
 import time
 from flask import url_for
-from .util import set_original_response, live_server_setup, extract_UUID_from_client, wait_for_all_checks
+from .util import set_original_response, live_server_setup, extract_UUID_from_client
 from changedetectionio.model import App
 
 
@@ -12,7 +12,7 @@ def set_response_with_filter():
      <p>Which is across multiple lines</p>
      <br>
      So let's see what happens.  <br>
-     <div id="nope-doesnt-exist">Some text thats the same</div>
+     <div id="nope-doesnt-exist">Some text thats the same</div>     
      </body>
      </html>
     """
@@ -37,14 +37,14 @@ def run_filter_test(client, content_filter):
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
         url_for("form_quick_watch_add"),
-        data={"url": test_url, "tags": ''},
+        data={"url": test_url, "tag": ''},
         follow_redirects=True
     )
 
     assert b"Watch added" in res.data
 
     # Give the thread time to pick up the first version
-    wait_for_all_checks(client)
+    time.sleep(3)
 
     # Goto the edit page, add our ignore text
     # Add our URL to the import page
@@ -66,14 +66,13 @@ def run_filter_test(client, content_filter):
                                                    "Snapshot: {{current_snapshot}}\n"
                                                    "Diff: {{diff}}\n"
                                                    "Diff Full: {{diff_full}}\n"
-                                                   "Diff as Patch: {{diff_patch}}\n"
                                                    ":-)",
                               "notification_format": "Text"}
 
     notification_form_data.update({
         "url": test_url,
-        "tags": "my tag",
-        "title": "my title 123",
+        "tag": "my tag",
+        "title": "my title",
         "headers": "",
         "filter_failure_notification_send": 'y',
         "include_filters": content_filter,
@@ -86,55 +85,43 @@ def run_filter_test(client, content_filter):
     )
 
     assert b"Updated watch." in res.data
-    wait_for_all_checks(client)
+    time.sleep(3)
 
     # Now the notification should not exist, because we didnt reach the threshold
     assert not os.path.isfile("test-datastore/notification.txt")
 
-    # -2 because we would have checked twice above (on adding and on edit)
-    for i in range(0, App._FILTER_FAILURE_THRESHOLD_ATTEMPTS_DEFAULT-2):
+    for i in range(0, App._FILTER_FAILURE_THRESHOLD_ATTEMPTS_DEFAULT):
         res = client.get(url_for("form_watch_checknow"), follow_redirects=True)
-        wait_for_all_checks(client)
-        assert not os.path.isfile("test-datastore/notification.txt"), f"test-datastore/notification.txt should not exist - Attempt {i}"
+        time.sleep(3)
 
     # We should see something in the frontend
     assert b'Warning, no filters were found' in res.data
 
-    # One more check should trigger it (see -2 above)
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    wait_for_all_checks(client)
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    wait_for_all_checks(client)
     # Now it should exist and contain our "filter not found" alert
     assert os.path.isfile("test-datastore/notification.txt")
-
+    notification = False
     with open("test-datastore/notification.txt", 'r') as f:
         notification = f.read()
-
     assert 'CSS/xPath filter was not present in the page' in notification
     assert content_filter.replace('"', '\\"') in notification
 
-    # Remove it and prove that it doesn't trigger when not expected
-    # It should register a change, but no 'filter not found'
+    # Remove it and prove that it doesnt trigger when not expected
     os.unlink("test-datastore/notification.txt")
     set_response_with_filter()
 
-    # Try several times, it should NOT have 'filter not found'
     for i in range(0, App._FILTER_FAILURE_THRESHOLD_ATTEMPTS_DEFAULT):
         client.get(url_for("form_watch_checknow"), follow_redirects=True)
-        wait_for_all_checks(client)
+        time.sleep(3)
 
     # It should have sent a notification, but..
     assert os.path.isfile("test-datastore/notification.txt")
-    # but it should not contain the info about a failed filter (because there was none in this case)
+    # but it should not contain the info about the failed filter
     with open("test-datastore/notification.txt", 'r') as f:
         notification = f.read()
     assert not 'CSS/xPath filter was not present in the page' in notification
 
-    # Re #1247 - All tokens got replaced correctly in the notification
-    res = client.get(url_for("index"))
+    # Re #1247 - All tokens got replaced
     uuid = extract_UUID_from_client(client)
-    # UUID is correct, but notification contains tag uuid as UUIID wtf
     assert uuid in notification
 
     # cleanup for the next
@@ -150,7 +137,7 @@ def test_setup(live_server):
 
 def test_check_include_filters_failure_notification(client, live_server):
     set_original_response()
-    wait_for_all_checks(client)
+    time.sleep(1)
     run_filter_test(client, '#nope-doesnt-exist')
 
 def test_check_xpath_filter_failure_notification(client, live_server):

changedetectionio/tests/test_group.py

@@ -1,323 +0,0 @@
-#!/usr/bin/python3
-
-import time
-from flask import url_for
-from .util import live_server_setup, wait_for_all_checks, extract_rss_token_from_UI, get_UUID_for_tag_name, extract_UUID_from_client
-import os
-
-
-def test_setup(client, live_server):
-    live_server_setup(live_server)
-
-def set_original_response():
-    test_return_data = """<html>
-       <body>
-     Some initial text<br>
-     <p id="only-this">Should be only this</p>
-     <br>
-     <p id="not-this">And never this</p>
-     </body>
-     </html>
-    """
-
-    with open("test-datastore/endpoint-content.txt", "w") as f:
-        f.write(test_return_data)
-    return None
-
-def set_modified_response():
-    test_return_data = """<html>
-       <body>
-     Some initial text<br>
-     <p id="only-this">Should be REALLY only this</p>
-     <br>
-     <p id="not-this">And never this</p>
-     </body>
-     </html>
-    """
-
-    with open("test-datastore/endpoint-content.txt", "w") as f:
-        f.write(test_return_data)
-    return None
-
-def test_setup_group_tag(client, live_server):
-    #live_server_setup(live_server)
-    set_original_response()
-
-    # Add a tag with some config, import a tag and it should roughly work
-    res = client.post(
-        url_for("tags.form_tag_add"),
-        data={"name": "test-tag"},
-        follow_redirects=True
-    )
-    assert b"Tag added" in res.data
-    assert b"test-tag" in res.data
-
-    res = client.post(
-        url_for("tags.form_tag_edit_submit", uuid="first"),
-        data={"name": "test-tag",
-              "include_filters": '#only-this',
-              "subtractive_selectors": '#not-this'},
-        follow_redirects=True
-    )
-    assert b"Updated" in res.data
-    tag_uuid = get_UUID_for_tag_name(client, name="test-tag")
-    res = client.get(
-        url_for("tags.form_tag_edit", uuid="first")
-    )
-    assert b"#only-this" in res.data
-    assert b"#not-this" in res.data
-
-    # Tag should be setup and ready, now add a watch
-
-    test_url = url_for('test_endpoint', _external=True)
-    res = client.post(
-        url_for("import_page"),
-        data={"urls": test_url + "?first-imported=1 test-tag, extra-import-tag"},
-        follow_redirects=True
-    )
-    assert b"1 Imported" in res.data
-
-    res = client.get(url_for("index"))
-    assert b'import-tag' in res.data
-    assert b'extra-import-tag' in res.data
-
-    res = client.get(
-        url_for("tags.tags_overview_page"),
-        follow_redirects=True
-    )
-    assert b'import-tag' in res.data
-    assert b'extra-import-tag' in res.data
-
-    wait_for_all_checks(client)
-
-    res = client.get(url_for("index"))
-    assert b'Warning, no filters were found' not in res.data
-
-    res = client.get(
-        url_for("preview_page", uuid="first"),
-        follow_redirects=True
-    )
-    assert b'Should be only this' in res.data
-    assert b'And never this' not in res.data
-
-
-    # RSS Group tag filter
-    # An extra one that should be excluded
-    res = client.post(
-        url_for("import_page"),
-        data={"urls": test_url + "?should-be-excluded=1 some-tag"},
-        follow_redirects=True
-    )
-    assert b"1 Imported" in res.data
-    wait_for_all_checks(client)
-    set_modified_response()
-    res = client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    wait_for_all_checks(client)
-    rss_token = extract_rss_token_from_UI(client)
-    res = client.get(
-        url_for("rss", token=rss_token, tag="extra-import-tag", _external=True),
-        follow_redirects=True
-    )
-    assert b"should-be-excluded" not in res.data
-    assert res.status_code == 200
-    assert b"first-imported=1" in res.data
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
-    assert b'Deleted' in res.data
-
-def test_tag_import_singular(client, live_server):
-    #live_server_setup(live_server)
-
-    test_url = url_for('test_endpoint', _external=True)
-    res = client.post(
-        url_for("import_page"),
-        data={"urls": test_url + " test-tag, test-tag\r\n"+ test_url + "?x=1 test-tag, test-tag\r\n"},
-        follow_redirects=True
-    )
-    assert b"2 Imported" in res.data
-
-    res = client.get(
-        url_for("tags.tags_overview_page"),
-        follow_redirects=True
-    )
-    # Should be only 1 tag because they both had the same
-    assert res.data.count(b'test-tag') == 1
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
-    assert b'Deleted' in res.data
-
-def test_tag_add_in_ui(client, live_server):
-    #live_server_setup(live_server)
-#
-    res = client.post(
-        url_for("tags.form_tag_add"),
-        data={"name": "new-test-tag"},
-        follow_redirects=True
-    )
-    assert b"Tag added" in res.data
-    assert b"new-test-tag" in res.data
-
-    res = client.get(url_for("tags.delete_all"), follow_redirects=True)
-    assert b'All tags deleted' in res.data
-
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
-    assert b'Deleted' in res.data
-
-def test_group_tag_notification(client, live_server):
-    #live_server_setup(live_server)
-    set_original_response()
-
-    test_url = url_for('test_endpoint', _external=True)
-    res = client.post(
-        url_for("form_quick_watch_add"),
-        data={"url": test_url, "tags": 'test-tag, other-tag'},
-        follow_redirects=True
-    )
-
-    assert b"Watch added" in res.data
-
-    notification_url = url_for('test_notification_endpoint', _external=True).replace('http', 'json')
-    notification_form_data = {"notification_urls": notification_url,
-                              "notification_title": "New GROUP TAG ChangeDetection.io Notification - {{watch_url}}",
-                              "notification_body": "BASE URL: {{base_url}}\n"
-                                                   "Watch URL: {{watch_url}}\n"
-                                                   "Watch UUID: {{watch_uuid}}\n"
-                                                   "Watch title: {{watch_title}}\n"
-                                                   "Watch tag: {{watch_tag}}\n"
-                                                   "Preview: {{preview_url}}\n"
-                                                   "Diff URL: {{diff_url}}\n"
-                                                   "Snapshot: {{current_snapshot}}\n"
-                                                   "Diff: {{diff}}\n"
-                                                   "Diff Added: {{diff_added}}\n"
-                                                   "Diff Removed: {{diff_removed}}\n"
-                                                   "Diff Full: {{diff_full}}\n"
-                                                   "Diff as Patch: {{diff_patch}}\n"
-                                                   ":-)",
-                              "notification_screenshot": True,
-                              "notification_format": "Text",
-                              "title": "test-tag"}
-
-    res = client.post(
-        url_for("tags.form_tag_edit_submit", uuid=get_UUID_for_tag_name(client, name="test-tag")),
-        data=notification_form_data,
-        follow_redirects=True
-    )
-    assert b"Updated" in res.data
-
-    wait_for_all_checks(client)
-
-    set_modified_response()
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    time.sleep(3)
-
-    assert os.path.isfile("test-datastore/notification.txt")
-
-    # Verify what was sent as a notification, this file should exist
-    with open("test-datastore/notification.txt", "r") as f:
-        notification_submission = f.read()
-    os.unlink("test-datastore/notification.txt")
-
-    # Did we see the URL that had a change, in the notification?
-    # Diff was correctly executed
-    assert test_url in notification_submission
-    assert ':-)' in notification_submission
-    assert "Diff Full: Some initial text" in notification_submission
-    assert "New GROUP TAG ChangeDetection.io" in notification_submission
-    assert "test-tag" in notification_submission
-    assert "other-tag" in notification_submission
-
-    #@todo Test that multiple notifications fired
-    #@todo Test that each of multiple notifications with different settings
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
-    assert b'Deleted' in res.data
-
-def test_limit_tag_ui(client, live_server):
-    #live_server_setup(live_server)
-
-    test_url = url_for('test_endpoint', _external=True)
-    urls=[]
-
-    for i in range(20):
-        urls.append(test_url+"?x="+str(i)+" test-tag")
-
-    for i in range(20):
-        urls.append(test_url+"?non-grouped="+str(i))
-
-    res = client.post(
-        url_for("import_page"),
-        data={"urls": "\r\n".join(urls)},
-        follow_redirects=True
-    )
-
-    assert b"40 Imported" in res.data
-
-    res = client.get(url_for("index"))
-    assert b'test-tag' in res.data
-
-    # All should be here
-    assert res.data.count(b'processor-text_json_diff') == 40
-
-    tag_uuid = get_UUID_for_tag_name(client, name="test-tag")
-
-    res = client.get(url_for("index", tag=tag_uuid))
-
-    # Just a subset should be here
-    assert b'test-tag' in res.data
-    assert res.data.count(b'processor-text_json_diff') == 20
-    assert b"object at" not in res.data
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
-    assert b'Deleted' in res.data
-    res = client.get(url_for("tags.delete_all"), follow_redirects=True)
-    assert b'All tags deleted' in res.data
-def test_clone_tag_on_import(client, live_server):
-    #live_server_setup(live_server)
-    test_url = url_for('test_endpoint', _external=True)
-    res = client.post(
-        url_for("import_page"),
-        data={"urls": test_url + " test-tag, another-tag\r\n"},
-        follow_redirects=True
-    )
-
-    assert b"1 Imported" in res.data
-
-    res = client.get(url_for("index"))
-    assert b'test-tag' in res.data
-    assert b'another-tag' in res.data
-
-    watch_uuid = extract_UUID_from_client(client)
-    res = client.get(url_for("form_clone", uuid=watch_uuid), follow_redirects=True)
-
-    assert b'Cloned' in res.data
-    # 2 times plus the top link to tag
-    assert res.data.count(b'test-tag') == 3
-    assert res.data.count(b'another-tag') == 3
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
-    assert b'Deleted' in res.data
-
-def test_clone_tag_on_quickwatchform_add(client, live_server):
-    #live_server_setup(live_server)
-
-    test_url = url_for('test_endpoint', _external=True)
-
-    res = client.post(
-        url_for("form_quick_watch_add"),
-        data={"url": test_url, "tags": ' test-tag, another-tag      '},
-        follow_redirects=True
-    )
-
-    assert b"Watch added" in res.data
-
-    res = client.get(url_for("index"))
-    assert b'test-tag' in res.data
-    assert b'another-tag' in res.data
-
-    watch_uuid = extract_UUID_from_client(client)
-    res = client.get(url_for("form_clone", uuid=watch_uuid), follow_redirects=True)
-
-    assert b'Cloned' in res.data
-    # 2 times plus the top link to tag
-    assert res.data.count(b'test-tag') == 3
-    assert res.data.count(b'another-tag') == 3
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
-    assert b'Deleted' in res.data
-
-    res = client.get(url_for("tags.delete_all"), follow_redirects=True)
-    assert b'All tags deleted' in res.data

changedetectionio/tests/test_ignore_regex_text.py

@@ -15,24 +15,11 @@ def test_strip_regex_text_func():
     but sometimes we want to remove the lines.
     
     but 1 lines
-    skip 5 lines
-    really? yes man
-#/not this tries weirdly formed regex or just strings starting with /
-/not this
     but including 1234 lines
     igNORe-cAse text we dont want to keep    
     but not always."""
 
-
-    ignore_lines = [
-        "sometimes",
-        "/\s\d{2,3}\s/",
-        "/ignore-case text/",
-        "really?",
-        "/skip \d lines/i",
-        "/not"
-    ]
-
+    ignore_lines = ["sometimes", "/\s\d{2,3}\s/", "/ignore-case text/"]
 
     fetcher = fetch_site_status.perform_site_check(datastore=False)
     stripped_content = html_tools.strip_ignore_text(test_content, ignore_lines)
@@ -40,10 +27,4 @@ def test_strip_regex_text_func():
     assert b"but 1 lines" in stripped_content
     assert b"igNORe-cAse text" not in stripped_content
     assert b"but 1234 lines" not in stripped_content
-    assert b"really" not in stripped_content
-    assert b"not this" not in stripped_content
-
-    # Check line number reporting
-    stripped_content = html_tools.strip_ignore_text(test_content, ignore_lines, mode="line numbers")
-    assert stripped_content == [2, 5, 6, 7, 8, 10]
 

changedetectionio/tests/test_ignore_text.py

@@ -2,7 +2,7 @@
 
 import time
 from flask import url_for
-from .util import live_server_setup, wait_for_all_checks
+from . util import live_server_setup
 from changedetectionio import html_tools
 
 def test_setup(live_server):
@@ -84,6 +84,7 @@ def set_modified_ignore_response():
 
 
 def test_check_ignore_text_functionality(client, live_server):
+    sleep_time_for_fetch_thread = 3
 
     # Use a mix of case in ZzZ to prove it works case-insensitive.
     ignore_text = "XXXXX\r\nYYYYY\r\nzZzZZ\r\nnew ignore stuff"
@@ -102,7 +103,7 @@ def test_check_ignore_text_functionality(client, live_server):
     assert b"1 Imported" in res.data
 
     # Give the thread time to pick it up
-    wait_for_all_checks(client)
+    time.sleep(sleep_time_for_fetch_thread)
 
     # Goto the edit page, add our ignore text
     # Add our URL to the import page
@@ -123,7 +124,7 @@ def test_check_ignore_text_functionality(client, live_server):
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
-    wait_for_all_checks(client)
+    time.sleep(sleep_time_for_fetch_thread)
 
     # It should report nothing found (no new 'unviewed' class)
     res = client.get(url_for("index"))
@@ -136,7 +137,7 @@ def test_check_ignore_text_functionality(client, live_server):
     # Trigger a check
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
     # Give the thread time to pick it up
-    wait_for_all_checks(client)
+    time.sleep(sleep_time_for_fetch_thread)
 
     # It should report nothing found (no new 'unviewed' class)
     res = client.get(url_for("index"))
@@ -150,7 +151,7 @@ def test_check_ignore_text_functionality(client, live_server):
     # Just to be sure.. set a regular modified change..
     set_modified_original_ignore_response()
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    wait_for_all_checks(client)
+    time.sleep(sleep_time_for_fetch_thread)
 
     res = client.get(url_for("index"))
     assert b'unviewed' in res.data
@@ -166,6 +167,7 @@ def test_check_ignore_text_functionality(client, live_server):
     assert b'Deleted' in res.data
 
 def test_check_global_ignore_text_functionality(client, live_server):
+    sleep_time_for_fetch_thread = 3
 
     # Give the endpoint time to spin up
     time.sleep(1)
@@ -196,7 +198,7 @@ def test_check_global_ignore_text_functionality(client, live_server):
     assert b"1 Imported" in res.data
 
     # Give the thread time to pick it up
-    wait_for_all_checks(client)
+    time.sleep(sleep_time_for_fetch_thread)
 
 
     # Goto the edit page of the item, add our ignore text
@@ -218,7 +220,7 @@ def test_check_global_ignore_text_functionality(client, live_server):
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
-    wait_for_all_checks(client)
+    time.sleep(sleep_time_for_fetch_thread)
 
     # so that we are sure everything is viewed and in a known 'nothing changed' state
     res = client.get(url_for("diff_history_page", uuid="first"))
@@ -235,7 +237,7 @@ def test_check_global_ignore_text_functionality(client, live_server):
     # Trigger a check
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
     # Give the thread time to pick it up
-    wait_for_all_checks(client)
+    time.sleep(sleep_time_for_fetch_thread)
 
     # It should report nothing found (no new 'unviewed' class)
     res = client.get(url_for("index"))
@@ -245,7 +247,7 @@ def test_check_global_ignore_text_functionality(client, live_server):
     # Just to be sure.. set a regular modified change that will trigger it
     set_modified_original_ignore_response()
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    wait_for_all_checks(client)
+    time.sleep(sleep_time_for_fetch_thread)
     res = client.get(url_for("index"))
     assert b'unviewed' in res.data
 

changedetectionio/tests/test_ignorehighlighter.py

@@ -1,57 +0,0 @@
-#!/usr/bin/python3
-
-import time
-from flask import url_for
-from .util import live_server_setup, wait_for_all_checks
-from changedetectionio import html_tools
-from . util import  extract_UUID_from_client
-
-def set_original_ignore_response():
-    test_return_data = """<html>
-       <body>
-     Some initial text<br>
-     <p>Which is across multiple lines</p>
-     <br>
-     So let's see what happens.  <br>
-     <p>oh yeah 456</p>
-     </body>
-     </html>
-
-    """
-
-    with open("test-datastore/endpoint-content.txt", "w") as f:
-        f.write(test_return_data)
-
-
-def test_highlight_ignore(client, live_server):
-    live_server_setup(live_server)
-    set_original_ignore_response()
-    test_url = url_for('test_endpoint', _external=True)
-    res = client.post(
-        url_for("import_page"),
-        data={"urls": test_url},
-        follow_redirects=True
-    )
-    assert b"1 Imported" in res.data
-
-    # Give the thread time to pick it up
-    wait_for_all_checks(client)
-    uuid = extract_UUID_from_client(client)
-    # use the highlighter endpoint
-    res = client.post(
-        url_for("highlight_submit_ignore_url", uuid=uuid),
-        data={"mode": 'digit-regex', 'selection': 'oh yeah 123'},
-        follow_redirects=True
-    )
-
-    res = client.get(url_for("edit_page", uuid=uuid))
-
-    # should be a regex now
-    assert b'/oh\ yeah\ \d+/' in res.data
-
-    # Should return a link
-    assert b'href' in res.data
-
-    # And it should register in the preview page
-    res = client.get(url_for("preview_page", uuid=uuid))
-    assert b'<div class="ignored">oh yeah 456' in res.data

changedetectionio/tests/test_ignorestatuscode.py

@@ -2,7 +2,7 @@
 
 import time
 from flask import url_for
-from .util import live_server_setup, wait_for_all_checks
+from . util import live_server_setup
 
 
 def test_setup(live_server):
@@ -40,7 +40,7 @@ def set_some_changed_response():
 
 
 def test_normal_page_check_works_with_ignore_status_code(client, live_server):
-
+    sleep_time_for_fetch_thread = 3
 
     # Give the endpoint time to spin up
     time.sleep(1)
@@ -68,15 +68,15 @@ def test_normal_page_check_works_with_ignore_status_code(client, live_server):
     )
     assert b"1 Imported" in res.data
 
-    wait_for_all_checks(client)
+    time.sleep(sleep_time_for_fetch_thread)
 
     set_some_changed_response()
-    wait_for_all_checks(client)
+    time.sleep(sleep_time_for_fetch_thread)
     # Trigger a check
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
-    wait_for_all_checks(client)
+    time.sleep(sleep_time_for_fetch_thread)
 
     # It should report nothing found (no new 'unviewed' class)
     res = client.get(url_for("index"))
@@ -109,13 +109,13 @@ def test_403_page_check_works_with_ignore_status_code(client, live_server):
     # Add our URL to the import page
     res = client.post(
         url_for("edit_page", uuid="first"),
-        data={"ignore_status_codes": "y", "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests"},
+        data={"ignore_status_codes": "y", "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
         follow_redirects=True
     )
     assert b"Updated watch." in res.data
 
     # Give the thread time to pick it up
-    wait_for_all_checks(client)
+    time.sleep(sleep_time_for_fetch_thread)
 
     #  Make a change
     set_some_changed_response()
@@ -123,7 +123,7 @@ def test_403_page_check_works_with_ignore_status_code(client, live_server):
     # Trigger a check
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
     # Give the thread time to pick it up
-    wait_for_all_checks(client)
+    time.sleep(sleep_time_for_fetch_thread)
 
     # It should have 'unviewed' still
     # Because it should be looking at only that 'sametext' id

changedetectionio/tests/test_import.py

@@ -112,7 +112,6 @@ def test_import_distillio(client, live_server):
     # did the tags work?
     res = client.get( url_for("index"))
 
-    # check tags
     assert b"nice stuff" in res.data
     assert b"nerd-news" in res.data
 

changedetectionio/tests/test_jinja2.py

@@ -20,7 +20,7 @@ def test_jinja2_in_url_query(client, live_server):
                               "date={% now 'Europe/Berlin', '%Y' %}.{% now 'Europe/Berlin', '%m' %}.{% now 'Europe/Berlin', '%d' %}", )
     res = client.post(
         url_for("form_quick_watch_add"),
-        data={"url": full_url, "tags": "test"},
+        data={"url": full_url, "tag": "test"},
         follow_redirects=True
     )
     assert b"Watch added" in res.data

changedetectionio/tests/test_jsonpath_jq_selector.py

@@ -208,7 +208,7 @@ def test_check_json_without_filter(client, live_server):
     )
 
     # Give the thread time to pick it up
-    wait_for_all_checks(client)
+    time.sleep(3)
 
     res = client.get(
         url_for("preview_page", uuid="first"),
@@ -238,7 +238,7 @@ def check_json_filter(json_filter, client, live_server):
     assert b"1 Imported" in res.data
 
     # Give the thread time to pick it up
-    wait_for_all_checks(client)
+    time.sleep(3)
 
     # Goto the edit page, add our ignore text
     # Add our URL to the import page
@@ -246,7 +246,7 @@ def check_json_filter(json_filter, client, live_server):
         url_for("edit_page", uuid="first"),
         data={"include_filters": json_filter,
               "url": test_url,
-              "tags": "",
+              "tag": "",
               "headers": "",
               "fetch_backend": "html_requests"
               },
@@ -261,14 +261,14 @@ def check_json_filter(json_filter, client, live_server):
     assert bytes(escape(json_filter).encode('utf-8')) in res.data
 
     # Give the thread time to pick it up
-    wait_for_all_checks(client)
+    time.sleep(3)
     #  Make a change
     set_modified_response()
 
     # Trigger a check
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
     # Give the thread time to pick it up
-    wait_for_all_checks(client)
+    time.sleep(4)
 
     # It should have 'unviewed' still
     res = client.get(url_for("index"))
@@ -306,14 +306,14 @@ def check_json_filter_bool_val(json_filter, client, live_server):
     )
     assert b"1 Imported" in res.data
 
-    wait_for_all_checks(client)
+    time.sleep(3)
     # Goto the edit page, add our ignore text
     # Add our URL to the import page
     res = client.post(
         url_for("edit_page", uuid="first"),
         data={"include_filters": json_filter,
               "url": test_url,
-              "tags": "",
+              "tag": "",
               "headers": "",
               "fetch_backend": "html_requests"
               },
@@ -322,14 +322,14 @@ def check_json_filter_bool_val(json_filter, client, live_server):
     assert b"Updated watch." in res.data
 
     # Give the thread time to pick it up
-    wait_for_all_checks(client)
+    time.sleep(3)
     #  Make a change
     set_modified_response()
 
     # Trigger a check
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
     # Give the thread time to pick it up
-    wait_for_all_checks(client)
+    time.sleep(3)
 
     res = client.get(url_for("diff_history_page", uuid="first"))
     # But the change should be there, tho its hard to test the change was detected because it will show old and new versions
@@ -366,7 +366,7 @@ def check_json_ext_filter(json_filter, client, live_server):
     assert b"1 Imported" in res.data
 
     # Give the thread time to pick it up
-    wait_for_all_checks(client)
+    time.sleep(3)
 
     # Goto the edit page, add our ignore text
     # Add our URL to the import page
@@ -374,7 +374,7 @@ def check_json_ext_filter(json_filter, client, live_server):
         url_for("edit_page", uuid="first"),
         data={"include_filters": json_filter,
               "url": test_url,
-              "tags": "",
+              "tag": "",
               "headers": "",
               "fetch_backend": "html_requests"
               },
@@ -389,14 +389,14 @@ def check_json_ext_filter(json_filter, client, live_server):
     assert bytes(escape(json_filter).encode('utf-8')) in res.data
 
     # Give the thread time to pick it up
-    wait_for_all_checks(client)
+    time.sleep(3)
     #  Make a change
     set_modified_ext_response()
 
     # Trigger a check
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
     # Give the thread time to pick it up
-    wait_for_all_checks(client)
+    time.sleep(4)
 
     # It should have 'unviewed'
     res = client.get(url_for("index"))
@@ -428,14 +428,14 @@ def test_ignore_json_order(client, live_server):
     )
     assert b"1 Imported" in res.data
 
-    wait_for_all_checks(client)
+    time.sleep(2)
 
     with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write('{"world" : 123, "hello": 123}')
 
     # Trigger a check
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    wait_for_all_checks(client)
+    time.sleep(2)
 
     res = client.get(url_for("index"))
     assert b'unviewed' not in res.data
@@ -446,7 +446,7 @@ def test_ignore_json_order(client, live_server):
 
     # Trigger a check
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    wait_for_all_checks(client)
+    time.sleep(2)
 
     res = client.get(url_for("index"))
     assert b'unviewed' in res.data

changedetectionio/tests/test_notification.py

@@ -3,8 +3,7 @@ import os
 import time
 import re
 from flask import url_for
-from .util import set_original_response, set_modified_response, set_more_modified_response, live_server_setup, wait_for_all_checks, \
-    set_longer_modified_response
+from . util import set_original_response, set_modified_response, set_more_modified_response, live_server_setup
 from . util import  extract_UUID_from_client
 import logging
 import base64
@@ -22,9 +21,11 @@ def test_setup(live_server):
 # Hard to just add more live server URLs when one test is already running (I think)
 # So we add our test here (was in a different file)
 def test_check_notification(client, live_server):
-    #live_server_setup(live_server)
     set_original_response()
 
+    # Give the endpoint time to spin up
+    time.sleep(3)
+
     # Re 360 - new install should have defaults set
     res = client.get(url_for("settings_page"))
     notification_url = url_for('test_notification_endpoint', _external=True).replace('http', 'json')
@@ -61,13 +62,13 @@ def test_check_notification(client, live_server):
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
         url_for("form_quick_watch_add"),
-        data={"url": test_url, "tags": ''},
+        data={"url": test_url, "tag": ''},
         follow_redirects=True
     )
     assert b"Watch added" in res.data
 
     # Give the thread time to pick up the first version
-    wait_for_all_checks(client)
+    time.sleep(3)
 
     # We write the PNG to disk, but a JPEG should appear in the notification
     # Write the last screenshot png
@@ -98,14 +99,13 @@ def test_check_notification(client, live_server):
                                                    "Diff Added: {{diff_added}}\n"
                                                    "Diff Removed: {{diff_removed}}\n"
                                                    "Diff Full: {{diff_full}}\n"
-                                                   "Diff as Patch: {{diff_patch}}\n"
                                                    ":-)",
                               "notification_screenshot": True,
                               "notification_format": "Text"}
 
     notification_form_data.update({
         "url": test_url,
-        "tags": "my tag, my second tag",
+        "tag": "my tag",
         "title": "my title",
         "headers": "",
         "fetch_backend": "html_requests"})
@@ -128,7 +128,7 @@ def test_check_notification(client, live_server):
 
 
     ## Now recheck, and it should have sent the notification
-    wait_for_all_checks(client)
+    time.sleep(3)
     set_modified_response()
 
     # Trigger a check
@@ -141,7 +141,8 @@ def test_check_notification(client, live_server):
 
     # Did we see the URL that had a change, in the notification?
     # Diff was correctly executed
-
+    assert test_url in notification_submission
+    assert ':-)' in notification_submission
     assert "Diff Full: Some initial text" in notification_submission
     assert "Diff: (changed) Which is across multiple lines" in notification_submission
     assert "(into) which has this one new line" in notification_submission
@@ -149,13 +150,12 @@ def test_check_notification(client, live_server):
     assert "b'" not in notification_submission
     assert re.search('Watch UUID: [0-9a-f]{8}(-[0-9a-f]{4}){3}-[0-9a-f]{12}', notification_submission, re.IGNORECASE)
     assert "Watch title: my title" in notification_submission
-    assert "Watch tag: my tag, my second tag" in notification_submission
+    assert "Watch tag: my tag" in notification_submission
     assert "diff/" in notification_submission
     assert "preview/" in notification_submission
     assert ":-)" in notification_submission
     assert "New ChangeDetection.io Notification - {}".format(test_url) in notification_submission
-    assert test_url in notification_submission
-    assert ':-)' in notification_submission
+
     # Check the attachment was added, and that it is a JPEG from the original PNG
     notification_submission_object = json.loads(notification_submission)
     # We keep PNG screenshots for now
@@ -193,11 +193,11 @@ def test_check_notification(client, live_server):
 
     # Trigger a check
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    wait_for_all_checks(client)
+    time.sleep(1)
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    wait_for_all_checks(client)
+    time.sleep(1)
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    wait_for_all_checks(client)
+    time.sleep(1)
     assert os.path.exists("test-datastore/notification.txt") == False
 
     res = client.get(url_for("notification_logs"))
@@ -209,7 +209,7 @@ def test_check_notification(client, live_server):
         url_for("edit_page", uuid="first"),
         data={
         "url": test_url,
-        "tags": "my tag",
+        "tag": "my tag",
         "title": "my title",
         "notification_urls": '',
         "notification_title": '',
@@ -243,7 +243,7 @@ def test_notification_validation(client, live_server):
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
         url_for("form_quick_watch_add"),
-        data={"url": test_url, "tags": 'nice one'},
+        data={"url": test_url, "tag": 'nice one'},
         follow_redirects=True
     )
 
@@ -274,7 +274,7 @@ def test_notification_validation(client, live_server):
 
 
 def test_notification_custom_endpoint_and_jinja2(client, live_server):
-    #live_server_setup(live_server)
+    time.sleep(1)
 
     # test_endpoint - that sends the contents of a file
     # test_notification_endpoint - that takes a POST and writes it to file (test-datastore/notification.txt)
@@ -285,14 +285,12 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server):
 
     res = client.post(
         url_for("settings_page"),
-        data={
-              "application-fetch_backend": "html_requests",
-              "application-minutes_between_check": 180,
+        data={"application-notification_title": "New ChangeDetection.io Notification - {{ watch_url }}",
               "application-notification_body": '{ "url" : "{{ watch_url }}", "secret": 444 }',
-              "application-notification_format": default_notification_format,
-              "application-notification_urls": test_notification_url,
               # https://github.com/caronc/apprise/wiki/Notify_Custom_JSON#get-parameter-manipulation
-              "application-notification_title": "New ChangeDetection.io Notification - {{ watch_url }}",
+              "application-notification_urls": test_notification_url,
+              "application-minutes_between_check": 180,
+              "application-fetch_backend": "html_requests"
               },
         follow_redirects=True
     )
@@ -305,20 +303,21 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server):
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
         url_for("form_quick_watch_add"),
-        data={"url": test_url, "tags": 'nice one'},
+        data={"url": test_url, "tag": 'nice one'},
         follow_redirects=True
     )
 
     assert b"Watch added" in res.data
 
-    wait_for_all_checks(client)
+    time.sleep(2)
     set_modified_response()
 
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
     time.sleep(2)
 
+
     with open("test-datastore/notification.txt", 'r') as f:
-        x = f.read()
+        x=f.read()
         j = json.loads(x)
         assert j['url'].startswith('http://localhost')
         assert j['secret'] == 444
@@ -329,9 +328,5 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server):
         notification_url = f.read()
         assert 'xxx=http' in notification_url
 
-    # Should always be automatically detected as JSON content type even when we set it as 'Text' (default)
-    assert os.path.isfile("test-datastore/notification-content-type.txt")
-    with open("test-datastore/notification-content-type.txt", 'r') as f:
-        assert 'application/json' in f.read()
-
     os.unlink("test-datastore/notification-url.txt")
+

changedetectionio/tests/test_notification_errors.py

@@ -17,7 +17,7 @@ def test_check_notification_error_handling(client, live_server):
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
         url_for("form_quick_watch_add"),
-        data={"url": test_url, "tags": ''},
+        data={"url": test_url, "tag": ''},
         follow_redirects=True
     )
     assert b"Watch added" in res.data
@@ -32,7 +32,7 @@ def test_check_notification_error_handling(client, live_server):
               "notification_body": "xxxxx",
               "notification_format": "Text",
               "url": test_url,
-              "tags": "",
+              "tag": "",
               "title": "",
               "headers": "",
               "time_between_check-minutes": "180",

changedetectionio/tests/test_request.py

@@ -25,7 +25,7 @@ def test_headers_in_request(client, live_server):
     )
     assert b"1 Imported" in res.data
 
-    wait_for_all_checks(client)
+    time.sleep(1)
 
     res = client.post(
         url_for("import_page"),
@@ -43,7 +43,7 @@ def test_headers_in_request(client, live_server):
         url_for("edit_page", uuid="first"),
         data={
               "url": test_url,
-              "tags": "",
+              "tag": "",
               "fetch_backend": 'html_webdriver' if os.getenv('PLAYWRIGHT_DRIVER_URL') else 'html_requests',
               "headers": "xxx:ooo\ncool:yeah\r\ncookie:"+cookie_header},
         follow_redirects=True
@@ -95,14 +95,14 @@ def test_body_in_request(client, live_server):
     )
     assert b"1 Imported" in res.data
 
-    wait_for_all_checks(client)
+    time.sleep(3)
 
     # add the first 'version'
     res = client.post(
         url_for("edit_page", uuid="first"),
         data={
               "url": test_url,
-              "tags": "",
+              "tag": "",
               "method": "POST",
               "fetch_backend": "html_requests",
               "body": "something something"},
@@ -110,7 +110,7 @@ def test_body_in_request(client, live_server):
     )
     assert b"Updated watch." in res.data
 
-    wait_for_all_checks(client)
+    time.sleep(3)
 
     # Now the change which should trigger a change
     body_value = 'Test Body Value'
@@ -118,7 +118,7 @@ def test_body_in_request(client, live_server):
         url_for("edit_page", uuid="first"),
         data={
               "url": test_url,
-              "tags": "",
+              "tag": "",
               "method": "POST",
               "fetch_backend": "html_requests",
               "body": body_value},
@@ -126,7 +126,7 @@ def test_body_in_request(client, live_server):
     )
     assert b"Updated watch." in res.data
 
-    wait_for_all_checks(client)
+    time.sleep(3)
 
     # The service should echo back the body
     res = client.get(
@@ -163,7 +163,7 @@ def test_body_in_request(client, live_server):
         url_for("edit_page", uuid="first"),
         data={
               "url": test_url,
-              "tags": "",
+              "tag": "",
               "method": "GET",
               "fetch_backend": "html_requests",
               "body": "invalid"},
@@ -187,7 +187,7 @@ def test_method_in_request(client, live_server):
     )
     assert b"1 Imported" in res.data
 
-    wait_for_all_checks(client)
+    time.sleep(2)
     res = client.post(
         url_for("import_page"),
         data={"urls": test_url},
@@ -195,14 +195,14 @@ def test_method_in_request(client, live_server):
     )
     assert b"1 Imported" in res.data
 
-    wait_for_all_checks(client)
+    time.sleep(2)
 
     # Attempt to add a method which is not valid
     res = client.post(
         url_for("edit_page", uuid="first"),
         data={
             "url": test_url,
-            "tags": "",
+            "tag": "",
             "fetch_backend": "html_requests",
             "method": "invalid"},
         follow_redirects=True
@@ -214,7 +214,7 @@ def test_method_in_request(client, live_server):
         url_for("edit_page", uuid="first"),
         data={
             "url": test_url,
-            "tags": "",
+            "tag": "",
             "fetch_backend": "html_requests",
             "method": "PATCH"},
         follow_redirects=True
@@ -222,7 +222,7 @@ def test_method_in_request(client, live_server):
     assert b"Updated watch." in res.data
 
     # Give the thread time to pick up the first version
-    wait_for_all_checks(client)
+    time.sleep(2)
 
     # The service should echo back the request verb
     res = client.get(
@@ -233,7 +233,7 @@ def test_method_in_request(client, live_server):
     # The test call service will return the verb as the body
     assert b"PATCH" in res.data
 
-    wait_for_all_checks(client)
+    time.sleep(2)
 
     watches_with_method = 0
     with open('test-datastore/url-watches.json') as f:
@@ -265,7 +265,7 @@ def test_headers_textfile_in_request(client, live_server):
     )
     assert b"1 Imported" in res.data
 
-    wait_for_all_checks(client)
+    time.sleep(1)
 
 
     # Add some headers to a request
@@ -273,7 +273,7 @@ def test_headers_textfile_in_request(client, live_server):
         url_for("edit_page", uuid="first"),
         data={
               "url": test_url,
-              "tags": "testtag",
+              "tag": "testtag",
               "fetch_backend": 'html_webdriver' if os.getenv('PLAYWRIGHT_DRIVER_URL') else 'html_requests',
               "headers": "xxx:ooo\ncool:yeah\r\n"},
         follow_redirects=True

changedetectionio/tests/test_search.py

@@ -1,74 +0,0 @@
-from flask import url_for
-from .util import set_original_response, set_modified_response, live_server_setup
-import time
-
-def test_setup(live_server):
-    live_server_setup(live_server)
-
-def test_basic_search(client, live_server):
-    #live_server_setup(live_server)
-
-    urls = ['https://localhost:12300?first-result=1',
-            'https://localhost:5000?second-result=1'
-            ]
-    res = client.post(
-        url_for("import_page"),
-        data={"urls": "\r\n".join(urls)},
-        follow_redirects=True
-    )
-
-    assert b"2 Imported" in res.data
-
-    # By URL
-    res = client.get(url_for("index") + "?q=first-res")
-    assert urls[0].encode('utf-8') in res.data
-    assert urls[1].encode('utf-8') not in res.data
-
-    # By Title
-
-    res = client.post(
-        url_for("edit_page", uuid="first"),
-        data={"title": "xxx-title", "url": urls[0], "tags": "", "headers": "", 'fetch_backend': "html_requests"},
-        follow_redirects=True
-    )
-    assert b"Updated watch." in res.data
-
-    res = client.get(url_for("index") + "?q=xxx-title")
-    assert urls[0].encode('utf-8') in res.data
-    assert urls[1].encode('utf-8') not in res.data
-
-
-def test_search_in_tag_limit(client, live_server):
-    #live_server_setup(live_server)
-
-    urls = ['https://localhost:12300?first-result=1 tag-one',
-            'https://localhost:5000?second-result=1 tag-two'
-            ]
-    res = client.post(
-        url_for("import_page"),
-        data={"urls": "\r\n".join(urls)},
-        follow_redirects=True
-    )
-
-    assert b"2 Imported" in res.data
-
-    # By URL
-
-    res = client.get(url_for("index") + "?q=first-res")
-    # Split because of the import tag separation
-    assert urls[0].split(' ')[0].encode('utf-8') in res.data, urls[0].encode('utf-8')
-    assert urls[1].split(' ')[0].encode('utf-8') not in res.data, urls[0].encode('utf-8')
-
-    # By Title
-    res = client.post(
-        url_for("edit_page", uuid="first"),
-        data={"title": "xxx-title", "url": urls[0].split(' ')[0], "tags": urls[0].split(' ')[1], "headers": "",
-              'fetch_backend': "html_requests"},
-        follow_redirects=True
-    )
-    assert b"Updated watch." in res.data
-
-    res = client.get(url_for("index") + "?q=xxx-title")
-    assert urls[0].split(' ')[0].encode('utf-8') in res.data, urls[0].encode('utf-8')
-    assert urls[1].split(' ')[0].encode('utf-8') not in res.data, urls[0].encode('utf-8')
-

changedetectionio/tests/test_security.py

@@ -18,7 +18,7 @@ def test_bad_access(client, live_server):
         url_for("edit_page", uuid="first"),
         data={
               "url": 'javascript:alert(document.domain)',
-              "tags": "",
+              "tag": "",
               "method": "GET",
               "fetch_backend": "html_requests",
               "body": ""},
@@ -29,7 +29,7 @@ def test_bad_access(client, live_server):
 
     res = client.post(
         url_for("form_quick_watch_add"),
-        data={"url": '            javascript:alert(123)', "tags": ''},
+        data={"url": '            javascript:alert(123)', "tag": ''},
         follow_redirects=True
     )
 
@@ -37,7 +37,7 @@ def test_bad_access(client, live_server):
 
     res = client.post(
         url_for("form_quick_watch_add"),
-        data={"url": '%20%20%20javascript:alert(123)%20%20', "tags": ''},
+        data={"url": '%20%20%20javascript:alert(123)%20%20', "tag": ''},
         follow_redirects=True
     )
 
@@ -46,7 +46,7 @@ def test_bad_access(client, live_server):
 
     res = client.post(
         url_for("form_quick_watch_add"),
-        data={"url": ' source:javascript:alert(document.domain)', "tags": ''},
+        data={"url": ' source:javascript:alert(document.domain)', "tag": ''},
         follow_redirects=True
     )
 
@@ -56,7 +56,7 @@ def test_bad_access(client, live_server):
 
     client.post(
         url_for("form_quick_watch_add"),
-        data={"url": 'file:///tasty/disk/drive', "tags": ''},
+        data={"url": 'file:///tasty/disk/drive', "tag": ''},
         follow_redirects=True
     )
     time.sleep(1)

changedetectionio/tests/test_share_watch.py

@@ -29,7 +29,7 @@ def test_share_watch(client, live_server):
     # Add our URL to the import page
     res = client.post(
         url_for("edit_page", uuid="first"),
-        data={"include_filters": include_filters, "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests"},
+        data={"include_filters": include_filters, "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
         follow_redirects=True
     )
     assert b"Updated watch." in res.data

changedetectionio/tests/test_source.py

@@ -3,7 +3,7 @@
 import time
 from flask import url_for
 from urllib.request import urlopen
-from .util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks
+from .util import set_original_response, set_modified_response, live_server_setup
 
 sleep_time_for_fetch_thread = 3
 
@@ -42,7 +42,7 @@ def test_check_basic_change_detection_functionality_source(client, live_server):
     res = client.get(url_for("form_watch_checknow"), follow_redirects=True)
     assert b'1 watches queued for rechecking.' in res.data
 
-    wait_for_all_checks(client)
+    time.sleep(5)
 
     # Now something should be ready, indicated by having a 'unviewed' class
     res = client.get(url_for("index"))
@@ -60,7 +60,7 @@ def test_check_basic_change_detection_functionality_source(client, live_server):
 # `subtractive_selectors` should still work in `source:` type requests
 def test_check_ignore_elements(client, live_server):
     set_original_response()
-    time.sleep(1)
+    time.sleep(2)
     test_url = 'source:'+url_for('test_endpoint', _external=True)
     # Add our URL to the import page
     res = client.post(
@@ -71,14 +71,14 @@ def test_check_ignore_elements(client, live_server):
 
     assert b"1 Imported" in res.data
 
-    wait_for_all_checks(client)
+    time.sleep(sleep_time_for_fetch_thread)
 
     #####################
     # We want <span> and <p> ONLY, but ignore span with .foobar-detection
 
     client.post(
         url_for("edit_page", uuid="first"),
-        data={"include_filters": 'span,p', "url": test_url, "tags": "", "subtractive_selectors": ".foobar-detection", 'fetch_backend': "html_requests"},
+        data={"include_filters": 'span,p', "url": test_url, "tag": "", "subtractive_selectors": ".foobar-detection", 'fetch_backend': "html_requests"},
         follow_redirects=True
     )
 

changedetectionio/tests/test_watch_fields_storage.py

@@ -26,7 +26,7 @@ def test_check_watch_field_storage(client, live_server):
                "title" : "My title",
                "ignore_text" : "ignore this",
                "url": test_url,
-               "tags": "woohoo",
+               "tag": "woohoo",
                "headers": "curl:foo",
                'fetch_backend': "html_requests"
                },

changedetectionio/tests/test_xpath_selector.py

@@ -89,7 +89,7 @@ def test_check_xpath_filter_utf8(client, live_server):
     time.sleep(1)
     res = client.post(
         url_for("edit_page", uuid="first"),
-        data={"include_filters": filter, "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests"},
+        data={"include_filters": filter, "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
         follow_redirects=True
     )
     assert b"Updated watch." in res.data
@@ -143,7 +143,7 @@ def test_check_xpath_text_function_utf8(client, live_server):
     time.sleep(1)
     res = client.post(
         url_for("edit_page", uuid="first"),
-        data={"include_filters": filter, "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests"},
+        data={"include_filters": filter, "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
         follow_redirects=True
     )
     assert b"Updated watch." in res.data
@@ -189,7 +189,7 @@ def test_check_markup_xpath_filter_restriction(client, live_server):
     # Add our URL to the import page
     res = client.post(
         url_for("edit_page", uuid="first"),
-        data={"include_filters": xpath_filter, "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests"},
+        data={"include_filters": xpath_filter, "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
         follow_redirects=True
     )
     assert b"Updated watch." in res.data
@@ -231,7 +231,7 @@ def test_xpath_validation(client, live_server):
 
     res = client.post(
         url_for("edit_page", uuid="first"),
-        data={"include_filters": "/something horrible", "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests"},
+        data={"include_filters": "/something horrible", "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
         follow_redirects=True
     )
     assert b"is not a valid XPath expression" in res.data
@@ -261,7 +261,7 @@ def test_check_with_prefix_include_filters(client, live_server):
 
     res = client.post(
         url_for("edit_page", uuid="first"),
-        data={"include_filters":  "xpath://*[contains(@class, 'sametext')]", "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests"},
+        data={"include_filters":  "xpath://*[contains(@class, 'sametext')]", "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
         follow_redirects=True
     )
 

changedetectionio/tests/unit/test_notification_diff.py

@@ -36,7 +36,7 @@ class TestDiffBuilder(unittest.TestCase):
         output = output.split("\n")
         self.assertIn('(removed) for having learned computerese,', output)
         self.assertIn('(removed) I continue to examine bits, bytes and words', output)
-
+        
         #diff_removed
         with open(base_dir + "/test-content/before.txt", 'r') as f:
             previous_version_file_contents = f.read()
@@ -49,7 +49,7 @@ class TestDiffBuilder(unittest.TestCase):
         self.assertIn('(into) xok', output)
         self.assertIn('(into) next-x-ok', output)
         self.assertNotIn('(added) and something new', output)
-
+        
         #diff_removed
         with open(base_dir + "/test-content/after-2.txt", 'r') as f:
             newest_version_file_contents = f.read()
@@ -57,25 +57,9 @@ class TestDiffBuilder(unittest.TestCase):
         output = output.split("\n")
         self.assertIn('(removed) for having learned computerese,', output)
         self.assertIn('(removed) I continue to examine bits, bytes and words', output)
-
-    def test_expected_diff_patch_output(self):
-        base_dir = os.path.dirname(__file__)
-        with open(base_dir + "/test-content/before.txt", 'r') as f:
-            before = f.read()
-        with open(base_dir + "/test-content/after.txt", 'r') as f:
-            after = f.read()
-
-        output = diff.render_diff(previous_version_file_contents=before,
-                                  newest_version_file_contents=after,
-                                  patch_format=True)
-        output = output.split("\n")
-
-        self.assertIn('-ok', output)
-        self.assertIn('+xok', output)
-        self.assertIn('+next-x-ok', output)
-        self.assertIn('+and something new', output)
+        
 
         # @todo test blocks of changed, blocks of added, blocks of removed
 
 if __name__ == '__main__':
-    unittest.main()
+    unittest.main()

changedetectionio/tests/util.py

@@ -38,25 +38,7 @@ def set_modified_response():
         f.write(test_return_data)
 
     return None
-def set_longer_modified_response():
-    test_return_data = """<html>
-    <head><title>modified head title</title></head>
-    <body>
-     Some initial text<br>
-     <p>which has this one new line</p>
-     <br>
-     So let's see what happens.  <br>
-     So let's see what happens.  <br>
-      So let's see what happens.  <br>
-     So let's see what happens.  <br>           
-     </body>
-     </html>
-    """
 
-    with open("test-datastore/endpoint-content.txt", "w") as f:
-        f.write(test_return_data)
-
-    return None
 def set_more_modified_response():
     test_return_data = """<html>
     <head><title>modified head title</title></head>
@@ -88,16 +70,6 @@ def extract_api_key_from_UI(client):
     api_key = m.group(1)
     return api_key.strip()
 
-
-# kinda funky, but works for now
-def get_UUID_for_tag_name(client, name):
-    app_config = client.application.config.get('DATASTORE').data
-    for uuid, tag in app_config['settings']['application'].get('tags', {}).items():
-        if name == tag.get('title', '').lower().strip():
-            return uuid
-    return None
-
-
 # kinda funky, but works for now
 def extract_rss_token_from_UI(client):
     import re
@@ -205,10 +177,6 @@ def live_server_setup(live_server):
         with open("test-datastore/notification-url.txt", "w") as f:
             f.write(request.url)
 
-        if request.content_type:
-            with open("test-datastore/notification-content-type.txt", "w") as f:
-                f.write(request.content_type)
-
         print("\n>> Test notification endpoint was hit.\n", data)
         return "Text was set"
 

changedetectionio/tests/visualselector/test_fetch_data.py

@@ -19,7 +19,7 @@ def test_visual_selector_content_ready(client, live_server):
 
     res = client.post(
         url_for("form_quick_watch_add"),
-        data={"url": test_url, "tags": '', 'edit_and_watch_submit_button': 'Edit > Watch'},
+        data={"url": test_url, "tag": '', 'edit_and_watch_submit_button': 'Edit > Watch'},
         follow_redirects=True
     )
     assert b"Watch added in Paused state, saving will unpause" in res.data
@@ -28,7 +28,7 @@ def test_visual_selector_content_ready(client, live_server):
         url_for("edit_page", uuid="first", unpause_on_save=1),
         data={
               "url": test_url,
-              "tags": "",
+              "tag": "",
               "headers": "",
               'fetch_backend': "html_webdriver",
               'webdriver_js_execute_code': 'document.querySelector("button[name=test-button]").click();'

changedetectionio/update_worker.py

@@ -26,90 +26,16 @@ class update_worker(threading.Thread):
         self.datastore = datastore
         super().__init__(*args, **kwargs)
 
-    def queue_notification_for_watch(self, n_object, watch):
+    def send_content_changed_notification(self, t, watch_uuid):
 
         from changedetectionio import diff
 
-        watch_history = watch.history
-        dates = list(watch_history.keys())
-        # Add text that was triggered
-        snapshot_contents = watch.get_history_snapshot(dates[-1])
-
-        # HTML needs linebreak, but MarkDown and Text can use a linefeed
-        if n_object['notification_format'] == 'HTML':
-            line_feed_sep = "<br>"
-            # Snapshot will be plaintext on the disk, convert to some kind of HTML
-            snapshot_contents = snapshot_contents.replace('\n', line_feed_sep)
-        else:
-            line_feed_sep = "\n"
-
-        trigger_text = watch.get('trigger_text', [])
-        triggered_text = ''
-
-        if len(trigger_text):
-            from . import html_tools
-            triggered_text = html_tools.get_triggered_text(content=snapshot_contents, trigger_text=trigger_text)
-            if triggered_text:
-                triggered_text = line_feed_sep.join(triggered_text)
-
-
-        n_object.update({
-            'current_snapshot': snapshot_contents,
-            'diff': diff.render_diff(watch.get_history_snapshot(dates[-2]), watch.get_history_snapshot(dates[-1]), line_feed_sep=line_feed_sep),
-            'diff_added': diff.render_diff(watch.get_history_snapshot(dates[-2]), watch.get_history_snapshot(dates[-1]), include_removed=False, line_feed_sep=line_feed_sep),
-            'diff_full': diff.render_diff(watch.get_history_snapshot(dates[-2]), watch.get_history_snapshot(dates[-1]), include_equal=True, line_feed_sep=line_feed_sep),
-            'diff_patch': diff.render_diff(watch.get_history_snapshot(dates[-2]), watch.get_history_snapshot(dates[-1]), line_feed_sep=line_feed_sep, patch_format=True),
-            'diff_removed': diff.render_diff(watch.get_history_snapshot(dates[-2]), watch.get_history_snapshot(dates[-1]), include_added=False, line_feed_sep=line_feed_sep),
-            'screenshot': watch.get_screenshot() if watch.get('notification_screenshot') else None,
-            'triggered_text': triggered_text,
-            'uuid': watch.get('uuid'),
-            'watch_url': watch.get('url'),
-        })
-        logging.info (">> SENDING NOTIFICATION")
-        self.notification_q.put(n_object)
-
-    # Prefer - Individual watch settings > Tag settings >  Global settings (in that order)
-    def _check_cascading_vars(self, var_name, watch):
-
         from changedetectionio.notification import (
-            default_notification_format_for_watch,
-            default_notification_body,
-            default_notification_title
+            default_notification_format_for_watch
         )
 
-
-        # Would be better if this was some kind of Object where Watch can reference the parent datastore etc
-        v = watch.get(var_name)
-        if v and not watch.get('notification_muted'):
-            if var_name == 'notification_format' and v == default_notification_format_for_watch:
-                return self.datastore.data['settings']['application'].get('notification_format')
-
-            return v
-
-        tags = self.datastore.get_all_tags_for_watch(uuid=watch.get('uuid'))
-        if tags:
-            for tag_uuid, tag in tags.items():
-                v = tag.get(var_name)
-                if v and not tag.get('notification_muted'):
-                    return v
-
-        if self.datastore.data['settings']['application'].get(var_name):
-            return self.datastore.data['settings']['application'].get(var_name)
-
-        # Otherwise could be defaults
-        if var_name == 'notification_format':
-            return default_notification_format_for_watch
-        if var_name == 'notification_body':
-            return default_notification_body
-        if var_name == 'notification_title':
-            return default_notification_title
-
-        return None
-
-    def send_content_changed_notification(self, watch_uuid):
-
         n_object = {}
-        watch = self.datastore.data['watching'].get(watch_uuid)
+        watch = self.datastore.data['watching'].get(watch_uuid, False)
         if not watch:
             return
 
@@ -122,27 +48,59 @@ class update_worker(threading.Thread):
                 "History index had 2 or more, but only 1 date loaded, timestamps were not unique? maybe two of the same timestamps got written, needs more delay?"
             )
 
-        # Should be a better parent getter in the model object
+        n_object['notification_urls'] = watch['notification_urls'] if len(watch['notification_urls']) else \
+            self.datastore.data['settings']['application']['notification_urls']
 
-        # Prefer - Individual watch settings > Tag settings >  Global settings (in that order)
-        n_object['notification_urls'] = self._check_cascading_vars('notification_urls', watch)
-        n_object['notification_title'] = self._check_cascading_vars('notification_title', watch)
-        n_object['notification_body'] = self._check_cascading_vars('notification_body', watch)
-        n_object['notification_format'] = self._check_cascading_vars('notification_format', watch)
+        n_object['notification_title'] = watch['notification_title'] if watch['notification_title'] else \
+            self.datastore.data['settings']['application']['notification_title']
 
-        # (Individual watch) Only prepare to notify if the rules above matched
-        queued = False
-        if n_object and n_object.get('notification_urls'):
-            queued = True
-            self.queue_notification_for_watch(n_object, watch)
+        n_object['notification_body'] = watch['notification_body'] if watch['notification_body'] else \
+            self.datastore.data['settings']['application']['notification_body']
 
-        return queued
+        n_object['notification_format'] = watch['notification_format'] if watch['notification_format'] != default_notification_format_for_watch else \
+            self.datastore.data['settings']['application']['notification_format']
 
 
+        # Only prepare to notify if the rules above matched
+        if 'notification_urls' in n_object and n_object['notification_urls']:
+            # HTML needs linebreak, but MarkDown and Text can use a linefeed
+            if n_object['notification_format'] == 'HTML':
+                line_feed_sep = "<br>"
+            else:
+                line_feed_sep = "\n"
+
+            # Add text that was triggered
+            snapshot_contents = watch.get_history_snapshot(dates[-1])
+            trigger_text = watch.get('trigger_text', [])
+            triggered_text = ''
+
+            if len(trigger_text):
+                from . import html_tools
+                triggered_text = html_tools.get_triggered_text(content=snapshot_contents, trigger_text=trigger_text)
+                if triggered_text:
+                    triggered_text = line_feed_sep.join(triggered_text)
+
+
+            n_object.update({
+                'current_snapshot': snapshot_contents,
+                'diff': diff.render_diff(watch.get_history_snapshot(dates[-2]), watch.get_history_snapshot(dates[-1]), line_feed_sep=line_feed_sep),
+                'diff_added': diff.render_diff(watch.get_history_snapshot(dates[-2]), watch.get_history_snapshot(dates[-1]), include_removed=False, line_feed_sep=line_feed_sep),
+                'diff_full': diff.render_diff(watch.get_history_snapshot(dates[-2]), watch.get_history_snapshot(dates[-1]), include_equal=True, line_feed_sep=line_feed_sep),
+                'diff_removed': diff.render_diff(watch.get_history_snapshot(dates[-2]), watch.get_history_snapshot(dates[-1]), include_added=False, line_feed_sep=line_feed_sep),
+                'screenshot': watch.get_screenshot() if watch.get('notification_screenshot') else None,
+                'triggered_text': triggered_text,
+                'uuid': watch_uuid,
+                'watch_url': watch['url'],
+            })
+            logging.info (">> SENDING NOTIFICATION")
+            self.notification_q.put(n_object)
+        else:
+            logging.info (">> NO Notification sent, notification_url was empty in both watch and system")
+
     def send_filter_failure_notification(self, watch_uuid):
 
         threshold = self.datastore.data['settings']['application'].get('filter_failure_notification_threshold_attempts')
-        watch = self.datastore.data['watching'].get(watch_uuid)
+        watch = self.datastore.data['watching'].get(watch_uuid, False)
         if not watch:
             return
 
@@ -219,7 +177,7 @@ class update_worker(threading.Thread):
                 uuid = queued_item_data.item.get('uuid')
                 self.current_uuid = uuid
 
-                if uuid in list(self.datastore.data['watching'].keys()) and self.datastore.data['watching'][uuid].get('url'):
+                if uuid in list(self.datastore.data['watching'].keys()):
                     changed_detected = False
                     contents = b''
                     process_changedetection_results = True
@@ -380,9 +338,6 @@ class update_worker(threading.Thread):
                         if not self.datastore.data['watching'][uuid].get('ignore_status_codes'):
                             update_obj['consecutive_filter_failures'] = 0
 
-                        # Everything ran OK, clean off any previous error
-                        update_obj['last_error'] = False
-
                         self.cleanup_error_artifacts(uuid)
 
                     #
@@ -405,7 +360,7 @@ class update_worker(threading.Thread):
                                 # Notifications should only trigger on the second time (first time, we gather the initial snapshot)
                                 if watch.history_n >= 2:
                                     if not self.datastore.data['watching'][uuid].get('notification_muted'):
-                                        self.send_content_changed_notification(watch_uuid=uuid)
+                                        self.send_content_changed_notification(self, watch_uuid=uuid)
 
 
                         except Exception as e:

docker-compose.yml

@@ -47,9 +47,6 @@ services:
   #
   #        Hides the `Referer` header so that monitored websites can't see the changedetection.io hostname.
   #      - HIDE_REFERER=true
-  #        
-  #        Default number of parallel/concurrent fetchers
-  #      - FETCH_WORKERS=10
 
       # Comment out ports: when using behind a reverse proxy , enable networks: etc.
       ports:

requirements.txt

@@ -10,8 +10,7 @@ flask~=2.0
 inscriptis~=2.2
 pytz
 timeago~=1.0
-validators~=0.21
-
+validators
 
 # Set these versions together to avoid a RequestsDependencyWarning
 # >= 2.26 also adds Brotli support if brotli is installed
@@ -33,7 +32,7 @@ dnspython<2.3.0
 # jq not available on Windows so must be installed manually
 
 # Notification library
-apprise~=1.5.0
+apprise~=1.3.0
 
 # apprise mqtt https://github.com/dgtlmoon/changedetection.io/issues/315
 paho-mqtt
@@ -72,6 +71,3 @@ pillow
 # Include pytest, so if theres a support issue we can ask them to run these tests on their setup
 pytest ~=7.2
 pytest-flask ~=1.2
-
-# Pin jsonschema version to prevent build errors on armv6 while rpds-py wheels aren't available (1708)
-jsonschema==4.17.3