dont use fetcher when browsersteps are defined

.github/workflows/test-only.yml

@@ -55,11 +55,6 @@ jobs:
           # Playwright/Browserless fetch
           docker run --rm -e "PLAYWRIGHT_DRIVER_URL=ws://browserless:3000" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio;pytest tests/fetchers/test_content.py && pytest tests/test_errorhandling.py && pytest tests/visualselector/test_fetch_data.py'
           
-          # Settings headers playwright tests - Call back in from Browserless, check headers
-          docker run --name "changedet" --hostname changedet --rm -e "FLASK_SERVER_NAME=changedet" -e "PLAYWRIGHT_DRIVER_URL=ws://browserless:3000?dumpio=true" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0  --live-server-port=5004 tests/test_request.py'
-          docker run --name "changedet" --hostname changedet --rm -e "FLASK_SERVER_NAME=changedet" -e "WEBDRIVER_URL=http://selenium:4444/wd/hub" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0  --live-server-port=5004 tests/test_request.py'
-          docker run --name "changedet" --hostname changedet --rm -e "FLASK_SERVER_NAME=changedet" -e "USE_EXPERIMENTAL_PUPPETEER_FETCH=yes" -e "PLAYWRIGHT_DRIVER_URL=ws://browserless:3000?dumpio=true" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0  --live-server-port=5004 tests/test_request.py'
-          
           # restock detection via playwright - added name=changedet here so that playwright/browserless can connect to it
           docker run --rm --name "changedet" -e "FLASK_SERVER_NAME=changedet" -e "PLAYWRIGHT_DRIVER_URL=ws://browserless:3000" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-port=5004 --live-server-host=0.0.0.0 tests/restock/test_restock.py'
 

Dockerfile

@@ -1,5 +1,5 @@
 # pip dependencies install stage
-FROM python:3.10-slim-bullseye as builder
+FROM python:3.10-slim as builder
 
 # See `cryptography` pin comment in requirements.txt
 ARG CRYPTOGRAPHY_DONT_BUILD_RUST=1
@@ -29,7 +29,7 @@ RUN pip install --target=/dependencies playwright~=1.27.1 \
     || echo "WARN: Failed to install Playwright. The application can still run, but the Playwright option will be disabled."
 
 # Final image stage
-FROM python:3.10-slim-bullseye
+FROM python:3.10-slim
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
     libssl1.1 \

README-pip.md

@@ -2,10 +2,10 @@
 
 Live your data-life pro-actively, track website content changes and receive notifications via Discord, Email, Slack, Telegram and 70+ more
 
-[<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/docs/screenshot.png" style="max-width:100%;" alt="Self-hosted web page change monitoring"  title="Self-hosted web page change monitoring"  />](https://changedetection.io)
+[<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/docs/screenshot.png" style="max-width:100%;" alt="Self-hosted web page change monitoring"  title="Self-hosted web page change monitoring"  />](https://lemonade.changedetection.io/start?src=pip)
 
 
-[**Don't have time? Let us host it for you! try our extremely affordable subscription use our proxies and support!**](https://changedetection.io) 
+[**Don't have time? Let us host it for you! try our extremely affordable subscription use our proxies and support!**](https://lemonade.changedetection.io/start) 
 
 
 #### Example use cases

README.md

@@ -5,13 +5,13 @@
 _Live your data-life pro-actively._ 
 
 
-[<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/docs/screenshot.png" style="max-width:100%;" alt="Self-hosted web page change monitoring"  title="Self-hosted web page change monitoring"  />](https://changedetection.io?src=github)
+[<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/docs/screenshot.png" style="max-width:100%;" alt="Self-hosted web page change monitoring"  title="Self-hosted web page change monitoring"  />](https://lemonade.changedetection.io/start?src=github)
 
 [![Release Version][release-shield]][release-link] [![Docker Pulls][docker-pulls]][docker-link] [![License][license-shield]](LICENSE.md)
 
 ![changedetection.io](https://github.com/dgtlmoon/changedetection.io/actions/workflows/test-only.yml/badge.svg?branch=master)
 
-[**Don't have time? Let us host it for you! try our $8.99/month subscription - use our proxies and support!**](https://changedetection.io) , _half the price of other website change monitoring services!_
+[**Don't have time? Let us host it for you! try our $8.99/month subscription - use our proxies and support!**](https://lemonade.changedetection.io/start) , _half the price of other website change monitoring services and comes with unlimited watches & checks!_
 
 - Chrome browser included.
 - Super fast, no registration needed setup.
@@ -22,11 +22,11 @@ _Live your data-life pro-actively._
 
 Available when connected to a <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Playwright-content-fetcher">playwright content fetcher</a> (included as part of our subscription service)
 
-[<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/docs/visualselector-anim.gif" style="max-width:100%;" alt="Self-hosted web page change monitoring context difference "  title="Self-hosted web page change monitoring context difference " />](https://changedetection.io?src=github)
+[<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/docs/visualselector-anim.gif" style="max-width:100%;" alt="Self-hosted web page change monitoring context difference "  title="Self-hosted web page change monitoring context difference " />](https://lemonade.changedetection.io/start?src=github)
 
 ### Easily see what changed, examine by word, line, or individual character.
 
-[<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/docs/screenshot-diff.png" style="max-width:100%;" alt="Self-hosted web page change monitoring context difference "  title="Self-hosted web page change monitoring context difference " />](https://changedetection.io?src=github)
+[<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/docs/screenshot-diff.png" style="max-width:100%;" alt="Self-hosted web page change monitoring context difference "  title="Self-hosted web page change monitoring context difference " />](https://lemonade.changedetection.io/start?src=github)
 
 
 ### Perform interactive browser steps
@@ -35,7 +35,7 @@ Fill in text boxes, click buttons and more, setup your changedetection scenario.
 
 Using the **Browser Steps** configuration, add basic steps before performing change detection, such as logging into websites, adding a product to a cart, accept cookie logins, entering dates and refining searches.
 
-[<img src="docs/browsersteps-anim.gif" style="max-width:100%;" alt="Self-hosted web page change monitoring context difference "  title="Website change detection with interactive browser steps, login, cookies etc" />](https://changedetection.io?src=github)
+[<img src="docs/browsersteps-anim.gif" style="max-width:100%;" alt="Self-hosted web page change monitoring context difference "  title="Website change detection with interactive browser steps, login, cookies etc" />](https://lemonade.changedetection.io/start?src=github)
 
 After **Browser Steps** have been run, then visit the **Visual Selector** tab to refine the content you're interested in.
 Requires Playwright to be enabled.
@@ -66,7 +66,6 @@ Requires Playwright to be enabled.
 - Proactively search for jobs, get notified when companies update their careers page, search job portals for keywords.
 - Get alerts when new job positions are open on Bamboo HR and other job platforms
 - Website defacement monitoring
-- Pokémon Card Restock Tracker / Pokémon TCG Tracker
 
 _Need an actual Chrome runner with Javascript support? We support fetching via WebDriver and Playwright!</a>_
 
@@ -145,7 +144,7 @@ See the wiki for more information https://github.com/dgtlmoon/changedetection.io
 ## Filters
 
 XPath, JSONPath, jq, and CSS support comes baked in! You can be as specific as you need, use XPath exported from various XPath element query creation tools. 
-(We support LXML `re:test`, `re:match` and `re:replace`.)
+(We support LXML `re:test`, `re:math` and `re:replace`.)
 
 ## Notifications
 
@@ -238,7 +237,7 @@ Supports managing the website watch list [via our API](https://changedetection.i
 Do you use changedetection.io to make money? does it save you time or money? Does it make your life easier? less stressful? Remember, we write this software when we should be doing actual paid work, we have to buy food and pay rent just like you.
 
 
-Firstly, consider taking out a [change detection monthly subscription - unlimited checks and watches](https://changedetection.io?src=github) , even if you don't use it, you still get the warm fuzzy feeling of helping out the project. (And who knows, you might just use it!)
+Firstly, consider taking out a [change detection monthly subscription - unlimited checks and watches](https://lemonade.changedetection.io/start) , even if you don't use it, you still get the warm fuzzy feeling of helping out the project. (And who knows, you might just use it!)
 
 Or directly donate an amount PayPal [![Donate](https://img.shields.io/badge/Donate-PayPal-green.svg)](https://www.paypal.com/donate/?hosted_button_id=7CP6HR9ZCNDYJ)
 

changedetectionio/__init__.py

@@ -38,7 +38,7 @@ from flask_paginate import Pagination, get_page_parameter
 from changedetectionio import html_tools
 from changedetectionio.api import api_v1
 
-__version__ = '0.43.1'
+__version__ = '0.41.1'
 
 datastore = None
 
@@ -124,15 +124,6 @@ def _jinja2_filter_datetimestamp(timestamp, format="%Y-%m-%d %H:%M:%S"):
 
     return timeago.format(timestamp, time.time())
 
-
-@app.template_filter('pagination_slice')
-def _jinja2_filter_pagination_slice(arr, skip):
-    per_page = datastore.data['settings']['application'].get('pager_size', 50)
-    if per_page:
-        return arr[skip:skip + per_page]
-
-    return arr
-
 @app.template_filter('format_seconds_ago')
 def _jinja2_filter_seconds_precise(timestamp):
     if timestamp == False:
@@ -317,21 +308,25 @@ def changedetection_app(config=None, datastore_o=None):
             return "Access denied, bad token", 403
 
         from . import diff
-        limit_tag = request.args.get('tag', '').lower().strip()
-        # Be sure limit_tag is a uuid
-        for uuid, tag in datastore.data['settings']['application'].get('tags', {}).items():
-            if limit_tag == tag.get('title', '').lower().strip():
-                limit_tag = uuid
+        limit_tag = request.args.get('tag')
 
         # Sort by last_changed and add the uuid which is usually the key..
         sorted_watches = []
 
         # @todo needs a .itemsWithTag() or something - then we can use that in Jinaj2 and throw this away
         for uuid, watch in datastore.data['watching'].items():
-            if limit_tag and not limit_tag in watch['tags']:
-                    continue
-            watch['uuid'] = uuid
-            sorted_watches.append(watch)
+
+            if limit_tag != None:
+                # Support for comma separated list of tags.
+                for tag_in_watch in watch['tag'].split(','):
+                    tag_in_watch = tag_in_watch.strip()
+                    if tag_in_watch == limit_tag:
+                        watch['uuid'] = uuid
+                        sorted_watches.append(watch)
+
+            else:
+                watch['uuid'] = uuid
+                sorted_watches.append(watch)
 
         sorted_watches.sort(key=lambda x: x.last_changed, reverse=False)
 
@@ -388,17 +383,9 @@ def changedetection_app(config=None, datastore_o=None):
     @app.route("/", methods=['GET'])
     @login_optionally_required
     def index():
-        global datastore
         from changedetectionio import forms
 
-        limit_tag = request.args.get('tag', '').lower().strip()
-
-        # Be sure limit_tag is a uuid
-        for uuid, tag in datastore.data['settings']['application'].get('tags', {}).items():
-            if limit_tag == tag.get('title', '').lower().strip():
-                limit_tag = uuid
-
-
+        limit_tag = request.args.get('tag')
         # Redirect for the old rss path which used the /?rss=true
         if request.args.get('rss'):
             return redirect(url_for('rss', tag=limit_tag))
@@ -416,32 +403,33 @@ def changedetection_app(config=None, datastore_o=None):
 
         # Sort by last_changed and add the uuid which is usually the key..
         sorted_watches = []
-        search_q = request.args.get('q').strip().lower() if request.args.get('q') else False
         for uuid, watch in datastore.data['watching'].items():
-            if limit_tag and not limit_tag in watch['tags']:
-                    continue
 
-            if search_q:
-                if (watch.get('title') and search_q in watch.get('title').lower()) or search_q in watch.get('url', '').lower():
-                    sorted_watches.append(watch)
+            if limit_tag != None:
+                # Support for comma separated list of tags.
+                if watch['tag'] is None:
+                    continue
+                for tag_in_watch in watch['tag'].split(','):
+                    tag_in_watch = tag_in_watch.strip()
+                    if tag_in_watch == limit_tag:
+                        watch['uuid'] = uuid
+                        sorted_watches.append(watch)
+
             else:
+                watch['uuid'] = uuid
                 sorted_watches.append(watch)
 
+        existing_tags = datastore.get_all_tags()
         form = forms.quickWatchForm(request.form)
         page = request.args.get(get_page_parameter(), type=int, default=1)
-        total_count = len(sorted_watches)
-
-        pagination = Pagination(page=page,
-                                total=total_count,
-                                per_page=datastore.data['settings']['application'].get('pager_size', 50), css_framework="semantic")
-
+        total_count = len(sorted_watches) if sorted_watches else len(datastore.data['watching'])
+        pagination = Pagination(page=page, total=total_count, per_page=int(os.getenv('pagination_per_page', 50)), css_framework = "semantic")
 
         output = render_template(
             "watch-overview.html",
                                  # Don't link to hosting when we're on the hosting environment
                                  active_tag=limit_tag,
                                  app_rss_token=datastore.data['settings']['application']['rss_access_token'],
-                                 datastore=datastore,
                                  form=form,
                                  guid=datastore.data['app_guid'],
                                  has_proxies=datastore.proxy_list,
@@ -449,11 +437,10 @@ def changedetection_app(config=None, datastore_o=None):
                                  hosted_sticky=os.getenv("SALTED_PASS", False) == False,
                                  pagination=pagination,
                                  queued_uuids=[q_uuid.item['uuid'] for q_uuid in update_q.queue],
-                                 search_q=request.args.get('q','').strip(),
                                  sort_attribute=request.args.get('sort') if request.args.get('sort') else request.cookies.get('sort'),
                                  sort_order=request.args.get('order') if request.args.get('order') else request.cookies.get('order'),
                                  system_default_fetcher=datastore.data['settings']['application'].get('fetch_backend'),
-                                 tags=datastore.data['settings']['application'].get('tags'),
+                                 tags=existing_tags,
                                  watches=sorted_watches
                                  )
 
@@ -596,13 +583,9 @@ def changedetection_app(config=None, datastore_o=None):
 
         # proxy_override set to the json/text list of the items
         form = forms.watchForm(formdata=request.form if request.method == 'POST' else None,
-                               data=default
+                               data=default,
                                )
 
-        # For the form widget tag uuid lookup
-        form.tags.datastore = datastore # in _value
-
-
         form.fetch_backend.choices.append(("system", 'System settings default'))
 
         # form.browser_steps[0] can be assumed that we 'goto url' first
@@ -653,16 +636,6 @@ def changedetection_app(config=None, datastore_o=None):
                 extra_update_obj['filter_text_replaced'] = True
                 extra_update_obj['filter_text_removed'] = True
 
-            # Because wtforms doesn't support accessing other data in process_ , but we convert the CSV list of tags back to a list of UUIDs
-            tag_uuids = []
-            if form.data.get('tags'):
-                # Sometimes in testing this can be list, dont know why
-                if type(form.data.get('tags')) == list:
-                    extra_update_obj['tags'] = form.data.get('tags')
-                else:
-                    for t in form.data.get('tags').split(','):
-                        tag_uuids.append(datastore.add_tag(name=t))
-                    extra_update_obj['tags'] = tag_uuids
 
             datastore.data['watching'][uuid].update(form.data)
             datastore.data['watching'][uuid].update(extra_update_obj)
@@ -717,7 +690,6 @@ def changedetection_app(config=None, datastore_o=None):
                                      form=form,
                                      has_default_notification_urls=True if len(datastore.data['settings']['application']['notification_urls']) else False,
                                      has_empty_checktime=using_default_check_time,
-                                     has_extra_headers_file=len(datastore.get_all_headers_in_textfile_for_watch(uuid=uuid)) > 0,
                                      is_html_webdriver=is_html_webdriver,
                                      jq_support=jq_support,
                                      playwright_enabled=os.getenv('PLAYWRIGHT_DRIVER_URL', False),
@@ -1114,8 +1086,8 @@ def changedetection_app(config=None, datastore_o=None):
                 os.path.join(datastore_o.datastore_path, list_with_tags_file), "w"
             ) as f:
                 for uuid in datastore.data["watching"]:
-                    url = datastore.data["watching"][uuid].get('url')
-                    tag = datastore.data["watching"][uuid].get('tags', {})
+                    url = datastore.data["watching"][uuid]["url"]
+                    tag = datastore.data["watching"][uuid]["tag"]
                     f.write("{} {}\r\n".format(url, tag))
 
             # Add it to the Zip
@@ -1203,7 +1175,7 @@ def changedetection_app(config=None, datastore_o=None):
 
         add_paused = request.form.get('edit_and_watch_submit_button') != None
         processor = request.form.get('processor', 'text_json_diff')
-        new_uuid = datastore.add_watch(url=url, tag=request.form.get('tags').strip(), extras={'paused': add_paused, 'processor': processor})
+        new_uuid = datastore.add_watch(url=url, tag=request.form.get('tag').strip(), extras={'paused': add_paused, 'processor': processor})
 
         if new_uuid:
             if add_paused:
@@ -1271,11 +1243,9 @@ def changedetection_app(config=None, datastore_o=None):
         elif tag != None:
             # Items that have this current tag
             for watch_uuid, watch in datastore.data['watching'].items():
-                if (tag != None and tag in watch.get('tags', {})):
+                if (tag != None and tag in watch['tag']):
                     if watch_uuid not in running_uuids and not datastore.data['watching'][watch_uuid]['paused']:
-                        update_q.put(
-                            queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': watch_uuid, 'skip_when_checksum_same': False})
-                        )
+                        update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': watch_uuid, 'skip_when_checksum_same': False}))
                         i += 1
 
         else:
@@ -1363,18 +1333,6 @@ def changedetection_app(config=None, datastore_o=None):
                     datastore.data['watching'][uuid.strip()]['notification_format'] = default_notification_format_for_watch
             flash("{} watches set to use default notification settings".format(len(uuids)))
 
-        elif (op == 'assign-tag'):
-            op_extradata = request.form.get('op_extradata', '').strip()
-            if op_extradata:
-                tag_uuid = datastore.add_tag(name=op_extradata)
-                if op_extradata and tag_uuid:
-                    for uuid in uuids:
-                        uuid = uuid.strip()
-                        if datastore.data['watching'].get(uuid):
-                            datastore.data['watching'][uuid]['tags'].append(tag_uuid)
-
-            flash("{} watches assigned tag".format(len(uuids)))
-
         return redirect(url_for('index'))
 
     @app.route("/api/share-url", methods=['GET'])
@@ -1384,6 +1342,7 @@ def changedetection_app(config=None, datastore_o=None):
            the share-link can be imported/added"""
         import requests
         import json
+        tag = request.args.get('tag')
         uuid = request.args.get('uuid')
 
         # more for testing
@@ -1436,8 +1395,6 @@ def changedetection_app(config=None, datastore_o=None):
     import changedetectionio.blueprint.price_data_follower as price_data_follower
     app.register_blueprint(price_data_follower.construct_blueprint(datastore, update_q), url_prefix='/price_data_follower')
 
-    import changedetectionio.blueprint.tags as tags
-    app.register_blueprint(tags.construct_blueprint(datastore), url_prefix='/tags')
 
     # @todo handle ctrl break
     ticker_thread = threading.Thread(target=ticker_thread_check_time_launch_checks).start()
@@ -1477,7 +1434,6 @@ def check_for_new_version():
         # Check daily
         app.config.exit.wait(86400)
 
-
 def notification_runner():
     global notification_debug_log
     from datetime import datetime

changedetectionio/api/api_v1.py

@@ -218,16 +218,9 @@ class CreateWatch(Resource):
                 return "Invalid proxy choice, currently supported proxies are '{}'".format(', '.join(plist)), 400
 
         extras = copy.deepcopy(json_data)
-
-        # Because we renamed 'tag' to 'tags' but don't want to change the API (can do this in v2 of the API)
-        tags = None
-        if extras.get('tag'):
-            tags = extras.get('tag')
-            del extras['tag']
-
         del extras['url']
 
-        new_uuid = self.datastore.add_watch(url=url, extras=extras, tag=tags)
+        new_uuid = self.datastore.add_watch(url=url, extras=extras)
         if new_uuid:
             self.update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': new_uuid, 'skip_when_checksum_same': True}))
             return {'uuid': new_uuid}, 201
@@ -266,16 +259,13 @@ class CreateWatch(Resource):
         """
         list = {}
 
-        tag_limit = request.args.get('tag', '').lower()
+        tag_limit = request.args.get('tag', None)
+        for k, watch in self.datastore.data['watching'].items():
+            if tag_limit:
+                if not tag_limit.lower() in watch.all_tags:
+                    continue
 
-
-        for uuid, watch in self.datastore.data['watching'].items():
-            # Watch tags by name (replace the other calls?)
-            tags = self.datastore.get_all_tags_for_watch(uuid=uuid)
-            if tag_limit and not any(v.get('title').lower() == tag_limit for k, v in tags.items()):
-                continue
-
-            list[uuid] = {'url': watch['url'],
+            list[k] = {'url': watch['url'],
                        'title': watch['title'],
                        'last_checked': watch['last_checked'],
                        'last_changed': watch.last_changed,

changedetectionio/blueprint/browser_steps/__init__.py

@@ -27,106 +27,58 @@ import os
 import logging
 from changedetectionio.store import ChangeDetectionStore
 from changedetectionio import login_optionally_required
+browsersteps_live_ui_o = {}
+browsersteps_playwright_browser_interface = None
+browsersteps_playwright_browser_interface_browser = None
+browsersteps_playwright_browser_interface_context = None
+browsersteps_playwright_browser_interface_end_time = None
+browsersteps_playwright_browser_interface_start_time = None
 
-browsersteps_sessions = {}
-io_interface_context = None
+def cleanup_playwright_session():
 
+    global browsersteps_live_ui_o
+    global browsersteps_playwright_browser_interface
+    global browsersteps_playwright_browser_interface_browser
+    global browsersteps_playwright_browser_interface_context
+    global browsersteps_playwright_browser_interface_end_time
+    global browsersteps_playwright_browser_interface_start_time
+
+    browsersteps_live_ui_o = {}
+    browsersteps_playwright_browser_interface = None
+    browsersteps_playwright_browser_interface_browser = None
+    browsersteps_playwright_browser_interface_end_time = None
+    browsersteps_playwright_browser_interface_start_time = None
+
+    print("Cleaning up old playwright session because time was up, calling .goodbye()")
+    try:
+        browsersteps_playwright_browser_interface_context.goodbye()
+    except Exception as e:
+        print ("Got exception in shutdown, probably OK")
+        print (str(e))
+
+    browsersteps_playwright_browser_interface_context = None
+
+    print ("Cleaning up old playwright session because time was up - done")
 
 def construct_blueprint(datastore: ChangeDetectionStore):
+
     browser_steps_blueprint = Blueprint('browser_steps', __name__, template_folder="templates")
 
-    def start_browsersteps_session(watch_uuid):
-        from . import nonContext
-        from . import browser_steps
-        import time
-        global browsersteps_sessions
-        global io_interface_context
-
-
-        # We keep the playwright session open for many minutes
-        seconds_keepalive = int(os.getenv('BROWSERSTEPS_MINUTES_KEEPALIVE', 10)) * 60
-
-        browsersteps_start_session = {'start_time': time.time()}
-
-        # You can only have one of these running
-        # This should be very fine to leave running for the life of the application
-        # @idea - Make it global so the pool of watch fetchers can use it also
-        if not io_interface_context:
-            io_interface_context = nonContext.c_sync_playwright()
-            # Start the Playwright context, which is actually a nodejs sub-process and communicates over STDIN/STDOUT pipes
-            io_interface_context = io_interface_context.start()
-
-
-        # keep it alive for 10 seconds more than we advertise, sometimes it helps to keep it shutting down cleanly
-        keepalive = "&timeout={}".format(((seconds_keepalive + 3) * 1000))
-        try:
-            browsersteps_start_session['browser'] = io_interface_context.chromium.connect_over_cdp(
-                os.getenv('PLAYWRIGHT_DRIVER_URL', '') + keepalive)
-        except Exception as e:
-            if 'ECONNREFUSED' in str(e):
-                return make_response('Unable to start the Playwright Browser session, is it running?', 401)
-            else:
-                return make_response(str(e), 401)
-
-        proxy_id = datastore.get_preferred_proxy_for_watch(uuid=watch_uuid)
-        proxy = None
-        if proxy_id:
-            proxy_url = datastore.proxy_list.get(proxy_id).get('url')
-            if proxy_url:
-
-                # Playwright needs separate username and password values
-                from urllib.parse import urlparse
-                parsed = urlparse(proxy_url)
-                proxy = {'server': proxy_url}
-
-                if parsed.username:
-                    proxy['username'] = parsed.username
-
-                if parsed.password:
-                    proxy['password'] = parsed.password
-
-                print("Browser Steps: UUID {} selected proxy {}".format(watch_uuid, proxy_url))
-
-        # Tell Playwright to connect to Chrome and setup a new session via our stepper interface
-        browsersteps_start_session['browserstepper'] = browser_steps.browsersteps_live_ui(
-            playwright_browser=browsersteps_start_session['browser'],
-            proxy=proxy)
-
-        # For test
-        #browsersteps_start_session['browserstepper'].action_goto_url(value="http://example.com?time="+str(time.time()))
-
-        return browsersteps_start_session
-
-
     @login_optionally_required
-    @browser_steps_blueprint.route("/browsersteps_start_session", methods=['GET'])
-    def browsersteps_start_session():
-        # A new session was requested, return sessionID
-
-        import uuid
-        global browsersteps_sessions
-
-        browsersteps_session_id = str(uuid.uuid4())
-        watch_uuid = request.args.get('uuid')
-
-        if not watch_uuid:
-            return make_response('No Watch UUID specified', 500)
-
-        print("Starting connection with playwright")
-        logging.debug("browser_steps.py connecting")
-        browsersteps_sessions[browsersteps_session_id] = start_browsersteps_session(watch_uuid)
-        print("Starting connection with playwright - done")
-        return {'browsersteps_session_id': browsersteps_session_id}
-
-    # A request for an action was received
-    @login_optionally_required
-    @browser_steps_blueprint.route("/browsersteps_update", methods=['POST'])
+    @browser_steps_blueprint.route("/browsersteps_update", methods=['GET', 'POST'])
     def browsersteps_ui_update():
         import base64
         import playwright._impl._api_types
-        global browsersteps_sessions
+        import time
+
         from changedetectionio.blueprint.browser_steps import browser_steps
 
+        global browsersteps_live_ui_o, browsersteps_playwright_browser_interface_end_time
+        global browsersteps_playwright_browser_interface_browser
+        global browsersteps_playwright_browser_interface
+        global browsersteps_playwright_browser_interface_start_time
+
+        step_n = None
         remaining =0
         uuid = request.args.get('uuid')
 
@@ -135,9 +87,13 @@ def construct_blueprint(datastore: ChangeDetectionStore):
         if not browsersteps_session_id:
             return make_response('No browsersteps_session_id specified', 500)
 
-        if not browsersteps_sessions.get(browsersteps_session_id):
-            return make_response('No session exists under that ID', 500)
-
+        # Because we don't "really" run in a context manager ( we make the playwright interface global/long-living )
+        # We need to manage the shutdown when the time is up
+        if browsersteps_playwright_browser_interface_end_time:
+            remaining = browsersteps_playwright_browser_interface_end_time-time.time()
+            if browsersteps_playwright_browser_interface_end_time and remaining <= 0:
+                cleanup_playwright_session()
+                return make_response('Browser session expired, please reload the Browser Steps interface', 401)
 
         # Actions - step/apply/etc, do the thing and return state
         if request.method == 'POST':
@@ -156,7 +112,12 @@ def construct_blueprint(datastore: ChangeDetectionStore):
             # @todo try.. accept.. nice errors not popups..
             try:
 
-                browsersteps_sessions[browsersteps_session_id]['browserstepper'].call_action(action_name=step_operation,
+                this_session = browsersteps_live_ui_o.get(browsersteps_session_id)
+                if not this_session:
+                    print("Browser exited")
+                    return make_response('Browser session ran out of time :( Please reload this page.', 401)
+
+                this_session.call_action(action_name=step_operation,
                                          selector=step_selector,
                                          optional_value=step_optional_value)
 
@@ -168,43 +129,108 @@ def construct_blueprint(datastore: ChangeDetectionStore):
             # Get visual selector ready/update its data (also use the current filter info from the page?)
             # When the last 'apply' button was pressed
             # @todo this adds overhead because the xpath selection is happening twice
-            u = browsersteps_sessions[browsersteps_session_id]['browserstepper'].page.url
+            u = this_session.page.url
             if is_last_step and u:
-                (screenshot, xpath_data) = browsersteps_sessions[browsersteps_session_id]['browserstepper'].request_visualselector_data()
+                (screenshot, xpath_data) = this_session.request_visualselector_data()
                 datastore.save_screenshot(watch_uuid=uuid, screenshot=screenshot)
                 datastore.save_xpath_data(watch_uuid=uuid, data=xpath_data)
 
-#        if not this_session.page:
-#            cleanup_playwright_session()
-#            return make_response('Browser session ran out of time :( Please reload this page.', 401)
+        # Setup interface
+        if request.method == 'GET':
 
-        # Screenshots and other info only needed on requesting a step (POST)
-        try:
-            state = browsersteps_sessions[browsersteps_session_id]['browserstepper'].get_current_state()
-        except playwright._impl._api_types.Error as e:
-            return make_response("Browser session ran out of time :( Please reload this page."+str(e), 401)
+            if not browsersteps_playwright_browser_interface:
+                print("Starting connection with playwright")
+                logging.debug("browser_steps.py connecting")
 
-        # Use send_file() which is way faster than read/write loop on bytes
-        import json
-        from tempfile import mkstemp
-        from flask import send_file
-        tmp_fd, tmp_file = mkstemp(text=True, suffix=".json", prefix="changedetectionio-")
+                global browsersteps_playwright_browser_interface_context
+                from . import nonContext
+                browsersteps_playwright_browser_interface_context = nonContext.c_sync_playwright()
+                browsersteps_playwright_browser_interface = browsersteps_playwright_browser_interface_context.start()
+                # At 20 minutes, some other variable is closing it
+                # @todo find out what it is and set it
+                seconds_keepalive = int(os.getenv('BROWSERSTEPS_MINUTES_KEEPALIVE', 10)) * 60
 
-        output = json.dumps({'screenshot': "data:image/jpeg;base64,{}".format(
-            base64.b64encode(state[0]).decode('ascii')),
-            'xpath_data': state[1],
-            'session_age_start': browsersteps_sessions[browsersteps_session_id]['browserstepper'].age_start,
-            'browser_time_remaining': round(remaining)
-        })
+                # keep it alive for 10 seconds more than we advertise, sometimes it helps to keep it shutting down cleanly
+                keepalive = "&timeout={}".format(((seconds_keepalive+3) * 1000))
+                try:
+                    browsersteps_playwright_browser_interface_browser = browsersteps_playwright_browser_interface.chromium.connect_over_cdp(
+                        os.getenv('PLAYWRIGHT_DRIVER_URL', '') + keepalive)
+                except Exception as e:
+                    if 'ECONNREFUSED' in str(e):
+                        return make_response('Unable to start the Playwright session properly, is it running?', 401)
 
-        with os.fdopen(tmp_fd, 'w') as f:
-            f.write(output)
+                browsersteps_playwright_browser_interface_end_time = time.time() + (seconds_keepalive-3)
+                print("Starting connection with playwright - done")
 
-        response = make_response(send_file(path_or_file=tmp_file,
-                                           mimetype='application/json; charset=UTF-8',
-                                           etag=True))
-        # No longer needed
-        os.unlink(tmp_file)
+            if not browsersteps_live_ui_o.get(browsersteps_session_id):
+                # Boot up a new session
+                proxy_id = datastore.get_preferred_proxy_for_watch(uuid=uuid)
+                proxy = None
+                if proxy_id:
+                    proxy_url = datastore.proxy_list.get(proxy_id).get('url')
+                    if proxy_url:
+
+                        # Playwright needs separate username and password values
+                        from urllib.parse import urlparse
+                        parsed = urlparse(proxy_url)
+                        proxy = {'server': proxy_url}
+
+                        if parsed.username:
+                            proxy['username'] = parsed.username
+
+                        if parsed.password:
+                            proxy['password'] = parsed.password
+
+                        print("Browser Steps: UUID {} Using proxy {}".format(uuid, proxy_url))
+
+                # Begin the new "Playwright Context" that re-uses the playwright interface
+                # Each session is a "Playwright Context" as a list, that uses the playwright interface
+                browsersteps_live_ui_o[browsersteps_session_id] = browser_steps.browsersteps_live_ui(
+                    playwright_browser=browsersteps_playwright_browser_interface_browser,
+                    proxy=proxy)
+                this_session = browsersteps_live_ui_o[browsersteps_session_id]
+
+        if not this_session.page:
+            cleanup_playwright_session()
+            return make_response('Browser session ran out of time :( Please reload this page.', 401)
+
+        response = None
+
+        if request.method == 'POST':
+            # Screenshots and other info only needed on requesting a step (POST)
+            try:
+                state = this_session.get_current_state()
+            except playwright._impl._api_types.Error as e:
+                return make_response("Browser session ran out of time :( Please reload this page."+str(e), 401)
+
+            # Use send_file() which is way faster than read/write loop on bytes
+            import json
+            from tempfile import mkstemp
+            from flask import send_file
+            tmp_fd, tmp_file = mkstemp(text=True, suffix=".json", prefix="changedetectionio-")
+
+            output = json.dumps({'screenshot': "data:image/jpeg;base64,{}".format(
+                base64.b64encode(state[0]).decode('ascii')),
+                'xpath_data': state[1],
+                'session_age_start': this_session.age_start,
+                'browser_time_remaining': round(remaining)
+            })
+
+            with os.fdopen(tmp_fd, 'w') as f:
+                f.write(output)
+
+            response = make_response(send_file(path_or_file=tmp_file,
+                                               mimetype='application/json; charset=UTF-8',
+                                               etag=True))
+            # No longer needed
+            os.unlink(tmp_file)
+
+        elif request.method == 'GET':
+            # Just enough to get the session rolling, it will call for goto-site via POST next
+            response = make_response({
+                'session_age_start': this_session.age_start,
+                'browser_time_remaining': round(remaining)
+            })
 
         return response
 

changedetectionio/blueprint/browser_steps/browser_steps.py

@@ -71,10 +71,10 @@ class steppable_browser_interface():
             optional_value = str(jinja2_env.from_string(optional_value).render())
 
         action_handler(selector, optional_value)
-        self.page.wait_for_timeout(1.5 * 1000)
+        self.page.wait_for_timeout(3 * 1000)
         print("Call action done in", time.time() - now)
 
-    def action_goto_url(self, selector=None, value=None):
+    def action_goto_url(self, selector, value):
         # self.page.set_viewport_size({"width": 1280, "height": 5000})
         now = time.time()
         response = self.page.goto(value, timeout=0, wait_until='commit')
@@ -105,8 +105,7 @@ class steppable_browser_interface():
         print("Clicking element")
         if not len(selector.strip()):
             return
-
-        self.page.click(selector=selector, timeout=30 * 1000, delay=randint(200, 500))
+        self.page.click(selector, timeout=10 * 1000, delay=randint(200, 500))
 
     def action_click_element_if_exists(self, selector, value):
         import playwright._impl._api_types as _api_types
@@ -133,18 +132,18 @@ class steppable_browser_interface():
         self.page.wait_for_timeout(1000)
 
     def action_wait_for_seconds(self, selector, value):
-        self.page.wait_for_timeout(float(value.strip()) * 1000)
+        self.page.wait_for_timeout(int(value) * 1000)
 
     def action_wait_for_text(self, selector, value):
         import json
         v = json.dumps(value)
-        self.page.wait_for_function(f'document.querySelector("body").innerText.includes({v});', timeout=90000)
+        self.page.wait_for_function(f'document.querySelector("body").innerText.includes({v});', timeout=30000)
 
     def action_wait_for_text_in_element(self, selector, value):
         import json
         s = json.dumps(selector)
         v = json.dumps(value)
-        self.page.wait_for_function(f'document.querySelector({s}).innerText.includes({v});', timeout=90000)
+        self.page.wait_for_function(f'document.querySelector({s}).innerText.includes({v});', timeout=30000)
 
     # @todo - in the future make some popout interface to capture what needs to be set
     # https://playwright.dev/python/docs/api/class-keyboard

changedetectionio/blueprint/tags/README.md

@@ -1,9 +0,0 @@
-# Groups tags
-
-## How it works
-
-Watch has a list() of tag UUID's, which relate to a config under application.settings.tags
-
-The 'tag' is actually a watch, because they basically will eventually share 90% of the same config.
-
-So a tag is like an abstract of a watch

changedetectionio/blueprint/tags/__init__.py

@@ -1,141 +0,0 @@
-from flask import Blueprint, request, make_response, render_template, flash, url_for, redirect
-from changedetectionio.store import ChangeDetectionStore
-from changedetectionio import login_optionally_required
-
-
-def construct_blueprint(datastore: ChangeDetectionStore):
-    tags_blueprint = Blueprint('tags', __name__, template_folder="templates")
-
-    @tags_blueprint.route("/list", methods=['GET'])
-    @login_optionally_required
-    def tags_overview_page():
-        from .form import SingleTag
-        add_form = SingleTag(request.form)
-        output = render_template("groups-overview.html",
-                                 form=add_form,
-                                 available_tags=datastore.data['settings']['application'].get('tags', {}),
-                                 )
-
-        return output
-
-    @tags_blueprint.route("/add", methods=['POST'])
-    @login_optionally_required
-    def form_tag_add():
-        from .form import SingleTag
-        add_form = SingleTag(request.form)
-
-        if not add_form.validate():
-            for widget, l in add_form.errors.items():
-                flash(','.join(l), 'error')
-            return redirect(url_for('tags.tags_overview_page'))
-
-        title = request.form.get('name').strip()
-
-        if datastore.tag_exists_by_name(title):
-            flash(f'The tag "{title}" already exists', "error")
-            return redirect(url_for('tags.tags_overview_page'))
-
-        datastore.add_tag(title)
-        flash("Tag added")
-
-
-        return redirect(url_for('tags.tags_overview_page'))
-
-    @tags_blueprint.route("/mute/<string:uuid>", methods=['GET'])
-    @login_optionally_required
-    def mute(uuid):
-        if datastore.data['settings']['application']['tags'].get(uuid):
-            datastore.data['settings']['application']['tags'][uuid]['notification_muted'] = not datastore.data['settings']['application']['tags'][uuid]['notification_muted']
-        return redirect(url_for('tags.tags_overview_page'))
-
-    @tags_blueprint.route("/delete/<string:uuid>", methods=['GET'])
-    @login_optionally_required
-    def delete(uuid):
-        removed = 0
-        # Delete the tag, and any tag reference
-        if datastore.data['settings']['application']['tags'].get(uuid):
-            del datastore.data['settings']['application']['tags'][uuid]
-
-        for watch_uuid, watch in datastore.data['watching'].items():
-            if watch.get('tags') and uuid in watch['tags']:
-                removed += 1
-                watch['tags'].remove(uuid)
-
-        flash(f"Tag deleted and removed from {removed} watches")
-        return redirect(url_for('tags.tags_overview_page'))
-
-    @tags_blueprint.route("/unlink/<string:uuid>", methods=['GET'])
-    @login_optionally_required
-    def unlink(uuid):
-        unlinked = 0
-        for watch_uuid, watch in datastore.data['watching'].items():
-            if watch.get('tags') and uuid in watch['tags']:
-                unlinked += 1
-                watch['tags'].remove(uuid)
-
-        flash(f"Tag unlinked removed from {unlinked} watches")
-        return redirect(url_for('tags.tags_overview_page'))
-
-    @tags_blueprint.route("/delete_all", methods=['GET'])
-    @login_optionally_required
-    def delete_all():
-        for watch_uuid, watch in datastore.data['watching'].items():
-            watch['tags'] = []
-        datastore.data['settings']['application']['tags'] = {}
-
-        flash(f"All tags deleted")
-        return redirect(url_for('tags.tags_overview_page'))
-
-    @tags_blueprint.route("/edit/<string:uuid>", methods=['GET'])
-    @login_optionally_required
-    def form_tag_edit(uuid):
-        from changedetectionio import forms
-
-        if uuid == 'first':
-            uuid = list(datastore.data['settings']['application']['tags'].keys()).pop()
-
-        default = datastore.data['settings']['application']['tags'].get(uuid)
-
-        form = forms.watchForm(formdata=request.form if request.method == 'POST' else None,
-                               data=default,
-                               )
-        form.datastore=datastore # needed?
-
-        output = render_template("edit-tag.html",
-                                 data=default,
-                                 form=form,
-                                 settings_application=datastore.data['settings']['application'],
-                                 )
-
-        return output
-
-
-    @tags_blueprint.route("/edit/<string:uuid>", methods=['POST'])
-    @login_optionally_required
-    def form_tag_edit_submit(uuid):
-        from changedetectionio import forms
-        if uuid == 'first':
-            uuid = list(datastore.data['settings']['application']['tags'].keys()).pop()
-
-        default = datastore.data['settings']['application']['tags'].get(uuid)
-
-        form = forms.watchForm(formdata=request.form if request.method == 'POST' else None,
-                               data=default,
-                               )
-        # @todo subclass form so validation works
-        #if not form.validate():
-#            for widget, l in form.errors.items():
-#                flash(','.join(l), 'error')
-#           return redirect(url_for('tags.form_tag_edit_submit', uuid=uuid))
-
-        datastore.data['settings']['application']['tags'][uuid].update(form.data)
-        datastore.needs_write_urgent = True
-        flash("Updated")
-
-        return redirect(url_for('tags.tags_overview_page'))
-
-
-    @tags_blueprint.route("/delete/<string:uuid>", methods=['GET'])
-    def form_tag_delete(uuid):
-        return redirect(url_for('tags.tags_overview_page'))
-    return tags_blueprint

changedetectionio/blueprint/tags/form.py

@@ -1,22 +0,0 @@
-from wtforms import (
-    BooleanField,
-    Form,
-    IntegerField,
-    RadioField,
-    SelectField,
-    StringField,
-    SubmitField,
-    TextAreaField,
-    validators,
-)
-
-
-
-class SingleTag(Form):
-
-    name = StringField('Tag name', [validators.InputRequired()], render_kw={"placeholder": "Name"})
-    save_button = SubmitField('Save', render_kw={"class": "pure-button pure-button-primary"})
-
-
-
-

changedetectionio/blueprint/tags/templates/edit-tag.html

@@ -1,131 +0,0 @@
-{% extends 'base.html' %}
-{% block content %}
-{% from '_helpers.jinja' import render_field, render_checkbox_field, render_button %}
-{% from '_common_fields.jinja' import render_common_settings_form %}
-<script src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
-<script>
-
-/*{% if emailprefix %}*/
-    /*const email_notification_prefix=JSON.parse('{{ emailprefix|tojson }}');*/
-/*{% endif %}*/
-
-
-</script>
-
-<script src="{{url_for('static_content', group='js', filename='watch-settings.js')}}" defer></script>
-<!--<script src="{{url_for('static_content', group='js', filename='limit.js')}}" defer></script>-->
-<script src="{{url_for('static_content', group='js', filename='notifications.js')}}" defer></script>
-
-<div class="edit-form monospaced-textarea">
-
-    <div class="tabs collapsable">
-        <ul>
-            <li class="tab" id=""><a href="#general">General</a></li>
-            <li class="tab"><a href="#filters-and-triggers">Filters &amp; Triggers</a></li>
-            <li class="tab"><a href="#notifications">Notifications</a></li>
-        </ul>
-    </div>
-
-    <div class="box-wrap inner">
-        <form class="pure-form pure-form-stacked"
-              action="{{ url_for('tags.form_tag_edit', uuid=data.uuid) }}" method="POST">
-             <input type="hidden" name="csrf_token" value="{{ csrf_token() }}">
-
-            <div class="tab-pane-inner" id="general">
-                <fieldset>
-                    <div class="pure-control-group">
-                        {{ render_field(form.title, placeholder="https://...", required=true, class="m-d") }}
-                    </div>
-                </fieldset>
-            </div>
-
-            <div class="tab-pane-inner" id="filters-and-triggers">
-                    <div class="pure-control-group">
-                        {% set field = render_field(form.include_filters,
-                            rows=5,
-                            placeholder="#example
-xpath://body/div/span[contains(@class, 'example-class')]",
-                            class="m-d")
-                        %}
-                        {{ field }}
-                        {% if '/text()' in  field %}
-                          <span class="pure-form-message-inline"><strong>Note!: //text() function does not work where the &lt;element&gt; contains &lt;![CDATA[]]&gt;</strong></span><br>
-                        {% endif %}
-                        <span class="pure-form-message-inline">One rule per line, <i>any</i> rules that matches will be used.<br>
-
-                    <ul>
-                        <li>CSS - Limit text to this CSS rule, only text matching this CSS rule is included.</li>
-                        <li>JSON - Limit text to this JSON rule, using either <a href="https://pypi.org/project/jsonpath-ng/" target="new">JSONPath</a> or <a href="https://stedolan.github.io/jq/" target="new">jq</a> (if installed).
-                            <ul>
-                                <li>JSONPath: Prefix with <code>json:</code>, use <code>json:$</code> to force re-formatting if required,  <a href="https://jsonpath.com/" target="new">test your JSONPath here</a>.</li>
-                                {% if jq_support %}
-                                <li>jq: Prefix with <code>jq:</code> and <a href="https://jqplay.org/" target="new">test your jq here</a>. Using <a href="https://stedolan.github.io/jq/" target="new">jq</a> allows for complex filtering and processing of JSON data with built-in functions, regex, filtering, and more. See examples and documentation <a href="https://stedolan.github.io/jq/manual/" target="new">here</a>.</li>
-                                {% else %}
-                                <li>jq support not installed</li>
-                                {% endif %}
-                            </ul>
-                        </li>
-                        <li>XPath - Limit text to this XPath rule, simply start with a forward-slash,
-                            <ul>
-                                <li>Example:  <code>//*[contains(@class, 'sametext')]</code> or <code>xpath://*[contains(@class, 'sametext')]</code>, <a
-                                href="http://xpather.com/" target="new">test your XPath here</a></li>
-                                <li>Example: Get all titles from an RSS feed <code>//title/text()</code></li>
-                            </ul>
-                            </li>
-                    </ul>
-                    Please be sure that you thoroughly understand how to write CSS, JSONPath, XPath{% if jq_support %}, or jq selector{%endif%} rules before filing an issue on GitHub! <a
-                                href="https://github.com/dgtlmoon/changedetection.io/wiki/CSS-Selector-help">here for more CSS selector help</a>.<br>
-                </span>
-                    </div>
-                <fieldset class="pure-control-group">
-                    {{ render_field(form.subtractive_selectors, rows=5, placeholder="header
-footer
-nav
-.stockticker") }}
-                    <span class="pure-form-message-inline">
-                        <ul>
-                          <li> Remove HTML element(s) by CSS selector before text conversion. </li>
-                          <li> Add multiple elements or CSS selectors per line to ignore multiple parts of the HTML. </li>
-                        </ul>
-                      </span>
-                </fieldset>
-
-            </div>
-
-            <div class="tab-pane-inner" id="notifications">
-                <fieldset>
-                    <div  class="pure-control-group inline-radio">
-                      {{ render_checkbox_field(form.notification_muted) }}
-                    </div>
-                    {% if is_html_webdriver %}
-                    <div class="pure-control-group inline-radio">
-                      {{ render_checkbox_field(form.notification_screenshot) }}
-                        <span class="pure-form-message-inline">
-                            <strong>Use with caution!</strong> This will easily fill up your email storage quota or flood other storages.
-                        </span>
-                    </div>
-                    {% endif %}
-                    <div class="field-group" id="notification-field-group">
-                        {% if has_default_notification_urls %}
-                        <div class="inline-warning">
-                            <img class="inline-warning-icon" src="{{url_for('static_content', group='images', filename='notice.svg')}}" alt="Look out!" title="Lookout!" >
-                            There are <a href="{{ url_for('settings_page')}}#notifications">system-wide notification URLs enabled</a>, this form will override notification settings for this watch only &dash; an empty Notification URL list here will still send notifications.
-                        </div>
-                        {% endif %}
-                        <a href="#notifications" id="notification-setting-reset-to-default" class="pure-button button-xsmall" style="right: 20px; top: 20px; position: absolute; background-color: #5f42dd; border-radius: 4px; font-size: 70%; color: #fff">Use system defaults</a>
-
-                        {{ render_common_settings_form(form, emailprefix, settings_application) }}
-                    </div>
-                </fieldset>
-            </div>
-
-            <div id="actions">
-                <div class="pure-control-group">
-                    {{ render_button(form.save_button) }}
-                </div>
-            </div>
-        </form>
-    </div>
-</div>
-
-{% endblock %}

changedetectionio/blueprint/tags/templates/groups-overview.html

@@ -1,60 +0,0 @@
-{% extends 'base.html' %}
-{% block content %}
-{% from '_helpers.jinja' import render_simple_field, render_field %}
-<script src="{{url_for('static_content', group='js', filename='jquery-3.6.0.min.js')}}"></script>
-
-<div class="box">
-    <form class="pure-form" action="{{ url_for('tags.form_tag_add') }}" method="POST" id="new-watch-form">
-        <input type="hidden" name="csrf_token" value="{{ csrf_token() }}" >
-        <fieldset>
-            <legend>Add a new organisational tag</legend>
-            <div id="watch-add-wrapper-zone">
-                <div>
-                    {{ render_simple_field(form.name, placeholder="watch label / tag") }}
-                </div>
-                <div>
-                    {{ render_simple_field(form.save_button, title="Save" ) }}
-                </div>
-            </div>
-            <br>
-            <div style="color: #fff;">Groups allows you to manage filters and notifications for multiple watches under a single organisational tag.</div>
-        </fieldset>
-    </form>
-    <!-- @todo maybe some overview matrix, 'tick' with which has notification, filter rules etc -->
-    <div id="watch-table-wrapper">
-
-        <table class="pure-table pure-table-striped watch-table group-overview-table">
-            <thead>
-            <tr>
-                <th></th>
-                <th>Tag / Label name</th>
-                <th></th>
-            </tr>
-            </thead>
-            <tbody>
-            <!--
-            @Todo - connect Last checked, Last Changed, Number of Watches etc
-            --->
-            {% if not available_tags|length %}
-            <tr>
-                <td colspan="3">No website organisational tags/groups configured</td>
-            </tr>
-            {% endif %}
-            {% for uuid, tag in available_tags.items()  %}
-            <tr id="{{ uuid }}" class="{{ loop.cycle('pure-table-odd', 'pure-table-even') }}">
-                <td class="watch-controls">
-                    <a class="link-mute state-{{'on' if tag.notification_muted else 'off'}}" href="{{url_for('tags.mute', uuid=tag.uuid)}}"><img src="{{url_for('static_content', group='images', filename='bell-off.svg')}}" alt="Mute notifications" title="Mute notifications" class="icon icon-mute" ></a>
-                </td>
-                <td class="title-col inline">{{tag.title}}</td>
-                <td>
-                    <a class="pure-button pure-button-primary" href="{{ url_for('tags.form_tag_edit', uuid=uuid) }}">Edit</a>&nbsp;
-                    <a class="pure-button pure-button-primary" href="{{ url_for('tags.delete', uuid=uuid) }}" title="Deletes and removes tag">Delete</a>
-                    <a class="pure-button pure-button-primary" href="{{ url_for('tags.unlink', uuid=uuid) }}" title="Keep the tag but unlink any watches">Unlink</a>
-                </td>
-            </tr>
-            {% endfor %}
-            </tbody>
-        </table>
-    </div>
-</div>
-{% endblock %}

changedetectionio/content_fetcher.py

@@ -147,13 +147,6 @@ class Fetcher():
     def is_ready(self):
         return True
 
-    def get_all_headers(self):
-        """
-        Get all headers but ensure all keys are lowercase
-        :return:
-        """
-        return {k.lower(): v for k, v in self.headers.items()}
-
     def iterate_browser_steps(self):
         from changedetectionio.blueprint.browser_steps.browser_steps import steppable_browser_interface
         from playwright._impl._api_types import TimeoutError
@@ -189,8 +182,7 @@ class Fetcher():
                                                       optional_value=optional_value)
                     self.screenshot_step(step_n)
                     self.save_step_html(step_n)
-                except TimeoutError as e:
-                    print(str(e))
+                except TimeoutError:
                     # Stop processing here
                     raise BrowserStepsStepTimout(step_n=step_n)
 
@@ -391,7 +383,6 @@ class base_html_playwright(Fetcher):
                 self.headers = x.get('headers')
                 self.instock_data = x.get('instock_data')
                 self.screenshot = base64.b64decode(x.get('screenshot'))
-                self.status_code = x.get('status_code')
                 self.xpath_data = x.get('xpath_data')
 
             else:

changedetectionio/forms.py

@@ -28,8 +28,6 @@ from changedetectionio.notification import (
 
 from wtforms.fields import FormField
 
-dictfilt = lambda x, y: dict([ (i,x[i]) for i in x if i in set(y) ])
-
 valid_method = {
     'GET',
     'POST',
@@ -92,29 +90,6 @@ class SaltyPasswordField(StringField):
         else:
             self.data = False
 
-class StringTagUUID(StringField):
-
-   # process_formdata(self, valuelist) handled manually in POST handler
-
-    # Is what is shown when field <input> is rendered
-    def _value(self):
-        # Tag UUID to name, on submit it will convert it back (in the submit handler of init.py)
-        if self.data and type(self.data) is list:
-            tag_titles = []
-            for i in self.data:
-                tag = self.datastore.data['settings']['application']['tags'].get(i)
-                if tag:
-                    tag_title = tag.get('title')
-                    if tag_title:
-                        tag_titles.append(tag_title)
-
-            return ', '.join(tag_titles)
-
-        if not self.data:
-            return ''
-
-        return 'error'
-
 class TimeBetweenCheckForm(Form):
     weeks = IntegerField('Weeks', validators=[validators.Optional(), validators.NumberRange(min=0, message="Should contain zero or more seconds")])
     days = IntegerField('Days', validators=[validators.Optional(), validators.NumberRange(min=0, message="Should contain zero or more seconds")])
@@ -372,7 +347,7 @@ class quickWatchForm(Form):
     from . import processors
 
     url = fields.URLField('URL', validators=[validateURL()])
-    tags = StringTagUUID('Group tag', [validators.Optional()])
+    tag = StringField('Group tag', [validators.Optional()])
     watch_submit_button = SubmitField('Watch', render_kw={"class": "pure-button pure-button-primary"})
     processor = RadioField(u'Processor', choices=processors.available_processors(), default="text_json_diff")
     edit_and_watch_submit_button = SubmitField('Edit > Watch', render_kw={"class": "pure-button pure-button-primary"})
@@ -380,7 +355,6 @@ class quickWatchForm(Form):
 
 # Common to a single watch and the global settings
 class commonSettingsForm(Form):
-
     notification_urls = StringListField('Notification URL List', validators=[validators.Optional(), ValidateAppRiseServers()])
     notification_title = StringField('Notification Title', default='ChangeDetection.io Notification - {{ watch_url }}', validators=[validators.Optional(), ValidateJinja2Template()])
     notification_body = TextAreaField('Notification Body', default='{{ watch_url }} had a change.', validators=[validators.Optional(), ValidateJinja2Template()])
@@ -408,7 +382,7 @@ class SingleBrowserStep(Form):
 class watchForm(commonSettingsForm):
 
     url = fields.URLField('URL', validators=[validateURL()])
-    tags = StringTagUUID('Group tag', [validators.Optional()], default='')
+    tag = StringField('Group tag', [validators.Optional()], default='')
 
     time_between_check = FormField(TimeBetweenCheckForm)
 
@@ -507,10 +481,6 @@ class globalSettingsApplicationForm(commonSettingsForm):
     global_subtractive_selectors = StringListField('Remove elements', [ValidateCSSJSONXPATHInput(allow_xpath=False, allow_json=False)])
     ignore_whitespace = BooleanField('Ignore whitespace')
     password = SaltyPasswordField()
-    pager_size = IntegerField('Pager size',
-                              render_kw={"style": "width: 5em;"},
-                              validators=[validators.NumberRange(min=0,
-                                                                 message="Should be atleast zero (disabled)")])
     removepassword_button = SubmitField('Remove password', render_kw={"class": "pure-button pure-button-primary"})
     render_anchor_tag_content = BooleanField('Render anchor tag content', default=False)
     shared_diff_access = BooleanField('Allow access to view diff page when password is enabled', default=False, validators=[validators.Optional()])

changedetectionio/html_tools.py

@@ -137,13 +137,12 @@ def _get_stripped_text_from_json_match(match):
 def extract_json_as_string(content, json_filter, ensure_is_ldjson_info_type=None):
     stripped_text_from_html = False
 
-    # Try to parse/filter out the JSON, if we get some parser error, then maybe it's embedded within HTML tags
+    # Try to parse/filter out the JSON, if we get some parser error, then maybe it's embedded <script type=ldjson>
     try:
         stripped_text_from_html = _parse_json(json.loads(content), json_filter)
     except json.JSONDecodeError:
 
         # Foreach <script json></script> blob.. just return the first that matches json_filter
-        # As a last resort, try to parse the whole <body>
         s = []
         soup = BeautifulSoup(content, 'html.parser')
 
@@ -151,34 +150,32 @@ def extract_json_as_string(content, json_filter, ensure_is_ldjson_info_type=None
             bs_result = soup.findAll('script', {"type": "application/ld+json"})
         else:
             bs_result = soup.findAll('script')
-        bs_result += soup.findAll('body')
 
-        bs_jsons = []
+
+        if not bs_result:
+            raise JSONNotFound("No parsable JSON found in this document")
+
         for result in bs_result:
             # Skip empty tags, and things that dont even look like JSON
-            if not result.text or '{' not in result.text:
+            if not result.string or not '{' in result.string:
                 continue
+                
             try:
-                json_data = json.loads(result.text)
-                bs_jsons.append(json_data)
+                json_data = json.loads(result.string)
             except json.JSONDecodeError:
-                # Skip objects which cannot be parsed
+                # Just skip it
                 continue
-
-        if not bs_jsons:
-            raise JSONNotFound("No parsable JSON found in this document")
-        
-        for json_data in bs_jsons:
-            stripped_text_from_html = _parse_json(json_data, json_filter)
-            if ensure_is_ldjson_info_type:
-                # Could sometimes be list, string or something else random
-                if isinstance(json_data, dict):
-                    # If it has LD JSON 'key' @type, and @type is 'product', and something was found for the search
-                    # (Some sites have multiple of the same ld+json @type='product', but some have the review part, some have the 'price' part)
-                    if json_data.get('@type', False) and json_data.get('@type','').lower() == ensure_is_ldjson_info_type.lower() and stripped_text_from_html:
-                        break
-            elif stripped_text_from_html:
-                break
+            else:
+                stripped_text_from_html = _parse_json(json_data, json_filter)
+                if ensure_is_ldjson_info_type:
+                    # Could sometimes be list, string or something else random
+                    if isinstance(json_data, dict):
+                        # If it has LD JSON 'key' @type, and @type is 'product', and something was found for the search
+                        # (Some sites have multiple of the same ld+json @type='product', but some have the review part, some have the 'price' part)
+                        if json_data.get('@type', False) and json_data.get('@type','').lower() == ensure_is_ldjson_info_type.lower() and stripped_text_from_html:
+                            break
+                elif stripped_text_from_html:
+                    break
 
     if not stripped_text_from_html:
         # Re 265 - Just return an empty string when filter not found

changedetectionio/importer.py

@@ -52,8 +52,7 @@ class import_url_list(Importer):
 
             # Flask wtform validators wont work with basic auth, use validators package
             # Up to 5000 per batch so we dont flood the server
-            # @todo validators.url failed on local hostnames (such as referring to ourself when using browserless)
-            if len(url) and 'http' in url.lower() and good < 5000:
+            if len(url) and validators.url(url.replace('source:', '')) and good < 5000:
                 extras = None
                 if processor:
                     extras = {'processor': processor}
@@ -85,8 +84,7 @@ class import_distill_io_json(Importer):
         now = time.time()
         self.new_uuids=[]
 
-        # @todo Use JSONSchema like in the API to validate here.
-        
+
         try:
             data = json.loads(data.strip())
         except json.decoder.JSONDecodeError:
@@ -121,8 +119,11 @@ class import_distill_io_json(Importer):
                 except IndexError:
                     pass
 
+
+                if d.get('tags', False):
+                    extras['tag'] = ", ".join(d['tags'])
+
                 new_uuid = datastore.add_watch(url=d['uri'].strip(),
-                                               tag=",".join(d.get('tags', [])),
                                                extras=extras,
                                                write_to_disk_now=False)
 

changedetectionio/model/App.py

@@ -23,27 +23,25 @@ class model(dict):
                     'workers': int(getenv("DEFAULT_SETTINGS_REQUESTS_WORKERS", "10")),  # Number of threads, lower is better for slow connections
                 },
                 'application': {
-                    # Custom notification content
                     'api_access_token_enabled': True,
+                    'password': False,
                     'base_url' : None,
-                    'empty_pages_are_a_change': False,
                     'extract_title_as_title': False,
+                    'empty_pages_are_a_change': False,
                     'fetch_backend': getenv("DEFAULT_FETCH_BACKEND", "html_requests"),
                     'filter_failure_notification_threshold_attempts': _FILTER_FAILURE_THRESHOLD_ATTEMPTS_DEFAULT,
                     'global_ignore_text': [], # List of text to ignore when calculating the comparison checksum
                     'global_subtractive_selectors': [],
                     'ignore_whitespace': True,
+                    'render_anchor_tag_content': False,
+                    'notification_urls': [], # Apprise URL list
+                    # Custom notification content
+                    'notification_title': default_notification_title,
                     'notification_body': default_notification_body,
                     'notification_format': default_notification_format,
-                    'notification_title': default_notification_title,
-                    'notification_urls': [], # Apprise URL list
-                    'pager_size': 50,
-                    'password': False,
-                    'render_anchor_tag_content': False,
                     'schema_version' : 0,
                     'shared_diff_access': False,
-                    'webdriver_delay': None , # Extra delay in seconds before extracting text
-                    'tags': {} #@todo use Tag.model initialisers
+                    'webdriver_delay': None  # Extra delay in seconds before extracting text
                 }
             }
         }
@@ -51,15 +49,3 @@ class model(dict):
     def __init__(self, *arg, **kw):
         super(model, self).__init__(*arg, **kw)
         self.update(self.base_config)
-
-
-def parse_headers_from_text_file(filepath):
-    headers = {}
-    with open(filepath, 'r') as f:
-        for l in f.readlines():
-            l = l.strip()
-            if not l.startswith('#') and ':' in l:
-                (k, v) = l.split(':')
-                headers[k.strip()] = v.strip()
-
-    return headers

changedetectionio/model/Tag.py

@@ -1,19 +0,0 @@
-from .Watch import base_config
-import uuid
-
-class model(dict):
-
-    def __init__(self, *arg, **kw):
-
-        self.update(base_config)
-
-        self['uuid'] = str(uuid.uuid4())
-
-        if kw.get('default'):
-            self.update(kw['default'])
-            del kw['default']
-
-
-        # Goes at the end so we update the default object with the initialiser
-        super(model, self).__init__(*arg, **kw)
-

changedetectionio/model/Watch.py

@@ -52,8 +52,7 @@ base_config = {
     'previous_md5_before_filters': False,  # Used for skipping changedetection entirely
     'proxy': None,  # Preferred proxy connection
     'subtractive_selectors': [],
-    'tag': '', # Old system of text name for a tag, to be removed
-    'tags': [], # list of UUIDs to App.Tags
+    'tag': None,
     'text_should_not_be_present': [],  # Text that should not present
     # Re #110, so then if this is set to None, we know to use the default value instead
     # Requires setting to None on submit if it's the same as the default
@@ -456,6 +455,10 @@ class model(dict):
 
         return csv_output_filename
 
+    @property
+    # Return list of tags, stripped and lowercase, used for searching
+    def all_tags(self):
+        return [s.strip().lower() for s in self.get('tag','').split(',')]
 
     def has_special_diff_filter_options_set(self):
 

changedetectionio/notification.py

@@ -92,12 +92,6 @@ def process_notification(n_object, datastore):
         n_object.get('notification_format', default_notification_format),
         valid_notification_formats[default_notification_format],
     )
-
-    # If we arrived with 'System default' then look it up
-    if n_format == default_notification_format_for_watch and datastore.data['settings']['application'].get('notification_format') != default_notification_format_for_watch:
-        # Initially text or whatever
-        n_format = datastore.data['settings']['application'].get('notification_format', valid_notification_formats[default_notification_format])
-
     
     # https://github.com/caronc/apprise/wiki/Development_LogCapture
     # Anything higher than or equal to WARNING (which covers things like Connection errors)
@@ -192,13 +186,8 @@ def create_notification_parameters(n_object, datastore):
     uuid = n_object['uuid'] if 'uuid' in n_object else ''
 
     if uuid != '':
-        watch_title = datastore.data['watching'][uuid].get('title', '')
-        tag_list = []
-        tags = datastore.get_all_tags_for_watch(uuid)
-        if tags:
-            for tag_uuid, tag in tags.items():
-                tag_list.append(tag.get('title'))
-        watch_tag = ', '.join(tag_list)
+        watch_title = datastore.data['watching'][uuid]['title']
+        watch_tag = datastore.data['watching'][uuid]['tag']
     else:
         watch_title = 'Change Detection'
         watch_tag = ''

changedetectionio/processors/restock_diff.py

@@ -12,12 +12,6 @@ urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 name = 'Re-stock detection for single product pages'
 description = 'Detects if the product goes back to in-stock'
 
-class UnableToExtractRestockData(Exception):
-    def __init__(self, status_code):
-        # Set this so we can use it in other parts of the app
-        self.status_code = status_code
-        return
-
 class perform_site_check(difference_detection_processor):
     screenshot = None
     xpath_data = None
@@ -42,10 +36,11 @@ class perform_site_check(difference_detection_processor):
 
         # Unset any existing notification error
         update_obj = {'last_notification_error': False, 'last_error': False}
+        extra_headers = watch.get('headers', [])
 
-        request_headers = watch.get('headers', [])
-        request_headers.update(self.datastore.get_all_base_headers())
-        request_headers.update(self.datastore.get_all_headers_in_textfile_for_watch(uuid=uuid))
+        # Tweak the base config with the per-watch ones
+        request_headers = deepcopy(self.datastore.data['settings']['headers'])
+        request_headers.update(extra_headers)
 
         # https://github.com/psf/requests/issues/4525
         # Requests doesnt yet support brotli encoding, so don't put 'br' here, be totally sure that the user cannot
@@ -110,8 +105,7 @@ class perform_site_check(difference_detection_processor):
             fetched_md5 = hashlib.md5(fetcher.instock_data.encode('utf-8')).hexdigest()
             # 'Possibly in stock' comes from stock-not-in-stock.js when no string found above the fold.
             update_obj["in_stock"] = True if fetcher.instock_data == 'Possibly in stock' else False
-        else:
-            raise UnableToExtractRestockData(status_code=fetcher.status_code)
+
 
         # The main thing that all this at the moment comes down to :)
         changed_detected = False

changedetectionio/processors/text_json_diff.py

@@ -57,6 +57,7 @@ class perform_site_check(difference_detection_processor):
 
         # DeepCopy so we can be sure we don't accidently change anything by reference
         watch = deepcopy(self.datastore.data['watching'].get(uuid))
+
         if not watch:
             raise Exception("Watch no longer exists.")
 
@@ -69,10 +70,11 @@ class perform_site_check(difference_detection_processor):
         # Unset any existing notification error
         update_obj = {'last_notification_error': False, 'last_error': False}
 
+        extra_headers = watch.get('headers', [])
+
         # Tweak the base config with the per-watch ones
-        request_headers = watch.get('headers', [])
-        request_headers.update(self.datastore.get_all_base_headers())
-        request_headers.update(self.datastore.get_all_headers_in_textfile_for_watch(uuid=uuid))
+        request_headers = deepcopy(self.datastore.data['settings']['headers'])
+        request_headers.update(extra_headers)
 
         # https://github.com/psf/requests/issues/4525
         # Requests doesnt yet support brotli encoding, so don't put 'br' here, be totally sure that the user cannot
@@ -138,7 +140,7 @@ class perform_site_check(difference_detection_processor):
         self.xpath_data = fetcher.xpath_data
 
         # Track the content type
-        update_obj['content_type'] = fetcher.get_all_headers().get('content-type', '').lower()
+        update_obj['content_type'] = fetcher.headers.get('Content-Type', '')
 
         # Watches added automatically in the queue manager will skip if its the same checksum as the previous run
         # Saves a lot of CPU
@@ -158,7 +160,7 @@ class perform_site_check(difference_detection_processor):
         # https://stackoverflow.com/questions/41817578/basic-method-chaining ?
         # return content().textfilter().jsonextract().checksumcompare() ?
 
-        is_json = 'application/json' in fetcher.get_all_headers().get('content-type', '').lower()
+        is_json = 'application/json' in fetcher.headers.get('Content-Type', '')
         is_html = not is_json
 
         # source: support, basically treat it as plaintext
@@ -166,7 +168,7 @@ class perform_site_check(difference_detection_processor):
             is_html = False
             is_json = False
 
-        if watch.is_pdf or 'application/pdf' in fetcher.get_all_headers().get('content-type', '').lower():
+        if watch.is_pdf or 'application/pdf' in fetcher.headers.get('Content-Type', '').lower():
             from shutil import which
             tool = os.getenv("PDF_TO_HTML_TOOL", "pdftohtml")
             if not which(tool):
@@ -190,23 +192,21 @@ class perform_site_check(difference_detection_processor):
 
             fetcher.content = fetcher.content.replace('</body>', metadata + '</body>')
 
-        # Better would be if Watch.model could access the global data also
-        # and then use getattr https://docs.python.org/3/reference/datamodel.html#object.__getitem__
-        # https://realpython.com/inherit-python-dict/ instead of doing it procedurely
-        include_filters_from_tags = self.datastore.get_tag_overrides_for_watch(uuid=uuid, attr='include_filters')
-        include_filters_rule = [*watch.get('include_filters', []), *include_filters_from_tags]
 
-        subtractive_selectors = [*self.datastore.get_tag_overrides_for_watch(uuid=uuid, attr='subtractive_selectors'),
-                                 *watch.get("subtractive_selectors", []),
-                                 *self.datastore.data["settings"]["application"].get("global_subtractive_selectors", [])
-                                 ]
+        include_filters_rule = deepcopy(watch.get('include_filters', []))
+        # include_filters_rule = watch['include_filters']
+        subtractive_selectors = watch.get(
+            "subtractive_selectors", []
+        ) + self.datastore.data["settings"]["application"].get(
+            "global_subtractive_selectors", []
+        )
 
         # Inject a virtual LD+JSON price tracker rule
         if watch.get('track_ldjson_price_data', '') == PRICE_DATA_TRACK_ACCEPT:
             include_filters_rule.append(html_tools.LD_JSON_PRODUCT_OFFER_SELECTOR)
 
-        has_filter_rule = len(include_filters_rule) and len(include_filters_rule[0].strip())
-        has_subtractive_selectors = len(subtractive_selectors) and len(subtractive_selectors[0].strip())
+        has_filter_rule = include_filters_rule and len("".join(include_filters_rule).strip())
+        has_subtractive_selectors = subtractive_selectors and len(subtractive_selectors[0].strip())
 
         if is_json and not has_filter_rule:
             include_filters_rule.append("json:$")
@@ -236,7 +236,7 @@ class perform_site_check(difference_detection_processor):
             html_content = fetcher.content
 
             # If not JSON,  and if it's not text/plain..
-            if 'text/plain' in fetcher.get_all_headers().get('content-type', '').lower():
+            if 'text/plain' in fetcher.headers.get('Content-Type', '').lower():
                 # Don't run get_text or xpath/css filters on plaintext
                 stripped_text_from_html = html_content
             else:

changedetectionio/res/puppeteer_fetch.js

@@ -97,10 +97,6 @@ module.exports = async ({page, context}) => {
                 console.log("Skipping (no_cache_list) - " + url);
                 return;
             }
-            if (url.toLowerCase().includes('data:')) {
-                console.log("Skipping (embedded-data) - " + url);
-                return;
-            }
             response.buffer().then(buffer => {
                 if (buffer.length > 100) {
                     console.log("Cache - Saving " + response.request().method() + " - " + url + " - " + response.request().resourceType());

changedetectionio/res/stock-not-in-stock.js

@@ -10,7 +10,6 @@ function isItemInStock() {
     'brak na stanie',
     'brak w magazynie',
     'coming soon',
-    'currently have any tickets for this',
     'currently unavailable',
     'en rupture de stock',
     'item is no longer available',
@@ -21,9 +20,7 @@ function isItemInStock() {
     'nicht zur verfügung',
     'no disponible temporalmente',
     'no longer in stock',
-    'no tickets available',
     'not available',
-    'not currently available',
     'not in stock',
     'notify me when available',
     'não estamos a aceitar encomendas',
@@ -33,8 +30,6 @@ function isItemInStock() {
     'sold out',
     'temporarily out of stock',
     'temporarily unavailable',
-    'tickets unavailable',
-    'unavailable tickets',
     'we do not currently have an estimate of when this product will be back in stock.',
     'zur zeit nicht an lager',
   ];

changedetectionio/static/js/browser-steps.js

@@ -114,11 +114,11 @@ $(document).ready(function () {
             e.preventDefault()
         });
 
-        // When the mouse moves we know which element it should be above
-        // mousedown will link that to the UI (select the right action, highlight etc)
         $('#browsersteps-selector-canvas').bind('mousedown', function (e) {
             // https://developer.mozilla.org/en-US/docs/Web/API/MouseEvent
             e.preventDefault()
+            console.log(e);
+            console.log("current xpath in index is " + current_selected_i);
             last_click_xy = {'x': parseInt((1 / x_scale) * e.offsetX), 'y': parseInt((1 / y_scale) * e.offsetY)}
             process_selected(current_selected_i);
             current_selected_i = false;
@@ -132,7 +132,6 @@ $(document).ready(function () {
             }
         });
 
-        // Debounce and find the current most 'interesting' element we are hovering above
         $('#browsersteps-selector-canvas').bind('mousemove', function (e) {
             if (!xpath_data) {
                 return;
@@ -152,40 +151,41 @@ $(document).ready(function () {
             current_selected_i = false;
             // Reverse order - the most specific one should be deeper/"laster"
             // Basically, find the most 'deepest'
-            var possible_elements = [];
-            xpath_data['size_pos'].forEach(function (item, index) {
+            //$('#browsersteps-selector-canvas').css('cursor', 'pointer');
+            for (var i = xpath_data['size_pos'].length; i !== 0; i--) {
+                // draw all of them? let them choose somehow?
+                var sel = xpath_data['size_pos'][i - 1];
                 // If we are in a bounding-box
-                if (e.offsetY > item.top * y_scale && e.offsetY < item.top * y_scale + item.height * y_scale
+                if (e.offsetY > sel.top * y_scale && e.offsetY < sel.top * y_scale + sel.height * y_scale
                     &&
-                    e.offsetX > item.left * y_scale && e.offsetX < item.left * y_scale + item.width * y_scale
+                    e.offsetX > sel.left * y_scale && e.offsetX < sel.left * y_scale + sel.width * y_scale
 
                 ) {
-                    // There could be many elements here, record them all and then we'll find out which is the most 'useful'
-                    // (input, textarea, button, A etc)
-                    if (item.width < xpath_data['browser_width']) {
-                        possible_elements.push(item);
+                    // Only highlight these interesting types
+                    if (1) {
+                        ctx.strokeRect(sel.left * x_scale, sel.top * y_scale, sel.width * x_scale, sel.height * y_scale);
+                        ctx.fillRect(sel.left * x_scale, sel.top * y_scale, sel.width * x_scale, sel.height * y_scale);
+                        current_selected_i = i - 1;
+                        break;
+
+                        // find the smallest one at this x,y
+                        // does it mean sort the xpath list by size (w*h) i think so!
+                    } else {
+
+                        if (include_text_elements[0].checked === true) {
+                            // blue one with background instead?
+                            ctx.fillStyle = 'rgba(0,0,255, 0.1)';
+                            ctx.strokeStyle = 'rgba(0,0,200, 0.7)';
+                            $('#browsersteps-selector-canvas').css('cursor', 'grab');
+                            ctx.strokeRect(sel.left * x_scale, sel.top * y_scale, sel.width * x_scale, sel.height * y_scale);
+                            ctx.fillRect(sel.left * x_scale, sel.top * y_scale, sel.width * x_scale, sel.height * y_scale);
+                            current_selected_i = i - 1;
+                            break;
+                        }
                     }
                 }
-            });
-
-            // Find the best one
-            if (possible_elements.length) {
-                possible_elements.forEach(function (item, index) {
-                  if (["a", "input", "textarea", "button"].includes(item['tagName'])) {
-                      current_selected_i = item;
-                  }
-                });
-
-                if (!current_selected_i) {
-                    current_selected_i = possible_elements[0];
-                }
-
-                sel = xpath_data['size_pos'][current_selected_i];
-                ctx.strokeRect(current_selected_i.left * x_scale, current_selected_i.top * y_scale, current_selected_i.width * x_scale, current_selected_i.height * y_scale);
-                ctx.fillRect(current_selected_i.left * x_scale, current_selected_i.top * y_scale, current_selected_i.width * x_scale, current_selected_i.height * y_scale);
             }
 
-
         }.debounce(10));
     });
 
@@ -195,16 +195,16 @@ $(document).ready(function () {
 
 
     // callback for clicking on an xpath on the canvas
-    function process_selected(selected_in_xpath_list) {
+    function process_selected(xpath_data_index) {
         found_something = false;
         var first_available = $("ul#browser_steps li.empty").first();
 
 
-        if (selected_in_xpath_list !== false) {
+        if (xpath_data_index !== false) {
             // Nothing focused, so fill in a new one
             // if inpt type button or <button>
             // from the top, find the next not used one and use it
-            var x = selected_in_xpath_list;
+            var x = xpath_data['size_pos'][xpath_data_index];
             console.log(x);
             if (x && first_available.length) {
                 // @todo will it let you click shit that has a layer ontop? probably not.
@@ -214,18 +214,26 @@ $(document).ready(function () {
                     $('input[placeholder="Value"]', first_available).addClass('ok').click().focus();
                     found_something = true;
                 } else {
-                    // There's no good way (that I know) to find if this
-                    // see https://stackoverflow.com/questions/446892/how-to-find-event-listeners-on-a-dom-node-in-javascript-or-in-debugging
-                    // https://codepen.io/azaslavsky/pen/DEJVWv
-
-                    // So we dont know if its really a clickable element or not :-(
-                    // Assume it is - then we dont fill the pages with unreliable "Click X,Y" selections
-                    // If you switch to "Click X,y" after an element here is setup, it will give the last co-ords anyway
-                    //if (x['isClickable'] || x['tagName'].startsWith('h') || x['tagName'] === 'a' || x['tagName'] === 'button' || x['tagtype'] === 'submit' || x['tagtype'] === 'checkbox' || x['tagtype'] === 'radio' || x['tagtype'] === 'li') {
+                    if (x['isClickable'] || x['tagName'].startsWith('h') || x['tagName'] === 'a' || x['tagName'] === 'button' || x['tagtype'] === 'submit' || x['tagtype'] === 'checkbox' || x['tagtype'] === 'radio' || x['tagtype'] === 'li') {
                         $('select', first_available).val('Click element').change();
                         $('input[type=text]', first_available).first().val(x['xpath']);
                         found_something = true;
-                    //}
+                    }
+                }
+
+                first_available.xpath_data_index = xpath_data_index;
+
+                if (!found_something) {
+                    if (include_text_elements[0].checked === true) {
+                        // Suggest that we use as filter?
+                        // @todo filters should always be in the last steps, nothing non-filter after it
+                        found_something = true;
+                        ctx.strokeStyle = 'rgba(0,0,255, 0.9)';
+                        ctx.fillStyle = 'rgba(0,0,255, 0.1)';
+                        $('select', first_available).val('Extract text and use as filter').change();
+                        $('input[type=text]', first_available).first().val(x['xpath']);
+                        include_text_elements[0].checked = false;
+                    }
                 }
             }
         }
@@ -240,7 +248,7 @@ $(document).ready(function () {
 
     function start() {
         console.log("Starting browser-steps UI");
-        browsersteps_session_id = false;
+        browsersteps_session_id = Date.now();
         // @todo This setting of the first one should be done at the datalayer but wtforms doesnt wanna play nice
         $('#browser_steps >li:first-child').removeClass('empty');
         set_first_gotosite_disabled();
@@ -248,7 +256,7 @@ $(document).ready(function () {
         $('.clear,.remove', $('#browser_steps >li:first-child')).hide();
         $.ajax({
             type: "GET",
-            url: browser_steps_start_url,
+            url: browser_steps_sync_url + "&browsersteps_session_id=" + browsersteps_session_id,
             statusCode: {
                 400: function () {
                     // More than likely the CSRF token was lost when the server restarted
@@ -256,12 +264,12 @@ $(document).ready(function () {
                 }
             }
         }).done(function (data) {
+            xpath_data = data.xpath_data;
             $("#loading-status-text").fadeIn();
-            browsersteps_session_id = data.browsersteps_session_id;
             // This should trigger 'Goto site'
             console.log("Got startup response, requesting Goto-Site (first) step fake click");
             $('#browser_steps >li:first-child .apply').click();
-            browserless_seconds_remaining = 500;
+            browserless_seconds_remaining = data.browser_time_remaining;
             set_first_gotosite_disabled();
         }).fail(function (data) {
             console.log(data);
@@ -422,6 +430,7 @@ $(document).ready(function () {
             apply_buttons_disabled = false;
             $("#browsersteps-img").css('opacity', 1);
             $('ul#browser_steps li .control .apply').css('opacity', 1);
+            browserless_seconds_remaining = data.browser_time_remaining;
             $("#loading-status-text").hide();
             set_first_gotosite_disabled();
         }).fail(function (data) {

changedetectionio/static/js/toggle-theme.js

@@ -3,45 +3,22 @@
  * Toggles theme between light and dark mode.
  */
 $(document).ready(function () {
-  const button = document.getElementById("toggle-light-mode");
+  const button = document.getElementsByClassName("toggle-theme")[0];
 
   button.onclick = () => {
     const htmlElement = document.getElementsByTagName("html");
     const isDarkMode = htmlElement[0].dataset.darkmode === "true";
     htmlElement[0].dataset.darkmode = !isDarkMode;
-    setCookieValue(!isDarkMode);
+    if (isDarkMode) {
+      button.classList.remove("dark");
+      setCookieValue(false);
+    } else {
+      button.classList.add("dark");
+      setCookieValue(true);
+    }
   };
 
   const setCookieValue = (value) => {
     document.cookie = `css_dark_mode=${value};max-age=31536000;path=/`
   }
-
-  // Search input box behaviour
-    const toggle_search = document.getElementById("toggle-search");
-  const search_q = document.getElementById("search-q");
-  window.addEventListener('keydown', function (e) {
-
-    if (e.altKey == true && e.keyCode == 83)
-      search_q.classList.toggle('expanded');
-      search_q.focus();
-  });
-
-
-  search_q.onkeydown = (e) => {
-    var key = e.keyCode || e.which;
-    if (key === 13) {
-      document.searchForm.submit();
-    }
-  };
-  toggle_search.onclick = () => {
-    // Could be that they want to search something once text is in there
-    if (search_q.value.length) {
-      document.searchForm.submit();
-    } else {
-      // If not..
-      search_q.classList.toggle('expanded');
-      search_q.focus();
-    }
-  };
-
 });

changedetectionio/static/js/visual-selector.js

@@ -61,12 +61,7 @@ $(document).ready(function () {
     function bootstrap_visualselector() {
         if (1) {
             // bootstrap it, this will trigger everything else
-            $("img#selector-background").on("error", function () {
-                $('.fetching-update-notice').html("<strong>Ooops!</strong> The VisualSelector tool needs atleast one fetched page, please unpause the watch and/or wait for the watch to complete fetching and then reload this page.");
-                $('.fetching-update-notice').css('color','#bb0000');
-                $('#selector-current-xpath').hide();
-                $('#clear-selector').hide();
-            }).bind('load', function () {
+            $("img#selector-background").bind('load', function () {
                 console.log("Loaded background...");
                 c = document.getElementById("selector-canvas");
                 // greyed out fill context
@@ -84,11 +79,10 @@ $(document).ready(function () {
             }).attr("src", screenshot_url);
         }
         // Tell visualSelector that the image should update
-        var s = $("img#selector-background").attr('src') + "?" + new Date().getTime();
-        $("img#selector-background").attr('src', s)
+        var s = $("img#selector-background").attr('src')+"?"+ new Date().getTime();
+        $("img#selector-background").attr('src',s)
     }
 
-    // This is fired once the img src is loaded in bootstrap_visualselector()
     function fetch_data() {
         // Image is ready
         $('.fetching-update-notice').html("Fetching element data..");
@@ -105,8 +99,7 @@ $(document).ready(function () {
             reflow_selector();
             $('.fetching-update-notice').fadeOut();
         });
-
-    }
+    };
 
 
     function set_scale() {

changedetectionio/static/js/watch-overview.js

@@ -1,45 +1,31 @@
 $(function () {
-    // Remove unviewed status when normally clicked
-    $('.diff-link').click(function () {
-        $(this).closest('.unviewed').removeClass('unviewed');
-    });
+  // Remove unviewed status when normally clicked
+  $('.diff-link').click(function () {
+    $(this).closest('.unviewed').removeClass('unviewed');
+  });
 
-    $("#checkbox-assign-tag").click(function (e) {
-        $('#op_extradata').val(prompt("Enter a tag name"));
-    });
 
-    $('.with-share-link > *').click(function () {
-        $("#copied-clipboard").remove();
+  $('.with-share-link > *').click(function () {
+      $("#copied-clipboard").remove();
 
-        var range = document.createRange();
-        var n = $("#share-link")[0];
-        range.selectNode(n);
-        window.getSelection().removeAllRanges();
-        window.getSelection().addRange(range);
-        document.execCommand("copy");
-        window.getSelection().removeAllRanges();
+      var range = document.createRange();
+      var n=$("#share-link")[0];
+      range.selectNode(n);
+      window.getSelection().removeAllRanges();
+      window.getSelection().addRange(range);
+      document.execCommand("copy");
+      window.getSelection().removeAllRanges();
 
-        $('.with-share-link').append('<span style="font-size: 80%; color: #fff;" id="copied-clipboard">Copied to clipboard</span>');
-        $("#copied-clipboard").fadeOut(2500, function () {
-            $(this).remove();
-        });
-    });
-
-    $(".watch-table tr").click(function (event) {
-        var tagName = event.target.tagName.toLowerCase();
-        if (tagName === 'tr' || tagName === 'td') {
-            var x = $('input[type=checkbox]', this);
-            if (x) {
-                $(x).click();
-            }
-        }
-    });
+      $('.with-share-link').append('<span style="font-size: 80%; color: #fff;" id="copied-clipboard">Copied to clipboard</span>');
+      $("#copied-clipboard").fadeOut(2500, function() {
+       $(this).remove();
+      });
+  });
 
     // checkboxes - check all
     $("#check-all").click(function (e) {
         $('input[type=checkbox]').not(this).prop('checked', this.checked);
     });
-
     // checkboxes - show/hide buttons
     $("input[type=checkbox]").click(function (e) {
         if ($('input[type=checkbox]:checked').length) {

changedetectionio/static/styles/scss/styles.scss

@@ -54,47 +54,8 @@ a.github-link {
   }
 }
 
-#toggle-light-mode {
-  width: 3rem;
-  .icon-dark {
-    display: none;
-  }
-
-  &.dark {
-    .icon-light {
-      display: none;
-    }
-
-    .icon-dark {
-      display: block;
-    }
-  }
-}
-
-#toggle-search {
-  width: 2rem;
-}
-
-#search-q {
-  opacity: 0;
-  -webkit-transition: all .9s ease;
-  -moz-transition: all .9s ease;
-  transition: all .9s ease;
-  width: 0;
-  display: none;
-  &.expanded {
-    width: auto;
-    display: inline-block;
-
-    opacity: 1;
-  }
-}
-#search-result-info {
-  color: #fff;
-}
-
-button.toggle-button {
-  vertical-align: middle;
+button.toggle-theme {
+  width: 4rem;
   background: transparent;
   border: none;
   cursor: pointer;
@@ -113,7 +74,19 @@ button.toggle-button {
     display: block;
   }
 
+  .icon-dark {
+    display: none;
+  }
 
+  &.dark {
+    .icon-light {
+      display: none;
+    }
+
+    .icon-dark {
+      display: block;
+    }
+  }
 }
 
 .pure-menu-horizontal {

changedetectionio/static/styles/styles.css

@@ -283,10 +283,6 @@ html[data-darkmode="true"] {
   --color-icon-github-hover: var(--color-grey-700);
   --color-watch-table-error: var(--color-light-red);
   --color-watch-table-row-text: var(--color-grey-800); }
-  html[data-darkmode="true"] #toggle-light-mode .icon-light {
-    display: none; }
-  html[data-darkmode="true"] #toggle-light-mode .icon-dark {
-    display: block; }
   html[data-darkmode="true"] .icon-spread {
     filter: hue-rotate(-10deg) brightness(1.5); }
   html[data-darkmode="true"] .watch-table .title-col a[target="_blank"]::after,
@@ -335,40 +331,23 @@ a.github-link {
   a.github-link:hover {
     color: var(--color-icon-github-hover); }
 
-#toggle-light-mode {
-  width: 3rem; }
-  #toggle-light-mode .icon-dark {
-    display: none; }
-
-#toggle-search {
-  width: 2rem; }
-
-#search-q {
-  opacity: 0;
-  -webkit-transition: all .9s ease;
-  -moz-transition: all .9s ease;
-  transition: all .9s ease;
-  width: 0;
-  display: none; }
-  #search-q.expanded {
-    width: auto;
-    display: inline-block;
-    opacity: 1; }
-
-#search-result-info {
-  color: #fff; }
-
-button.toggle-button {
-  vertical-align: middle;
+button.toggle-theme {
+  width: 4rem;
   background: transparent;
   border: none;
   cursor: pointer;
   color: var(--color-icon-github); }
-  button.toggle-button:hover {
+  button.toggle-theme:hover {
     color: var(--color-icon-github-hover); }
-  button.toggle-button svg {
+  button.toggle-theme svg {
     fill: currentColor; }
-  button.toggle-button .icon-light {
+  button.toggle-theme .icon-light {
+    display: block; }
+  button.toggle-theme .icon-dark {
+    display: none; }
+  button.toggle-theme.dark .icon-light {
+    display: none; }
+  button.toggle-theme.dark .icon-dark {
     display: block; }
 
 .pure-menu-horizontal {

changedetectionio/store.py

@@ -3,7 +3,7 @@ from flask import (
 )
 
 from . model import App, Watch
-from copy import deepcopy, copy
+from copy import deepcopy
 from os import path, unlink
 from threading import Lock
 import json
@@ -16,8 +16,6 @@ import threading
 import time
 import uuid as uuid_builder
 
-dictfilt = lambda x, y: dict([ (i,x[i]) for i in x if i in set(y) ])
-
 # Is there an existing library to ensure some data store (JSON etc) is in sync with CRUD methods?
 # Open a github issue if you know something :)
 # https://stackoverflow.com/questions/6190468/how-to-trigger-function-on-value-change
@@ -180,6 +178,20 @@ class ChangeDetectionStore:
 
         return self.__data
 
+    def get_all_tags(self):
+        tags = []
+        for uuid, watch in self.data['watching'].items():
+            if watch['tag'] is None:
+                continue
+            # Support for comma separated list of tags.
+            for tag in watch['tag'].split(','):
+                tag = tag.strip()
+                if tag not in tags:
+                    tags.append(tag)
+
+        tags.sort()
+        return tags
+
     # Delete a single watch by UUID
     def delete(self, uuid):
         import pathlib
@@ -192,22 +204,22 @@ class ChangeDetectionStore:
                 # GitHub #30 also delete history records
                 for uuid in self.data['watching']:
                     path = pathlib.Path(os.path.join(self.datastore_path, uuid))
-                    if os.path.exists(path):
-                        shutil.rmtree(path)
+                    shutil.rmtree(path)
+                    self.needs_write_urgent = True
 
             else:
                 path = pathlib.Path(os.path.join(self.datastore_path, uuid))
-                if os.path.exists(path):
-                    shutil.rmtree(path)
+                shutil.rmtree(path)
                 del self.data['watching'][uuid]
 
-        self.needs_write_urgent = True
+            self.needs_write_urgent = True
 
     # Clone a watch by UUID
     def clone(self, uuid):
-        url = self.data['watching'][uuid].get('url')
+        url = self.data['watching'][uuid]['url']
+        tag = self.data['watching'][uuid]['tag']
         extras = self.data['watching'][uuid]
-        new_uuid = self.add_watch(url=url, extras=extras)
+        new_uuid = self.add_watch(url=url, tag=tag, extras=extras)
         return new_uuid
 
     def url_exists(self, url):
@@ -242,14 +254,16 @@ class ChangeDetectionStore:
 
         self.needs_write_urgent = True
 
-    def add_watch(self, url, tag='', extras=None, tag_uuids=None, write_to_disk_now=True):
+    def add_watch(self, url, tag="", extras=None, write_to_disk_now=True):
 
         if extras is None:
             extras = {}
+        # should always be str
+        if tag is None or not tag:
+            tag = ''
 
         # Incase these are copied across, assume it's a reference and deepcopy()
         apply_extras = deepcopy(extras)
-        apply_extras['tags'] = [] if not apply_extras.get('tags') else apply_extras.get('tags')
 
         # Was it a share link? try to fetch the data
         if (url.startswith("https://changedetection.io/share/")):
@@ -276,7 +290,6 @@ class ChangeDetectionStore:
                     'processor',
                     'subtractive_selectors',
                     'tag',
-                    'tags',
                     'text_should_not_be_present',
                     'title',
                     'trigger_text',
@@ -299,36 +312,25 @@ class ChangeDetectionStore:
             flash('Watch protocol is not permitted by SAFE_PROTOCOL_REGEX', 'error')
             return None
 
-        if tag and type(tag) == str:
-            # Then it's probably a string of the actual tag by name, split and add it
-            for t in tag.split(','):
-                # for each stripped tag, add tag as UUID
-                for a_t in t.split(','):
-                    tag_uuid = self.add_tag(a_t)
-                    apply_extras['tags'].append(tag_uuid)
+        with self.lock:
+            # #Re 569
+            new_watch = Watch.model(datastore_path=self.datastore_path, default={
+                'url': url,
+                'tag': tag,
+                'date_created': int(time.time())
+            })
 
-        # Or if UUIDs given directly
-        if tag_uuids:
-            apply_extras['tags'] = list(set(apply_extras['tags'] + tag_uuids))
+            new_uuid = new_watch['uuid']
+            logging.debug("Added URL {} - {}".format(url, new_uuid))
 
-        # Make any uuids unique
-        if apply_extras.get('tags'):
-            apply_extras['tags'] = list(set(apply_extras.get('tags')))
+            for k in ['uuid', 'history', 'last_checked', 'last_changed', 'newest_history_key', 'previous_md5', 'viewed']:
+                if k in apply_extras:
+                    del apply_extras[k]
 
-        new_watch = Watch.model(datastore_path=self.datastore_path, url=url)
-
-        new_uuid = new_watch.get('uuid')
-
-        logging.debug("Added URL {} - {}".format(url, new_uuid))
-
-        for k in ['uuid', 'history', 'last_checked', 'last_changed', 'newest_history_key', 'previous_md5', 'viewed']:
-            if k in apply_extras:
-                del apply_extras[k]
-
-        new_watch.update(apply_extras)
-        new_watch.ensure_data_dir_exists()
-        self.__data['watching'][new_uuid] = new_watch
+            new_watch.update(apply_extras)
+            self.__data['watching'][new_uuid] = new_watch
 
+        self.__data['watching'][new_uuid].ensure_data_dir_exists()
 
         if write_to_disk_now:
             self.sync_to_json()
@@ -364,21 +366,19 @@ class ChangeDetectionStore:
     def save_error_text(self, watch_uuid, contents):
         if not self.data['watching'].get(watch_uuid):
             return
-
-        self.data['watching'][watch_uuid].ensure_data_dir_exists()
         target_path = os.path.join(self.datastore_path, watch_uuid, "last-error.txt")
+
         with open(target_path, 'w') as f:
             f.write(contents)
 
     def save_xpath_data(self, watch_uuid, data, as_error=False):
-
         if not self.data['watching'].get(watch_uuid):
             return
         if as_error:
             target_path = os.path.join(self.datastore_path, watch_uuid, "elements-error.json")
         else:
             target_path = os.path.join(self.datastore_path, watch_uuid, "elements.json")
-        self.data['watching'][watch_uuid].ensure_data_dir_exists()
+
         with open(target_path, 'w') as f:
             f.write(json.dumps(data))
             f.close()
@@ -472,6 +472,8 @@ class ChangeDetectionStore:
         return proxy_list if len(proxy_list) else None
 
 
+
+
     def get_preferred_proxy_for_watch(self, uuid):
         """
         Returns the preferred proxy by ID key
@@ -503,105 +505,6 @@ class ChangeDetectionStore:
 
         return None
 
-    @property
-    def has_extra_headers_file(self):
-        filepath = os.path.join(self.datastore_path, 'headers.txt')
-        return os.path.isfile(filepath)
-
-    def get_all_base_headers(self):
-        from .model.App import parse_headers_from_text_file
-        headers = {}
-        # Global app settings
-        headers.update(self.data['settings'].get('headers', {}))
-
-        return headers
-
-    def get_all_headers_in_textfile_for_watch(self, uuid):
-        from .model.App import parse_headers_from_text_file
-        headers = {}
-
-        # Global in /datastore/headers.txt
-        filepath = os.path.join(self.datastore_path, 'headers.txt')
-        try:
-            if os.path.isfile(filepath):
-                headers.update(parse_headers_from_text_file(filepath))
-        except Exception as e:
-            print(f"ERROR reading headers.txt at {filepath}", str(e))
-
-        watch = self.data['watching'].get(uuid)
-        if watch:
-
-            # In /datastore/xyz-xyz/headers.txt
-            filepath = os.path.join(watch.watch_data_dir, 'headers.txt')
-            try:
-                if os.path.isfile(filepath):
-                    headers.update(parse_headers_from_text_file(filepath))
-            except Exception as e:
-                print(f"ERROR reading headers.txt at {filepath}", str(e))
-
-            # In /datastore/tag-name.txt
-            tags = self.get_all_tags_for_watch(uuid=uuid)
-            for tag_uuid, tag in tags.items():
-                fname = "headers-"+re.sub(r'[\W_]', '', tag.get('title')).lower().strip() + ".txt"
-                filepath = os.path.join(self.datastore_path, fname)
-                try:
-                    if os.path.isfile(filepath):
-                        headers.update(parse_headers_from_text_file(filepath))
-                except Exception as e:
-                    print(f"ERROR reading headers.txt at {filepath}", str(e))
-
-        return headers
-
-    def get_tag_overrides_for_watch(self, uuid, attr):
-        tags = self.get_all_tags_for_watch(uuid=uuid)
-        ret = []
-
-        if tags:
-            for tag_uuid, tag in tags.items():
-                if attr in tag and tag[attr]:
-                    ret=[*ret, *tag[attr]]
-
-        return ret
-
-    def add_tag(self, name):
-        # If name exists, return that
-        n = name.strip().lower()
-        print (f">>> Adding new tag - '{n}'")
-        if not n:
-            return False
-
-        for uuid, tag in self.__data['settings']['application'].get('tags', {}).items():
-            if n == tag.get('title', '').lower().strip():
-                print (f">>> Tag {name} already exists")
-                return uuid
-
-        # Eventually almost everything todo with a watch will apply as a Tag
-        # So we use the same model as a Watch
-        with self.lock:
-            new_tag = Watch.model(datastore_path=self.datastore_path, default={
-                'title': name.strip(),
-                'date_created': int(time.time())
-            })
-
-            new_uuid = new_tag.get('uuid')
-
-            self.__data['settings']['application']['tags'][new_uuid] = new_tag
-
-        return new_uuid
-
-    def get_all_tags_for_watch(self, uuid):
-        """This should be in Watch model but Watch doesn't have access to datastore, not sure how to solve that yet"""
-        watch = self.data['watching'].get(uuid)
-
-        # Should return a dict of full tag info linked by UUID
-        if watch:
-            return dictfilt(self.__data['settings']['application']['tags'], watch.get('tags', []))
-
-        return {}
-
-    def tag_exists_by_name(self, tag_name):
-        return any(v.get('title', '').lower() == tag_name.lower() for k, v in self.__data['settings']['application']['tags'].items())
-
     # Run all updates
     # IMPORTANT - Each update could be run even when they have a new install and the schema is correct
     #             So therefor - each `update_n` should be very careful about checking if it needs to actually run
@@ -787,16 +690,3 @@ class ChangeDetectionStore:
             i+=1
         return
 
-    # Create tag objects and their references from existing tag text
-    def update_12(self):
-        i = 0
-        for uuid, watch in self.data['watching'].items():
-            # Split out and convert old tag string
-            tag = watch.get('tag')
-            if tag:
-                tag_uuids = []
-                for t in tag.split(','):
-                    tag_uuids.append(self.add_tag(name=t))
-
-                self.data['watching'][uuid]['tags'] = tag_uuids
-

changedetectionio/templates/_common_fields.jinja

@@ -115,7 +115,7 @@
                                     URLs generated by changedetection.io (such as <code>{{ '{{diff_url}}' }}</code>) require the <code>BASE_URL</code> environment variable set.<br>
                                     Your <code>BASE_URL</code> var is currently "{{settings_application['current_base_url']}}"
 									<br>
-									Warning: Contents of <code>{{ '{{diff}}' }}</code>, <code>{{ '{{diff_removed}}' }}</code>, and <code>{{ '{{diff_added}}' }}</code> depend on how the difference algorithm perceives the change. For example, an addition or removal could be perceived as a change in some cases. <a target="_new" href="https://github.com/dgtlmoon/changedetection.io/wiki/Using-the-%7B%7Bdiff%7D%7D,-%7B%7Bdiff_added%7D%7D,-and-%7B%7Bdiff_removed%7D%7D-notification-tokens">More Here</a> <br>
+									Warning: Contents of <code>{{ '{{diff}}' }}</code>, <code>{{ '{{diff_removed}}' }}</code>, and <code>{{ '{{diff_added}}' }}</code> depend on how the difference algorithm perceives the change. For example, an addition or removal could be perceived as a change in some cases. <a target="_new" href="https://github.com/dgtlmoon/changedetection.io/wiki/Using-the-%7B%7Bdiff%7D%7D,-%7B%7Bdiff_added%7D%7D,-and-%7B%7Bdiff_removal%7D%7D-notification-tokens">More Here</a> <br>
                                 </div>
                             </div>
                         </div>

changedetectionio/templates/base.html

@@ -58,9 +58,6 @@
           {% if current_user.is_authenticated or not has_password %}
             {% if not
             current_diff_url %}
-              <li class="pure-menu-item">
-                <a href="{{ url_for('tags.tags_overview_page')}}" class="pure-menu-link">GROUPS</a>
-              </li>
               <li class="pure-menu-item">
                 <a href="{{ url_for('settings_page')}}" class="pure-menu-link">SETTINGS</a>
               </li>
@@ -85,18 +82,11 @@
               <a href="{{url_for('logout')}}" class="pure-menu-link">LOG OUT</a>
             </li>
           {% endif %}
-          <li class="pure-menu-item pure-form" id="search-menu-item">
-            <!-- We use GET here so it offers people a chance to set bookmarks etc -->
-            <form name="searchForm" action="" method="GET">
-              <input id="search-q" class="" name="q" placeholder="URL or Title {% if active_tag %}in '{{ active_tag }}'{% endif %}" required="" type="text" value="">
-              <input name="tags" type="hidden" value="{% if active_tag %}{{active_tag}}{% endif %}">
-              <button class="toggle-button " id="toggle-search" type="button" title="Search, or Use Alt+S Key" >
-                {% include "svgs/search-icon.svg" %}
-              </button>
-            </form>
-          </li>
           <li class="pure-menu-item">
-            <button class="toggle-button" id ="toggle-light-mode" type="button" title="Toggle Light/Dark Mode">
+            {% if dark_mode %}
+            {% set darkClass = 'dark' %}
+            {% endif %}
+            <button class="toggle-theme {{darkClass}}" type="button" title="Toggle Light/Dark Mode">
               <span class="visually-hidden">Toggle light/dark mode</span>
               <span class="icon-light">
                 {% include "svgs/light-mode-toggle-icon.svg" %}
@@ -116,7 +106,7 @@
     </div>
     {% if hosted_sticky %}
       <div class="sticky-tab" id="hosted-sticky">
-        <a href="https://changedetection.io/?ref={{guid}}">Let us host your instance!</a>
+        <a href="https://lemonade.changedetection.io/start?ref={{guid}}">Let us host your instance!</a>
       </div>
     {% endif %}
     {% if left_sticky %}

changedetectionio/templates/edit.html

@@ -14,9 +14,7 @@
 {% endif %}
 
     const browser_steps_config=JSON.parse('{{ browser_steps_config|tojson }}');
-    const browser_steps_start_url="{{url_for('browser_steps.browsersteps_start_session', uuid=uuid)}}";
     const browser_steps_sync_url="{{url_for('browser_steps.browsersteps_ui_update', uuid=uuid)}}";
-
 </script>
 
 <script src="{{url_for('static_content', group='js', filename='watch-settings.js')}}" defer></script>
@@ -75,7 +73,7 @@
                         {{ render_field(form.title, class="m-d") }}
                     </div>
                     <div class="pure-control-group">
-                        {{ render_field(form.tags) }}
+                        {{ render_field(form.tag) }}
                         <span class="pure-form-message-inline">Organisational tag/group name used in the main listing page</span>
                     </div>
                     <div class="pure-control-group">
@@ -152,17 +150,6 @@
 {{ render_field(form.headers, rows=5, placeholder="Example
 Cookie: foobar
 User-Agent: wonderbra 1.0") }}
-
-                        <div class="pure-form-message-inline">
-                            {% if has_extra_headers_file %}
-                                <strong>Alert! Extra headers file found and will be added to this watch!</strong>
-                            {% else %}
-                                Headers can be also read from a file in your data-directory <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Adding-headers-from-an-external-file">Read more here</a>
-                            {% endif %}
-                            <br>
-                            (Not supported by Selenium browser)
-                        </div>
-
                     </div>
                     <div class="pure-control-group" id="request-body">
                                         {{ render_field(form.body, rows=5, placeholder="Example
@@ -199,8 +186,7 @@ User-Agent: wonderbra 1.0") }}
                                     <span class="loader" >
                                         <span id="browsersteps-click-start">
                                             <h2 >Click here to Start</h2>
-                                            <svg style="height: 3.5rem;" version="1.1" viewBox="0 0 32 32"  xml:space="preserve" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink"><g id="Layer_1"/><g id="play_x5F_alt"><path d="M16,0C7.164,0,0,7.164,0,16s7.164,16,16,16s16-7.164,16-16S24.836,0,16,0z M10,24V8l16.008,8L10,24z" style="fill: var(--color-grey-400);"/></g></svg><br>
-                                            Please allow 10-15 seconds for the browser to connect.<br>
+                                            Please allow 10-15 seconds for the browser to connect.
                                         </span>
                                         <div class="spinner"  style="display: none;"></div>
                                     </span>

changedetectionio/templates/settings.html

@@ -70,10 +70,6 @@
                             <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Configurable-BASE_URL-setting">read more here</a>.
                         </span>
                     </div>
-                    <div class="pure-control-group">
-                        {{ render_field(form.application.form.pager_size) }}
-                        <span class="pure-form-message-inline">Number of items per page in the watch overview list, 0 to disable.</span>
-                    </div>
 
                     <div class="pure-control-group">
                         {{ render_checkbox_field(form.application.form.extract_title_as_title) }}

changedetectionio/templates/svgs/search-icon.svg

@@ -1 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?><svg version="1.1" id="Layer_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 0 122.879 119.799" enable-background="new 0 0 122.879 119.799" xml:space="preserve"><g><path d="M49.988,0h0.016v0.007C63.803,0.011,76.298,5.608,85.34,14.652c9.027,9.031,14.619,21.515,14.628,35.303h0.007v0.033v0.04 h-0.007c-0.005,5.557-0.917,10.905-2.594,15.892c-0.281,0.837-0.575,1.641-0.877,2.409v0.007c-1.446,3.66-3.315,7.12-5.547,10.307 l29.082,26.139l0.018,0.016l0.157,0.146l0.011,0.011c1.642,1.563,2.536,3.656,2.649,5.78c0.11,2.1-0.543,4.248-1.979,5.971 l-0.011,0.016l-0.175,0.203l-0.035,0.035l-0.146,0.16l-0.016,0.021c-1.565,1.642-3.654,2.534-5.78,2.646 c-2.097,0.111-4.247-0.54-5.971-1.978l-0.015-0.011l-0.204-0.175l-0.029-0.024L78.761,90.865c-0.88,0.62-1.778,1.209-2.687,1.765 c-1.233,0.755-2.51,1.466-3.813,2.115c-6.699,3.342-14.269,5.222-22.272,5.222v0.007h-0.016v-0.007 c-13.799-0.004-26.296-5.601-35.338-14.645C5.605,76.291,0.016,63.805,0.007,50.021H0v-0.033v-0.016h0.007 c0.004-13.799,5.601-26.296,14.645-35.338C23.683,5.608,36.167,0.016,49.955,0.007V0H49.988L49.988,0z M50.004,11.21v0.007h-0.016 h-0.033V11.21c-10.686,0.007-20.372,4.35-27.384,11.359C15.56,29.578,11.213,39.274,11.21,49.973h0.007v0.016v0.033H11.21 c0.007,10.686,4.347,20.367,11.359,27.381c7.009,7.012,16.705,11.359,27.403,11.361v-0.007h0.016h0.033v0.007 c10.686-0.007,20.368-4.348,27.382-11.359c7.011-7.009,11.358-16.702,11.36-27.4h-0.006v-0.016v-0.033h0.006 c-0.006-10.686-4.35-20.372-11.358-27.384C70.396,15.56,60.703,11.213,50.004,11.21L50.004,11.21z"/></g></svg>

changedetectionio/templates/watch-overview.html

@@ -13,7 +13,7 @@
             <div id="watch-add-wrapper-zone">
                 <div>
                     {{ render_simple_field(form.url, placeholder="https://...", required=true) }}
-                    {{ render_simple_field(form.tags, value=tags[active_tag].title if active_tag else '', placeholder="watch label / tag") }}
+                    {{ render_simple_field(form.tag, value=active_tag if active_tag else '', placeholder="watch label / tag") }}
                 </div>
                 <div>
                     {{ render_simple_field(form.watch_submit_button, title="Watch this URL!" ) }}
@@ -30,14 +30,12 @@
 
     <form class="pure-form" action="{{ url_for('form_watch_list_checkbox_operations') }}" method="POST" id="watch-list-form">
     <input type="hidden" name="csrf_token" value="{{ csrf_token() }}" >
-    <input type="hidden" id="op_extradata" name="op_extradata" value="" >
     <div id="checkbox-operations">
         <button class="pure-button button-secondary button-xsmall"  name="op" value="pause">Pause</button>
         <button class="pure-button button-secondary button-xsmall"  name="op" value="unpause">UnPause</button>
         <button class="pure-button button-secondary button-xsmall"  name="op" value="mute">Mute</button>
         <button class="pure-button button-secondary button-xsmall"  name="op" value="unmute">UnMute</button>
         <button class="pure-button button-secondary button-xsmall" name="op" value="recheck">Recheck</button>
-        <button class="pure-button button-secondary button-xsmall" name="op" value="assign-tag" id="checkbox-assign-tag">Tag</button>
         <button class="pure-button button-secondary button-xsmall" name="op" value="mark-viewed">Mark viewed</button>
         <button class="pure-button button-secondary button-xsmall" name="op" value="notification-default">Use default notification</button>
         <button class="pure-button button-secondary button-xsmall" style="background: #dd4242;" name="op" value="clear-history">Clear/reset history</button>
@@ -46,12 +44,11 @@
     {% if watches|length >= pagination.per_page %}
         {{ pagination.info }}
     {% endif %}
-    {% if search_q %}<div id="search-result-info">Searching "<strong><i>{{search_q}}</i></strong>"</div>{% endif %}
     <div>
         <a href="{{url_for('index')}}" class="pure-button button-tag {{'active' if not active_tag }}">All</a>
-        {% for uuid, tag in tags.items() %}
+        {% for tag in tags %}
             {% if tag != "" %}
-                <a href="{{url_for('index', tag=uuid) }}" class="pure-button button-tag {{'active' if active_tag == uuid }}">{{ tag.title }}</a>
+                <a href="{{url_for('index', tag=tag) }}" class="pure-button button-tag {{'active' if active_tag == tag }}">{{ tag }}</a>
             {% endif %}
         {% endfor %}
     </div>
@@ -76,12 +73,8 @@
             </tr>
             </thead>
             <tbody>
-            {% if not watches|length %}
-            <tr>
-                <td colspan="6">No website watches configured, please add a URL in the box above, or <a href="{{ url_for('import_page')}}" >import a list</a>.</td>
-            </tr>
-            {% endif %}
-            {% for watch in (watches|sort(attribute=sort_attribute, reverse=sort_order == 'asc'))|pagination_slice(skip=pagination.skip) %}
+
+            {% for watch in (watches|sort(attribute=sort_attribute, reverse=sort_order == 'asc'))[pagination.skip:pagination.skip+pagination.per_page] %}
             <tr id="{{ watch.uuid }}"
                 class="{{ loop.cycle('pure-table-odd', 'pure-table-even') }} processor-{{ watch['processor'] }}
                 {% if watch.last_error is defined and watch.last_error != False %}error{% endif %}
@@ -145,11 +138,9 @@
                     </span>
                     {% endif %}
 
-
-                    {% for watch_tag_uuid, watch_tag in datastore.get_all_tags_for_watch(watch['uuid']).items() %}
-                      <span class="watch-tag-list">{{ watch_tag.title }}</span>
-                    {% endfor %}
-
+                    {% if not active_tag %}
+                    <span class="watch-tag-list">{{ watch.tag}}</span>
+                    {% endif %}
                 </td>
                 <td class="last-checked">{{watch|format_last_checked_time|safe}}</td>
                 <td class="last-changed">{% if watch.history_n >=2 and watch.last_changed >0 %}
@@ -182,7 +173,7 @@
             {% endif %}
             <li>
                <a href="{{ url_for('form_watch_checknow', tag=active_tag) }}" class="pure-button button-tag ">Recheck
-                all {% if active_tag%} in "{{tags[active_tag].title}}"{%endif%}</a>
+                all {% if active_tag%}in "{{active_tag}}"{%endif%}</a>
             </li>
             <li>
                 <a href="{{ url_for('rss', tag=active_tag , token=app_rss_token)}}"><img alt="RSS Feed" id="feed-icon" src="{{url_for('static_content', group='images', filename='Generic_Feed-icon.svg')}}" height="15"></a>

changedetectionio/tests/conftest.py

@@ -14,16 +14,13 @@ global app
 
 def cleanup(datastore_path):
     # Unlink test output files
-    files = [
-        'count.txt',
-        'endpoint-content.txt'
-        'headers.txt',
-        'headers-testtag.txt',
-        'notification.txt',
-        'secret.txt',
-        'url-watches.json',
-        'output.txt',
-    ]
+    files = ['output.txt',
+             'url-watches.json',
+             'secret.txt',
+             'notification.txt',
+             'count.txt',
+             'endpoint-content.txt'
+                 ]
     for file in files:
         try:
             os.unlink("{}/{}".format(datastore_path, file))

changedetectionio/tests/proxy_list/test_multiple_proxy.py

@@ -28,7 +28,7 @@ def test_preferred_proxy(client, live_server):
                 "fetch_backend": "html_requests",
                 "headers": "",
                 "proxy": "proxy-two",
-                "tags": "",
+                "tag": "",
                 "url": url,
               },
         follow_redirects=True

changedetectionio/tests/restock/test_restock.py

@@ -77,7 +77,7 @@ def test_restock_detection(client, live_server):
 
     client.post(
         url_for("form_quick_watch_add"),
-        data={"url": test_url, "tags": '', 'processor': 'restock_diff'},
+        data={"url": test_url, "tag": '', 'processor': 'restock_diff'},
         follow_redirects=True
     )
 

changedetectionio/tests/test_access_control.py

@@ -45,15 +45,6 @@ def test_check_access_control(app, client, live_server):
         res = client.get(url_for("diff_history_page", uuid="first"))
         assert b'Random content' in res.data
 
-        # Check wrong password does not let us in
-        res = c.post(
-            url_for("login"),
-            data={"password": "WRONG PASSWORD"},
-            follow_redirects=True
-        )
-
-        assert b"LOG OUT" not in res.data
-        assert b"Incorrect password" in res.data
 
 
         # Menu should not be available yet

changedetectionio/tests/test_add_replace_remove_filter.py

@@ -2,7 +2,7 @@
 
 import time
 from flask import url_for
-from .util import live_server_setup, wait_for_all_checks
+from .util import live_server_setup
 from changedetectionio import html_tools
 
 
@@ -39,6 +39,7 @@ def test_setup(client, live_server):
     live_server_setup(live_server)
 
 def test_check_removed_line_contains_trigger(client, live_server):
+    sleep_time_for_fetch_thread = 3
 
     # Give the endpoint time to spin up
     time.sleep(1)
@@ -53,7 +54,7 @@ def test_check_removed_line_contains_trigger(client, live_server):
     assert b"1 Imported" in res.data
 
     # Give the thread time to pick it up
-    wait_for_all_checks(client)
+    time.sleep(sleep_time_for_fetch_thread)
 
     # Goto the edit page, add our ignore text
     # Add our URL to the import page
@@ -66,20 +67,20 @@ def test_check_removed_line_contains_trigger(client, live_server):
         follow_redirects=True
     )
     assert b"Updated watch." in res.data
-    wait_for_all_checks(client)
+    time.sleep(sleep_time_for_fetch_thread)
     set_original(excluding='Something irrelevant')
 
     # A line thats not the trigger should not trigger anything
     res = client.get(url_for("form_watch_checknow"), follow_redirects=True)
     assert b'1 watches queued for rechecking.' in res.data
-    wait_for_all_checks(client)
+    time.sleep(sleep_time_for_fetch_thread)
     res = client.get(url_for("index"))
     assert b'unviewed' not in res.data
 
     # The trigger line is REMOVED,  this should trigger
     set_original(excluding='The golden line')
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    wait_for_all_checks(client)
+    time.sleep(sleep_time_for_fetch_thread)
     res = client.get(url_for("index"))
     assert b'unviewed' in res.data
 
@@ -88,14 +89,14 @@ def test_check_removed_line_contains_trigger(client, live_server):
     client.get(url_for("mark_all_viewed"), follow_redirects=True)
     set_original(excluding=None)
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    wait_for_all_checks(client)
+    time.sleep(sleep_time_for_fetch_thread)
     res = client.get(url_for("index"))
     assert b'unviewed' not in res.data
 
     # Remove it again, and we should get a trigger
     set_original(excluding='The golden line')
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    wait_for_all_checks(client)
+    time.sleep(sleep_time_for_fetch_thread)
     res = client.get(url_for("index"))
     assert b'unviewed' in res.data
 
@@ -104,7 +105,8 @@ def test_check_removed_line_contains_trigger(client, live_server):
 
 
 def test_check_add_line_contains_trigger(client, live_server):
-    #live_server_setup(live_server)
+
+    sleep_time_for_fetch_thread = 3
 
     # Give the endpoint time to spin up
     time.sleep(1)
@@ -134,7 +136,8 @@ def test_check_add_line_contains_trigger(client, live_server):
     assert b"1 Imported" in res.data
 
     # Give the thread time to pick it up
-    wait_for_all_checks(client)
+    time.sleep(sleep_time_for_fetch_thread)
+
     # Goto the edit page, add our ignore text
     # Add our URL to the import page
     res = client.post(
@@ -147,25 +150,23 @@ def test_check_add_line_contains_trigger(client, live_server):
         follow_redirects=True
     )
     assert b"Updated watch." in res.data
-    wait_for_all_checks(client)
+    time.sleep(sleep_time_for_fetch_thread)
     set_original(excluding='Something irrelevant')
 
     # A line thats not the trigger should not trigger anything
     res = client.get(url_for("form_watch_checknow"), follow_redirects=True)
     assert b'1 watches queued for rechecking.' in res.data
-    wait_for_all_checks(client)
+    time.sleep(sleep_time_for_fetch_thread)
     res = client.get(url_for("index"))
     assert b'unviewed' not in res.data
 
     # The trigger line is ADDED,  this should trigger
     set_original(add_line='<p>Oh yes please</p>')
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    wait_for_all_checks(client)
+    time.sleep(sleep_time_for_fetch_thread)
     res = client.get(url_for("index"))
     assert b'unviewed' in res.data
 
-    # Takes a moment for apprise to fire
-    time.sleep(3)
     with open("test-datastore/notification.txt", 'r') as f:
         response= f.read()
         assert '-Oh yes please-' in response

changedetectionio/tests/test_api.py

@@ -2,7 +2,7 @@
 
 import time
 from flask import url_for
-from .util import live_server_setup, extract_api_key_from_UI, wait_for_all_checks
+from .util import live_server_setup, extract_api_key_from_UI
 
 import json
 import uuid
@@ -57,7 +57,6 @@ def test_setup(client, live_server):
     live_server_setup(live_server)
 
 def test_api_simple(client, live_server):
-    #live_server_setup(live_server)
 
     api_key = extract_api_key_from_UI(client)
 
@@ -87,7 +86,7 @@ def test_api_simple(client, live_server):
     watch_uuid = res.json.get('uuid')
     assert res.status_code == 201
 
-    wait_for_all_checks(client)
+    time.sleep(3)
 
     # Verify its in the list and that recheck worked
     res = client.get(
@@ -108,7 +107,7 @@ def test_api_simple(client, live_server):
     )
     assert len(res.json) == 0
 
-    wait_for_all_checks(client)
+    time.sleep(2)
 
     set_modified_response()
     # Trigger recheck of all ?recheck_all=1
@@ -116,7 +115,7 @@ def test_api_simple(client, live_server):
         url_for("createwatch", recheck_all='1'),
         headers={'x-api-key': api_key},
     )
-    wait_for_all_checks(client)
+    time.sleep(3)
 
     # Did the recheck fire?
     res = client.get(
@@ -267,7 +266,7 @@ def test_api_watch_PUT_update(client, live_server):
 
     #live_server_setup(live_server)
     api_key = extract_api_key_from_UI(client)
-
+    time.sleep(1)
     # Create a watch
     set_original_response()
     test_url = url_for('test_endpoint', _external=True,
@@ -283,6 +282,7 @@ def test_api_watch_PUT_update(client, live_server):
 
     assert res.status_code == 201
 
+    time.sleep(1)
 
     # Get a listing, it will be the first one
     res = client.get(
@@ -297,8 +297,6 @@ def test_api_watch_PUT_update(client, live_server):
         url_for("edit_page", uuid=watch_uuid),
     )
     assert b"cookie: yum" in res.data, "'cookie: yum' found in 'headers' section"
-    assert b"One" in res.data, "Tag 'One' was found"
-    assert b"Two" in res.data, "Tag 'Two' was found"
 
     # HTTP PUT ( UPDATE an existing watch )
     res = client.put(
@@ -321,8 +319,7 @@ def test_api_watch_PUT_update(client, live_server):
     )
     assert b"new title" in res.data, "new title found in edit page"
     assert b"552" in res.data, "552 minutes found in edit page"
-    assert b"One" in res.data, "Tag 'One' was found"
-    assert b"Two" in res.data, "Tag 'Two' was found"
+    assert b"One, Two" in res.data, "Tag 'One, Two' was found"
     assert b"cookie: all eaten" in res.data, "'cookie: all eaten' found in 'headers' section"
 
     ######################################################

changedetectionio/tests/test_auth.py

@@ -24,7 +24,7 @@ def test_basic_auth(client, live_server):
     # Check form validation
     res = client.post(
         url_for("edit_page", uuid="first"),
-        data={"include_filters": "", "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests"},
+        data={"include_filters": "", "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
         follow_redirects=True
     )
     assert b"Updated watch." in res.data

changedetectionio/tests/test_block_while_text_present.py

@@ -2,7 +2,7 @@
 
 import time
 from flask import url_for
-from .util import live_server_setup, wait_for_all_checks
+from . util import live_server_setup
 from changedetectionio import html_tools
 
 def set_original_ignore_response():
@@ -61,7 +61,7 @@ def set_modified_response_minus_block_text():
 
 
 def test_check_block_changedetection_text_NOT_present(client, live_server):
-
+    sleep_time_for_fetch_thread = 3
     live_server_setup(live_server)
     # Use a mix of case in ZzZ to prove it works case-insensitive.
     ignore_text = "out of stoCk\r\nfoobar"
@@ -81,7 +81,7 @@ def test_check_block_changedetection_text_NOT_present(client, live_server):
     assert b"1 Imported" in res.data
 
     # Give the thread time to pick it up
-    wait_for_all_checks(client)
+    time.sleep(sleep_time_for_fetch_thread)
 
     # Goto the edit page, add our ignore text
     # Add our URL to the import page
@@ -96,7 +96,7 @@ def test_check_block_changedetection_text_NOT_present(client, live_server):
     assert b"Updated watch." in res.data
 
     # Give the thread time to pick it up
-    wait_for_all_checks(client)
+    time.sleep(sleep_time_for_fetch_thread)
     # Check it saved
     res = client.get(
         url_for("edit_page", uuid="first"),
@@ -107,7 +107,7 @@ def test_check_block_changedetection_text_NOT_present(client, live_server):
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
-    wait_for_all_checks(client)
+    time.sleep(sleep_time_for_fetch_thread)
 
     # It should report nothing found (no new 'unviewed' class)
     res = client.get(url_for("index"))
@@ -120,7 +120,7 @@ def test_check_block_changedetection_text_NOT_present(client, live_server):
     # Trigger a check
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
     # Give the thread time to pick it up
-    wait_for_all_checks(client)
+    time.sleep(sleep_time_for_fetch_thread)
 
     # It should report nothing found (no new 'unviewed' class)
     res = client.get(url_for("index"))
@@ -131,7 +131,7 @@ def test_check_block_changedetection_text_NOT_present(client, live_server):
     # Now we set a change where the text is gone, it should now trigger
     set_modified_response_minus_block_text()
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    wait_for_all_checks(client)
+    time.sleep(sleep_time_for_fetch_thread)
     res = client.get(url_for("index"))
     assert b'unviewed' in res.data
 

changedetectionio/tests/test_css_selector.py

@@ -96,7 +96,7 @@ def test_check_markup_include_filters_restriction(client, live_server):
     # Add our URL to the import page
     res = client.post(
         url_for("edit_page", uuid="first"),
-        data={"include_filters": include_filters, "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests"},
+        data={"include_filters": include_filters, "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
         follow_redirects=True
     )
     assert b"Updated watch." in res.data
@@ -157,7 +157,7 @@ def test_check_multiple_filters(client, live_server):
         url_for("edit_page", uuid="first"),
         data={"include_filters": include_filters,
               "url": test_url,
-              "tags": "",
+              "tag": "",
               "headers": "",
               'fetch_backend': "html_requests"},
         follow_redirects=True

changedetectionio/tests/test_element_removal.py

@@ -129,7 +129,7 @@ def test_element_removal_full(client, live_server):
         data={
             "subtractive_selectors": subtractive_selectors_data,
             "url": test_url,
-            "tags": "",
+            "tag": "",
             "headers": "",
             "fetch_backend": "html_requests",
         },

changedetectionio/tests/test_extract_regex.py

@@ -91,7 +91,7 @@ def test_check_filter_multiline(client, live_server):
         data={"include_filters": '',
               'extract_text': '/something.+?6 billion.+?lines/si',
               "url": test_url,
-              "tags": "",
+              "tag": "",
               "headers": "",
               'fetch_backend': "html_requests"
               },
@@ -146,7 +146,7 @@ def test_check_filter_and_regex_extract(client, live_server):
         data={"include_filters": include_filters,
               'extract_text': '\d+ online\r\n\d+ guests\r\n/somecase insensitive \d+/i\r\n/somecase insensitive (345\d)/i',
               "url": test_url,
-              "tags": "",
+              "tag": "",
               "headers": "",
               'fetch_backend': "html_requests"
               },

changedetectionio/tests/test_filter_exist_changes.py

@@ -56,7 +56,7 @@ def test_filter_doesnt_exist_then_exists_should_get_notification(client, live_se
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
         url_for("form_quick_watch_add"),
-        data={"url": test_url, "tags": 'cinema'},
+        data={"url": test_url, "tag": 'cinema'},
         follow_redirects=True
     )
     assert b"Watch added" in res.data
@@ -89,7 +89,7 @@ def test_filter_doesnt_exist_then_exists_should_get_notification(client, live_se
 
     notification_form_data.update({
         "url": test_url,
-        "tags": "my tag",
+        "tag": "my tag",
         "title": "my title",
         "headers": "",
         "include_filters": '.ticket-available',

changedetectionio/tests/test_filter_failure_notification.py

@@ -1,7 +1,7 @@
 import os
 import time
 from flask import url_for
-from .util import set_original_response, live_server_setup, extract_UUID_from_client, wait_for_all_checks
+from .util import set_original_response, live_server_setup, extract_UUID_from_client
 from changedetectionio.model import App
 
 
@@ -37,14 +37,14 @@ def run_filter_test(client, content_filter):
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
         url_for("form_quick_watch_add"),
-        data={"url": test_url, "tags": ''},
+        data={"url": test_url, "tag": ''},
         follow_redirects=True
     )
 
     assert b"Watch added" in res.data
 
     # Give the thread time to pick up the first version
-    wait_for_all_checks(client)
+    time.sleep(3)
 
     # Goto the edit page, add our ignore text
     # Add our URL to the import page
@@ -71,8 +71,8 @@ def run_filter_test(client, content_filter):
 
     notification_form_data.update({
         "url": test_url,
-        "tags": "my tag",
-        "title": "my title 123",
+        "tag": "my tag",
+        "title": "my title",
         "headers": "",
         "filter_failure_notification_send": 'y',
         "include_filters": content_filter,
@@ -85,55 +85,43 @@ def run_filter_test(client, content_filter):
     )
 
     assert b"Updated watch." in res.data
-    wait_for_all_checks(client)
+    time.sleep(3)
 
     # Now the notification should not exist, because we didnt reach the threshold
     assert not os.path.isfile("test-datastore/notification.txt")
 
-    # -2 because we would have checked twice above (on adding and on edit)
-    for i in range(0, App._FILTER_FAILURE_THRESHOLD_ATTEMPTS_DEFAULT-2):
+    for i in range(0, App._FILTER_FAILURE_THRESHOLD_ATTEMPTS_DEFAULT):
         res = client.get(url_for("form_watch_checknow"), follow_redirects=True)
-        wait_for_all_checks(client)
-        assert not os.path.isfile("test-datastore/notification.txt"), f"test-datastore/notification.txt should not exist - Attempt {i}"
+        time.sleep(3)
 
     # We should see something in the frontend
     assert b'Warning, no filters were found' in res.data
 
-    # One more check should trigger it (see -2 above)
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    wait_for_all_checks(client)
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    wait_for_all_checks(client)
     # Now it should exist and contain our "filter not found" alert
     assert os.path.isfile("test-datastore/notification.txt")
-
+    notification = False
     with open("test-datastore/notification.txt", 'r') as f:
         notification = f.read()
-
     assert 'CSS/xPath filter was not present in the page' in notification
     assert content_filter.replace('"', '\\"') in notification
 
-    # Remove it and prove that it doesn't trigger when not expected
-    # It should register a change, but no 'filter not found'
+    # Remove it and prove that it doesnt trigger when not expected
     os.unlink("test-datastore/notification.txt")
     set_response_with_filter()
 
-    # Try several times, it should NOT have 'filter not found'
     for i in range(0, App._FILTER_FAILURE_THRESHOLD_ATTEMPTS_DEFAULT):
         client.get(url_for("form_watch_checknow"), follow_redirects=True)
-        wait_for_all_checks(client)
+        time.sleep(3)
 
     # It should have sent a notification, but..
     assert os.path.isfile("test-datastore/notification.txt")
-    # but it should not contain the info about a failed filter (because there was none in this case)
+    # but it should not contain the info about the failed filter
     with open("test-datastore/notification.txt", 'r') as f:
         notification = f.read()
     assert not 'CSS/xPath filter was not present in the page' in notification
 
-    # Re #1247 - All tokens got replaced correctly in the notification
-    res = client.get(url_for("index"))
+    # Re #1247 - All tokens got replaced
     uuid = extract_UUID_from_client(client)
-    # UUID is correct, but notification contains tag uuid as UUIID wtf
     assert uuid in notification
 
     # cleanup for the next
@@ -149,7 +137,7 @@ def test_setup(live_server):
 
 def test_check_include_filters_failure_notification(client, live_server):
     set_original_response()
-    wait_for_all_checks(client)
+    time.sleep(1)
     run_filter_test(client, '#nope-doesnt-exist')
 
 def test_check_xpath_filter_failure_notification(client, live_server):

changedetectionio/tests/test_group.py

@@ -1,322 +0,0 @@
-#!/usr/bin/python3
-
-import time
-from flask import url_for
-from .util import live_server_setup, wait_for_all_checks, extract_rss_token_from_UI, get_UUID_for_tag_name, extract_UUID_from_client
-import os
-
-
-def test_setup(client, live_server):
-    live_server_setup(live_server)
-
-def set_original_response():
-    test_return_data = """<html>
-       <body>
-     Some initial text<br>
-     <p id="only-this">Should be only this</p>
-     <br>
-     <p id="not-this">And never this</p>     
-     </body>
-     </html>
-    """
-
-    with open("test-datastore/endpoint-content.txt", "w") as f:
-        f.write(test_return_data)
-    return None
-
-def set_modified_response():
-    test_return_data = """<html>
-       <body>
-     Some initial text<br>
-     <p id="only-this">Should be REALLY only this</p>
-     <br>
-     <p id="not-this">And never this</p>     
-     </body>
-     </html>
-    """
-
-    with open("test-datastore/endpoint-content.txt", "w") as f:
-        f.write(test_return_data)
-    return None
-
-def test_setup_group_tag(client, live_server):
-    #live_server_setup(live_server)
-    set_original_response()
-
-    # Add a tag with some config, import a tag and it should roughly work
-    res = client.post(
-        url_for("tags.form_tag_add"),
-        data={"name": "test-tag"},
-        follow_redirects=True
-    )
-    assert b"Tag added" in res.data
-    assert b"test-tag" in res.data
-
-    res = client.post(
-        url_for("tags.form_tag_edit_submit", uuid="first"),
-        data={"name": "test-tag",
-              "include_filters": '#only-this',
-              "subtractive_selectors": '#not-this'},
-        follow_redirects=True
-    )
-    assert b"Updated" in res.data
-    tag_uuid = get_UUID_for_tag_name(client, name="test-tag")
-    res = client.get(
-        url_for("tags.form_tag_edit", uuid="first")
-    )
-    assert b"#only-this" in res.data
-    assert b"#not-this" in res.data
-
-    # Tag should be setup and ready, now add a watch
-
-    test_url = url_for('test_endpoint', _external=True)
-    res = client.post(
-        url_for("import_page"),
-        data={"urls": test_url + "?first-imported=1 test-tag, extra-import-tag"},
-        follow_redirects=True
-    )
-    assert b"1 Imported" in res.data
-
-    res = client.get(url_for("index"))
-    assert b'import-tag' in res.data
-    assert b'extra-import-tag' in res.data
-
-    res = client.get(
-        url_for("tags.tags_overview_page"),
-        follow_redirects=True
-    )
-    assert b'import-tag' in res.data
-    assert b'extra-import-tag' in res.data
-
-    wait_for_all_checks(client)
-
-    res = client.get(url_for("index"))
-    assert b'Warning, no filters were found' not in res.data
-
-    res = client.get(
-        url_for("preview_page", uuid="first"),
-        follow_redirects=True
-    )
-    assert b'Should be only this' in res.data
-    assert b'And never this' not in res.data
-
-
-    # RSS Group tag filter
-    # An extra one that should be excluded
-    res = client.post(
-        url_for("import_page"),
-        data={"urls": test_url + "?should-be-excluded=1 some-tag"},
-        follow_redirects=True
-    )
-    assert b"1 Imported" in res.data
-    wait_for_all_checks(client)
-    set_modified_response()
-    res = client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    wait_for_all_checks(client)
-    rss_token = extract_rss_token_from_UI(client)
-    res = client.get(
-        url_for("rss", token=rss_token, tag="extra-import-tag", _external=True),
-        follow_redirects=True
-    )
-    assert b"should-be-excluded" not in res.data
-    assert res.status_code == 200
-    assert b"first-imported=1" in res.data
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
-    assert b'Deleted' in res.data
-
-def test_tag_import_singular(client, live_server):
-    #live_server_setup(live_server)
-
-    test_url = url_for('test_endpoint', _external=True)
-    res = client.post(
-        url_for("import_page"),
-        data={"urls": test_url + " test-tag, test-tag\r\n"+ test_url + "?x=1 test-tag, test-tag\r\n"},
-        follow_redirects=True
-    )
-    assert b"2 Imported" in res.data
-
-    res = client.get(
-        url_for("tags.tags_overview_page"),
-        follow_redirects=True
-    )
-    # Should be only 1 tag because they both had the same
-    assert res.data.count(b'test-tag') == 1
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
-    assert b'Deleted' in res.data
-
-def test_tag_add_in_ui(client, live_server):
-    #live_server_setup(live_server)
-#
-    res = client.post(
-        url_for("tags.form_tag_add"),
-        data={"name": "new-test-tag"},
-        follow_redirects=True
-    )
-    assert b"Tag added" in res.data
-    assert b"new-test-tag" in res.data
-
-    res = client.get(url_for("tags.delete_all"), follow_redirects=True)
-    assert b'All tags deleted' in res.data
-
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
-    assert b'Deleted' in res.data
-
-def test_group_tag_notification(client, live_server):
-    #live_server_setup(live_server)
-    set_original_response()
-
-    test_url = url_for('test_endpoint', _external=True)
-    res = client.post(
-        url_for("form_quick_watch_add"),
-        data={"url": test_url, "tags": 'test-tag, other-tag'},
-        follow_redirects=True
-    )
-
-    assert b"Watch added" in res.data
-
-    notification_url = url_for('test_notification_endpoint', _external=True).replace('http', 'json')
-    notification_form_data = {"notification_urls": notification_url,
-                              "notification_title": "New GROUP TAG ChangeDetection.io Notification - {{watch_url}}",
-                              "notification_body": "BASE URL: {{base_url}}\n"
-                                                   "Watch URL: {{watch_url}}\n"
-                                                   "Watch UUID: {{watch_uuid}}\n"
-                                                   "Watch title: {{watch_title}}\n"
-                                                   "Watch tag: {{watch_tag}}\n"
-                                                   "Preview: {{preview_url}}\n"
-                                                   "Diff URL: {{diff_url}}\n"
-                                                   "Snapshot: {{current_snapshot}}\n"
-                                                   "Diff: {{diff}}\n"
-                                                   "Diff Added: {{diff_added}}\n"
-                                                   "Diff Removed: {{diff_removed}}\n"
-                                                   "Diff Full: {{diff_full}}\n"
-                                                   ":-)",
-                              "notification_screenshot": True,
-                              "notification_format": "Text",
-                              "title": "test-tag"}
-
-    res = client.post(
-        url_for("tags.form_tag_edit_submit", uuid=get_UUID_for_tag_name(client, name="test-tag")),
-        data=notification_form_data,
-        follow_redirects=True
-    )
-    assert b"Updated" in res.data
-
-    wait_for_all_checks(client)
-
-    set_modified_response()
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    time.sleep(3)
-
-    assert os.path.isfile("test-datastore/notification.txt")
-
-    # Verify what was sent as a notification, this file should exist
-    with open("test-datastore/notification.txt", "r") as f:
-        notification_submission = f.read()
-    os.unlink("test-datastore/notification.txt")
-
-    # Did we see the URL that had a change, in the notification?
-    # Diff was correctly executed
-    assert test_url in notification_submission
-    assert ':-)' in notification_submission
-    assert "Diff Full: Some initial text" in notification_submission
-    assert "New GROUP TAG ChangeDetection.io" in notification_submission
-    assert "test-tag" in notification_submission
-    assert "other-tag" in notification_submission
-
-    #@todo Test that multiple notifications fired
-    #@todo Test that each of multiple notifications with different settings
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
-    assert b'Deleted' in res.data
-
-def test_limit_tag_ui(client, live_server):
-    #live_server_setup(live_server)
-
-    test_url = url_for('test_endpoint', _external=True)
-    urls=[]
-
-    for i in range(20):
-        urls.append(test_url+"?x="+str(i)+" test-tag")
-
-    for i in range(20):
-        urls.append(test_url+"?non-grouped="+str(i))
-
-    res = client.post(
-        url_for("import_page"),
-        data={"urls": "\r\n".join(urls)},
-        follow_redirects=True
-    )
-
-    assert b"40 Imported" in res.data
-
-    res = client.get(url_for("index"))
-    assert b'test-tag' in res.data
-
-    # All should be here
-    assert res.data.count(b'processor-text_json_diff') == 40
-
-    tag_uuid = get_UUID_for_tag_name(client, name="test-tag")
-
-    res = client.get(url_for("index", tag=tag_uuid))
-
-    # Just a subset should be here
-    assert b'test-tag' in res.data
-    assert res.data.count(b'processor-text_json_diff') == 20
-    assert b"object at" not in res.data
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
-    assert b'Deleted' in res.data
-    res = client.get(url_for("tags.delete_all"), follow_redirects=True)
-    assert b'All tags deleted' in res.data
-def test_clone_tag_on_import(client, live_server):
-    #live_server_setup(live_server)
-    test_url = url_for('test_endpoint', _external=True)
-    res = client.post(
-        url_for("import_page"),
-        data={"urls": test_url + " test-tag, another-tag\r\n"},
-        follow_redirects=True
-    )
-
-    assert b"1 Imported" in res.data
-
-    res = client.get(url_for("index"))
-    assert b'test-tag' in res.data
-    assert b'another-tag' in res.data
-
-    watch_uuid = extract_UUID_from_client(client)
-    res = client.get(url_for("form_clone", uuid=watch_uuid), follow_redirects=True)
-
-    assert b'Cloned' in res.data
-    # 2 times plus the top link to tag
-    assert res.data.count(b'test-tag') == 3
-    assert res.data.count(b'another-tag') == 3
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
-    assert b'Deleted' in res.data
-
-def test_clone_tag_on_quickwatchform_add(client, live_server):
-    #live_server_setup(live_server)
-
-    test_url = url_for('test_endpoint', _external=True)
-
-    res = client.post(
-        url_for("form_quick_watch_add"),
-        data={"url": test_url, "tags": ' test-tag, another-tag      '},
-        follow_redirects=True
-    )
-
-    assert b"Watch added" in res.data
-
-    res = client.get(url_for("index"))
-    assert b'test-tag' in res.data
-    assert b'another-tag' in res.data
-
-    watch_uuid = extract_UUID_from_client(client)
-    res = client.get(url_for("form_clone", uuid=watch_uuid), follow_redirects=True)
-
-    assert b'Cloned' in res.data
-    # 2 times plus the top link to tag
-    assert res.data.count(b'test-tag') == 3
-    assert res.data.count(b'another-tag') == 3
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
-    assert b'Deleted' in res.data
-
-    res = client.get(url_for("tags.delete_all"), follow_redirects=True)
-    assert b'All tags deleted' in res.data

changedetectionio/tests/test_ignore_text.py

@@ -2,7 +2,7 @@
 
 import time
 from flask import url_for
-from .util import live_server_setup, wait_for_all_checks
+from . util import live_server_setup
 from changedetectionio import html_tools
 
 def test_setup(live_server):
@@ -84,6 +84,7 @@ def set_modified_ignore_response():
 
 
 def test_check_ignore_text_functionality(client, live_server):
+    sleep_time_for_fetch_thread = 3
 
     # Use a mix of case in ZzZ to prove it works case-insensitive.
     ignore_text = "XXXXX\r\nYYYYY\r\nzZzZZ\r\nnew ignore stuff"
@@ -102,7 +103,7 @@ def test_check_ignore_text_functionality(client, live_server):
     assert b"1 Imported" in res.data
 
     # Give the thread time to pick it up
-    wait_for_all_checks(client)
+    time.sleep(sleep_time_for_fetch_thread)
 
     # Goto the edit page, add our ignore text
     # Add our URL to the import page
@@ -123,7 +124,7 @@ def test_check_ignore_text_functionality(client, live_server):
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
-    wait_for_all_checks(client)
+    time.sleep(sleep_time_for_fetch_thread)
 
     # It should report nothing found (no new 'unviewed' class)
     res = client.get(url_for("index"))
@@ -136,7 +137,7 @@ def test_check_ignore_text_functionality(client, live_server):
     # Trigger a check
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
     # Give the thread time to pick it up
-    wait_for_all_checks(client)
+    time.sleep(sleep_time_for_fetch_thread)
 
     # It should report nothing found (no new 'unviewed' class)
     res = client.get(url_for("index"))
@@ -150,7 +151,7 @@ def test_check_ignore_text_functionality(client, live_server):
     # Just to be sure.. set a regular modified change..
     set_modified_original_ignore_response()
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    wait_for_all_checks(client)
+    time.sleep(sleep_time_for_fetch_thread)
 
     res = client.get(url_for("index"))
     assert b'unviewed' in res.data
@@ -166,6 +167,7 @@ def test_check_ignore_text_functionality(client, live_server):
     assert b'Deleted' in res.data
 
 def test_check_global_ignore_text_functionality(client, live_server):
+    sleep_time_for_fetch_thread = 3
 
     # Give the endpoint time to spin up
     time.sleep(1)
@@ -196,7 +198,7 @@ def test_check_global_ignore_text_functionality(client, live_server):
     assert b"1 Imported" in res.data
 
     # Give the thread time to pick it up
-    wait_for_all_checks(client)
+    time.sleep(sleep_time_for_fetch_thread)
 
 
     # Goto the edit page of the item, add our ignore text
@@ -218,7 +220,7 @@ def test_check_global_ignore_text_functionality(client, live_server):
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
-    wait_for_all_checks(client)
+    time.sleep(sleep_time_for_fetch_thread)
 
     # so that we are sure everything is viewed and in a known 'nothing changed' state
     res = client.get(url_for("diff_history_page", uuid="first"))
@@ -235,7 +237,7 @@ def test_check_global_ignore_text_functionality(client, live_server):
     # Trigger a check
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
     # Give the thread time to pick it up
-    wait_for_all_checks(client)
+    time.sleep(sleep_time_for_fetch_thread)
 
     # It should report nothing found (no new 'unviewed' class)
     res = client.get(url_for("index"))
@@ -245,7 +247,7 @@ def test_check_global_ignore_text_functionality(client, live_server):
     # Just to be sure.. set a regular modified change that will trigger it
     set_modified_original_ignore_response()
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    wait_for_all_checks(client)
+    time.sleep(sleep_time_for_fetch_thread)
     res = client.get(url_for("index"))
     assert b'unviewed' in res.data
 

changedetectionio/tests/test_ignorestatuscode.py

@@ -2,7 +2,7 @@
 
 import time
 from flask import url_for
-from .util import live_server_setup, wait_for_all_checks
+from . util import live_server_setup
 
 
 def test_setup(live_server):
@@ -40,7 +40,7 @@ def set_some_changed_response():
 
 
 def test_normal_page_check_works_with_ignore_status_code(client, live_server):
-
+    sleep_time_for_fetch_thread = 3
 
     # Give the endpoint time to spin up
     time.sleep(1)
@@ -68,15 +68,15 @@ def test_normal_page_check_works_with_ignore_status_code(client, live_server):
     )
     assert b"1 Imported" in res.data
 
-    wait_for_all_checks(client)
+    time.sleep(sleep_time_for_fetch_thread)
 
     set_some_changed_response()
-    wait_for_all_checks(client)
+    time.sleep(sleep_time_for_fetch_thread)
     # Trigger a check
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
-    wait_for_all_checks(client)
+    time.sleep(sleep_time_for_fetch_thread)
 
     # It should report nothing found (no new 'unviewed' class)
     res = client.get(url_for("index"))
@@ -109,13 +109,13 @@ def test_403_page_check_works_with_ignore_status_code(client, live_server):
     # Add our URL to the import page
     res = client.post(
         url_for("edit_page", uuid="first"),
-        data={"ignore_status_codes": "y", "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests"},
+        data={"ignore_status_codes": "y", "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
         follow_redirects=True
     )
     assert b"Updated watch." in res.data
 
     # Give the thread time to pick it up
-    wait_for_all_checks(client)
+    time.sleep(sleep_time_for_fetch_thread)
 
     #  Make a change
     set_some_changed_response()
@@ -123,7 +123,7 @@ def test_403_page_check_works_with_ignore_status_code(client, live_server):
     # Trigger a check
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
     # Give the thread time to pick it up
-    wait_for_all_checks(client)
+    time.sleep(sleep_time_for_fetch_thread)
 
     # It should have 'unviewed' still
     # Because it should be looking at only that 'sametext' id

changedetectionio/tests/test_import.py

@@ -112,7 +112,6 @@ def test_import_distillio(client, live_server):
     # did the tags work?
     res = client.get( url_for("index"))
 
-    # check tags
     assert b"nice stuff" in res.data
     assert b"nerd-news" in res.data
 

changedetectionio/tests/test_jinja2.py

@@ -20,7 +20,7 @@ def test_jinja2_in_url_query(client, live_server):
                               "date={% now 'Europe/Berlin', '%Y' %}.{% now 'Europe/Berlin', '%m' %}.{% now 'Europe/Berlin', '%d' %}", )
     res = client.post(
         url_for("form_quick_watch_add"),
-        data={"url": full_url, "tags": "test"},
+        data={"url": full_url, "tag": "test"},
         follow_redirects=True
     )
     assert b"Watch added" in res.data

changedetectionio/tests/test_jsonpath_jq_selector.py

@@ -3,7 +3,7 @@
 
 import time
 from flask import url_for, escape
-from . util import live_server_setup, wait_for_all_checks
+from . util import live_server_setup
 import pytest
 jq_support = True
 
@@ -64,24 +64,6 @@ and it can also be repeated
         with pytest.raises(html_tools.JSONNotFound) as e_info:
             html_tools.extract_json_as_string('COMPLETE GIBBERISH, NO JSON!', "jq:.id")
 
-
-def test_unittest_inline_extract_body():
-    content = """
-    <html>
-        <head></head>
-        <body>
-            <pre style="word-wrap: break-word; white-space: pre-wrap;">
-                {"testKey": 42}
-            </pre>
-        </body>
-    </html>
-    """
-    from .. import html_tools
-
-    # See that we can find the second <script> one, which is not broken, and matches our filter
-    text = html_tools.extract_json_as_string(content, "json:$.testKey")
-    assert text == '42'
-
 def set_original_ext_response():
     data = """
         [
@@ -208,7 +190,7 @@ def test_check_json_without_filter(client, live_server):
     )
 
     # Give the thread time to pick it up
-    wait_for_all_checks(client)
+    time.sleep(3)
 
     res = client.get(
         url_for("preview_page", uuid="first"),
@@ -238,7 +220,7 @@ def check_json_filter(json_filter, client, live_server):
     assert b"1 Imported" in res.data
 
     # Give the thread time to pick it up
-    wait_for_all_checks(client)
+    time.sleep(3)
 
     # Goto the edit page, add our ignore text
     # Add our URL to the import page
@@ -246,7 +228,7 @@ def check_json_filter(json_filter, client, live_server):
         url_for("edit_page", uuid="first"),
         data={"include_filters": json_filter,
               "url": test_url,
-              "tags": "",
+              "tag": "",
               "headers": "",
               "fetch_backend": "html_requests"
               },
@@ -261,14 +243,14 @@ def check_json_filter(json_filter, client, live_server):
     assert bytes(escape(json_filter).encode('utf-8')) in res.data
 
     # Give the thread time to pick it up
-    wait_for_all_checks(client)
+    time.sleep(3)
     #  Make a change
     set_modified_response()
 
     # Trigger a check
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
     # Give the thread time to pick it up
-    wait_for_all_checks(client)
+    time.sleep(4)
 
     # It should have 'unviewed' still
     res = client.get(url_for("index"))
@@ -306,14 +288,14 @@ def check_json_filter_bool_val(json_filter, client, live_server):
     )
     assert b"1 Imported" in res.data
 
-    wait_for_all_checks(client)
+    time.sleep(3)
     # Goto the edit page, add our ignore text
     # Add our URL to the import page
     res = client.post(
         url_for("edit_page", uuid="first"),
         data={"include_filters": json_filter,
               "url": test_url,
-              "tags": "",
+              "tag": "",
               "headers": "",
               "fetch_backend": "html_requests"
               },
@@ -322,14 +304,14 @@ def check_json_filter_bool_val(json_filter, client, live_server):
     assert b"Updated watch." in res.data
 
     # Give the thread time to pick it up
-    wait_for_all_checks(client)
+    time.sleep(3)
     #  Make a change
     set_modified_response()
 
     # Trigger a check
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
     # Give the thread time to pick it up
-    wait_for_all_checks(client)
+    time.sleep(3)
 
     res = client.get(url_for("diff_history_page", uuid="first"))
     # But the change should be there, tho its hard to test the change was detected because it will show old and new versions
@@ -366,7 +348,7 @@ def check_json_ext_filter(json_filter, client, live_server):
     assert b"1 Imported" in res.data
 
     # Give the thread time to pick it up
-    wait_for_all_checks(client)
+    time.sleep(3)
 
     # Goto the edit page, add our ignore text
     # Add our URL to the import page
@@ -374,7 +356,7 @@ def check_json_ext_filter(json_filter, client, live_server):
         url_for("edit_page", uuid="first"),
         data={"include_filters": json_filter,
               "url": test_url,
-              "tags": "",
+              "tag": "",
               "headers": "",
               "fetch_backend": "html_requests"
               },
@@ -389,14 +371,14 @@ def check_json_ext_filter(json_filter, client, live_server):
     assert bytes(escape(json_filter).encode('utf-8')) in res.data
 
     # Give the thread time to pick it up
-    wait_for_all_checks(client)
+    time.sleep(3)
     #  Make a change
     set_modified_ext_response()
 
     # Trigger a check
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
     # Give the thread time to pick it up
-    wait_for_all_checks(client)
+    time.sleep(4)
 
     # It should have 'unviewed'
     res = client.get(url_for("index"))
@@ -428,14 +410,14 @@ def test_ignore_json_order(client, live_server):
     )
     assert b"1 Imported" in res.data
 
-    wait_for_all_checks(client)
+    time.sleep(2)
 
     with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write('{"world" : 123, "hello": 123}')
 
     # Trigger a check
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    wait_for_all_checks(client)
+    time.sleep(2)
 
     res = client.get(url_for("index"))
     assert b'unviewed' not in res.data
@@ -446,7 +428,7 @@ def test_ignore_json_order(client, live_server):
 
     # Trigger a check
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    wait_for_all_checks(client)
+    time.sleep(2)
 
     res = client.get(url_for("index"))
     assert b'unviewed' in res.data
@@ -454,37 +436,6 @@ def test_ignore_json_order(client, live_server):
     res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
 
-def test_correct_header_detect(client, live_server):
-    # Like in https://github.com/dgtlmoon/changedetection.io/pull/1593
-    # Specify extra html that JSON is sometimes wrapped in - when using Browserless/Puppeteer etc
-    with open("test-datastore/endpoint-content.txt", "w") as f:
-        f.write('<html><body>{"hello" : 123, "world": 123}')
-
-    # Add our URL to the import page
-    # Check weird casing is cleaned up and detected also
-    test_url = url_for('test_endpoint', content_type="aPPlication/JSon", uppercase_headers=True, _external=True)
-    res = client.post(
-        url_for("import_page"),
-        data={"urls": test_url},
-        follow_redirects=True
-    )
-    assert b"1 Imported" in res.data
-    wait_for_all_checks(client)
-    res = client.get(url_for("index"))
-
-    # Fixed in #1593
-    assert b'No parsable JSON found in this document' not in res.data
-
-    res = client.get(
-        url_for("preview_page", uuid="first"),
-        follow_redirects=True
-    )
-    assert b'&#34;world&#34;:' in res.data
-    assert res.data.count(b'{') >= 2
-
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
-    assert b'Deleted' in res.data
-
 def test_check_jsonpath_ext_filter(client, live_server):
     check_json_ext_filter('json:$[?(@.status==Sold)]', client, live_server)
 

changedetectionio/tests/test_notification.py

@@ -3,7 +3,7 @@ import os
 import time
 import re
 from flask import url_for
-from .util import set_original_response, set_modified_response, set_more_modified_response, live_server_setup, wait_for_all_checks
+from . util import set_original_response, set_modified_response, set_more_modified_response, live_server_setup
 from . util import  extract_UUID_from_client
 import logging
 import base64
@@ -21,9 +21,11 @@ def test_setup(live_server):
 # Hard to just add more live server URLs when one test is already running (I think)
 # So we add our test here (was in a different file)
 def test_check_notification(client, live_server):
-    #live_server_setup(live_server)
     set_original_response()
 
+    # Give the endpoint time to spin up
+    time.sleep(3)
+
     # Re 360 - new install should have defaults set
     res = client.get(url_for("settings_page"))
     notification_url = url_for('test_notification_endpoint', _external=True).replace('http', 'json')
@@ -60,13 +62,13 @@ def test_check_notification(client, live_server):
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
         url_for("form_quick_watch_add"),
-        data={"url": test_url, "tags": ''},
+        data={"url": test_url, "tag": ''},
         follow_redirects=True
     )
     assert b"Watch added" in res.data
 
     # Give the thread time to pick up the first version
-    wait_for_all_checks(client)
+    time.sleep(3)
 
     # We write the PNG to disk, but a JPEG should appear in the notification
     # Write the last screenshot png
@@ -103,7 +105,7 @@ def test_check_notification(client, live_server):
 
     notification_form_data.update({
         "url": test_url,
-        "tags": "my tag, my second tag",
+        "tag": "my tag",
         "title": "my title",
         "headers": "",
         "fetch_backend": "html_requests"})
@@ -126,7 +128,7 @@ def test_check_notification(client, live_server):
 
 
     ## Now recheck, and it should have sent the notification
-    wait_for_all_checks(client)
+    time.sleep(3)
     set_modified_response()
 
     # Trigger a check
@@ -139,7 +141,8 @@ def test_check_notification(client, live_server):
 
     # Did we see the URL that had a change, in the notification?
     # Diff was correctly executed
-
+    assert test_url in notification_submission
+    assert ':-)' in notification_submission
     assert "Diff Full: Some initial text" in notification_submission
     assert "Diff: (changed) Which is across multiple lines" in notification_submission
     assert "(into) which has this one new line" in notification_submission
@@ -147,13 +150,12 @@ def test_check_notification(client, live_server):
     assert "b'" not in notification_submission
     assert re.search('Watch UUID: [0-9a-f]{8}(-[0-9a-f]{4}){3}-[0-9a-f]{12}', notification_submission, re.IGNORECASE)
     assert "Watch title: my title" in notification_submission
-    assert "Watch tag: my tag, my second tag" in notification_submission
+    assert "Watch tag: my tag" in notification_submission
     assert "diff/" in notification_submission
     assert "preview/" in notification_submission
     assert ":-)" in notification_submission
     assert "New ChangeDetection.io Notification - {}".format(test_url) in notification_submission
-    assert test_url in notification_submission
-    assert ':-)' in notification_submission
+
     # Check the attachment was added, and that it is a JPEG from the original PNG
     notification_submission_object = json.loads(notification_submission)
     # We keep PNG screenshots for now
@@ -191,11 +193,11 @@ def test_check_notification(client, live_server):
 
     # Trigger a check
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    wait_for_all_checks(client)
+    time.sleep(1)
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    wait_for_all_checks(client)
+    time.sleep(1)
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    wait_for_all_checks(client)
+    time.sleep(1)
     assert os.path.exists("test-datastore/notification.txt") == False
 
     res = client.get(url_for("notification_logs"))
@@ -207,7 +209,7 @@ def test_check_notification(client, live_server):
         url_for("edit_page", uuid="first"),
         data={
         "url": test_url,
-        "tags": "my tag",
+        "tag": "my tag",
         "title": "my title",
         "notification_urls": '',
         "notification_title": '',
@@ -241,7 +243,7 @@ def test_notification_validation(client, live_server):
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
         url_for("form_quick_watch_add"),
-        data={"url": test_url, "tags": 'nice one'},
+        data={"url": test_url, "tag": 'nice one'},
         follow_redirects=True
     )
 
@@ -301,13 +303,13 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server):
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
         url_for("form_quick_watch_add"),
-        data={"url": test_url, "tags": 'nice one'},
+        data={"url": test_url, "tag": 'nice one'},
         follow_redirects=True
     )
 
     assert b"Watch added" in res.data
 
-    wait_for_all_checks(client)
+    time.sleep(2)
     set_modified_response()
 
     client.get(url_for("form_watch_checknow"), follow_redirects=True)

changedetectionio/tests/test_notification_errors.py

@@ -17,7 +17,7 @@ def test_check_notification_error_handling(client, live_server):
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
         url_for("form_quick_watch_add"),
-        data={"url": test_url, "tags": ''},
+        data={"url": test_url, "tag": ''},
         follow_redirects=True
     )
     assert b"Watch added" in res.data
@@ -32,7 +32,7 @@ def test_check_notification_error_handling(client, live_server):
               "notification_body": "xxxxx",
               "notification_format": "Text",
               "url": test_url,
-              "tags": "",
+              "tag": "",
               "title": "",
               "headers": "",
               "time_between_check-minutes": "180",

changedetectionio/tests/test_request.py

@@ -1,8 +1,7 @@
 import json
-import os
 import time
 from flask import url_for
-from . util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks, extract_UUID_from_client
+from . util import set_original_response, set_modified_response, live_server_setup
 
 def test_setup(live_server):
     live_server_setup(live_server)
@@ -10,12 +9,8 @@ def test_setup(live_server):
 # Hard to just add more live server URLs when one test is already running (I think)
 # So we add our test here (was in a different file)
 def test_headers_in_request(client, live_server):
-    #live_server_setup(live_server)
     # Add our URL to the import page
     test_url = url_for('test_headers', _external=True)
-    if os.getenv('PLAYWRIGHT_DRIVER_URL'):
-        # Because its no longer calling back to localhost but from browserless, set in test-only.yml
-        test_url = test_url.replace('localhost', 'changedet')
 
     # Add the test URL twice, we will check
     res = client.post(
@@ -25,7 +20,7 @@ def test_headers_in_request(client, live_server):
     )
     assert b"1 Imported" in res.data
 
-    wait_for_all_checks(client)
+    time.sleep(1)
 
     res = client.post(
         url_for("import_page"),
@@ -34,7 +29,7 @@ def test_headers_in_request(client, live_server):
     )
     assert b"1 Imported" in res.data
 
-    wait_for_all_checks(client)
+    time.sleep(3)
     cookie_header = '_ga=GA1.2.1022228332; cookie-preferences=analytics:accepted;'
 
 
@@ -43,8 +38,8 @@ def test_headers_in_request(client, live_server):
         url_for("edit_page", uuid="first"),
         data={
               "url": test_url,
-              "tags": "",
-              "fetch_backend": 'html_webdriver' if os.getenv('PLAYWRIGHT_DRIVER_URL') else 'html_requests',
+              "tag": "",
+              "fetch_backend": "html_requests",
               "headers": "xxx:ooo\ncool:yeah\r\ncookie:"+cookie_header},
         follow_redirects=True
     )
@@ -52,7 +47,7 @@ def test_headers_in_request(client, live_server):
 
 
     # Give the thread time to pick up the first version
-    wait_for_all_checks(client)
+    time.sleep(5)
 
     # The service should echo back the request headers
     res = client.get(
@@ -68,7 +63,7 @@ def test_headers_in_request(client, live_server):
     from html import escape
     assert escape(cookie_header).encode('utf-8') in res.data
 
-    wait_for_all_checks(client)
+    time.sleep(5)
 
     # Re #137 -  Examine the JSON index file, it should have only one set of headers entered
     watches_with_headers = 0
@@ -84,9 +79,6 @@ def test_headers_in_request(client, live_server):
 def test_body_in_request(client, live_server):
     # Add our URL to the import page
     test_url = url_for('test_body', _external=True)
-    if os.getenv('PLAYWRIGHT_DRIVER_URL'):
-        # Because its no longer calling back to localhost but from browserless, set in test-only.yml
-        test_url = test_url.replace('localhost', 'cdio')
 
     res = client.post(
         url_for("import_page"),
@@ -95,14 +87,14 @@ def test_body_in_request(client, live_server):
     )
     assert b"1 Imported" in res.data
 
-    wait_for_all_checks(client)
+    time.sleep(3)
 
     # add the first 'version'
     res = client.post(
         url_for("edit_page", uuid="first"),
         data={
               "url": test_url,
-              "tags": "",
+              "tag": "",
               "method": "POST",
               "fetch_backend": "html_requests",
               "body": "something something"},
@@ -110,7 +102,7 @@ def test_body_in_request(client, live_server):
     )
     assert b"Updated watch." in res.data
 
-    wait_for_all_checks(client)
+    time.sleep(3)
 
     # Now the change which should trigger a change
     body_value = 'Test Body Value'
@@ -118,7 +110,7 @@ def test_body_in_request(client, live_server):
         url_for("edit_page", uuid="first"),
         data={
               "url": test_url,
-              "tags": "",
+              "tag": "",
               "method": "POST",
               "fetch_backend": "html_requests",
               "body": body_value},
@@ -126,7 +118,7 @@ def test_body_in_request(client, live_server):
     )
     assert b"Updated watch." in res.data
 
-    wait_for_all_checks(client)
+    time.sleep(3)
 
     # The service should echo back the body
     res = client.get(
@@ -163,7 +155,7 @@ def test_body_in_request(client, live_server):
         url_for("edit_page", uuid="first"),
         data={
               "url": test_url,
-              "tags": "",
+              "tag": "",
               "method": "GET",
               "fetch_backend": "html_requests",
               "body": "invalid"},
@@ -175,9 +167,6 @@ def test_body_in_request(client, live_server):
 def test_method_in_request(client, live_server):
     # Add our URL to the import page
     test_url = url_for('test_method', _external=True)
-    if os.getenv('PLAYWRIGHT_DRIVER_URL'):
-        # Because its no longer calling back to localhost but from browserless, set in test-only.yml
-        test_url = test_url.replace('localhost', 'cdio')
 
     # Add the test URL twice, we will check
     res = client.post(
@@ -187,7 +176,7 @@ def test_method_in_request(client, live_server):
     )
     assert b"1 Imported" in res.data
 
-    wait_for_all_checks(client)
+    time.sleep(2)
     res = client.post(
         url_for("import_page"),
         data={"urls": test_url},
@@ -195,14 +184,14 @@ def test_method_in_request(client, live_server):
     )
     assert b"1 Imported" in res.data
 
-    wait_for_all_checks(client)
+    time.sleep(2)
 
     # Attempt to add a method which is not valid
     res = client.post(
         url_for("edit_page", uuid="first"),
         data={
             "url": test_url,
-            "tags": "",
+            "tag": "",
             "fetch_backend": "html_requests",
             "method": "invalid"},
         follow_redirects=True
@@ -214,7 +203,7 @@ def test_method_in_request(client, live_server):
         url_for("edit_page", uuid="first"),
         data={
             "url": test_url,
-            "tags": "",
+            "tag": "",
             "fetch_backend": "html_requests",
             "method": "PATCH"},
         follow_redirects=True
@@ -222,7 +211,7 @@ def test_method_in_request(client, live_server):
     assert b"Updated watch." in res.data
 
     # Give the thread time to pick up the first version
-    wait_for_all_checks(client)
+    time.sleep(2)
 
     # The service should echo back the request verb
     res = client.get(
@@ -233,7 +222,7 @@ def test_method_in_request(client, live_server):
     # The test call service will return the verb as the body
     assert b"PATCH" in res.data
 
-    wait_for_all_checks(client)
+    time.sleep(2)
 
     watches_with_method = 0
     with open('test-datastore/url-watches.json') as f:
@@ -245,76 +234,3 @@ def test_method_in_request(client, live_server):
     # Should be only one with method set to PATCH
     assert watches_with_method == 1
 
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
-    assert b'Deleted' in res.data
-
-def test_headers_textfile_in_request(client, live_server):
-    #live_server_setup(live_server)
-    # Add our URL to the import page
-    test_url = url_for('test_headers', _external=True)
-    if os.getenv('PLAYWRIGHT_DRIVER_URL'):
-        # Because its no longer calling back to localhost but from browserless, set in test-only.yml
-        test_url = test_url.replace('localhost', 'cdio')
-
-    print ("TEST URL IS ",test_url)
-    # Add the test URL twice, we will check
-    res = client.post(
-        url_for("import_page"),
-        data={"urls": test_url},
-        follow_redirects=True
-    )
-    assert b"1 Imported" in res.data
-
-    wait_for_all_checks(client)
-
-
-    # Add some headers to a request
-    res = client.post(
-        url_for("edit_page", uuid="first"),
-        data={
-              "url": test_url,
-              "tags": "testtag",
-              "fetch_backend": 'html_webdriver' if os.getenv('PLAYWRIGHT_DRIVER_URL') else 'html_requests',
-              "headers": "xxx:ooo\ncool:yeah\r\n"},
-        follow_redirects=True
-    )
-    assert b"Updated watch." in res.data
-    wait_for_all_checks(client)
-
-    with open('test-datastore/headers-testtag.txt', 'w') as f:
-        f.write("tag-header: test")
-
-    with open('test-datastore/headers.txt', 'w') as f:
-        f.write("global-header: nice\r\nnext-global-header: nice")
-
-    with open('test-datastore/'+extract_UUID_from_client(client)+'/headers.txt', 'w') as f:
-        f.write("watch-header: nice")
-
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
-
-    # Give the thread time to pick it up
-    wait_for_all_checks(client)
-
-    res = client.get(url_for("edit_page", uuid="first"))
-    assert b"Extra headers file found and will be added to this watch" in res.data
-
-    # Not needed anymore
-    os.unlink('test-datastore/headers.txt')
-    os.unlink('test-datastore/headers-testtag.txt')
-    os.unlink('test-datastore/'+extract_UUID_from_client(client)+'/headers.txt')
-    # The service should echo back the request verb
-    res = client.get(
-        url_for("preview_page", uuid="first"),
-        follow_redirects=True
-    )
-
-    assert b"Global-Header:nice" in res.data
-    assert b"Next-Global-Header:nice" in res.data
-    assert b"Xxx:ooo" in res.data
-    assert b"Watch-Header:nice" in res.data
-    assert b"Tag-Header:test" in res.data
-
-
-    #unlink headers.txt on start/stop
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
-    assert b'Deleted' in res.data

changedetectionio/tests/test_search.py

@@ -1,74 +0,0 @@
-from flask import url_for
-from .util import set_original_response, set_modified_response, live_server_setup
-import time
-
-def test_setup(live_server):
-    live_server_setup(live_server)
-
-def test_basic_search(client, live_server):
-    #live_server_setup(live_server)
-
-    urls = ['https://localhost:12300?first-result=1',
-            'https://localhost:5000?second-result=1'
-            ]
-    res = client.post(
-        url_for("import_page"),
-        data={"urls": "\r\n".join(urls)},
-        follow_redirects=True
-    )
-
-    assert b"2 Imported" in res.data
-
-    # By URL
-    res = client.get(url_for("index") + "?q=first-res")
-    assert urls[0].encode('utf-8') in res.data
-    assert urls[1].encode('utf-8') not in res.data
-
-    # By Title
-
-    res = client.post(
-        url_for("edit_page", uuid="first"),
-        data={"title": "xxx-title", "url": urls[0], "tags": "", "headers": "", 'fetch_backend': "html_requests"},
-        follow_redirects=True
-    )
-    assert b"Updated watch." in res.data
-
-    res = client.get(url_for("index") + "?q=xxx-title")
-    assert urls[0].encode('utf-8') in res.data
-    assert urls[1].encode('utf-8') not in res.data
-
-
-def test_search_in_tag_limit(client, live_server):
-    #live_server_setup(live_server)
-
-    urls = ['https://localhost:12300?first-result=1 tag-one',
-            'https://localhost:5000?second-result=1 tag-two'
-            ]
-    res = client.post(
-        url_for("import_page"),
-        data={"urls": "\r\n".join(urls)},
-        follow_redirects=True
-    )
-
-    assert b"2 Imported" in res.data
-
-    # By URL
-
-    res = client.get(url_for("index") + "?q=first-res")
-    # Split because of the import tag separation
-    assert urls[0].split(' ')[0].encode('utf-8') in res.data, urls[0].encode('utf-8')
-    assert urls[1].split(' ')[0].encode('utf-8') not in res.data, urls[0].encode('utf-8')
-
-    # By Title
-    res = client.post(
-        url_for("edit_page", uuid="first"),
-        data={"title": "xxx-title", "url": urls[0].split(' ')[0], "tags": urls[0].split(' ')[1], "headers": "",
-              'fetch_backend': "html_requests"},
-        follow_redirects=True
-    )
-    assert b"Updated watch." in res.data
-
-    res = client.get(url_for("index") + "?q=xxx-title")
-    assert urls[0].split(' ')[0].encode('utf-8') in res.data, urls[0].encode('utf-8')
-    assert urls[1].split(' ')[0].encode('utf-8') not in res.data, urls[0].encode('utf-8')
-

changedetectionio/tests/test_security.py

@@ -18,7 +18,7 @@ def test_bad_access(client, live_server):
         url_for("edit_page", uuid="first"),
         data={
               "url": 'javascript:alert(document.domain)',
-              "tags": "",
+              "tag": "",
               "method": "GET",
               "fetch_backend": "html_requests",
               "body": ""},
@@ -29,7 +29,7 @@ def test_bad_access(client, live_server):
 
     res = client.post(
         url_for("form_quick_watch_add"),
-        data={"url": '            javascript:alert(123)', "tags": ''},
+        data={"url": '            javascript:alert(123)', "tag": ''},
         follow_redirects=True
     )
 
@@ -37,7 +37,7 @@ def test_bad_access(client, live_server):
 
     res = client.post(
         url_for("form_quick_watch_add"),
-        data={"url": '%20%20%20javascript:alert(123)%20%20', "tags": ''},
+        data={"url": '%20%20%20javascript:alert(123)%20%20', "tag": ''},
         follow_redirects=True
     )
 
@@ -46,7 +46,7 @@ def test_bad_access(client, live_server):
 
     res = client.post(
         url_for("form_quick_watch_add"),
-        data={"url": ' source:javascript:alert(document.domain)', "tags": ''},
+        data={"url": ' source:javascript:alert(document.domain)', "tag": ''},
         follow_redirects=True
     )
 
@@ -56,7 +56,7 @@ def test_bad_access(client, live_server):
 
     client.post(
         url_for("form_quick_watch_add"),
-        data={"url": 'file:///tasty/disk/drive', "tags": ''},
+        data={"url": 'file:///tasty/disk/drive', "tag": ''},
         follow_redirects=True
     )
     time.sleep(1)

changedetectionio/tests/test_share_watch.py

@@ -29,7 +29,7 @@ def test_share_watch(client, live_server):
     # Add our URL to the import page
     res = client.post(
         url_for("edit_page", uuid="first"),
-        data={"include_filters": include_filters, "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests"},
+        data={"include_filters": include_filters, "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
         follow_redirects=True
     )
     assert b"Updated watch." in res.data

changedetectionio/tests/test_source.py

@@ -3,7 +3,7 @@
 import time
 from flask import url_for
 from urllib.request import urlopen
-from .util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks
+from .util import set_original_response, set_modified_response, live_server_setup
 
 sleep_time_for_fetch_thread = 3
 
@@ -42,7 +42,7 @@ def test_check_basic_change_detection_functionality_source(client, live_server):
     res = client.get(url_for("form_watch_checknow"), follow_redirects=True)
     assert b'1 watches queued for rechecking.' in res.data
 
-    wait_for_all_checks(client)
+    time.sleep(5)
 
     # Now something should be ready, indicated by having a 'unviewed' class
     res = client.get(url_for("index"))
@@ -60,7 +60,7 @@ def test_check_basic_change_detection_functionality_source(client, live_server):
 # `subtractive_selectors` should still work in `source:` type requests
 def test_check_ignore_elements(client, live_server):
     set_original_response()
-    time.sleep(1)
+    time.sleep(2)
     test_url = 'source:'+url_for('test_endpoint', _external=True)
     # Add our URL to the import page
     res = client.post(
@@ -71,14 +71,14 @@ def test_check_ignore_elements(client, live_server):
 
     assert b"1 Imported" in res.data
 
-    wait_for_all_checks(client)
+    time.sleep(sleep_time_for_fetch_thread)
 
     #####################
     # We want <span> and <p> ONLY, but ignore span with .foobar-detection
 
     client.post(
         url_for("edit_page", uuid="first"),
-        data={"include_filters": 'span,p', "url": test_url, "tags": "", "subtractive_selectors": ".foobar-detection", 'fetch_backend': "html_requests"},
+        data={"include_filters": 'span,p', "url": test_url, "tag": "", "subtractive_selectors": ".foobar-detection", 'fetch_backend': "html_requests"},
         follow_redirects=True
     )
 

changedetectionio/tests/test_watch_fields_storage.py

@@ -26,7 +26,7 @@ def test_check_watch_field_storage(client, live_server):
                "title" : "My title",
                "ignore_text" : "ignore this",
                "url": test_url,
-               "tags": "woohoo",
+               "tag": "woohoo",
                "headers": "curl:foo",
                'fetch_backend': "html_requests"
                },

changedetectionio/tests/test_xpath_selector.py

@@ -89,7 +89,7 @@ def test_check_xpath_filter_utf8(client, live_server):
     time.sleep(1)
     res = client.post(
         url_for("edit_page", uuid="first"),
-        data={"include_filters": filter, "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests"},
+        data={"include_filters": filter, "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
         follow_redirects=True
     )
     assert b"Updated watch." in res.data
@@ -143,7 +143,7 @@ def test_check_xpath_text_function_utf8(client, live_server):
     time.sleep(1)
     res = client.post(
         url_for("edit_page", uuid="first"),
-        data={"include_filters": filter, "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests"},
+        data={"include_filters": filter, "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
         follow_redirects=True
     )
     assert b"Updated watch." in res.data
@@ -189,7 +189,7 @@ def test_check_markup_xpath_filter_restriction(client, live_server):
     # Add our URL to the import page
     res = client.post(
         url_for("edit_page", uuid="first"),
-        data={"include_filters": xpath_filter, "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests"},
+        data={"include_filters": xpath_filter, "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
         follow_redirects=True
     )
     assert b"Updated watch." in res.data
@@ -231,7 +231,7 @@ def test_xpath_validation(client, live_server):
 
     res = client.post(
         url_for("edit_page", uuid="first"),
-        data={"include_filters": "/something horrible", "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests"},
+        data={"include_filters": "/something horrible", "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
         follow_redirects=True
     )
     assert b"is not a valid XPath expression" in res.data
@@ -261,7 +261,7 @@ def test_check_with_prefix_include_filters(client, live_server):
 
     res = client.post(
         url_for("edit_page", uuid="first"),
-        data={"include_filters":  "xpath://*[contains(@class, 'sametext')]", "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests"},
+        data={"include_filters":  "xpath://*[contains(@class, 'sametext')]", "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
         follow_redirects=True
     )
 

changedetectionio/tests/util.py

@@ -70,16 +70,6 @@ def extract_api_key_from_UI(client):
     api_key = m.group(1)
     return api_key.strip()
 
-
-# kinda funky, but works for now
-def get_UUID_for_tag_name(client, name):
-    app_config = client.application.config.get('DATASTORE').data
-    for uuid, tag in app_config['settings']['application'].get('tags', {}).items():
-        if name == tag.get('title', '').lower().strip():
-            return uuid
-    return None
-
-
 # kinda funky, but works for now
 def extract_rss_token_from_UI(client):
     import re
@@ -129,26 +119,16 @@ def live_server_setup(live_server):
         status_code = request.args.get('status_code')
         content = request.args.get('content') or None
 
-        # Used to just try to break the header detection
-        uppercase_headers = request.args.get('uppercase_headers')
-
         try:
             if content is not None:
                 resp = make_response(content, status_code)
-                if uppercase_headers:
-                    ctype=ctype.upper()
-                    resp.headers['CONTENT-TYPE'] = ctype if ctype else 'text/html'
-                else:
-                    resp.headers['Content-Type'] = ctype if ctype else 'text/html'
+                resp.headers['Content-Type'] = ctype if ctype else 'text/html'
                 return resp
 
             # Tried using a global var here but didn't seem to work, so reading from a file instead.
             with open("test-datastore/endpoint-content.txt", "r") as f:
                 resp = make_response(f.read(), status_code)
-                if uppercase_headers:
-                    resp.headers['CONTENT-TYPE'] = ctype if ctype else 'text/html'
-                else:
-                    resp.headers['Content-Type'] = ctype if ctype else 'text/html'
+                resp.headers['Content-Type'] = ctype if ctype else 'text/html'
                 return resp
         except FileNotFoundError:
             return make_response('', status_code)

changedetectionio/tests/visualselector/test_fetch_data.py

@@ -19,7 +19,7 @@ def test_visual_selector_content_ready(client, live_server):
 
     res = client.post(
         url_for("form_quick_watch_add"),
-        data={"url": test_url, "tags": '', 'edit_and_watch_submit_button': 'Edit > Watch'},
+        data={"url": test_url, "tag": '', 'edit_and_watch_submit_button': 'Edit > Watch'},
         follow_redirects=True
     )
     assert b"Watch added in Paused state, saving will unpause" in res.data
@@ -28,7 +28,7 @@ def test_visual_selector_content_ready(client, live_server):
         url_for("edit_page", uuid="first", unpause_on_save=1),
         data={
               "url": test_url,
-              "tags": "",
+              "tag": "",
               "headers": "",
               'fetch_backend': "html_webdriver",
               'webdriver_js_execute_code': 'document.querySelector("button[name=test-button]").click();'

changedetectionio/update_worker.py

@@ -5,7 +5,7 @@ import time
 
 from changedetectionio import content_fetcher
 from .processors.text_json_diff import FilterNotFoundInResponse
-from .processors.restock_diff import UnableToExtractRestockData
+
 
 # A single update worker
 #
@@ -26,84 +26,16 @@ class update_worker(threading.Thread):
         self.datastore = datastore
         super().__init__(*args, **kwargs)
 
-    def queue_notification_for_watch(self, n_object, watch):
+    def send_content_changed_notification(self, t, watch_uuid):
 
         from changedetectionio import diff
 
-        watch_history = watch.history
-        dates = list(watch_history.keys())
-
-        # HTML needs linebreak, but MarkDown and Text can use a linefeed
-        if n_object['notification_format'] == 'HTML':
-            line_feed_sep = "<br>"
-        else:
-            line_feed_sep = "\n"
-
-        # Add text that was triggered
-        snapshot_contents = watch.get_history_snapshot(dates[-1])
-        trigger_text = watch.get('trigger_text', [])
-        triggered_text = ''
-
-        if len(trigger_text):
-            from . import html_tools
-            triggered_text = html_tools.get_triggered_text(content=snapshot_contents, trigger_text=trigger_text)
-            if triggered_text:
-                triggered_text = line_feed_sep.join(triggered_text)
-
-
-        n_object.update({
-            'current_snapshot': snapshot_contents,
-            'diff': diff.render_diff(watch.get_history_snapshot(dates[-2]), watch.get_history_snapshot(dates[-1]), line_feed_sep=line_feed_sep),
-            'diff_added': diff.render_diff(watch.get_history_snapshot(dates[-2]), watch.get_history_snapshot(dates[-1]), include_removed=False, line_feed_sep=line_feed_sep),
-            'diff_full': diff.render_diff(watch.get_history_snapshot(dates[-2]), watch.get_history_snapshot(dates[-1]), include_equal=True, line_feed_sep=line_feed_sep),
-            'diff_removed': diff.render_diff(watch.get_history_snapshot(dates[-2]), watch.get_history_snapshot(dates[-1]), include_added=False, line_feed_sep=line_feed_sep),
-            'screenshot': watch.get_screenshot() if watch.get('notification_screenshot') else None,
-            'triggered_text': triggered_text,
-            'uuid': watch.get('uuid'),
-            'watch_url': watch.get('url'),
-        })
-        logging.info (">> SENDING NOTIFICATION")
-        self.notification_q.put(n_object)
-
-    # Prefer - Individual watch settings > Tag settings >  Global settings (in that order)
-    def _check_cascading_vars(self, var_name, watch):
-
         from changedetectionio.notification import (
-            default_notification_format_for_watch,
-            default_notification_body,
-            default_notification_title
+            default_notification_format_for_watch
         )
 
-
-        # Would be better if this was some kind of Object where Watch can reference the parent datastore etc
-        v = watch.get(var_name)
-        if v and not watch.get('notification_muted'):
-            return v
-
-        tags = self.datastore.get_all_tags_for_watch(uuid=watch.get('uuid'))
-        if tags:
-            for tag_uuid, tag in tags.items():
-                v = tag.get(var_name)
-                if v and not tag.get('notification_muted'):
-                    return v
-
-        if self.datastore.data['settings']['application'].get(var_name):
-            return self.datastore.data['settings']['application'].get(var_name)
-
-        # Otherwise could be defaults
-        if var_name == 'notification_format':
-            return default_notification_format_for_watch
-        if var_name == 'notification_body':
-            return default_notification_body
-        if var_name == 'notification_title':
-            return default_notification_title
-
-        return None
-
-    def send_content_changed_notification(self, watch_uuid):
-
         n_object = {}
-        watch = self.datastore.data['watching'].get(watch_uuid)
+        watch = self.datastore.data['watching'].get(watch_uuid, False)
         if not watch:
             return
 
@@ -116,27 +48,59 @@ class update_worker(threading.Thread):
                 "History index had 2 or more, but only 1 date loaded, timestamps were not unique? maybe two of the same timestamps got written, needs more delay?"
             )
 
-        # Should be a better parent getter in the model object
+        n_object['notification_urls'] = watch['notification_urls'] if len(watch['notification_urls']) else \
+            self.datastore.data['settings']['application']['notification_urls']
 
-        # Prefer - Individual watch settings > Tag settings >  Global settings (in that order)
-        n_object['notification_urls'] = self._check_cascading_vars('notification_urls', watch)
-        n_object['notification_title'] = self._check_cascading_vars('notification_title', watch)
-        n_object['notification_body'] = self._check_cascading_vars('notification_body', watch)
-        n_object['notification_format'] = self._check_cascading_vars('notification_format', watch)
+        n_object['notification_title'] = watch['notification_title'] if watch['notification_title'] else \
+            self.datastore.data['settings']['application']['notification_title']
 
-        # (Individual watch) Only prepare to notify if the rules above matched
-        queued = False
-        if n_object and n_object.get('notification_urls'):
-            queued = True
-            self.queue_notification_for_watch(n_object, watch)
+        n_object['notification_body'] = watch['notification_body'] if watch['notification_body'] else \
+            self.datastore.data['settings']['application']['notification_body']
 
-        return queued
+        n_object['notification_format'] = watch['notification_format'] if watch['notification_format'] != default_notification_format_for_watch else \
+            self.datastore.data['settings']['application']['notification_format']
 
 
+        # Only prepare to notify if the rules above matched
+        if 'notification_urls' in n_object and n_object['notification_urls']:
+            # HTML needs linebreak, but MarkDown and Text can use a linefeed
+            if n_object['notification_format'] == 'HTML':
+                line_feed_sep = "<br>"
+            else:
+                line_feed_sep = "\n"
+
+            # Add text that was triggered
+            snapshot_contents = watch.get_history_snapshot(dates[-1])
+            trigger_text = watch.get('trigger_text', [])
+            triggered_text = ''
+
+            if len(trigger_text):
+                from . import html_tools
+                triggered_text = html_tools.get_triggered_text(content=snapshot_contents, trigger_text=trigger_text)
+                if triggered_text:
+                    triggered_text = line_feed_sep.join(triggered_text)
+
+
+            n_object.update({
+                'current_snapshot': snapshot_contents,
+                'diff': diff.render_diff(watch.get_history_snapshot(dates[-2]), watch.get_history_snapshot(dates[-1]), line_feed_sep=line_feed_sep),
+                'diff_added': diff.render_diff(watch.get_history_snapshot(dates[-2]), watch.get_history_snapshot(dates[-1]), include_removed=False, line_feed_sep=line_feed_sep),
+                'diff_full': diff.render_diff(watch.get_history_snapshot(dates[-2]), watch.get_history_snapshot(dates[-1]), include_equal=True, line_feed_sep=line_feed_sep),
+                'diff_removed': diff.render_diff(watch.get_history_snapshot(dates[-2]), watch.get_history_snapshot(dates[-1]), include_added=False, line_feed_sep=line_feed_sep),
+                'screenshot': watch.get_screenshot() if watch.get('notification_screenshot') else None,
+                'triggered_text': triggered_text,
+                'uuid': watch_uuid,
+                'watch_url': watch['url'],
+            })
+            logging.info (">> SENDING NOTIFICATION")
+            self.notification_q.put(n_object)
+        else:
+            logging.info (">> NO Notification sent, notification_url was empty in both watch and system")
+
     def send_filter_failure_notification(self, watch_uuid):
 
         threshold = self.datastore.data['settings']['application'].get('filter_failure_notification_threshold_attempts')
-        watch = self.datastore.data['watching'].get(watch_uuid)
+        watch = self.datastore.data['watching'].get(watch_uuid, False)
         if not watch:
             return
 
@@ -213,7 +177,7 @@ class update_worker(threading.Thread):
                 uuid = queued_item_data.item.get('uuid')
                 self.current_uuid = uuid
 
-                if uuid in list(self.datastore.data['watching'].keys()) and self.datastore.data['watching'][uuid].get('url'):
+                if uuid in list(self.datastore.data['watching'].keys()):
                     changed_detected = False
                     contents = b''
                     process_changedetection_results = True
@@ -274,7 +238,7 @@ class update_worker(threading.Thread):
                         if not self.datastore.data['watching'].get(uuid):
                             continue
 
-                        err_text = "Warning, no filters were found, no change detection ran - Did the page change layout? update your Visual Filter if necessary."
+                        err_text = "Warning, no filters were found, no change detection ran."
                         self.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text})
 
                         # Only when enabled, send the notification
@@ -298,7 +262,6 @@ class update_worker(threading.Thread):
                         # Yes fine, so nothing todo, don't continue to process.
                         process_changedetection_results = False
                         changed_detected = False
-                        self.datastore.update_watch(uuid=uuid, update_obj={'last_error': False})
 
                     except content_fetcher.BrowserStepsStepTimout as e:
 
@@ -355,11 +318,6 @@ class update_worker(threading.Thread):
                                                                            'last_check_status': e.status_code,
                                                                            'has_ldjson_price_data': None})
                         process_changedetection_results = False
-                    except UnableToExtractRestockData as e:
-                        # Usually when fetcher.instock_data returns empty
-                        self.app.logger.error("Exception reached processing watch UUID: %s - %s", uuid, str(e))
-                        self.datastore.update_watch(uuid=uuid, update_obj={'last_error': f"Unable to extract restock data for this page unfortunately. (Got code {e.status_code} from server)"})
-                        process_changedetection_results = False
                     except Exception as e:
                         self.app.logger.error("Exception reached processing watch UUID: %s - %s", uuid, str(e))
                         self.datastore.update_watch(uuid=uuid, update_obj={'last_error': str(e)})
@@ -396,7 +354,7 @@ class update_worker(threading.Thread):
                                 # Notifications should only trigger on the second time (first time, we gather the initial snapshot)
                                 if watch.history_n >= 2:
                                     if not self.datastore.data['watching'][uuid].get('notification_muted'):
-                                        self.send_content_changed_notification(watch_uuid=uuid)
+                                        self.send_content_changed_notification(self, watch_uuid=uuid)
 
 
                         except Exception as e: