Filter - unique lines fix for empty set

changedetectionio/api/api_schema.py

@@ -112,6 +112,35 @@ def build_watch_json_schema(d):
 
     schema['properties']['time_between_check'] = build_time_between_check_json_schema()
 
+    schema['properties']['browser_steps'] = {
+        "anyOf": [
+            {
+                "type": "array",
+                "items": {
+                    "type": "object",
+                    "properties": {
+                        "operation": {
+                            "type": ["string", "null"],
+                            "maxLength": 5000  # Allows null and any string up to 5000 chars (including "")
+                        },
+                        "selector": {
+                            "type": ["string", "null"],
+                            "maxLength": 5000
+                        },
+                        "optional_value": {
+                            "type": ["string", "null"],
+                            "maxLength": 5000
+                        }
+                    },
+                    "required": ["operation", "selector", "optional_value"],
+                    "additionalProperties": False  # No extra keys allowed
+                }
+            },
+            {"type": "null"},  # Allows null for `browser_steps`
+            {"type": "array", "maxItems": 0}  # Allows empty array []
+        ]
+    }
+
     # headers ?
     return schema
 

changedetectionio/blueprint/browser_steps/browser_steps.py

@@ -52,6 +52,8 @@ class steppable_browser_interface():
     page = None
     start_url = None
 
+    action_timeout = 10 * 1000
+
     def __init__(self, start_url):
         self.start_url = start_url
 
@@ -102,7 +104,7 @@ class steppable_browser_interface():
             return
         elem = self.page.get_by_text(value)
         if elem.count():
-            elem.first.click(delay=randint(200, 500), timeout=3000)
+            elem.first.click(delay=randint(200, 500), timeout=self.action_timeout)
 
     def action_click_element_containing_text_if_exists(self, selector=None, value=''):
         logger.debug("Clicking element containing text if exists")
@@ -111,7 +113,7 @@ class steppable_browser_interface():
         elem = self.page.get_by_text(value)
         logger.debug(f"Clicking element containing text - {elem.count()} elements found")
         if elem.count():
-            elem.first.click(delay=randint(200, 500), timeout=3000)
+            elem.first.click(delay=randint(200, 500), timeout=self.action_timeout)
         else:
             return
 
@@ -119,7 +121,7 @@ class steppable_browser_interface():
         if not len(selector.strip()):
             return
 
-        self.page.fill(selector, value, timeout=10 * 1000)
+        self.page.fill(selector, value, timeout=self.action_timeout)
 
     def action_execute_js(self, selector, value):
         response = self.page.evaluate(value)
@@ -130,7 +132,7 @@ class steppable_browser_interface():
         if not len(selector.strip()):
             return
 
-        self.page.click(selector=selector, timeout=30 * 1000, delay=randint(200, 500))
+        self.page.click(selector=selector, timeout=self.action_timeout + 20 * 1000, delay=randint(200, 500))
 
     def action_click_element_if_exists(self, selector, value):
         import playwright._impl._errors as _api_types
@@ -138,7 +140,7 @@ class steppable_browser_interface():
         if not len(selector.strip()):
             return
         try:
-            self.page.click(selector, timeout=10 * 1000, delay=randint(200, 500))
+            self.page.click(selector, timeout=self.action_timeout, delay=randint(200, 500))
         except _api_types.TimeoutError as e:
             return
         except _api_types.Error as e:
@@ -185,10 +187,10 @@ class steppable_browser_interface():
         self.page.keyboard.press("PageDown", delay=randint(200, 500))
 
     def action_check_checkbox(self, selector, value):
-        self.page.locator(selector).check(timeout=1000)
+        self.page.locator(selector).check(timeout=self.action_timeout)
 
     def action_uncheck_checkbox(self, selector, value):
-        self.page.locator(selector, timeout=1000).uncheck(timeout=1000)
+        self.page.locator(selector).uncheck(timeout=self.action_timeout)
 
 
 # Responsible for maintaining a live 'context' with the chrome CDP

changedetectionio/model/Watch.py

@@ -352,7 +352,7 @@ class model(watch_base):
     # Iterate over all history texts and see if something new exists
     # Always applying .strip() to start/end but optionally replace any other whitespace
     def lines_contain_something_unique_compared_to_history(self, lines: list, ignore_whitespace=False):
-        local_lines = []
+        local_lines = set([])
         if lines:
             if ignore_whitespace:
                 if isinstance(lines[0], str): # Can be either str or bytes depending on what was on the disk

changedetectionio/templates/edit.html

@@ -45,9 +45,8 @@
             {% if extra_tab_content %}
             <li class="tab"><a href="#extras_tab">{{ extra_tab_content }}</a></li>
             {% endif %}
-            {% if playwright_enabled %}
             <li class="tab"><a id="browsersteps-tab" href="#browser-steps">Browser Steps</a></li>
-            {% endif %}
+        <!-- should goto extra forms? -->
             {% if watch['processor'] == 'text_json_diff' %}
             <li class="tab"><a id="visualselector-tab" href="#visualselector">Visual Filter Selector</a></li>
             <li class="tab" id="filters-and-triggers-tab"><a href="#filters-and-triggers">Filters &amp; Triggers</a></li>
@@ -199,8 +198,9 @@ Math: {{ 1 + 1 }}") }}
                     </div>
             </fieldset>
             </div>
-            {% if playwright_enabled %}
+
             <div class="tab-pane-inner" id="browser-steps">
+            {% if playwright_enabled %}
                 <img class="beta-logo" src="{{url_for('static_content', group='images', filename='beta-logo.png')}}" alt="New beta functionality">
                 <fieldset>
                     <div class="pure-control-group">
@@ -240,8 +240,16 @@ Math: {{ 1 + 1 }}") }}
                         </div>
                     </div>
                 </fieldset>
+                {% else %}
+                    <span class="pure-form-message-inline">
+                        <p>Sorry, this functionality only works with Playwright/Chrome enabled watches.</p>
+                        <p>Enable the Playwright Chrome fetcher, or alternatively try our <a href="https://lemonade.changedetection.io/start">very affordable subscription based service</a>.</p>
+                        <p>This is because Selenium/WebDriver can not extract full page screenshots reliably.</p>
+                        <p>You may need to <a href="https://github.com/dgtlmoon/changedetection.io/blob/09ebc6ec6338545bdd694dc6eee57f2e9d2b8075/docker-compose.yml#L31">Enable playwright environment variable</a> and uncomment the <strong>sockpuppetbrowser</strong> from the docker-compose.yml file.</p>
+                    </span>
+                {% endif %}
             </div>
-            {% endif %}
+
 
             <div class="tab-pane-inner" id="notifications">
                 <fieldset>
@@ -493,6 +501,7 @@ keyword") }}
                                 <p>Sorry, this functionality only works with Playwright/Chrome enabled watches.</p>
                                 <p>Enable the Playwright Chrome fetcher, or alternatively try our <a href="https://lemonade.changedetection.io/start">very affordable subscription based service</a>.</p>
                                 <p>This is because Selenium/WebDriver can not extract full page screenshots reliably.</p>
+                                <p>You may need to <a href="https://github.com/dgtlmoon/changedetection.io/blob/09ebc6ec6338545bdd694dc6eee57f2e9d2b8075/docker-compose.yml#L31">Enable playwright environment variable</a> and uncomment the <strong>sockpuppetbrowser</strong> from the docker-compose.yml file.</p>
                             </span>
                         {% endif %}
                     </div>

changedetectionio/templates/settings.html

@@ -280,9 +280,7 @@ nav
                         
                     </div>
                 </div>
-                <p>
-                    Your proxy provider may need to whitelist our IP of <code>204.15.192.195</code>
-                </p>
+
                <p><strong>Tip</strong>: "Residential" and "Mobile" proxy type can be more successfull than "Data Center" for blocked websites.
 
                 <div class="pure-control-group" id="extra-proxies-setting">

changedetectionio/templates/watch-overview.html

@@ -108,7 +108,8 @@
                     {% else %}
                     <a class="state-on" href="{{url_for('index', op='pause', uuid=watch.uuid, tag=active_tag_uuid)}}"><img src="{{url_for('static_content', group='images', filename='play.svg')}}" alt="UnPause checks" title="UnPause checks" class="icon icon-unpause" ></a>
                     {% endif %}
-                    <a class="link-mute state-{{'on' if watch.notification_muted else 'off'}}" href="{{url_for('index', op='mute', uuid=watch.uuid, tag=active_tag_uuid)}}"><img src="{{url_for('static_content', group='images', filename='bell-off.svg')}}" alt="Mute notifications" title="Mute notifications" class="icon icon-mute" ></a>
+                    {% set mute_label = 'UnMute notification' if watch.notification_muted else 'Mute notification' %}
+                    <a class="link-mute state-{{'on' if watch.notification_muted else 'off'}}" href="{{url_for('index', op='mute', uuid=watch.uuid, tag=active_tag_uuid)}}"><img src="{{url_for('static_content', group='images', filename='bell-off.svg')}}" alt="{{ mute_label }}" title="{{ mute_label }}" class="icon icon-mute" ></a>
                 </td>
                 <td class="title-col inline">{{watch.title if watch.title is not none and watch.title|length > 0 else watch.url}}
                     <a class="external" target="_blank" rel="noopener" href="{{ watch.link.replace('source:','') }}"></a>