Merge branch 'master' into browserstep-refactor

changedetectionio/__init__.py

@@ -38,7 +38,7 @@ from flask_paginate import Pagination, get_page_parameter
 from changedetectionio import html_tools
 from changedetectionio.api import api_v1
 
-__version__ = '0.42.1'
+__version__ = '0.41.1'
 
 datastore = None
 
@@ -403,35 +403,26 @@ def changedetection_app(config=None, datastore_o=None):
 
         # Sort by last_changed and add the uuid which is usually the key..
         sorted_watches = []
-        search_q = request.args.get('q').strip().lower() if request.args.get('q') else False
         for uuid, watch in datastore.data['watching'].items():
 
-            if limit_tag:
+            if limit_tag != None:
                 # Support for comma separated list of tags.
-                if not watch.get('tag'):
+                if watch['tag'] is None:
                     continue
-                for tag_in_watch in watch.get('tag', '').split(','):
+                for tag_in_watch in watch['tag'].split(','):
                     tag_in_watch = tag_in_watch.strip()
                     if tag_in_watch == limit_tag:
                         watch['uuid'] = uuid
-                        if search_q:
-                            if (watch.get('title') and search_q in watch.get('title').lower()) or search_q in watch.get('url', '').lower():
-                                sorted_watches.append(watch)
-                        else:
-                            sorted_watches.append(watch)
+                        sorted_watches.append(watch)
 
             else:
-                #watch['uuid'] = uuid
-                if search_q:
-                    if (watch.get('title') and search_q in watch.get('title').lower()) or search_q in watch.get('url', '').lower():
-                        sorted_watches.append(watch)
-                else:
-                    sorted_watches.append(watch)
+                watch['uuid'] = uuid
+                sorted_watches.append(watch)
 
         existing_tags = datastore.get_all_tags()
         form = forms.quickWatchForm(request.form)
         page = request.args.get(get_page_parameter(), type=int, default=1)
-        total_count = len(sorted_watches)
+        total_count = len(sorted_watches) if sorted_watches else len(datastore.data['watching'])
         pagination = Pagination(page=page, total=total_count, per_page=int(os.getenv('pagination_per_page', 50)), css_framework = "semantic")
 
         output = render_template(
@@ -446,7 +437,6 @@ def changedetection_app(config=None, datastore_o=None):
                                  hosted_sticky=os.getenv("SALTED_PASS", False) == False,
                                  pagination=pagination,
                                  queued_uuids=[q_uuid.item['uuid'] for q_uuid in update_q.queue],
-                                 search_q=request.args.get('q','').strip(),
                                  sort_attribute=request.args.get('sort') if request.args.get('sort') else request.cookies.get('sort'),
                                  sort_order=request.args.get('order') if request.args.get('order') else request.cookies.get('order'),
                                  system_default_fetcher=datastore.data['settings']['application'].get('fetch_backend'),

changedetectionio/blueprint/browser_steps/__init__.py

@@ -104,17 +104,14 @@ def construct_blueprint(datastore: ChangeDetectionStore):
         # A new session was requested, return sessionID
 
         import uuid
-        global browsersteps_sessions
-
         browsersteps_session_id = str(uuid.uuid4())
-        watch_uuid = request.args.get('uuid')
 
-        if not watch_uuid:
-            return make_response('No Watch UUID specified', 500)
+        watch_uuid = request.args.get('uuid')
+        global browsersteps_sessions
 
         print("Starting connection with playwright")
         logging.debug("browser_steps.py connecting")
-        browsersteps_sessions[browsersteps_session_id] = start_browsersteps_session(watch_uuid)
+        browsersteps_sessions[browsersteps_session_id] = start_browsersteps_session()
         print("Starting connection with playwright - done")
         return {'browsersteps_session_id': browsersteps_session_id}
 

changedetectionio/blueprint/browser_steps/browser_steps.py

@@ -133,7 +133,7 @@ class steppable_browser_interface():
         self.page.wait_for_timeout(1000)
 
     def action_wait_for_seconds(self, selector, value):
-        self.page.wait_for_timeout(float(value.strip()) * 1000)
+        self.page.wait_for_timeout(int(value) * 1000)
 
     def action_wait_for_text(self, selector, value):
         import json

changedetectionio/processors/restock_diff.py

@@ -12,12 +12,6 @@ urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 name = 'Re-stock detection for single product pages'
 description = 'Detects if the product goes back to in-stock'
 
-class UnableToExtractRestockData(Exception):
-    def __init__(self, status_code):
-        # Set this so we can use it in other parts of the app
-        self.status_code = status_code
-        return
-
 class perform_site_check(difference_detection_processor):
     screenshot = None
     xpath_data = None
@@ -111,8 +105,7 @@ class perform_site_check(difference_detection_processor):
             fetched_md5 = hashlib.md5(fetcher.instock_data.encode('utf-8')).hexdigest()
             # 'Possibly in stock' comes from stock-not-in-stock.js when no string found above the fold.
             update_obj["in_stock"] = True if fetcher.instock_data == 'Possibly in stock' else False
-        else:
-            raise UnableToExtractRestockData(status_code=fetcher.status_code)
+
 
         # The main thing that all this at the moment comes down to :)
         changed_detected = False

changedetectionio/static/js/browser-steps.js

@@ -162,9 +162,7 @@ $(document).ready(function () {
                 ) {
                     // There could be many elements here, record them all and then we'll find out which is the most 'useful'
                     // (input, textarea, button, A etc)
-                    if (item.width < xpath_data['browser_width']) {
-                        possible_elements.push(item);
-                    }
+                    possible_elements.push(item);
                 }
             });
 
@@ -422,6 +420,7 @@ $(document).ready(function () {
             apply_buttons_disabled = false;
             $("#browsersteps-img").css('opacity', 1);
             $('ul#browser_steps li .control .apply').css('opacity', 1);
+            browserless_seconds_remaining = data.browser_time_remaining;
             $("#loading-status-text").hide();
             set_first_gotosite_disabled();
         }).fail(function (data) {

changedetectionio/static/js/toggle-theme.js

@@ -3,7 +3,7 @@
  * Toggles theme between light and dark mode.
  */
 $(document).ready(function () {
-  const button = document.getElementById("toggle-light-mode");
+  const button = document.getElementsByClassName("toggle-theme")[0];
 
   button.onclick = () => {
     const htmlElement = document.getElementsByTagName("html");
@@ -21,33 +21,4 @@ $(document).ready(function () {
   const setCookieValue = (value) => {
     document.cookie = `css_dark_mode=${value};max-age=31536000;path=/`
   }
-
-  // Search input box behaviour
-    const toggle_search = document.getElementById("toggle-search");
-  const search_q = document.getElementById("search-q");
-  window.addEventListener('keydown', function (e) {
-
-    if (e.altKey == true && e.keyCode == 83)
-      search_q.classList.toggle('expanded');
-      search_q.focus();
-  });
-
-
-  search_q.onkeydown = (e) => {
-    var key = e.keyCode || e.which;
-    if (key === 13) {
-      document.searchForm.submit();
-    }
-  };
-  toggle_search.onclick = () => {
-    // Could be that they want to search something once text is in there
-    if (search_q.value.length) {
-      document.searchForm.submit();
-    } else {
-      // If not..
-      search_q.classList.toggle('expanded');
-      search_q.focus();
-    }
-  };
-
 });

changedetectionio/static/js/visual-selector.js

@@ -61,12 +61,7 @@ $(document).ready(function () {
     function bootstrap_visualselector() {
         if (1) {
             // bootstrap it, this will trigger everything else
-            $("img#selector-background").on("error", function () {
-                $('.fetching-update-notice').html("<strong>Ooops!</strong> The VisualSelector tool needs atleast one fetched page, please unpause the watch and/or wait for the watch to complete fetching and then reload this page.");
-                $('.fetching-update-notice').css('color','#bb0000');
-                $('#selector-current-xpath').hide();
-                $('#clear-selector').hide();
-            }).bind('load', function () {
+            $("img#selector-background").bind('load', function () {
                 console.log("Loaded background...");
                 c = document.getElementById("selector-canvas");
                 // greyed out fill context
@@ -84,11 +79,10 @@ $(document).ready(function () {
             }).attr("src", screenshot_url);
         }
         // Tell visualSelector that the image should update
-        var s = $("img#selector-background").attr('src') + "?" + new Date().getTime();
-        $("img#selector-background").attr('src', s)
+        var s = $("img#selector-background").attr('src')+"?"+ new Date().getTime();
+        $("img#selector-background").attr('src',s)
     }
 
-    // This is fired once the img src is loaded in bootstrap_visualselector()
     function fetch_data() {
         // Image is ready
         $('.fetching-update-notice').html("Fetching element data..");
@@ -105,8 +99,7 @@ $(document).ready(function () {
             reflow_selector();
             $('.fetching-update-notice').fadeOut();
         });
-
-    }
+    };
 
 
     function set_scale() {

changedetectionio/static/styles/scss/styles.scss

@@ -54,47 +54,8 @@ a.github-link {
   }
 }
 
-#toggle-light-mode {
-  width: 3rem;
-  .icon-dark {
-    display: none;
-  }
-
-  &.dark {
-    .icon-light {
-      display: none;
-    }
-
-    .icon-dark {
-      display: block;
-    }
-  }
-}
-
-#toggle-search {
-  width: 2rem;
-}
-
-#search-q {
-  opacity: 0;
-  -webkit-transition: all .9s ease;
-  -moz-transition: all .9s ease;
-  transition: all .9s ease;
-  width: 0;
-  display: none;
-  &.expanded {
-    width: auto;
-    display: inline-block;
-
-    opacity: 1;
-  }
-}
-#search-result-info {
-  color: #fff;
-}
-
-button.toggle-button {
-  vertical-align: middle;
+button.toggle-theme {
+  width: 4rem;
   background: transparent;
   border: none;
   cursor: pointer;
@@ -113,7 +74,19 @@ button.toggle-button {
     display: block;
   }
 
+  .icon-dark {
+    display: none;
+  }
 
+  &.dark {
+    .icon-light {
+      display: none;
+    }
+
+    .icon-dark {
+      display: block;
+    }
+  }
 }
 
 .pure-menu-horizontal {

changedetectionio/static/styles/styles.css

@@ -331,44 +331,23 @@ a.github-link {
   a.github-link:hover {
     color: var(--color-icon-github-hover); }
 
-#toggle-light-mode {
-  width: 3rem; }
-  #toggle-light-mode .icon-dark {
-    display: none; }
-  #toggle-light-mode.dark .icon-light {
-    display: none; }
-  #toggle-light-mode.dark .icon-dark {
-    display: block; }
-
-#toggle-search {
-  width: 2rem; }
-
-#search-q {
-  opacity: 0;
-  -webkit-transition: all .9s ease;
-  -moz-transition: all .9s ease;
-  transition: all .9s ease;
-  width: 0;
-  display: none; }
-  #search-q.expanded {
-    width: auto;
-    display: inline-block;
-    opacity: 1; }
-
-#search-result-info {
-  color: #fff; }
-
-button.toggle-button {
-  vertical-align: middle;
+button.toggle-theme {
+  width: 4rem;
   background: transparent;
   border: none;
   cursor: pointer;
   color: var(--color-icon-github); }
-  button.toggle-button:hover {
+  button.toggle-theme:hover {
     color: var(--color-icon-github-hover); }
-  button.toggle-button svg {
+  button.toggle-theme svg {
     fill: currentColor; }
-  button.toggle-button .icon-light {
+  button.toggle-theme .icon-light {
+    display: block; }
+  button.toggle-theme .icon-dark {
+    display: none; }
+  button.toggle-theme.dark .icon-light {
+    display: none; }
+  button.toggle-theme.dark .icon-dark {
     display: block; }
 
 .pure-menu-horizontal {

changedetectionio/store.py

@@ -366,21 +366,19 @@ class ChangeDetectionStore:
     def save_error_text(self, watch_uuid, contents):
         if not self.data['watching'].get(watch_uuid):
             return
-
-        self.data['watching'][watch_uuid].ensure_data_dir_exists()
         target_path = os.path.join(self.datastore_path, watch_uuid, "last-error.txt")
+
         with open(target_path, 'w') as f:
             f.write(contents)
 
     def save_xpath_data(self, watch_uuid, data, as_error=False):
-
         if not self.data['watching'].get(watch_uuid):
             return
         if as_error:
             target_path = os.path.join(self.datastore_path, watch_uuid, "elements-error.json")
         else:
             target_path = os.path.join(self.datastore_path, watch_uuid, "elements.json")
-        self.data['watching'][watch_uuid].ensure_data_dir_exists()
+
         with open(target_path, 'w') as f:
             f.write(json.dumps(data))
             f.close()

changedetectionio/templates/_common_fields.jinja

@@ -115,7 +115,7 @@
                                     URLs generated by changedetection.io (such as <code>{{ '{{diff_url}}' }}</code>) require the <code>BASE_URL</code> environment variable set.<br>
                                     Your <code>BASE_URL</code> var is currently "{{settings_application['current_base_url']}}"
 									<br>
-									Warning: Contents of <code>{{ '{{diff}}' }}</code>, <code>{{ '{{diff_removed}}' }}</code>, and <code>{{ '{{diff_added}}' }}</code> depend on how the difference algorithm perceives the change. For example, an addition or removal could be perceived as a change in some cases. <a target="_new" href="https://github.com/dgtlmoon/changedetection.io/wiki/Using-the-%7B%7Bdiff%7D%7D,-%7B%7Bdiff_added%7D%7D,-and-%7B%7Bdiff_removed%7D%7D-notification-tokens">More Here</a> <br>
+									Warning: Contents of <code>{{ '{{diff}}' }}</code>, <code>{{ '{{diff_removed}}' }}</code>, and <code>{{ '{{diff_added}}' }}</code> depend on how the difference algorithm perceives the change. For example, an addition or removal could be perceived as a change in some cases. <a target="_new" href="https://github.com/dgtlmoon/changedetection.io/wiki/Using-the-%7B%7Bdiff%7D%7D,-%7B%7Bdiff_added%7D%7D,-and-%7B%7Bdiff_removal%7D%7D-notification-tokens">More Here</a> <br>
                                 </div>
                             </div>
                         </div>

changedetectionio/templates/base.html

@@ -82,21 +82,11 @@
               <a href="{{url_for('logout')}}" class="pure-menu-link">LOG OUT</a>
             </li>
           {% endif %}
-          <li class="pure-menu-item pure-form" id="search-menu-item">
-            <!-- We use GET here so it offers people a chance to set bookmarks etc -->
-            <form name="searchForm" action="" method="GET">
-              <input id="search-q" class="" name="q" placeholder="URL or Title {% if active_tag %}in '{{ active_tag }}'{% endif %}" required="" type="text" value="">
-              <input name="tag" type="hidden" value="{% if active_tag %}{{active_tag}}{% endif %}">
-              <button class="toggle-button " id="toggle-search" type="button" title="Search, or Use Alt+S Key" >
-                {% include "svgs/search-icon.svg" %}
-              </button>
-            </form>
-          </li>
           <li class="pure-menu-item">
             {% if dark_mode %}
             {% set darkClass = 'dark' %}
             {% endif %}
-            <button class="toggle-button {{darkClass}}"  id ="toggle-light-mode" type="button" title="Toggle Light/Dark Mode">
+            <button class="toggle-theme {{darkClass}}" type="button" title="Toggle Light/Dark Mode">
               <span class="visually-hidden">Toggle light/dark mode</span>
               <span class="icon-light">
                 {% include "svgs/light-mode-toggle-icon.svg" %}
@@ -116,7 +106,7 @@
     </div>
     {% if hosted_sticky %}
       <div class="sticky-tab" id="hosted-sticky">
-        <a href="https://changedetection.io/?ref={{guid}}">Let us host your instance!</a>
+        <a href="https://lemonade.changedetection.io/start?ref={{guid}}">Let us host your instance!</a>
       </div>
     {% endif %}
     {% if left_sticky %}

changedetectionio/templates/svgs/search-icon.svg

@@ -1 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?><svg version="1.1" id="Layer_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 0 122.879 119.799" enable-background="new 0 0 122.879 119.799" xml:space="preserve"><g><path d="M49.988,0h0.016v0.007C63.803,0.011,76.298,5.608,85.34,14.652c9.027,9.031,14.619,21.515,14.628,35.303h0.007v0.033v0.04 h-0.007c-0.005,5.557-0.917,10.905-2.594,15.892c-0.281,0.837-0.575,1.641-0.877,2.409v0.007c-1.446,3.66-3.315,7.12-5.547,10.307 l29.082,26.139l0.018,0.016l0.157,0.146l0.011,0.011c1.642,1.563,2.536,3.656,2.649,5.78c0.11,2.1-0.543,4.248-1.979,5.971 l-0.011,0.016l-0.175,0.203l-0.035,0.035l-0.146,0.16l-0.016,0.021c-1.565,1.642-3.654,2.534-5.78,2.646 c-2.097,0.111-4.247-0.54-5.971-1.978l-0.015-0.011l-0.204-0.175l-0.029-0.024L78.761,90.865c-0.88,0.62-1.778,1.209-2.687,1.765 c-1.233,0.755-2.51,1.466-3.813,2.115c-6.699,3.342-14.269,5.222-22.272,5.222v0.007h-0.016v-0.007 c-13.799-0.004-26.296-5.601-35.338-14.645C5.605,76.291,0.016,63.805,0.007,50.021H0v-0.033v-0.016h0.007 c0.004-13.799,5.601-26.296,14.645-35.338C23.683,5.608,36.167,0.016,49.955,0.007V0H49.988L49.988,0z M50.004,11.21v0.007h-0.016 h-0.033V11.21c-10.686,0.007-20.372,4.35-27.384,11.359C15.56,29.578,11.213,39.274,11.21,49.973h0.007v0.016v0.033H11.21 c0.007,10.686,4.347,20.367,11.359,27.381c7.009,7.012,16.705,11.359,27.403,11.361v-0.007h0.016h0.033v0.007 c10.686-0.007,20.368-4.348,27.382-11.359c7.011-7.009,11.358-16.702,11.36-27.4h-0.006v-0.016v-0.033h0.006 c-0.006-10.686-4.35-20.372-11.358-27.384C70.396,15.56,60.703,11.213,50.004,11.21L50.004,11.21z"/></g></svg>

changedetectionio/templates/watch-overview.html

@@ -44,7 +44,6 @@
     {% if watches|length >= pagination.per_page %}
         {{ pagination.info }}
     {% endif %}
-    {% if search_q %}<div id="search-result-info">Searching "<strong><i>{{search_q}}</i></strong>"</div>{% endif %}
     <div>
         <a href="{{url_for('index')}}" class="pure-button button-tag {{'active' if not active_tag }}">All</a>
         {% for tag in tags %}
@@ -74,11 +73,7 @@
             </tr>
             </thead>
             <tbody>
-            {% if not watches|length %}
-            <tr>
-                <td colspan="6">No website watches configured, please add a URL in the box above, or <a href="{{ url_for('import_page')}}" >import a list</a>.</td>
-            </tr>
-            {% endif %}
+
             {% for watch in (watches|sort(attribute=sort_attribute, reverse=sort_order == 'asc'))[pagination.skip:pagination.skip+pagination.per_page] %}
             <tr id="{{ watch.uuid }}"
                 class="{{ loop.cycle('pure-table-odd', 'pure-table-even') }} processor-{{ watch['processor'] }}

changedetectionio/update_worker.py

@@ -5,7 +5,7 @@ import time
 
 from changedetectionio import content_fetcher
 from .processors.text_json_diff import FilterNotFoundInResponse
-from .processors.restock_diff import UnableToExtractRestockData
+
 
 # A single update worker
 #
@@ -262,7 +262,6 @@ class update_worker(threading.Thread):
                         # Yes fine, so nothing todo, don't continue to process.
                         process_changedetection_results = False
                         changed_detected = False
-                        self.datastore.update_watch(uuid=uuid, update_obj={'last_error': False})
 
                     except content_fetcher.BrowserStepsStepTimout as e:
 
@@ -319,11 +318,6 @@ class update_worker(threading.Thread):
                                                                            'last_check_status': e.status_code,
                                                                            'has_ldjson_price_data': None})
                         process_changedetection_results = False
-                    except UnableToExtractRestockData as e:
-                        # Usually when fetcher.instock_data returns empty
-                        self.app.logger.error("Exception reached processing watch UUID: %s - %s", uuid, str(e))
-                        self.datastore.update_watch(uuid=uuid, update_obj={'last_error': f"Unable to extract restock data for this page unfortunately. (Got code {e.status_code} from server)"})
-                        process_changedetection_results = False
                     except Exception as e:
                         self.app.logger.error("Exception reached processing watch UUID: %s - %s", uuid, str(e))
                         self.datastore.update_watch(uuid=uuid, update_obj={'last_error': str(e)})