Fixing access to assets #3053

changedetectionio/__init__.py

@@ -2,7 +2,7 @@
 
 # Read more https://github.com/dgtlmoon/changedetection.io/wiki
 
-__version__ = '0.49.8'
+__version__ = '0.49.7'
 
 from changedetectionio.strtobool import strtobool
 from json.decoder import JSONDecodeError

changedetectionio/api/Search.py

@@ -1,51 +0,0 @@
-from flask_restful import Resource, abort
-from flask import request
-from . import auth
-
-class Search(Resource):
-    def __init__(self, **kwargs):
-        # datastore is a black box dependency
-        self.datastore = kwargs['datastore']
-
-    @auth.check_token
-    def get(self):
-        """
-        @api {get} /api/v1/search Search for watches
-        @apiDescription Search watches by URL or title text
-        @apiExample {curl} Example usage:
-            curl "http://localhost:5000/api/v1/search?q=https://example.com/page1" -H"x-api-key:813031b16330fe25e3780cf0325daa45"
-            curl "http://localhost:5000/api/v1/search?q=https://example.com/page1?tag=Favourites" -H"x-api-key:813031b16330fe25e3780cf0325daa45"
-            curl "http://localhost:5000/api/v1/search?q=https://example.com?partial=true" -H"x-api-key:813031b16330fe25e3780cf0325daa45"
-        @apiName Search
-        @apiGroup Watch Management
-        @apiQuery {String} q Search query to match against watch URLs and titles
-        @apiQuery {String} [tag] Optional name of tag to limit results (name not UUID)
-        @apiQuery {String} [partial] Allow partial matching of URL query
-        @apiSuccess (200) {Object} JSON Object containing matched watches
-        """
-        query = request.args.get('q', '').strip()
-        tag_limit = request.args.get('tag', '').strip()
-        from changedetectionio.strtobool import strtobool
-        partial = bool(strtobool(request.args.get('partial', '0'))) if 'partial' in request.args else False
-
-        # Require a search query
-        if not query:
-            abort(400, message="Search query 'q' parameter is required")
-
-        # Use the search function from the datastore
-        matching_uuids = self.datastore.search_watches_for_url(query=query, tag_limit=tag_limit, partial=partial)
-
-        # Build the response with watch details
-        results = {}
-        for uuid in matching_uuids:
-            watch = self.datastore.data['watching'].get(uuid)
-            results[uuid] = {
-                'last_changed': watch.last_changed,
-                'last_checked': watch['last_checked'],
-                'last_error': watch['last_error'],
-                'title': watch['title'],
-                'url': watch['url'],
-                'viewed': watch.viewed
-            }
-
-        return results, 200

changedetectionio/blueprint/watchlist/__init__.py

@@ -1,5 +1,7 @@
+import flask_login
 import os
 import time
+import timeago
 
 from flask import Blueprint, request, make_response, render_template, redirect, url_for, flash, session
 from flask_login import current_user
@@ -8,6 +10,7 @@ from flask_paginate import Pagination, get_page_parameter
 from changedetectionio import forms
 from changedetectionio.store import ChangeDetectionStore
 from changedetectionio.auth_decorator import login_optionally_required
+from changedetectionio.strtobool import strtobool
 
 def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMetaData):
     watchlist_blueprint = Blueprint('watchlist', __name__, template_folder="templates")
@@ -74,6 +77,7 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMe
         sorted_tags = sorted(datastore.data['settings']['application'].get('tags').items(), key=lambda x: x[1]['title'])
         output = render_template(
             "watch-overview.html",
+                                 # Don't link to hosting when we're on the hosting environment
                                  active_tag=active_tag,
                                  active_tag_uuid=active_tag_uuid,
                                  app_rss_token=datastore.data['settings']['application'].get('rss_access_token'),
@@ -84,10 +88,9 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMe
                                  has_proxies=datastore.proxy_list,
                                  has_unviewed=datastore.has_unviewed,
                                  hosted_sticky=os.getenv("SALTED_PASS", False) == False,
-                                 now_time_server=time.time(),
                                  pagination=pagination,
                                  queued_uuids=[q_uuid.item['uuid'] for q_uuid in update_q.queue],
-                                 search_q=request.args.get('q', '').strip(),
+                                 search_q=request.args.get('q','').strip(),
                                  sort_attribute=request.args.get('sort') if request.args.get('sort') else request.cookies.get('sort'),
                                  sort_order=request.args.get('order') if request.args.get('order') else request.cookies.get('order'),
                                  system_default_fetcher=datastore.data['settings']['application'].get('fetch_backend'),

changedetectionio/blueprint/watchlist/templates/watch-overview.html

@@ -3,16 +3,7 @@
 {% from '_helpers.html' import render_simple_field, render_field, render_nolabel_field, sort_by_title %}
 <script src="{{url_for('static_content', group='js', filename='jquery-3.6.0.min.js')}}"></script>
 <script src="{{url_for('static_content', group='js', filename='watch-overview.js')}}" defer></script>
-<script>let nowtimeserver={{ now_time_server }};</script>
 
-<style>
-.checking-now .last-checked {
-    background-image: linear-gradient(to bottom, transparent 0%, rgba(0,0,0,0.05) 40%, rgba(0,0,0,0.1) 100%);
-    background-size: 0 100%;
-    background-repeat: no-repeat;
-    transition: background-size 0.9s ease
-}
-</style>
 <div class="box">
 
     <form class="pure-form" action="{{ url_for('ui.ui_views.form_quick_watch_add', tag=active_tag_uuid) }}" method="POST" id="new-watch-form">
@@ -100,8 +91,8 @@
             {% endif %}
             {% for watch in (watches|sort(attribute=sort_attribute, reverse=sort_order == 'asc'))|pagination_slice(skip=pagination.skip) %}
 
-                {% set is_unviewed = watch.newest_history_key| int > watch.last_viewed and watch.history_n>=2 %}
-                {% set checking_now = is_checking_now(watch) %}
+                {% set is_unviewed =  watch.newest_history_key| int > watch.last_viewed and watch.history_n>=2 %}
+
             <tr id="{{ watch.uuid }}"
                 class="{{ loop.cycle('pure-table-odd', 'pure-table-even') }} processor-{{ watch['processor'] }}
                 {% if watch.last_error is defined and watch.last_error != False %}error{% endif %}
@@ -109,9 +100,7 @@
                 {% if watch.paused is defined and watch.paused != False %}paused{% endif %}
                 {% if is_unviewed %}unviewed{% endif %}
                 {% if watch.has_restock_info %} has-restock-info {% if watch['restock']['in_stock'] %}in-stock{% else %}not-in-stock{% endif %} {% else %}no-restock-info{% endif %}
-                {% if watch.uuid in queued_uuids %}queued{% endif %}
-                {% if checking_now %}checking-now{% endif %}
-                ">
+                {% if watch.uuid in queued_uuids %}queued{% endif %}">
                 <td class="inline checkbox-uuid" ><input name="uuids"  type="checkbox" value="{{ watch.uuid}} " > <span>{{ loop.index+pagination.skip }}</span></td>
                 <td class="inline watch-controls">
                     {% if not watch.paused %}
@@ -189,14 +178,7 @@
                     {% endif %}
                 </td>
 {% endif %}
-            {#last_checked becomes fetch-start-time#}
-                <td class="last-checked" data-timestamp="{{ watch.last_checked }}" {% if checking_now %} data-fetchduration={{ watch.fetch_time }} data-eta_complete="{{ watch.last_checked+watch.fetch_time }}" {% endif %} >
-                    {% if checking_now %}
-                        <span class="spinner"></span><span> Checking now</span>
-                    {% else %}
-                        {{watch|format_last_checked_time|safe}}</td>
-                    {% endif %}
-
+                <td class="last-checked" data-timestamp="{{ watch.last_checked }}">{{watch|format_last_checked_time|safe}}</td>
                 <td class="last-changed" data-timestamp="{{ watch.last_changed }}">{% if watch.history_n >=2 and watch.last_changed >0 %}
                     {{watch.last_changed|format_timestamp_timeago}}
                     {% else %}

changedetectionio/flask_app.py

@@ -34,7 +34,6 @@ from loguru import logger
 from changedetectionio import __version__
 from changedetectionio import queuedWatchMetaData
 from changedetectionio.api import Watch, WatchHistory, WatchSingleHistory, CreateWatch, Import, SystemInfo, Tag, Tags
-from changedetectionio.api.Search import Search
 from .time_handler import is_within_schedule
 
 datastore = None
@@ -123,18 +122,14 @@ def _jinja2_filter_format_number_locale(value: float) -> str:
 
     return formatted_value
 
-@app.template_global('is_checking_now')
-def _watch_is_checking_now(watch_obj, format="%Y-%m-%d %H:%M:%S"):
-    # Worker thread tells us which UUID it is currently processing.
-    for t in running_update_threads:
-        if t.current_uuid == watch_obj['uuid']:
-            return True
-
-
 # We use the whole watch object from the store/JSON so we can see if there's some related status in terms of a thread
 # running or something similar.
 @app.template_filter('format_last_checked_time')
 def _jinja2_filter_datetime(watch_obj, format="%Y-%m-%d %H:%M:%S"):
+    # Worker thread tells us which UUID it is currently processing.
+    for t in running_update_threads:
+        if t.current_uuid == watch_obj['uuid']:
+            return '<span class="spinner"></span><span> Checking now</span>'
 
     if watch_obj['last_checked'] == 0:
         return 'Not yet'
@@ -280,9 +275,6 @@ def changedetection_app(config=None, datastore_o=None):
 
     watch_api.add_resource(Tag, '/api/v1/tag', '/api/v1/tag/<string:uuid>',
                            resource_class_kwargs={'datastore': datastore})
-                           
-    watch_api.add_resource(Search, '/api/v1/search',
-                           resource_class_kwargs={'datastore': datastore})
 
 
 

changedetectionio/static/js/watch-overview.js

@@ -48,8 +48,6 @@ $(function () {
         $('input[type=checkbox]').not(this).prop('checked', this.checked);
     });
 
-    const time_check_step_size_seconds=1;
-
     // checkboxes - show/hide buttons
     $("input[type=checkbox]").click(function (e) {
         if ($('input[type=checkbox]:checked').length) {
@@ -59,30 +57,5 @@ $(function () {
         }
     });
 
-    setInterval(function () {
-        // Background ETA completion for 'checking now'
-        $(".watch-table .checking-now .last-checked").each(function () {
-            const eta_complete = parseFloat($(this).data('eta_complete'));
-            const fetch_duration = parseInt($(this).data('fetchduration'));
-
-            if (eta_complete + 2 > nowtimeserver && fetch_duration > 3) {
-                const remaining_seconds = Math.abs(eta_complete) - nowtimeserver - 1;
-
-                let r = (1.0 - (remaining_seconds / fetch_duration)) * 100;
-                if (r < 10) {
-                    r = 10;
-                }
-                if (r >= 90) {
-                    r = 100;
-                }
-                $(this).css('background-size', `${r}% 100%`);
-                //$(this).text(`${r}% remain ${remaining_seconds}`);
-            } else {
-                $(this).css('background-size', `100% 100%`);
-            }
-        });
-
-        nowtimeserver = nowtimeserver + time_check_step_size_seconds;
-    }, time_check_step_size_seconds * 1000);
 });
 

changedetectionio/store.py

@@ -631,41 +631,6 @@ class ChangeDetectionStore:
             if watch.get('processor') == processor_name:
                 return True
         return False
-        
-    def search_watches_for_url(self, query, tag_limit=None, partial=False):
-        """Search watches by URL, title, or error messages
-        
-        Args:
-            query (str): Search term to match against watch URLs, titles, and error messages
-            tag_limit (str, optional): Optional tag name to limit search results
-            partial: (bool, optional): sub-string matching
-
-        Returns:
-            list: List of UUIDs of watches that match the search criteria
-        """
-        matching_uuids = []
-        query = query.lower().strip()
-        tag = self.tag_exists_by_name(tag_limit) if tag_limit else False
-
-        for uuid, watch in self.data['watching'].items():
-            # Filter by tag if requested
-            if tag_limit:
-                if not tag.get('uuid') in watch.get('tags', []):
-                    continue
-
-            # Search in URL, title, or error messages
-            if partial:
-                if ((watch.get('title') and query in watch.get('title').lower()) or
-                    query in watch.get('url', '').lower() or
-                    (watch.get('last_error') and query in watch.get('last_error').lower())):
-                    matching_uuids.append(uuid)
-            else:
-                if ((watch.get('title') and query == watch.get('title').lower()) or
-                        query == watch.get('url', '').lower() or
-                        (watch.get('last_error') and query == watch.get('last_error').lower())):
-                    matching_uuids.append(uuid)
-
-        return matching_uuids
 
     def get_unique_notification_tokens_available(self):
         # Ask each type of watch if they have any extra notification token to add to the validation

changedetectionio/tests/test_api_search.py

@@ -1,101 +0,0 @@
-from copy import copy
-
-from flask import url_for
-import json
-import time
-from .util import live_server_setup, wait_for_all_checks
-
-
-def test_api_search(client, live_server):
-    live_server_setup(live_server)
-    api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')
-
-    watch_data = {}
-    # Add some test watches
-    urls = [
-        'https://example.com/page1',
-        'https://example.org/testing',
-        'https://test-site.com/example'
-    ]
-
-    # Import the test URLs
-    res = client.post(
-        url_for("imports.import_page"),
-        data={"urls": "\r\n".join(urls)},
-        follow_redirects=True
-    )
-    assert b"3 Imported" in res.data
-    wait_for_all_checks(client)
-
-    # Get a listing, it will be the first one
-    watches_response = client.get(
-        url_for("createwatch"),
-        headers={'x-api-key': api_key}
-    )
-
-
-    # Add a title to one watch for title search testing
-    for uuid, watch in watches_response.json.items():
-
-        watch_data = client.get(url_for("watch", uuid=uuid),
-                                follow_redirects=True,
-                                headers={'x-api-key': api_key}
-                                )
-
-        if urls[0] == watch_data.json['url']:
-            # HTTP PUT ( UPDATE an existing watch )
-            client.put(
-                url_for("watch", uuid=uuid),
-                headers={'x-api-key': api_key, 'content-type': 'application/json'},
-                data=json.dumps({'title': 'Example Title Test'}),
-            )
-
-    # Test search by URL
-    res = client.get(url_for("search")+"?q=https://example.com/page1", headers={'x-api-key': api_key, 'content-type': 'application/json'})
-    assert len(res.json) == 1
-    assert list(res.json.values())[0]['url'] == urls[0]
-
-    # Test search by URL - partial should NOT match without ?partial=true flag
-    res = client.get(url_for("search")+"?q=https://example", headers={'x-api-key': api_key, 'content-type': 'application/json'})
-    assert len(res.json) == 0
-
-
-    # Test search by title
-    res = client.get(url_for("search")+"?q=Example Title Test", headers={'x-api-key': api_key, 'content-type': 'application/json'})
-    assert len(res.json) == 1
-    assert list(res.json.values())[0]['url'] == urls[0]
-    assert list(res.json.values())[0]['title'] == 'Example Title Test'
-
-    # Test search that should return multiple results (partial = true)
-    res = client.get(url_for("search")+"?q=https://example&partial=true", headers={'x-api-key': api_key, 'content-type': 'application/json'})
-    assert len(res.json) == 2
-
-    # Test empty search
-    res = client.get(url_for("search")+"?q=", headers={'x-api-key': api_key, 'content-type': 'application/json'})
-    assert res.status_code == 400
-
-    # Add a tag to test search with tag filter
-    tag_name = 'test-tag'
-    res = client.post(
-        url_for("tag"),
-        data=json.dumps({"title": tag_name}),
-        headers={'content-type': 'application/json', 'x-api-key': api_key}
-    )
-    assert res.status_code == 201
-    tag_uuid = res.json['uuid']
-
-    # Add the tag to one watch
-    for uuid, watch in watches_response.json.items():
-        if urls[2] == watch['url']:
-            client.put(
-                url_for("watch", uuid=uuid),
-                headers={'x-api-key': api_key, 'content-type': 'application/json'},
-                data=json.dumps({'tags': [tag_uuid]}),
-            )
-
-
-    # Test search with tag filter and q
-    res = client.get(url_for("search") + f"?q={urls[2]}&tag={tag_name}", headers={'x-api-key': api_key, 'content-type': 'application/json'})
-    assert len(res.json) == 1
-    assert list(res.json.values())[0]['url'] == urls[2]
-

changedetectionio/update_worker.py

@@ -253,9 +253,8 @@ class update_worker(threading.Thread):
                 pass
 
             else:
-
+                fetch_start_time = time.time()
                 uuid = queued_item_data.item.get('uuid')
-                fetch_start_time = round(time.time())  # Also used for a unique history key for now
                 self.current_uuid = uuid
                 if uuid in list(self.datastore.data['watching'].keys()) and self.datastore.data['watching'][uuid].get('url'):
                     changed_detected = False
@@ -263,10 +262,8 @@ class update_worker(threading.Thread):
                     process_changedetection_results = True
                     update_obj = {}
 
-
                     # Clear last errors (move to preflight func?)
                     self.datastore.data['watching'][uuid]['browser_steps_last_error_step'] = None
-                    self.datastore.data['watching'][uuid]['last_checked'] = fetch_start_time
 
                     watch = self.datastore.data['watching'].get(uuid)
 
@@ -290,6 +287,10 @@ class update_worker(threading.Thread):
 
                         update_handler.call_browser()
 
+                        # In reality, the actual time of when the change was detected could be a few seconds after this
+                        # For example it should include when the page stopped rendering if using a playwright/chrome type fetch
+                        fetch_start_time = time.time()
+
                         changed_detected, update_obj, contents = update_handler.run_changedetection(watch=watch)
 
                         # Re #342
@@ -586,6 +587,7 @@ class update_worker(threading.Thread):
                         pass
 
                     self.datastore.update_watch(uuid=uuid, update_obj={'fetch_time': round(time.time() - fetch_start_time, 3),
+                                                                       'last_checked': int(fetch_start_time),
                                                                        'check_count': count
                                                                        })
 

requirements.txt

@@ -1,3 +1,6 @@
+# Used by Pyppeteer
+pyee
+
 eventlet>=0.38.0
 feedgen~=0.9
 flask-compress
@@ -70,8 +73,7 @@ jq~=1.3; python_version >= "3.8" and sys_platform == "linux"
 
 # playwright is installed at Dockerfile build time because it's not available on all platforms
 
-pyppeteer-ng==2.0.0rc9
-
+pyppeteer-ng==2.0.0rc5
 pyppeteerstealth>=0.0.4
 
 # Include pytest, so if theres a support issue we can ask them to run these tests on their setup