mirror of
https://github.com/dgtlmoon/changedetection.io.git
synced 2025-12-04 15:15:32 +00:00
Compare commits
5 Commits
custom-res
...
3270-histo
| Author | SHA1 | Date | |
|---|---|---|---|
|
c5433dd0ed | ||
|
|
1a496add5b | ||
|
|
fd650955a8 | ||
|
|
27743ba818 | ||
|
|
df2e7ef034 |
1
.github/test/Dockerfile-alpine
vendored
1
.github/test/Dockerfile-alpine
vendored
@@ -18,7 +18,6 @@ RUN \
|
||||
libxslt-dev \
|
||||
openssl-dev \
|
||||
python3-dev \
|
||||
file \
|
||||
zip \
|
||||
zlib-dev && \
|
||||
apk add --update --no-cache \
|
||||
|
||||
@@ -16,7 +16,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
libssl-dev \
|
||||
libxslt-dev \
|
||||
make \
|
||||
patch \
|
||||
zlib1g-dev
|
||||
|
||||
RUN mkdir /install
|
||||
@@ -54,8 +53,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
locales \
|
||||
# For pdftohtml
|
||||
poppler-utils \
|
||||
# favicon type detection and other uses
|
||||
file \
|
||||
zlib1g \
|
||||
&& apt-get clean && rm -rf /var/lib/apt/lists/*
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
# Read more https://github.com/dgtlmoon/changedetection.io/wiki
|
||||
|
||||
__version__ = '0.50.7'
|
||||
__version__ = '0.50.5'
|
||||
|
||||
from changedetectionio.strtobool import strtobool
|
||||
from json.decoder import JSONDecodeError
|
||||
|
||||
@@ -5,7 +5,7 @@ from flask_expects_json import expects_json
|
||||
from changedetectionio import queuedWatchMetaData
|
||||
from changedetectionio import worker_handler
|
||||
from flask_restful import abort, Resource
|
||||
from flask import request, make_response, send_from_directory
|
||||
from flask import request, make_response
|
||||
import validators
|
||||
from . import auth
|
||||
import copy
|
||||
@@ -191,47 +191,6 @@ class WatchSingleHistory(Resource):
|
||||
|
||||
return response
|
||||
|
||||
class WatchFavicon(Resource):
|
||||
def __init__(self, **kwargs):
|
||||
# datastore is a black box dependency
|
||||
self.datastore = kwargs['datastore']
|
||||
|
||||
@auth.check_token
|
||||
def get(self, uuid):
|
||||
"""
|
||||
@api {get} /api/v1/watch/<string:uuid>/favicon Get Favicon for a watch
|
||||
@apiDescription Requires watch `uuid`
|
||||
@apiExample {curl} Example usage:
|
||||
curl http://localhost:5000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091/favicon -H"x-api-key:813031b16330fe25e3780cf0325daa45"
|
||||
@apiName Get latest Favicon
|
||||
@apiGroup Watch History
|
||||
@apiSuccess (200) {String} OK
|
||||
@apiSuccess (404) {String} ERR Not found
|
||||
"""
|
||||
watch = self.datastore.data['watching'].get(uuid)
|
||||
if not watch:
|
||||
abort(404, message=f"No watch exists with the UUID of {uuid}")
|
||||
|
||||
favicon_filename = watch.get_favicon_filename()
|
||||
if favicon_filename:
|
||||
try:
|
||||
import magic
|
||||
mime = magic.from_file(
|
||||
os.path.join(watch.watch_data_dir, favicon_filename),
|
||||
mime=True
|
||||
)
|
||||
except ImportError:
|
||||
# Fallback, no python-magic
|
||||
import mimetypes
|
||||
mime, encoding = mimetypes.guess_type(favicon_filename)
|
||||
|
||||
response = make_response(send_from_directory(watch.watch_data_dir, favicon_filename))
|
||||
response.headers['Content-type'] = mime
|
||||
response.headers['Cache-Control'] = 'max-age=300, must-revalidate' # Cache for 5 minutes, then revalidate
|
||||
return response
|
||||
|
||||
abort(404, message=f'No Favicon available for {uuid}')
|
||||
|
||||
|
||||
class CreateWatch(Resource):
|
||||
def __init__(self, **kwargs):
|
||||
|
||||
@@ -26,7 +26,7 @@ schema_delete_notification_urls = copy.deepcopy(schema_notification_urls)
|
||||
schema_delete_notification_urls['required'] = ['notification_urls']
|
||||
|
||||
# Import all API resources
|
||||
from .Watch import Watch, WatchHistory, WatchSingleHistory, CreateWatch, WatchFavicon
|
||||
from .Watch import Watch, WatchHistory, WatchSingleHistory, CreateWatch
|
||||
from .Tags import Tags, Tag
|
||||
from .Import import Import
|
||||
from .SystemInfo import SystemInfo
|
||||
|
||||
@@ -353,12 +353,6 @@ async def async_update_worker(worker_id, q, notification_q, app, datastore):
|
||||
except Exception as e:
|
||||
pass
|
||||
|
||||
# Store favicon if necessary
|
||||
if update_handler.fetcher.favicon_blob and update_handler.fetcher.favicon_blob.get('base64'):
|
||||
watch.bump_favicon(url=update_handler.fetcher.favicon_blob.get('url'),
|
||||
favicon_base_64=update_handler.fetcher.favicon_blob.get('base64')
|
||||
)
|
||||
|
||||
datastore.update_watch(uuid=uuid, update_obj={'fetch_time': round(time.time() - fetch_start_time, 3),
|
||||
'check_count': count})
|
||||
|
||||
|
||||
@@ -199,14 +199,6 @@ nav
|
||||
</ul>
|
||||
</span>
|
||||
</fieldset>
|
||||
<fieldset class="pure-group">
|
||||
{{ render_field(form.application.form.custom_outofstock_strings) }}
|
||||
<span class="pure-form-message-inline">Additional custom out-of-stock detection strings (one per line).</span>
|
||||
</fieldset>
|
||||
<fieldset class="pure-group">
|
||||
{{ render_field(form.application.form.custom_instock_strings) }}
|
||||
<span class="pure-form-message-inline">Additional custom in-stock detection strings (one per line).</span>
|
||||
</fieldset>
|
||||
</div>
|
||||
|
||||
<div class="tab-pane-inner" id="api">
|
||||
@@ -264,11 +256,6 @@ nav
|
||||
{{ render_checkbox_field(form.application.form.ui.form.socket_io_enabled, class="socket_io_enabled") }}
|
||||
<span class="pure-form-message-inline">Realtime UI Updates Enabled - (Restart required if this is changed)</span>
|
||||
</div>
|
||||
<div class="pure-control-group">
|
||||
{{ render_checkbox_field(form.application.form.ui.form.favicons_enabled, class="") }}
|
||||
<span class="pure-form-message-inline">Enable or Disable Favicons next to the watch list</span>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
<div class="tab-pane-inner" id="proxies">
|
||||
<div id="recommended-proxy">
|
||||
|
||||
@@ -4,7 +4,6 @@
|
||||
<script src="{{url_for('static_content', group='js', filename='jquery-3.6.0.min.js')}}"></script>
|
||||
<script src="{{url_for('static_content', group='js', filename='watch-overview.js')}}" defer></script>
|
||||
<script>let nowtimeserver={{ now_time_server }};</script>
|
||||
<script>let favicon_baseURL="{{ url_for('static_content', group='favicon', filename="PLACEHOLDER")}}";</script>
|
||||
<script>
|
||||
// Initialize Feather icons after the page loads
|
||||
document.addEventListener('DOMContentLoaded', function() {
|
||||
@@ -81,23 +80,16 @@ document.addEventListener('DOMContentLoaded', function() {
|
||||
{%- if any_has_restock_price_processor -%}
|
||||
{%- set cols_required = cols_required + 1 -%}
|
||||
{%- endif -%}
|
||||
{%- set ui_settings = datastore.data['settings']['application']['ui'] -%}
|
||||
|
||||
<div id="watch-table-wrapper">
|
||||
{%- set table_classes = [
|
||||
'favicon-enabled' if 'favicons_enabled' not in ui_settings or ui_settings['favicons_enabled'] else 'favicon-not-enabled',
|
||||
] -%}
|
||||
<table class="pure-table pure-table-striped watch-table {{ table_classes | reject('equalto', '') | join(' ') }}">
|
||||
|
||||
<table class="pure-table pure-table-striped watch-table">
|
||||
<thead>
|
||||
<tr>
|
||||
{%- set link_order = "desc" if sort_order == 'asc' else "asc" -%}
|
||||
{%- set arrow_span = "" -%}
|
||||
<th><input style="vertical-align: middle" type="checkbox" id="check-all" > <a class="{{ 'active '+link_order if sort_attribute == 'date_created' else 'inactive' }}" href="{{url_for('watchlist.index', sort='date_created', order=link_order, tag=active_tag_uuid)}}"># <span class='arrow {{link_order}}'></span></a></th>
|
||||
<th>
|
||||
<a class="{{ 'active '+link_order if sort_attribute == 'paused' else 'inactive' }}" href="{{url_for('watchlist.index', sort='paused', order=link_order, tag=active_tag_uuid)}}"><i data-feather="pause" style="vertical-align: bottom; width: 14px; height: 14px; margin-right: 4px;"></i><span class='arrow {{link_order}}'></span></a>
|
||||
|
||||
<a class="{{ 'active '+link_order if sort_attribute == 'notification_muted' else 'inactive' }}" href="{{url_for('watchlist.index', sort='notification_muted', order=link_order, tag=active_tag_uuid)}}"><i data-feather="volume-2" style="vertical-align: bottom; width: 14px; height: 14px; margin-right: 4px;"></i><span class='arrow {{link_order}}'></span></a>
|
||||
</th>
|
||||
<th class="empty-cell"></th>
|
||||
<th><a class="{{ 'active '+link_order if sort_attribute == 'label' else 'inactive' }}" href="{{url_for('watchlist.index', sort='label', order=link_order, tag=active_tag_uuid)}}">Website <span class='arrow {{link_order}}'></span></a></th>
|
||||
{%- if any_has_restock_price_processor -%}
|
||||
<th>Restock & Price</th>
|
||||
@@ -113,11 +105,9 @@ document.addEventListener('DOMContentLoaded', function() {
|
||||
<td colspan="{{ cols_required }}" style="text-wrap: wrap;">No website watches configured, please add a URL in the box above, or <a href="{{ url_for('imports.import_page')}}" >import a list</a>.</td>
|
||||
</tr>
|
||||
{%- endif -%}
|
||||
|
||||
{%- for watch in (watches|sort(attribute=sort_attribute, reverse=sort_order == 'asc'))|pagination_slice(skip=pagination.skip) -%}
|
||||
{%- set checking_now = is_checking_now(watch) -%}
|
||||
{%- set history_n = watch.history_n -%}
|
||||
{%- set favicon = watch.get_favicon_filename() -%}
|
||||
{# Mirror in changedetectionio/static/js/realtime.js for the frontend #}
|
||||
{%- set row_classes = [
|
||||
loop.cycle('pure-table-odd', 'pure-table-even'),
|
||||
@@ -126,63 +116,49 @@ document.addEventListener('DOMContentLoaded', function() {
|
||||
'paused' if watch.paused is defined and watch.paused != False else '',
|
||||
'unviewed' if watch.has_unviewed else '',
|
||||
'has-restock-info' if watch.has_restock_info else 'no-restock-info',
|
||||
'has-favicon' if favicon else '',
|
||||
'in-stock' if watch.has_restock_info and watch['restock']['in_stock'] else '',
|
||||
'not-in-stock' if watch.has_restock_info and not watch['restock']['in_stock'] else '',
|
||||
'queued' if watch.uuid in queued_uuids else '',
|
||||
'checking-now' if checking_now else '',
|
||||
'notification_muted' if watch.notification_muted else '',
|
||||
'single-history' if history_n == 1 else '',
|
||||
'multiple-history' if history_n >= 2 else '',
|
||||
'multiple-history' if history_n >= 2 else ''
|
||||
] -%}
|
||||
<tr id="{{ watch.uuid }}" data-watch-uuid="{{ watch.uuid }}" class="{{ row_classes | reject('equalto', '') | join(' ') }}">
|
||||
<td class="inline checkbox-uuid" ><div><input name="uuids" type="checkbox" value="{{ watch.uuid}} " > <span class="counter-i">{{ loop.index+pagination.skip }}</span></div></td>
|
||||
<td class="inline checkbox-uuid" ><input name="uuids" type="checkbox" value="{{ watch.uuid}} " > <span>{{ loop.index+pagination.skip }}</span></td>
|
||||
<td class="inline watch-controls">
|
||||
<div>
|
||||
<a class="ajax-op state-off pause-toggle" data-op="pause" href="{{url_for('watchlist.index', op='pause', uuid=watch.uuid, tag=active_tag_uuid)}}"><img src="{{url_for('static_content', group='images', filename='pause.svg')}}" alt="Pause checks" title="Pause checks" class="icon icon-pause" ></a>
|
||||
<a class="ajax-op state-on pause-toggle" data-op="pause" style="display: none" href="{{url_for('watchlist.index', op='pause', uuid=watch.uuid, tag=active_tag_uuid)}}"><img src="{{url_for('static_content', group='images', filename='play.svg')}}" alt="UnPause checks" title="UnPause checks" class="icon icon-unpause" ></a>
|
||||
<a class="ajax-op state-off mute-toggle" data-op="mute" href="{{url_for('watchlist.index', op='mute', uuid=watch.uuid, tag=active_tag_uuid)}}"><img src="{{url_for('static_content', group='images', filename='bell-off.svg')}}" alt="Mute notification" title="Mute notification" class="icon icon-mute" ></a>
|
||||
<a class="ajax-op state-on mute-toggle" data-op="mute" style="display: none" href="{{url_for('watchlist.index', op='mute', uuid=watch.uuid, tag=active_tag_uuid)}}"><img src="{{url_for('static_content', group='images', filename='bell-off.svg')}}" alt="UnMute notification" title="UnMute notification" class="icon icon-mute" ></a>
|
||||
</div>
|
||||
</td>
|
||||
<td class="title-col inline">{{watch.title if watch.title is not none and watch.title|length > 0 else watch.url}}
|
||||
<a class="external" target="_blank" rel="noopener" href="{{ watch.link.replace('source:','') }}"> </a>
|
||||
<a class="link-spread" href="{{url_for('ui.form_share_put_watch', uuid=watch.uuid)}}"><img src="{{url_for('static_content', group='images', filename='spread.svg')}}" class="status-icon icon icon-spread" title="Create a link to share watch config with others" ></a>
|
||||
|
||||
<td class="title-col inline">
|
||||
<div class="flex-wrapper">
|
||||
{% if 'favicons_enabled' not in ui_settings or ui_settings['favicons_enabled'] %}
|
||||
<div>{# A page might have hundreds of these images, set IMG options for lazy loading, don't set SRC if we dont have it so it doesnt fetch the placeholder' #}
|
||||
<img alt="Favicon thumbnail" class="favicon" loading="lazy" decoding="async" fetchpriority="low" {% if favicon %} src="{{url_for('static_content', group='favicon', filename=watch.uuid)}}" {% else %} src='data:image/svg+xml;utf8,%3Csvg xmlns="http://www.w3.org/2000/svg" width="7.087" height="7.087" viewBox="0 0 7.087 7.087"%3E%3Ccircle cx="3.543" cy="3.543" r="3.279" stroke="%23e1e1e1" stroke-width="0.45" fill="none" opacity="0.74"/%3E%3C/svg%3E' {% endif %} />
|
||||
</div>
|
||||
{% endif %}
|
||||
<div>
|
||||
<span class="watch-title">
|
||||
{{watch.title if watch.title is not none and watch.title|length > 0 else watch.url}} <a class="external" target="_blank" rel="noopener" href="{{ watch.link.replace('source:','') }}"> </a>
|
||||
</span>
|
||||
<div class="error-text" style="display:none;">{{ watch.compile_error_texts(has_proxies=datastore.proxy_list) }}</div>
|
||||
{%- if watch['processor'] == 'text_json_diff' -%}
|
||||
{%- if watch['has_ldjson_price_data'] and not watch['track_ldjson_price_data'] -%}
|
||||
<div class="ldjson-price-track-offer">Switch to Restock & Price watch mode? <a href="{{url_for('price_data_follower.accept', uuid=watch.uuid)}}" class="pure-button button-xsmall">Yes</a> <a href="{{url_for('price_data_follower.reject', uuid=watch.uuid)}}" class="">No</a></div>
|
||||
{%- endif -%}
|
||||
{%- endif -%}
|
||||
{%- if watch['processor'] == 'restock_diff' -%}
|
||||
<span class="tracking-ldjson-price-data" title="Automatically following embedded price information"><img src="{{url_for('static_content', group='images', filename='price-tag-icon.svg')}}" class="status-icon price-follow-tag-icon" > Price</span>
|
||||
{%- endif -%}
|
||||
{%- for watch_tag_uuid, watch_tag in datastore.get_all_tags_for_watch(watch['uuid']).items() -%}
|
||||
<span class="watch-tag-list">{{ watch_tag.title }}</span>
|
||||
{%- endfor -%}
|
||||
</div>
|
||||
<div class="status-icons">
|
||||
<a class="link-spread" href="{{url_for('ui.form_share_put_watch', uuid=watch.uuid)}}"><img src="{{url_for('static_content', group='images', filename='spread.svg')}}" class="status-icon icon icon-spread" title="Create a link to share watch config with others" ></a>
|
||||
{%- if watch.get_fetch_backend == "html_webdriver"
|
||||
or ( watch.get_fetch_backend == "system" and system_default_fetcher == 'html_webdriver' )
|
||||
or "extra_browser_" in watch.get_fetch_backend
|
||||
-%}
|
||||
<img class="status-icon" src="{{url_for('static_content', group='images', filename='google-chrome-icon.png')}}" alt="Using a Chrome browser" title="Using a Chrome browser" >
|
||||
{%- endif -%}
|
||||
{%- if watch.is_pdf -%}<img class="status-icon" src="{{url_for('static_content', group='images', filename='pdf-icon.svg')}}" alt="Converting PDF to text" >{%- endif -%}
|
||||
{%- if watch.has_browser_steps -%}<img class="status-icon status-browsersteps" src="{{url_for('static_content', group='images', filename='steps.svg')}}" alt="Browser Steps is enabled" >{%- endif -%}
|
||||
{%- if watch.get_fetch_backend == "html_webdriver"
|
||||
or ( watch.get_fetch_backend == "system" and system_default_fetcher == 'html_webdriver' )
|
||||
or "extra_browser_" in watch.get_fetch_backend
|
||||
-%}
|
||||
<img class="status-icon" src="{{url_for('static_content', group='images', filename='google-chrome-icon.png')}}" alt="Using a Chrome browser" title="Using a Chrome browser" >
|
||||
{%- endif -%}
|
||||
|
||||
</div>
|
||||
</div>
|
||||
{%- if watch.is_pdf -%}<img class="status-icon" src="{{url_for('static_content', group='images', filename='pdf-icon.svg')}}" alt="Converting PDF to text" >{%- endif -%}
|
||||
{%- if watch.has_browser_steps -%}<img class="status-icon status-browsersteps" src="{{url_for('static_content', group='images', filename='steps.svg')}}" alt="Browser Steps is enabled" >{%- endif -%}
|
||||
|
||||
<div class="error-text" style="display:none;">{{ watch.compile_error_texts(has_proxies=datastore.proxy_list) }}</div>
|
||||
|
||||
{%- if watch['processor'] == 'text_json_diff' -%}
|
||||
{%- if watch['has_ldjson_price_data'] and not watch['track_ldjson_price_data'] -%}
|
||||
<div class="ldjson-price-track-offer">Switch to Restock & Price watch mode? <a href="{{url_for('price_data_follower.accept', uuid=watch.uuid)}}" class="pure-button button-xsmall">Yes</a> <a href="{{url_for('price_data_follower.reject', uuid=watch.uuid)}}" class="">No</a></div>
|
||||
{%- endif -%}
|
||||
{%- endif -%}
|
||||
{%- if watch['processor'] == 'restock_diff' -%}
|
||||
<span class="tracking-ldjson-price-data" title="Automatically following embedded price information"><img src="{{url_for('static_content', group='images', filename='price-tag-icon.svg')}}" class="status-icon price-follow-tag-icon" > Price</span>
|
||||
{%- endif -%}
|
||||
{%- for watch_tag_uuid, watch_tag in datastore.get_all_tags_for_watch(watch['uuid']).items() -%}
|
||||
<span class="watch-tag-list">{{ watch_tag.title }}</span>
|
||||
{%- endfor -%}
|
||||
</td>
|
||||
{%- if any_has_restock_price_processor -%}
|
||||
<td class="restock-and-price">
|
||||
@@ -219,15 +195,13 @@ document.addEventListener('DOMContentLoaded', function() {
|
||||
Not yet
|
||||
{%- endif -%}
|
||||
</td>
|
||||
<td class="buttons">
|
||||
<div>
|
||||
<td>
|
||||
{%- set target_attr = ' target="' ~ watch.uuid ~ '"' if datastore.data['settings']['application']['ui'].get('open_diff_in_new_tab') else '' -%}
|
||||
<a href="" class="already-in-queue-button recheck pure-button pure-button-primary" style="display: none;" disabled="disabled">Queued</a>
|
||||
<a href="{{ url_for('ui.form_watch_checknow', uuid=watch.uuid, tag=request.args.get('tag')) }}" data-op='recheck' class="ajax-op recheck pure-button pure-button-primary">Recheck</a>
|
||||
<a href="{{ url_for('ui.ui_edit.edit_page', uuid=watch.uuid, tag=active_tag_uuid)}}#general" class="pure-button pure-button-primary">Edit</a>
|
||||
<a href="{{ url_for('ui.ui_views.diff_history_page', uuid=watch.uuid)}}" {{target_attr}} class="pure-button pure-button-primary history-link" style="display: none;">History</a>
|
||||
<a href="{{ url_for('ui.ui_views.preview_page', uuid=watch.uuid)}}" {{target_attr}} class="pure-button pure-button-primary preview-link" style="display: none;">Preview</a>
|
||||
</div>
|
||||
</td>
|
||||
</tr>
|
||||
{%- endfor -%}
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
from flask import Blueprint
|
||||
|
||||
from json_logic.builtins import BUILTINS
|
||||
|
||||
from .exceptions import EmptyConditionRuleRowNotUsable
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
import pluggy
|
||||
from loguru import logger
|
||||
|
||||
LEVENSHTEIN_MAX_LEN_FOR_EDIT_STATS=100000
|
||||
|
||||
# Support both plugin systems
|
||||
conditions_hookimpl = pluggy.HookimplMarker("changedetectionio_conditions")
|
||||
global_hookimpl = pluggy.HookimplMarker("changedetectionio")
|
||||
@@ -74,17 +72,7 @@ def ui_edit_stats_extras(watch):
|
||||
"""Generate the HTML for Levenshtein stats - shared by both plugin systems"""
|
||||
if len(watch.history.keys()) < 2:
|
||||
return "<p>Not enough history to calculate Levenshtein metrics</p>"
|
||||
|
||||
|
||||
# Protection against the algorithm getting stuck on huge documents
|
||||
k = list(watch.history.keys())
|
||||
if any(
|
||||
len(watch.get_history_snapshot(timestamp=k[idx])) > LEVENSHTEIN_MAX_LEN_FOR_EDIT_STATS
|
||||
for idx in (-1, -2)
|
||||
if len(k) >= abs(idx)
|
||||
):
|
||||
return "<p>Snapshot too large for edit statistics, skipping.</p>"
|
||||
|
||||
|
||||
try:
|
||||
lev_data = levenshtein_ratio_recent_history(watch)
|
||||
if not lev_data or not isinstance(lev_data, dict):
|
||||
|
||||
@@ -28,7 +28,6 @@ from changedetectionio.content_fetchers.requests import fetcher as html_requests
|
||||
import importlib.resources
|
||||
XPATH_ELEMENT_JS = importlib.resources.files("changedetectionio.content_fetchers.res").joinpath('xpath_element_scraper.js').read_text(encoding='utf-8')
|
||||
INSTOCK_DATA_JS = importlib.resources.files("changedetectionio.content_fetchers.res").joinpath('stock-not-in-stock.js').read_text(encoding='utf-8')
|
||||
FAVICON_FETCHER_JS = importlib.resources.files("changedetectionio.content_fetchers.res").joinpath('favicon-fetcher.js').read_text(encoding='utf-8')
|
||||
|
||||
|
||||
def available_fetchers():
|
||||
|
||||
@@ -48,7 +48,6 @@ class Fetcher():
|
||||
error = None
|
||||
fetcher_description = "No description"
|
||||
headers = {}
|
||||
favicon_blob = None
|
||||
instock_data = None
|
||||
instock_data_js = ""
|
||||
status_code = None
|
||||
|
||||
@@ -5,7 +5,7 @@ from urllib.parse import urlparse
|
||||
from loguru import logger
|
||||
|
||||
from changedetectionio.content_fetchers import SCREENSHOT_MAX_HEIGHT_DEFAULT, visualselector_xpath_selectors, \
|
||||
SCREENSHOT_SIZE_STITCH_THRESHOLD, SCREENSHOT_MAX_TOTAL_HEIGHT, XPATH_ELEMENT_JS, INSTOCK_DATA_JS, FAVICON_FETCHER_JS
|
||||
SCREENSHOT_SIZE_STITCH_THRESHOLD, SCREENSHOT_MAX_TOTAL_HEIGHT, XPATH_ELEMENT_JS, INSTOCK_DATA_JS
|
||||
from changedetectionio.content_fetchers.base import Fetcher, manage_user_agent
|
||||
from changedetectionio.content_fetchers.exceptions import PageUnloadable, Non200ErrorCodeReceived, EmptyReply, ScreenshotUnavailable
|
||||
|
||||
@@ -234,12 +234,6 @@ class fetcher(Fetcher):
|
||||
await browser.close()
|
||||
raise PageUnloadable(url=url, status_code=None, message=str(e))
|
||||
|
||||
try:
|
||||
self.favicon_blob = await self.page.evaluate(FAVICON_FETCHER_JS)
|
||||
await self.page.request_gc()
|
||||
except Exception as e:
|
||||
logger.error(f"Error fetching FavIcon info {str(e)}, continuing.")
|
||||
|
||||
if self.status_code != 200 and not ignore_status_codes:
|
||||
screenshot = await capture_full_page_async(self.page)
|
||||
raise Non200ErrorCodeReceived(url=url, status_code=self.status_code, screenshot=screenshot)
|
||||
@@ -280,7 +274,6 @@ class fetcher(Fetcher):
|
||||
await self.page.request_gc()
|
||||
logger.debug(f"Scrape xPath element data in browser done in {time.time() - now:.2f}s")
|
||||
|
||||
|
||||
# Bug 3 in Playwright screenshot handling
|
||||
# Some bug where it gives the wrong screenshot size, but making a request with the clip set first seems to solve it
|
||||
# JPEG is better here because the screenshots can be very very large
|
||||
|
||||
@@ -8,7 +8,7 @@ from loguru import logger
|
||||
|
||||
from changedetectionio.content_fetchers import SCREENSHOT_MAX_HEIGHT_DEFAULT, visualselector_xpath_selectors, \
|
||||
SCREENSHOT_SIZE_STITCH_THRESHOLD, SCREENSHOT_DEFAULT_QUALITY, XPATH_ELEMENT_JS, INSTOCK_DATA_JS, \
|
||||
SCREENSHOT_MAX_TOTAL_HEIGHT, FAVICON_FETCHER_JS
|
||||
SCREENSHOT_MAX_TOTAL_HEIGHT
|
||||
from changedetectionio.content_fetchers.base import Fetcher, manage_user_agent
|
||||
from changedetectionio.content_fetchers.exceptions import PageUnloadable, Non200ErrorCodeReceived, EmptyReply, BrowserFetchTimedOut, \
|
||||
BrowserConnectError
|
||||
@@ -179,8 +179,10 @@ class fetcher(Fetcher):
|
||||
except Exception as e:
|
||||
raise BrowserConnectError(msg=f"Error connecting to the browser - Exception '{str(e)}'")
|
||||
|
||||
# more reliable is to just request a new page
|
||||
self.page = await browser.newPage()
|
||||
# Better is to launch chrome with the URL as arg
|
||||
# non-headless - newPage() will launch an extra tab/window, .browser should already contain 1 page/tab
|
||||
# headless - ask a new page
|
||||
self.page = (pages := await browser.pages) and len(pages) or await browser.newPage()
|
||||
|
||||
if '--window-size' in self.browser_connection_url:
|
||||
# Be sure the viewport is always the window-size, this is often not the same thing
|
||||
@@ -290,11 +292,6 @@ class fetcher(Fetcher):
|
||||
await browser.close()
|
||||
raise PageUnloadable(url=url, status_code=None, message=str(e))
|
||||
|
||||
try:
|
||||
self.favicon_blob = await self.page.evaluate(FAVICON_FETCHER_JS)
|
||||
except Exception as e:
|
||||
logger.error(f"Error fetching FavIcon info {str(e)}, continuing.")
|
||||
|
||||
if self.status_code != 200 and not ignore_status_codes:
|
||||
screenshot = await capture_full_page(page=self.page)
|
||||
|
||||
|
||||
@@ -1,79 +0,0 @@
|
||||
(async () => {
|
||||
const links = Array.from(document.querySelectorAll(
|
||||
'link[rel~="apple-touch-icon"], link[rel~="icon"]'
|
||||
));
|
||||
|
||||
const icons = links.map(link => {
|
||||
const sizesStr = link.getAttribute('sizes');
|
||||
let size = 0;
|
||||
if (sizesStr) {
|
||||
const [w] = sizesStr.split('x').map(Number);
|
||||
if (!isNaN(w)) size = w;
|
||||
} else {
|
||||
size = 16;
|
||||
}
|
||||
return {
|
||||
size,
|
||||
rel: link.getAttribute('rel'),
|
||||
href: link.href
|
||||
};
|
||||
});
|
||||
|
||||
// If no icons found, add fallback favicon.ico
|
||||
if (icons.length === 0) {
|
||||
icons.push({
|
||||
size: 16,
|
||||
rel: 'icon',
|
||||
href: '/favicon.ico'
|
||||
});
|
||||
}
|
||||
|
||||
// sort preference
|
||||
icons.sort((a, b) => {
|
||||
const isAppleA = /apple-touch-icon/.test(a.rel);
|
||||
const isAppleB = /apple-touch-icon/.test(b.rel);
|
||||
if (isAppleA && !isAppleB) return -1;
|
||||
if (!isAppleA && isAppleB) return 1;
|
||||
return b.size - a.size;
|
||||
});
|
||||
|
||||
const timeoutMs = 2000;
|
||||
|
||||
for (const icon of icons) {
|
||||
try {
|
||||
const controller = new AbortController();
|
||||
const timeout = setTimeout(() => controller.abort(), timeoutMs);
|
||||
|
||||
const resp = await fetch(icon.href, {
|
||||
signal: controller.signal,
|
||||
redirect: 'follow'
|
||||
});
|
||||
|
||||
clearTimeout(timeout);
|
||||
|
||||
if (!resp.ok) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const blob = await resp.blob();
|
||||
|
||||
// Convert blob to base64
|
||||
const reader = new FileReader();
|
||||
return await new Promise(resolve => {
|
||||
reader.onloadend = () => {
|
||||
resolve({
|
||||
url: icon.href,
|
||||
base64: reader.result.split(",")[1]
|
||||
});
|
||||
};
|
||||
reader.readAsDataURL(blob);
|
||||
});
|
||||
|
||||
} catch (e) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
// nothing found
|
||||
return null;
|
||||
})();
|
||||
@@ -1,8 +1,8 @@
|
||||
async (customOutOfStockStrings = []) => {
|
||||
async () => {
|
||||
|
||||
function isItemInStock() {
|
||||
// @todo Pass these in so the same list can be used in non-JS fetchers
|
||||
const builtInOutOfStockTexts = [
|
||||
const outOfStockTexts = [
|
||||
' أخبرني عندما يتوفر',
|
||||
'0 in stock',
|
||||
'actuellement indisponible',
|
||||
@@ -17,7 +17,6 @@ async (customOutOfStockStrings = []) => {
|
||||
'back in stock soon',
|
||||
'back-order or out of stock',
|
||||
'backordered',
|
||||
'backorder',
|
||||
'benachrichtigt mich', // notify me
|
||||
'binnenkort leverbaar', // coming soon
|
||||
'brak na stanie',
|
||||
@@ -40,7 +39,6 @@ async (customOutOfStockStrings = []) => {
|
||||
'mail me when available',
|
||||
'message if back in stock',
|
||||
'mevcut değil',
|
||||
'more on order',
|
||||
'nachricht bei',
|
||||
'nicht auf lager',
|
||||
'nicht lagernd',
|
||||
@@ -110,9 +108,6 @@ async (customOutOfStockStrings = []) => {
|
||||
'품절'
|
||||
];
|
||||
|
||||
// Combine built-in strings with custom strings provided by user
|
||||
const outOfStockTexts = [...builtInOutOfStockTexts, ...customOutOfStockStrings];
|
||||
|
||||
|
||||
const vh = Math.max(document.documentElement.clientHeight || 0, window.innerHeight || 0);
|
||||
|
||||
|
||||
@@ -19,10 +19,12 @@ from flask import (
|
||||
Flask,
|
||||
abort,
|
||||
flash,
|
||||
make_response,
|
||||
redirect,
|
||||
render_template,
|
||||
request,
|
||||
send_from_directory,
|
||||
session,
|
||||
url_for,
|
||||
)
|
||||
from flask_compress import Compress as FlaskCompress
|
||||
@@ -38,7 +40,7 @@ from loguru import logger
|
||||
|
||||
from changedetectionio import __version__
|
||||
from changedetectionio import queuedWatchMetaData
|
||||
from changedetectionio.api import Watch, WatchHistory, WatchSingleHistory, CreateWatch, Import, SystemInfo, Tag, Tags, Notifications, WatchFavicon
|
||||
from changedetectionio.api import Watch, WatchHistory, WatchSingleHistory, CreateWatch, Import, SystemInfo, Tag, Tags, Notifications
|
||||
from changedetectionio.api.Search import Search
|
||||
from .time_handler import is_within_schedule
|
||||
|
||||
@@ -305,9 +307,7 @@ def changedetection_app(config=None, datastore_o=None):
|
||||
watch_api.add_resource(WatchSingleHistory,
|
||||
'/api/v1/watch/<string:uuid>/history/<string:timestamp>',
|
||||
resource_class_kwargs={'datastore': datastore, 'update_q': update_q})
|
||||
watch_api.add_resource(WatchFavicon,
|
||||
'/api/v1/watch/<string:uuid>/favicon',
|
||||
resource_class_kwargs={'datastore': datastore})
|
||||
|
||||
watch_api.add_resource(WatchHistory,
|
||||
'/api/v1/watch/<string:uuid>/history',
|
||||
resource_class_kwargs={'datastore': datastore})
|
||||
@@ -427,32 +427,6 @@ def changedetection_app(config=None, datastore_o=None):
|
||||
except FileNotFoundError:
|
||||
abort(404)
|
||||
|
||||
if group == 'favicon':
|
||||
# Could be sensitive, follow password requirements
|
||||
if datastore.data['settings']['application']['password'] and not flask_login.current_user.is_authenticated:
|
||||
abort(403)
|
||||
# Get the watch object
|
||||
watch = datastore.data['watching'].get(filename)
|
||||
if not watch:
|
||||
abort(404)
|
||||
|
||||
favicon_filename = watch.get_favicon_filename()
|
||||
if favicon_filename:
|
||||
try:
|
||||
import magic
|
||||
mime = magic.from_file(
|
||||
os.path.join(watch.watch_data_dir, favicon_filename),
|
||||
mime=True
|
||||
)
|
||||
except ImportError:
|
||||
# Fallback, no python-magic
|
||||
import mimetypes
|
||||
mime, encoding = mimetypes.guess_type(favicon_filename)
|
||||
|
||||
response = make_response(send_from_directory(watch.watch_data_dir, favicon_filename))
|
||||
response.headers['Content-type'] = mime
|
||||
response.headers['Cache-Control'] = 'max-age=300, must-revalidate' # Cache for 5 minutes, then revalidate
|
||||
return response
|
||||
|
||||
if group == 'visual_selector_data':
|
||||
# Could be sensitive, follow password requirements
|
||||
|
||||
@@ -740,7 +740,6 @@ class globalSettingsRequestForm(Form):
|
||||
class globalSettingsApplicationUIForm(Form):
|
||||
open_diff_in_new_tab = BooleanField("Open 'History' page in a new tab", default=True, validators=[validators.Optional()])
|
||||
socket_io_enabled = BooleanField('Realtime UI Updates Enabled', default=True, validators=[validators.Optional()])
|
||||
favicons_enabled = BooleanField('Favicons Enabled', default=True, validators=[validators.Optional()])
|
||||
|
||||
# datastore.data['settings']['application']..
|
||||
class globalSettingsApplicationForm(commonSettingsForm):
|
||||
@@ -774,20 +773,6 @@ class globalSettingsApplicationForm(commonSettingsForm):
|
||||
message="Should contain zero or more attempts")])
|
||||
ui = FormField(globalSettingsApplicationUIForm)
|
||||
|
||||
#@todo better validations?
|
||||
|
||||
custom_outofstock_strings = StringListField('Custom out-of-stock detection strings',
|
||||
[validators.Optional()],
|
||||
render_kw={
|
||||
"placeholder": "Enter custom out-of-stock strings, one per line\nExample:\nPronto estarán en stock!\nTemporarily out of stock",
|
||||
"rows": "3"})
|
||||
|
||||
custom_instock_strings = StringListField('Custom in-stock detection strings',
|
||||
[validators.Optional()],
|
||||
render_kw={
|
||||
"placeholder": "Enter custom in-stock strings, one per line\nExample:\nDisponible ahora\nIn voorraad",
|
||||
"rows": "3"})
|
||||
|
||||
|
||||
class globalSettingsForm(Form):
|
||||
# Define these as FormFields/"sub forms", this way it matches the JSON storage
|
||||
|
||||
@@ -38,8 +38,6 @@ class model(dict):
|
||||
# Custom notification content
|
||||
'api_access_token_enabled': True,
|
||||
'base_url' : None,
|
||||
'custom_instock_strings': [],
|
||||
'custom_outofstock_strings' : [],
|
||||
'empty_pages_are_a_change': False,
|
||||
'extract_title_as_title': False,
|
||||
'fetch_backend': getenv("DEFAULT_FETCH_BACKEND", "html_requests"),
|
||||
@@ -64,8 +62,7 @@ class model(dict):
|
||||
'timezone': None, # Default IANA timezone name
|
||||
'ui': {
|
||||
'open_diff_in_new_tab': True,
|
||||
'socket_io_enabled': True,
|
||||
'favicons_enabled': True
|
||||
'socket_io_enabled': True
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -103,13 +103,6 @@ class model(watch_base):
|
||||
return 'DISABLED'
|
||||
return ready_url
|
||||
|
||||
@property
|
||||
def domain_only_from_link(self):
|
||||
from urllib.parse import urlparse
|
||||
parsed = urlparse(self.link)
|
||||
domain = parsed.hostname
|
||||
return domain
|
||||
|
||||
def clear_watch(self):
|
||||
import pathlib
|
||||
|
||||
@@ -420,132 +413,6 @@ class model(watch_base):
|
||||
# False is not an option for AppRise, must be type None
|
||||
return None
|
||||
|
||||
def bump_favicon(self, url, favicon_base_64: str) -> None:
|
||||
from urllib.parse import urlparse
|
||||
import base64
|
||||
import binascii
|
||||
decoded = None
|
||||
|
||||
if url:
|
||||
try:
|
||||
parsed = urlparse(url)
|
||||
filename = os.path.basename(parsed.path)
|
||||
(base, extension) = filename.lower().strip().rsplit('.', 1)
|
||||
except ValueError:
|
||||
logger.error(f"UUID: {self.get('uuid')} Cant work out file extension from '{url}'")
|
||||
return None
|
||||
else:
|
||||
# Assume favicon.ico
|
||||
base = "favicon"
|
||||
extension = "ico"
|
||||
|
||||
fname = os.path.join(self.watch_data_dir, f"favicon.{extension}")
|
||||
|
||||
try:
|
||||
# validate=True makes sure the string only contains valid base64 chars
|
||||
decoded = base64.b64decode(favicon_base_64, validate=True)
|
||||
except (binascii.Error, ValueError) as e:
|
||||
logger.warning(f"UUID: {self.get('uuid')} FavIcon save data (Base64) corrupt? {str(e)}")
|
||||
else:
|
||||
if decoded:
|
||||
try:
|
||||
with open(fname, 'wb') as f:
|
||||
f.write(decoded)
|
||||
# A signal that could trigger the socket server to update the browser also
|
||||
watch_check_update = signal('watch_favicon_bump')
|
||||
if watch_check_update:
|
||||
watch_check_update.send(watch_uuid=self.get('uuid'))
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"UUID: {self.get('uuid')} error saving FavIcon to {fname} - {str(e)}")
|
||||
|
||||
# @todo - Store some checksum and only write when its different
|
||||
logger.debug(f"UUID: {self.get('uuid')} updated favicon to at {fname}")
|
||||
|
||||
def get_favicon_filename(self) -> str | None:
|
||||
"""
|
||||
Find any favicon.* file in the current working directory
|
||||
and return the contents of the newest one.
|
||||
|
||||
Returns:
|
||||
bytes: Contents of the newest favicon file, or None if not found.
|
||||
"""
|
||||
import glob
|
||||
|
||||
# Search for all favicon.* files
|
||||
files = glob.glob(os.path.join(self.watch_data_dir, "favicon.*"))
|
||||
|
||||
if not files:
|
||||
return None
|
||||
|
||||
# Find the newest by modification time
|
||||
newest_file = max(files, key=os.path.getmtime)
|
||||
return os.path.basename(newest_file)
|
||||
|
||||
def get_screenshot_as_thumbnail(self, max_age=3200):
|
||||
"""Return path to a square thumbnail of the most recent screenshot.
|
||||
|
||||
Creates a 150x150 pixel thumbnail from the top portion of the screenshot.
|
||||
|
||||
Args:
|
||||
max_age: Maximum age in seconds before recreating thumbnail
|
||||
|
||||
Returns:
|
||||
Path to thumbnail or None if no screenshot exists
|
||||
"""
|
||||
import os
|
||||
import time
|
||||
|
||||
thumbnail_path = os.path.join(self.watch_data_dir, "thumbnail.jpeg")
|
||||
top_trim = 500 # Pixels from top of screenshot to use
|
||||
|
||||
screenshot_path = self.get_screenshot()
|
||||
if not screenshot_path:
|
||||
return None
|
||||
|
||||
# Reuse thumbnail if it's fresh and screenshot hasn't changed
|
||||
if os.path.isfile(thumbnail_path):
|
||||
thumbnail_mtime = os.path.getmtime(thumbnail_path)
|
||||
screenshot_mtime = os.path.getmtime(screenshot_path)
|
||||
|
||||
if screenshot_mtime <= thumbnail_mtime and time.time() - thumbnail_mtime < max_age:
|
||||
return thumbnail_path
|
||||
|
||||
try:
|
||||
from PIL import Image
|
||||
|
||||
with Image.open(screenshot_path) as img:
|
||||
# Crop top portion first (full width, top_trim height)
|
||||
top_crop_height = min(top_trim, img.height)
|
||||
img = img.crop((0, 0, img.width, top_crop_height))
|
||||
|
||||
# Create a smaller intermediate image (to reduce memory usage)
|
||||
aspect = img.width / img.height
|
||||
interim_width = min(top_trim, img.width)
|
||||
interim_height = int(interim_width / aspect) if aspect > 0 else top_trim
|
||||
img = img.resize((interim_width, interim_height), Image.NEAREST)
|
||||
|
||||
# Convert to RGB if needed
|
||||
if img.mode != 'RGB':
|
||||
img = img.convert('RGB')
|
||||
|
||||
# Crop to square from top center
|
||||
square_size = min(img.width, img.height)
|
||||
left = (img.width - square_size) // 2
|
||||
img = img.crop((left, 0, left + square_size, square_size))
|
||||
|
||||
# Final resize to exact thumbnail size with better filter
|
||||
img = img.resize((350, 350), Image.BILINEAR)
|
||||
|
||||
# Save with optimized settings
|
||||
img.save(thumbnail_path, "JPEG", quality=75, optimize=True)
|
||||
|
||||
return thumbnail_path
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error creating thumbnail for {self.get('uuid')}: {str(e)}")
|
||||
return None
|
||||
|
||||
def __get_file_ctime(self, filename):
|
||||
fname = os.path.join(self.watch_data_dir, filename)
|
||||
if os.path.isfile(fname):
|
||||
|
||||
@@ -1,8 +1,7 @@
|
||||
from wtforms import (
|
||||
BooleanField,
|
||||
validators,
|
||||
FloatField,
|
||||
TextAreaField
|
||||
FloatField
|
||||
)
|
||||
from wtforms.fields.choices import RadioField
|
||||
from wtforms.fields.form import FormField
|
||||
@@ -30,7 +29,6 @@ class RestockSettingsForm(Form):
|
||||
|
||||
follow_price_changes = BooleanField('Follow price changes', default=True)
|
||||
|
||||
|
||||
class processor_settings_form(processor_text_json_diff_form):
|
||||
restock_settings = FormField(RestockSettingsForm)
|
||||
|
||||
@@ -76,7 +74,7 @@ class processor_settings_form(processor_text_json_diff_form):
|
||||
{{ render_field(form.restock_settings.price_change_threshold_percent) }}
|
||||
<span class="pure-form-message-inline">Price must change more than this % to trigger a change since the first check.</span><br>
|
||||
<span class="pure-form-message-inline">For example, If the product is $1,000 USD originally, <strong>2%</strong> would mean it has to change more than $20 since the first check.</span><br>
|
||||
</fieldset>
|
||||
</fieldset>
|
||||
</div>
|
||||
</fieldset>
|
||||
"""
|
||||
|
||||
@@ -143,89 +143,6 @@ def is_between(number, lower=None, upper=None):
|
||||
class perform_site_check(difference_detection_processor):
|
||||
screenshot = None
|
||||
xpath_data = None
|
||||
|
||||
def _normalize_text_for_matching(self, text):
|
||||
"""
|
||||
Normalize text for more robust matching:
|
||||
- Convert to lowercase
|
||||
- Remove accents/diacritics
|
||||
- Normalize whitespace
|
||||
"""
|
||||
import unicodedata
|
||||
import re
|
||||
|
||||
if not text:
|
||||
return ""
|
||||
|
||||
# Convert to lowercase
|
||||
text = text.lower()
|
||||
|
||||
# Remove accents/diacritics (NFD normalization + filter)
|
||||
# This converts "é" to "e", "ñ" to "n", etc.
|
||||
text = unicodedata.normalize('NFD', text)
|
||||
text = ''.join(char for char in text if unicodedata.category(char) != 'Mn')
|
||||
|
||||
# Normalize whitespace (replace multiple spaces/tabs/newlines with single space)
|
||||
text = re.sub(r'\s+', ' ', text).strip()
|
||||
|
||||
return text
|
||||
|
||||
def _check_custom_strings(self, text_to_check, custom_strings, string_type="out-of-stock"):
|
||||
"""
|
||||
Check text against custom strings (either in-stock or out-of-stock).
|
||||
Uses normalized matching for better international support.
|
||||
Returns the matched string if found, None otherwise.
|
||||
"""
|
||||
if not custom_strings:
|
||||
return None
|
||||
|
||||
# Split custom strings by newlines and clean them up
|
||||
raw_custom_list = [s.strip() for s in custom_strings.split('\n') if s.strip()]
|
||||
|
||||
if not raw_custom_list:
|
||||
return None
|
||||
|
||||
# Normalize both the page text and custom strings for matching
|
||||
normalized_text = self._normalize_text_for_matching(text_to_check)
|
||||
|
||||
# Check each custom string against the text
|
||||
for original_custom_text in raw_custom_list:
|
||||
normalized_custom_text = self._normalize_text_for_matching(original_custom_text)
|
||||
|
||||
if normalized_custom_text and normalized_custom_text in normalized_text:
|
||||
logger.debug(f"Custom {string_type} string found: '{original_custom_text}' (normalized: '{normalized_custom_text}')")
|
||||
return original_custom_text # Return the original user-provided string
|
||||
|
||||
return None
|
||||
|
||||
def _get_combined_instock_strings(self, restock_settings):
|
||||
"""
|
||||
Get combined list of built-in and custom in-stock strings.
|
||||
Custom strings are normalized for better matching.
|
||||
"""
|
||||
# Built-in in-stock strings (from the TODO line)
|
||||
builtin_instock_strings = [
|
||||
'instock',
|
||||
'instoreonly',
|
||||
'limitedavailability',
|
||||
'onlineonly',
|
||||
'presale'
|
||||
]
|
||||
|
||||
# Add custom in-stock strings if provided
|
||||
custom_strings = restock_settings.get('custom_instock_strings', '').strip()
|
||||
if custom_strings:
|
||||
# Normalize custom strings for better matching
|
||||
custom_list = []
|
||||
for s in custom_strings.split('\n'):
|
||||
s = s.strip()
|
||||
if s:
|
||||
normalized = self._normalize_text_for_matching(s)
|
||||
if normalized:
|
||||
custom_list.append(normalized)
|
||||
builtin_instock_strings.extend(custom_list)
|
||||
|
||||
return builtin_instock_strings
|
||||
|
||||
def run_changedetection(self, watch):
|
||||
import hashlib
|
||||
@@ -288,7 +205,6 @@ class perform_site_check(difference_detection_processor):
|
||||
|
||||
if itemprop_availability.get('availability'):
|
||||
# @todo: Configurable?
|
||||
|
||||
if any(substring.lower() in itemprop_availability['availability'].lower() for substring in [
|
||||
'instock',
|
||||
'instoreonly',
|
||||
@@ -322,8 +238,6 @@ class perform_site_check(difference_detection_processor):
|
||||
if self.fetcher.instock_data and itemprop_availability.get('availability') is None:
|
||||
# 'Possibly in stock' comes from stock-not-in-stock.js when no string found above the fold.
|
||||
# Careful! this does not really come from chrome/js when the watch is set to plaintext
|
||||
stock_detection_result = self.fetcher.instock_data
|
||||
|
||||
update_obj['restock']["in_stock"] = True if self.fetcher.instock_data == 'Possibly in stock' else False
|
||||
logger.debug(f"Watch UUID {watch.get('uuid')} restock check returned instock_data - '{self.fetcher.instock_data}' from JS scraper.")
|
||||
|
||||
|
||||
@@ -29,9 +29,6 @@ class SignalHandler:
|
||||
watch_delete_signal = signal('watch_deleted')
|
||||
watch_delete_signal.connect(self.handle_deleted_signal, weak=False)
|
||||
|
||||
watch_favicon_bumped_signal = signal('watch_favicon_bump')
|
||||
watch_favicon_bumped_signal.connect(self.handle_watch_bumped_favicon_signal, weak=False)
|
||||
|
||||
# Connect to the notification_event signal
|
||||
notification_event_signal = signal('notification_event')
|
||||
notification_event_signal.connect(self.handle_notification_event, weak=False)
|
||||
@@ -40,7 +37,7 @@ class SignalHandler:
|
||||
# Create and start the queue update thread using standard threading
|
||||
import threading
|
||||
self.polling_emitter_thread = threading.Thread(
|
||||
target=self.polling_emit_running_or_queued_watches_threaded,
|
||||
target=self.polling_emit_running_or_queued_watches_threaded,
|
||||
daemon=True
|
||||
)
|
||||
self.polling_emitter_thread.start()
|
||||
@@ -72,16 +69,6 @@ class SignalHandler:
|
||||
else:
|
||||
logger.warning(f"Watch UUID {watch_uuid} not found in datastore")
|
||||
|
||||
def handle_watch_bumped_favicon_signal(self, *args, **kwargs):
|
||||
watch_uuid = kwargs.get('watch_uuid')
|
||||
if watch_uuid:
|
||||
# Emit the queue size to all connected clients
|
||||
self.socketio_instance.emit("watch_bumped_favicon", {
|
||||
"uuid": watch_uuid,
|
||||
"event_timestamp": time.time()
|
||||
})
|
||||
logger.debug(f"Watch UUID {watch_uuid} got its favicon updated")
|
||||
|
||||
def handle_deleted_signal(self, *args, **kwargs):
|
||||
watch_uuid = kwargs.get('watch_uuid')
|
||||
if watch_uuid:
|
||||
@@ -118,38 +105,39 @@ class SignalHandler:
|
||||
"watch_uuid": watch_uuid,
|
||||
"event_timestamp": time.time()
|
||||
})
|
||||
|
||||
|
||||
logger.trace(f"Socket.IO: Emitted notification_event for watch UUID {watch_uuid}")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Socket.IO error in handle_notification_event: {str(e)}")
|
||||
|
||||
|
||||
def polling_emit_running_or_queued_watches_threaded(self):
|
||||
"""Threading version of polling for Windows compatibility"""
|
||||
import time
|
||||
import threading
|
||||
logger.info("Queue update thread started (threading mode)")
|
||||
|
||||
|
||||
# Import here to avoid circular imports
|
||||
from changedetectionio.flask_app import app
|
||||
from changedetectionio import worker_handler
|
||||
watch_check_update = signal('watch_check_update')
|
||||
|
||||
|
||||
# Track previous state to avoid unnecessary emissions
|
||||
previous_running_uuids = set()
|
||||
|
||||
|
||||
# Run until app shutdown - check exit flag more frequently for fast shutdown
|
||||
exit_event = getattr(app.config, 'exit', threading.Event())
|
||||
|
||||
|
||||
while not exit_event.is_set():
|
||||
try:
|
||||
# Get current running UUIDs from async workers
|
||||
running_uuids = set(worker_handler.get_running_uuids())
|
||||
|
||||
|
||||
# Only send updates for UUIDs that changed state
|
||||
newly_running = running_uuids - previous_running_uuids
|
||||
no_longer_running = previous_running_uuids - running_uuids
|
||||
|
||||
|
||||
# Send updates for newly running UUIDs (but exit fast if shutdown requested)
|
||||
for uuid in newly_running:
|
||||
if exit_event.is_set():
|
||||
@@ -158,7 +146,7 @@ class SignalHandler:
|
||||
with app.app_context():
|
||||
watch_check_update.send(app_context=app, watch_uuid=uuid)
|
||||
time.sleep(0.01) # Small yield
|
||||
|
||||
|
||||
# Send updates for UUIDs that finished processing (but exit fast if shutdown requested)
|
||||
if not exit_event.is_set():
|
||||
for uuid in no_longer_running:
|
||||
@@ -168,16 +156,16 @@ class SignalHandler:
|
||||
with app.app_context():
|
||||
watch_check_update.send(app_context=app, watch_uuid=uuid)
|
||||
time.sleep(0.01) # Small yield
|
||||
|
||||
|
||||
# Update tracking for next iteration
|
||||
previous_running_uuids = running_uuids
|
||||
|
||||
|
||||
# Sleep between polling cycles, but check exit flag every 0.5 seconds for fast shutdown
|
||||
for _ in range(20): # 20 * 0.5 = 10 seconds total
|
||||
if exit_event.is_set():
|
||||
break
|
||||
time.sleep(0.5)
|
||||
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error in threading polling: {str(e)}")
|
||||
# Even during error recovery, check for exit quickly
|
||||
@@ -185,11 +173,11 @@ class SignalHandler:
|
||||
if exit_event.is_set():
|
||||
break
|
||||
time.sleep(0.5)
|
||||
|
||||
|
||||
# Check if we're in pytest environment - if so, be more gentle with logging
|
||||
import sys
|
||||
in_pytest = "pytest" in sys.modules or "PYTEST_CURRENT_TEST" in os.environ
|
||||
|
||||
|
||||
if not in_pytest:
|
||||
logger.info("Queue update thread stopped (threading mode)")
|
||||
|
||||
@@ -220,20 +208,20 @@ def handle_watch_update(socketio, **kwargs):
|
||||
|
||||
watch_data = {
|
||||
'checking_now': True if watch.get('uuid') in running_uuids else False,
|
||||
'error_text': error_texts,
|
||||
'event_timestamp': time.time(),
|
||||
'fetch_time': watch.get('fetch_time'),
|
||||
'has_error': True if error_texts else False,
|
||||
'has_favicon': True if watch.get_favicon_filename() else False,
|
||||
'history_n': watch.history_n,
|
||||
'last_changed_text': timeago.format(int(watch.last_changed), time.time()) if watch.history_n >= 2 and int(watch.last_changed) > 0 else 'Not yet',
|
||||
'last_changed': watch.get('last_changed'),
|
||||
'last_checked': watch.get('last_checked'),
|
||||
'error_text': error_texts,
|
||||
'history_n': watch.history_n,
|
||||
'last_checked_text': _jinja2_filter_datetime(watch),
|
||||
'notification_muted': True if watch.get('notification_muted') else False,
|
||||
'paused': True if watch.get('paused') else False,
|
||||
'last_changed_text': timeago.format(int(watch.last_changed), time.time()) if watch.history_n >= 2 and int(watch.last_changed) > 0 else 'Not yet',
|
||||
'queued': True if watch.get('uuid') in queue_list else False,
|
||||
'paused': True if watch.get('paused') else False,
|
||||
'notification_muted': True if watch.get('notification_muted') else False,
|
||||
'unviewed': watch.has_unviewed,
|
||||
'uuid': watch.get('uuid'),
|
||||
'event_timestamp': time.time()
|
||||
}
|
||||
|
||||
errored_count = 0
|
||||
@@ -263,15 +251,15 @@ def init_socketio(app, datastore):
|
||||
"""Initialize SocketIO with the main Flask app"""
|
||||
import platform
|
||||
import sys
|
||||
|
||||
|
||||
# Platform-specific async_mode selection for better stability
|
||||
system = platform.system().lower()
|
||||
python_version = sys.version_info
|
||||
|
||||
|
||||
# Check for SocketIO mode configuration via environment variable
|
||||
# Default is 'threading' for best cross-platform compatibility
|
||||
socketio_mode = os.getenv('SOCKETIO_MODE', 'threading').lower()
|
||||
|
||||
|
||||
if socketio_mode == 'gevent':
|
||||
# Use gevent mode (higher concurrency but platform limitations)
|
||||
try:
|
||||
@@ -289,7 +277,7 @@ def init_socketio(app, datastore):
|
||||
# Invalid mode specified, use default
|
||||
async_mode = 'threading'
|
||||
logger.warning(f"Invalid SOCKETIO_MODE='{socketio_mode}', using default {async_mode} mode for Socket.IO")
|
||||
|
||||
|
||||
# Log platform info for debugging
|
||||
logger.info(f"Platform: {system}, Python: {python_version.major}.{python_version.minor}, Socket.IO mode: {async_mode}")
|
||||
|
||||
@@ -327,6 +315,7 @@ def init_socketio(app, datastore):
|
||||
emit_flash=False
|
||||
)
|
||||
|
||||
|
||||
@socketio.on('connect')
|
||||
def handle_connect():
|
||||
"""Handle client connection"""
|
||||
@@ -404,4 +393,4 @@ def init_socketio(app, datastore):
|
||||
|
||||
logger.info("Socket.IO initialized and attached to main Flask app")
|
||||
logger.info(f"Socket.IO: Registered event handlers: {socketio.handlers if hasattr(socketio, 'handlers') else 'No handlers found'}")
|
||||
return socketio
|
||||
return socketio
|
||||
|
||||
@@ -159,7 +159,6 @@
|
||||
// Return the current request in case it's needed
|
||||
return requests[namespace];
|
||||
};
|
||||
|
||||
})(jQuery);
|
||||
|
||||
|
||||
|
||||
@@ -104,18 +104,8 @@ $(document).ready(function () {
|
||||
});
|
||||
});
|
||||
|
||||
// So that the favicon is only updated when the server has written the scraped favicon to disk.
|
||||
socket.on('watch_bumped_favicon', function (watch) {
|
||||
const $watchRow = $(`tr[data-watch-uuid="${watch.uuid}"]`);
|
||||
if ($watchRow.length) {
|
||||
$watchRow.addClass('has-favicon');
|
||||
// Because the event could be emitted from a process that is outside the app context, url_for() might not work.
|
||||
// Lets use url_for at template generation time to give us a PLACEHOLDER instead
|
||||
let favicon_url = favicon_baseURL.replace('/PLACEHOLDER', `/${watch.uuid}?cache=${watch.event_timestamp}`);
|
||||
console.log(`Setting favicon for UUID - ${watch.uuid} - ${favicon_url}`);
|
||||
$('img.favicon', $watchRow).attr('src', favicon_url);
|
||||
}
|
||||
})
|
||||
// Listen for periodically emitted watch data
|
||||
console.log('Adding watch_update event listener');
|
||||
|
||||
socket.on('watch_update', function (data) {
|
||||
const watch = data.watch;
|
||||
@@ -126,28 +116,29 @@ $(document).ready(function () {
|
||||
console.log(`${watch.event_timestamp} - Watch update ${watch.uuid} - Checking now - ${watch.checking_now} - UUID in URL ${window.location.href.includes(watch.uuid)}`);
|
||||
console.log('Watch data:', watch);
|
||||
console.log('General stats:', general_stats);
|
||||
|
||||
|
||||
// Updating watch table rows
|
||||
const $watchRow = $('tr[data-watch-uuid="' + watch.uuid + '"]');
|
||||
console.log('Found watch row elements:', $watchRow.length);
|
||||
|
||||
|
||||
if ($watchRow.length) {
|
||||
$($watchRow).toggleClass('checking-now', watch.checking_now);
|
||||
$($watchRow).toggleClass('queued', watch.queued);
|
||||
$($watchRow).toggleClass('unviewed', watch.unviewed);
|
||||
$($watchRow).toggleClass('has-error', watch.has_error);
|
||||
$($watchRow).toggleClass('has-favicon', watch.has_favicon);
|
||||
$($watchRow).toggleClass('notification_muted', watch.notification_muted);
|
||||
$($watchRow).toggleClass('paused', watch.paused);
|
||||
$($watchRow).toggleClass('single-history', watch.history_n === 1);
|
||||
$($watchRow).toggleClass('multiple-history', watch.history_n >= 2);
|
||||
|
||||
$('td.title-col .error-text', $watchRow).html(watch.error_text)
|
||||
|
||||
$('td.last-changed', $watchRow).text(watch.last_changed_text)
|
||||
|
||||
$('td.last-checked .innertext', $watchRow).text(watch.last_checked_text)
|
||||
$('td.last-checked', $watchRow).data('timestamp', watch.last_checked).data('fetchduration', watch.fetch_time);
|
||||
$('td.last-checked', $watchRow).data('eta_complete', watch.last_checked + watch.fetch_time);
|
||||
|
||||
|
||||
console.log('Updated UI for watch:', watch.uuid);
|
||||
}
|
||||
|
||||
|
||||
File diff suppressed because one or more lines are too long
@@ -3,16 +3,15 @@
|
||||
"version": "0.0.3",
|
||||
"description": "",
|
||||
"main": "index.js",
|
||||
"engines": {
|
||||
"node": ">=18.0.0"
|
||||
},
|
||||
"scripts": {
|
||||
"watch": "sass --watch scss:. --style=compressed --no-source-map",
|
||||
"build": "sass scss:. --style=compressed --no-source-map"
|
||||
"watch": "node-sass -w scss -o .",
|
||||
"build": "node-sass scss -o ."
|
||||
},
|
||||
"author": "Leigh Morresi / Web Technologies s.r.o.",
|
||||
"license": "Apache",
|
||||
"author": "",
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
"sass": "^1.77.8"
|
||||
"node-sass": "^7.0.0",
|
||||
"tar": "^6.1.9",
|
||||
"trim-newlines": "^3.0.1"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
@use "parts/variables";
|
||||
@import "parts/_variables.scss";
|
||||
|
||||
#diff-ui {
|
||||
|
||||
|
||||
@@ -64,17 +64,17 @@ body.proxy-check-active {
|
||||
#recommended-proxy {
|
||||
display: grid;
|
||||
gap: 2rem;
|
||||
padding-bottom: 1em;
|
||||
|
||||
@media (min-width: 991px) {
|
||||
grid-template-columns: repeat(2, 1fr);
|
||||
}
|
||||
@media (min-width: 991px) {
|
||||
grid-template-columns: repeat(2, 1fr);
|
||||
}
|
||||
|
||||
> div {
|
||||
border: 1px #aaa solid;
|
||||
border-radius: 4px;
|
||||
padding: 1em;
|
||||
}
|
||||
|
||||
padding-bottom: 1em;
|
||||
}
|
||||
|
||||
#extra-proxies-setting {
|
||||
|
||||
@@ -1,92 +0,0 @@
|
||||
.watch-table {
|
||||
&.favicon-not-enabled {
|
||||
tr {
|
||||
.favicon {
|
||||
display: none;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
tr {
|
||||
/* make the icons and the text inline-ish */
|
||||
td.inline.title-col {
|
||||
.flex-wrapper {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 4px;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
td,
|
||||
th {
|
||||
vertical-align: middle;
|
||||
}
|
||||
|
||||
tr.has-favicon {
|
||||
&.unviewed {
|
||||
img.favicon {
|
||||
opacity: 1.0 !important;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
.status-icons {
|
||||
white-space: nowrap;
|
||||
display: flex;
|
||||
align-items: center; /* Vertical centering */
|
||||
gap: 4px; /* Space between image and text */
|
||||
> * {
|
||||
vertical-align: middle;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
.title-col {
|
||||
/* Optional, for spacing */
|
||||
padding: 10px;
|
||||
}
|
||||
|
||||
.title-wrapper {
|
||||
display: flex;
|
||||
align-items: center; /* Vertical centering */
|
||||
gap: 10px; /* Space between image and text */
|
||||
}
|
||||
|
||||
/* Make sure .title-col-inner doesn't collapse or misalign */
|
||||
.title-col-inner {
|
||||
display: inline-block;
|
||||
vertical-align: middle;
|
||||
}
|
||||
|
||||
/* favicon styling */
|
||||
.watch-table {
|
||||
img.favicon {
|
||||
vertical-align: middle;
|
||||
max-width: 25px;
|
||||
max-height: 25px;
|
||||
height: 25px;
|
||||
padding-right: 4px;
|
||||
}
|
||||
|
||||
// Reserved for future use
|
||||
/* &.thumbnail-type-screenshot {
|
||||
tr.has-favicon {
|
||||
td.inline.title-col {
|
||||
img.thumbnail {
|
||||
background-color: #fff; !* fallback bg for SVGs without bg *!
|
||||
border-radius: 4px; !* subtle rounded corners *!
|
||||
border: 1px solid #ddd; !* light border for contrast *!
|
||||
box-shadow: 0 2px 6px rgba(0, 0, 0, 0.15); !* soft shadow *!
|
||||
filter: contrast(1.05) saturate(1.1) drop-shadow(0 0 0.5px rgba(0, 0, 0, 0.2));
|
||||
object-fit: cover; !* crop/fill if needed *!
|
||||
opacity: 0.8;
|
||||
max-width: 30px;
|
||||
max-height: 30px;
|
||||
height: 30px;
|
||||
}
|
||||
}
|
||||
}
|
||||
}*/
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
@use "minitabs";
|
||||
@import "minitabs";
|
||||
|
||||
body.preview-text-enabled {
|
||||
|
||||
|
||||
@@ -1,178 +0,0 @@
|
||||
$grid-col-checkbox: 20px;
|
||||
$grid-col-watch: 100px;
|
||||
$grid-gap: 0.5rem;
|
||||
|
||||
|
||||
@media (max-width: 767px) {
|
||||
|
||||
/*
|
||||
Max width before this PARTICULAR table gets nasty
|
||||
This query will take effect for any screen smaller than 760px
|
||||
and also iPads specifically.
|
||||
*/
|
||||
.watch-table {
|
||||
/* make headings work on mobile */
|
||||
thead {
|
||||
display: block;
|
||||
|
||||
tr {
|
||||
th {
|
||||
display: inline-block;
|
||||
// Hide the "Last" text for smaller screens
|
||||
@media (max-width: 768px) {
|
||||
.hide-on-mobile {
|
||||
display: none;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
.empty-cell {
|
||||
display: none;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
.last-checked {
|
||||
margin-left: calc($grid-col-checkbox + $grid-gap);
|
||||
|
||||
> span {
|
||||
vertical-align: middle;
|
||||
}
|
||||
}
|
||||
|
||||
.last-changed {
|
||||
margin-left: calc($grid-col-checkbox + $grid-gap);
|
||||
}
|
||||
|
||||
.last-checked::before {
|
||||
color: var(--color-text);
|
||||
content: "Last Checked ";
|
||||
}
|
||||
|
||||
.last-changed::before {
|
||||
color: var(--color-text);
|
||||
content: "Last Changed ";
|
||||
}
|
||||
|
||||
/* Force table to not be like tables anymore */
|
||||
td.inline {
|
||||
display: inline-block;
|
||||
}
|
||||
|
||||
.pure-table td,
|
||||
.pure-table th {
|
||||
border: none;
|
||||
}
|
||||
|
||||
td {
|
||||
/* Behave like a "row" */
|
||||
border: none;
|
||||
border-bottom: 1px solid var(--color-border-watch-table-cell);
|
||||
vertical-align: middle;
|
||||
|
||||
&:before {
|
||||
/* Top/left values mimic padding */
|
||||
top: 6px;
|
||||
left: 6px;
|
||||
width: 45%;
|
||||
padding-right: 10px;
|
||||
white-space: nowrap;
|
||||
}
|
||||
}
|
||||
|
||||
&.pure-table-striped {
|
||||
tr {
|
||||
background-color: var(--color-table-background);
|
||||
}
|
||||
|
||||
tr:nth-child(2n-1) {
|
||||
background-color: var(--color-table-stripe);
|
||||
}
|
||||
|
||||
tr:nth-child(2n-1) td {
|
||||
background-color: inherit;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@media (max-width: 767px) {
|
||||
.watch-table {
|
||||
tbody {
|
||||
tr {
|
||||
padding-bottom: 10px;
|
||||
padding-top: 10px;
|
||||
display: grid;
|
||||
grid-template-columns: $grid-col-checkbox 1fr $grid-col-watch;
|
||||
grid-template-rows: auto auto auto auto;
|
||||
gap: $grid-gap;
|
||||
|
||||
.counter-i {
|
||||
display: none;
|
||||
}
|
||||
|
||||
td.checkbox-uuid {
|
||||
display: grid;
|
||||
place-items: center;
|
||||
}
|
||||
|
||||
td.inline {
|
||||
/* display: block !important;;*/
|
||||
}
|
||||
|
||||
> td {
|
||||
border-bottom: none;
|
||||
}
|
||||
|
||||
> td.title-col {
|
||||
grid-column: 1 / -1;
|
||||
grid-row: 1;
|
||||
.watch-title {
|
||||
font-size: 0.92rem;
|
||||
}
|
||||
.link-spread {
|
||||
display: none;
|
||||
}
|
||||
}
|
||||
|
||||
> td.last-checked {
|
||||
grid-column: 1 / -1;
|
||||
grid-row: 2;
|
||||
}
|
||||
|
||||
> td.last-changed {
|
||||
grid-column: 1 / -1;
|
||||
grid-row: 3;
|
||||
}
|
||||
|
||||
> td.checkbox-uuid {
|
||||
grid-column: 1;
|
||||
grid-row: 4;
|
||||
}
|
||||
|
||||
> td.buttons {
|
||||
grid-column: 2;
|
||||
grid-row: 4;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: flex-start;
|
||||
}
|
||||
|
||||
> td.watch-controls {
|
||||
grid-column: 3;
|
||||
grid-row: 4;
|
||||
display: grid;
|
||||
place-items: center;
|
||||
|
||||
a img {
|
||||
padding: 10px;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
.pure-table td {
|
||||
padding: 3px !important;
|
||||
}
|
||||
}
|
||||
@@ -7,7 +7,6 @@
|
||||
&.unviewed {
|
||||
font-weight: bold;
|
||||
}
|
||||
|
||||
color: var(--color-watch-table-row-text);
|
||||
}
|
||||
|
||||
@@ -49,17 +48,17 @@
|
||||
/* Row with 'checking-now' */
|
||||
tr.checking-now {
|
||||
td:first-child {
|
||||
position: relative;
|
||||
position: relative;
|
||||
}
|
||||
|
||||
td:first-child::before {
|
||||
content: "";
|
||||
position: absolute;
|
||||
top: 0;
|
||||
bottom: 0;
|
||||
left: 0;
|
||||
width: 3px;
|
||||
background-color: #293eff;
|
||||
content: "";
|
||||
position: absolute;
|
||||
top: 0;
|
||||
bottom: 0;
|
||||
left: 0;
|
||||
width: 3px;
|
||||
background-color: #293eff;
|
||||
}
|
||||
|
||||
td.last-checked {
|
||||
@@ -110,7 +109,6 @@
|
||||
|
||||
tr.has-error {
|
||||
color: var(--color-watch-table-error);
|
||||
|
||||
.error-text {
|
||||
display: block !important;
|
||||
}
|
||||
@@ -121,7 +119,6 @@
|
||||
display: inline-block !important;
|
||||
}
|
||||
}
|
||||
|
||||
tr.multiple-history {
|
||||
a.history-link {
|
||||
display: inline-block !important;
|
||||
@@ -129,3 +126,5 @@
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -2,25 +2,21 @@
|
||||
* -- BASE STYLES --
|
||||
*/
|
||||
|
||||
@use "parts/variables";
|
||||
@use "parts/arrows";
|
||||
@use "parts/browser-steps";
|
||||
@use "parts/extra_proxies";
|
||||
@use "parts/extra_browsers";
|
||||
@use "parts/pagination";
|
||||
@use "parts/spinners";
|
||||
@use "parts/darkmode";
|
||||
@use "parts/menu";
|
||||
@use "parts/love";
|
||||
@use "parts/preview_text_filter";
|
||||
@use "parts/watch_table";
|
||||
@use "parts/watch_table-mobile";
|
||||
@use "parts/edit";
|
||||
@use "parts/conditions_table";
|
||||
@use "parts/lister_extra";
|
||||
@use "parts/socket";
|
||||
@use "parts/visualselector";
|
||||
|
||||
@import "parts/_arrows";
|
||||
@import "parts/_browser-steps";
|
||||
@import "parts/_extra_proxies";
|
||||
@import "parts/_extra_browsers";
|
||||
@import "parts/_pagination";
|
||||
@import "parts/_spinners";
|
||||
@import "parts/_variables";
|
||||
@import "parts/_darkmode";
|
||||
@import "parts/_menu";
|
||||
@import "parts/_love";
|
||||
@import "parts/preview_text_filter";
|
||||
@import "parts/_watch_table";
|
||||
@import "parts/_edit";
|
||||
@import "parts/_conditions_table";
|
||||
@import "parts/_socket";
|
||||
|
||||
body {
|
||||
color: var(--color-text);
|
||||
@@ -188,15 +184,9 @@ code {
|
||||
@extend .inline-tag;
|
||||
}
|
||||
|
||||
@media (min-width: 768px) {
|
||||
.box {
|
||||
margin: 0 1em !important;
|
||||
}
|
||||
}
|
||||
|
||||
.box {
|
||||
max-width: 100%;
|
||||
margin: 0 0.3em;
|
||||
margin: 0 1em;
|
||||
flex-direction: column;
|
||||
display: flex;
|
||||
justify-content: center;
|
||||
@@ -704,6 +694,114 @@ footer {
|
||||
width: 100%;
|
||||
}
|
||||
|
||||
/*
|
||||
Max width before this PARTICULAR table gets nasty
|
||||
This query will take effect for any screen smaller than 760px
|
||||
and also iPads specifically.
|
||||
*/
|
||||
.watch-table {
|
||||
/* make headings work on mobile */
|
||||
thead {
|
||||
display: block;
|
||||
tr {
|
||||
th {
|
||||
display: inline-block;
|
||||
// Hide the "Last" text for smaller screens
|
||||
@media (max-width: 768px) {
|
||||
.hide-on-mobile {
|
||||
display: none;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
.empty-cell {
|
||||
display: none;
|
||||
}
|
||||
}
|
||||
|
||||
/* Force table to not be like tables anymore */
|
||||
tbody {
|
||||
td,
|
||||
tr {
|
||||
display: block;
|
||||
}
|
||||
}
|
||||
|
||||
tbody {
|
||||
tr {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
|
||||
// The third child of each row will take up the remaining space
|
||||
// This is useful for the URL column, which should expand to fill the remaining space
|
||||
:nth-child(3) {
|
||||
flex-grow: 1;
|
||||
}
|
||||
// The last three children (from the end) of each row will take up the full width
|
||||
// This is useful for the "Last Checked", "Last Changed", and the action buttons columns, which should each take up the full width
|
||||
:nth-last-child(-n+3) {
|
||||
flex-basis: 100%;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
.last-checked {
|
||||
>span {
|
||||
vertical-align: middle;
|
||||
}
|
||||
}
|
||||
|
||||
.last-checked::before {
|
||||
color: var(--color-last-checked);
|
||||
content: "Last Checked ";
|
||||
}
|
||||
|
||||
.last-changed::before {
|
||||
color: var(--color-last-checked);
|
||||
content: "Last Changed ";
|
||||
}
|
||||
|
||||
/* Force table to not be like tables anymore */
|
||||
td.inline {
|
||||
display: inline-block;
|
||||
}
|
||||
|
||||
.pure-table td,
|
||||
.pure-table th {
|
||||
border: none;
|
||||
}
|
||||
|
||||
td {
|
||||
/* Behave like a "row" */
|
||||
border: none;
|
||||
border-bottom: 1px solid var(--color-border-watch-table-cell);
|
||||
vertical-align: middle;
|
||||
|
||||
&:before {
|
||||
/* Top/left values mimic padding */
|
||||
top: 6px;
|
||||
left: 6px;
|
||||
width: 45%;
|
||||
padding-right: 10px;
|
||||
white-space: nowrap;
|
||||
}
|
||||
}
|
||||
|
||||
&.pure-table-striped {
|
||||
tr {
|
||||
background-color: var(--color-table-background);
|
||||
}
|
||||
|
||||
tr:nth-child(2n-1) {
|
||||
background-color: var(--color-table-stripe);
|
||||
}
|
||||
|
||||
tr:nth-child(2n-1) td {
|
||||
background-color: inherit;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
.pure-table {
|
||||
@@ -958,6 +1056,8 @@ ul {
|
||||
}
|
||||
}
|
||||
|
||||
@import "parts/_visualselector";
|
||||
|
||||
#webdriver_delay {
|
||||
width: 5em;
|
||||
}
|
||||
@@ -1075,23 +1175,17 @@ ul {
|
||||
|
||||
|
||||
#quick-watch-processor-type {
|
||||
ul#processor {
|
||||
color: #fff;
|
||||
padding-left: 0px;
|
||||
color: #fff;
|
||||
ul {
|
||||
padding: 0.3rem;
|
||||
li {
|
||||
list-style: none;
|
||||
font-size: 0.9rem;
|
||||
display: grid;
|
||||
grid-template-columns: auto 1fr;
|
||||
align-items: center;
|
||||
gap: 0.5rem;
|
||||
margin-bottom: 0.5rem;
|
||||
> * {
|
||||
display: inline-block;
|
||||
}
|
||||
}
|
||||
}
|
||||
label, input {
|
||||
padding: 0;
|
||||
margin: 0;
|
||||
}
|
||||
}
|
||||
|
||||
.restock-label {
|
||||
|
||||
File diff suppressed because one or more lines are too long
@@ -2,24 +2,19 @@
|
||||
|
||||
import time
|
||||
from flask import url_for
|
||||
import os
|
||||
from ..util import live_server_setup, wait_for_all_checks
|
||||
import logging
|
||||
|
||||
|
||||
# Requires playwright to be installed
|
||||
def test_fetch_webdriver_content(client, live_server, measure_memory_usage):
|
||||
# live_server_setup(live_server) # Setup on conftest per function
|
||||
# live_server_setup(live_server) # Setup on conftest per function
|
||||
|
||||
#####################
|
||||
res = client.post(
|
||||
url_for("settings.settings_page"),
|
||||
data={
|
||||
"application-empty_pages_are_a_change": "",
|
||||
"requests-time_between_check-minutes": 180,
|
||||
'application-fetch_backend': "html_webdriver",
|
||||
'application-ui-favicons_enabled': "y",
|
||||
},
|
||||
data={"application-empty_pages_are_a_change": "",
|
||||
"requests-time_between_check-minutes": 180,
|
||||
'application-fetch_backend': "html_webdriver"},
|
||||
follow_redirects=True
|
||||
)
|
||||
|
||||
@@ -35,51 +30,11 @@ def test_fetch_webdriver_content(client, live_server, measure_memory_usage):
|
||||
assert b"1 Imported" in res.data
|
||||
wait_for_all_checks(client)
|
||||
|
||||
|
||||
res = client.get(
|
||||
url_for("ui.ui_views.preview_page", uuid="first"),
|
||||
follow_redirects=True
|
||||
)
|
||||
logging.getLogger().info("Looking for correct fetched HTML (text) from server")
|
||||
|
||||
assert b'cool it works' in res.data
|
||||
|
||||
# Favicon scraper check, favicon only so far is fetched when in browser mode (not requests mode)
|
||||
if os.getenv("PLAYWRIGHT_DRIVER_URL"):
|
||||
uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
|
||||
res = client.get(
|
||||
url_for("watchlist.index"),
|
||||
)
|
||||
# The UI can access it here
|
||||
assert f'src="/static/favicon/{uuid}'.encode('utf8') in res.data
|
||||
|
||||
# Attempt to fetch it, make sure that works
|
||||
res = client.get(url_for('static_content', group='favicon', filename=uuid))
|
||||
assert res.status_code == 200
|
||||
assert len(res.data) > 10
|
||||
|
||||
# Check the API also returns it
|
||||
api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')
|
||||
res = client.get(
|
||||
url_for("watchfavicon", uuid=uuid),
|
||||
headers={'x-api-key': api_key}
|
||||
)
|
||||
assert res.status_code == 200
|
||||
assert len(res.data) > 10
|
||||
|
||||
##################### disable favicons check
|
||||
res = client.post(
|
||||
url_for("settings.settings_page"),
|
||||
data={
|
||||
"requests-time_between_check-minutes": 180,
|
||||
'application-ui-favicons_enabled': "",
|
||||
"application-empty_pages_are_a_change": "",
|
||||
},
|
||||
follow_redirects=True
|
||||
)
|
||||
|
||||
assert b"Settings updated." in res.data
|
||||
|
||||
res = client.get(
|
||||
url_for("watchlist.index"),
|
||||
)
|
||||
# The UI can access it here
|
||||
assert f'src="/static/favicon'.encode('utf8') not in res.data
|
||||
|
||||
@@ -111,130 +111,3 @@ def test_restock_detection(client, live_server, measure_memory_usage):
|
||||
res = client.get(url_for("watchlist.index"))
|
||||
assert b'not-in-stock' in res.data, "Correctly showing NOT IN STOCK in the list after it changed from IN STOCK"
|
||||
|
||||
|
||||
def test_restock_custom_strings(client, live_server):
|
||||
"""Test custom out-of-stock strings feature"""
|
||||
|
||||
# Set up a response with custom out-of-stock text
|
||||
test_return_data = """<html>
|
||||
<body>
|
||||
Some initial text<br>
|
||||
<p>Which is across multiple lines</p>
|
||||
<br>
|
||||
So let's see what happens. <br>
|
||||
<div>price: $10.99</div>
|
||||
<div id="custom">Pronto estarán en stock!</div>
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
|
||||
with open("test-datastore/endpoint-content.txt", "w") as f:
|
||||
f.write(test_return_data)
|
||||
|
||||
test_url = url_for('test_endpoint', _external=True).replace('http://localhost', 'http://changedet')
|
||||
|
||||
# Add watch with custom out-of-stock strings
|
||||
res = client.post(
|
||||
url_for("ui.ui_views.form_quick_watch_add"),
|
||||
data={"url": test_url, "tags": '', 'processor': 'restock_diff'},
|
||||
follow_redirects=True
|
||||
)
|
||||
|
||||
# Get the UUID so we can configure the watch
|
||||
uuid = extract_UUID_from_client(client)
|
||||
|
||||
# Configure custom out-of-stock strings
|
||||
res = client.post(
|
||||
url_for("ui.ui_edit.edit_page", uuid=uuid, unpause_on_save=1),
|
||||
data={
|
||||
"url": test_url,
|
||||
'processor': 'restock_diff',
|
||||
'restock_settings-custom_outofstock_strings': 'Pronto estarán en stock!\nCustom unavailable message'
|
||||
},
|
||||
follow_redirects=True
|
||||
)
|
||||
assert b"Updated watch." in res.data
|
||||
|
||||
# Check that it detects as out of stock
|
||||
wait_for_all_checks(client)
|
||||
res = client.get(url_for("watchlist.index"))
|
||||
assert b'not-in-stock' in res.data, "Should detect custom out-of-stock string"
|
||||
|
||||
# Test custom in-stock strings by changing the content
|
||||
test_return_data_instock = """<html>
|
||||
<body>
|
||||
Some initial text<br>
|
||||
<p>Which is across multiple lines</p>
|
||||
<br>
|
||||
So let's see what happens. <br>
|
||||
<div>price: $10.99</div>
|
||||
<div id="custom">Disponible ahora</div>
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
|
||||
with open("test-datastore/endpoint-content.txt", "w") as f:
|
||||
f.write(test_return_data_instock)
|
||||
|
||||
# Update the watch to include custom in-stock strings
|
||||
res = client.post(
|
||||
url_for("ui.ui_edit.edit_page", uuid=uuid, unpause_on_save=1),
|
||||
data={
|
||||
"url": test_url,
|
||||
'processor': 'restock_diff',
|
||||
'restock_settings-custom_outofstock_strings': 'Pronto estarán en stock!\nCustom unavailable message',
|
||||
'restock_settings-custom_instock_strings': 'Disponible ahora\nIn voorraad'
|
||||
},
|
||||
follow_redirects=True
|
||||
)
|
||||
assert b"Updated watch." in res.data
|
||||
|
||||
# Check again - should be detected as in stock now
|
||||
client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
|
||||
wait_for_all_checks(client)
|
||||
res = client.get(url_for("watchlist.index"))
|
||||
assert b'not-in-stock' not in res.data, "Should detect custom in-stock string and show as available"
|
||||
|
||||
|
||||
def test_restock_custom_strings_normalization(client, live_server):
|
||||
"""Test key normalization scenarios: accents, case, and spaces"""
|
||||
|
||||
# Test page with Spanish text with accents and mixed case
|
||||
test_return_data = """<html>
|
||||
<body>
|
||||
<div>price: $10.99</div>
|
||||
<div id="status">¡TEMPORALMENTE AGOTADO!</div>
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
|
||||
with open("test-datastore/endpoint-content.txt", "w") as f:
|
||||
f.write(test_return_data)
|
||||
|
||||
test_url = url_for('test_endpoint', _external=True).replace('http://localhost', 'http://changedet')
|
||||
|
||||
# Add watch
|
||||
res = client.post(
|
||||
url_for("ui.ui_views.form_quick_watch_add"),
|
||||
data={"url": test_url, "tags": '', 'processor': 'restock_diff'},
|
||||
follow_redirects=True
|
||||
)
|
||||
|
||||
uuid = extract_UUID_from_client(client)
|
||||
|
||||
# Configure custom string without accents, lowercase, no extra spaces
|
||||
res = client.post(
|
||||
url_for("ui.ui_edit.edit_page", uuid=uuid, unpause_on_save=1),
|
||||
data={
|
||||
"url": test_url,
|
||||
'processor': 'restock_diff',
|
||||
'restock_settings-custom_outofstock_strings': 'temporalmente agotado'
|
||||
},
|
||||
follow_redirects=True
|
||||
)
|
||||
|
||||
# Should detect as out of stock despite text differences
|
||||
wait_for_all_checks(client)
|
||||
res = client.get(url_for("watchlist.index"))
|
||||
assert b'not-in-stock' in res.data, "Should match despite accents, case, and spacing differences"
|
||||
|
||||
|
||||
@@ -1,95 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import unittest
|
||||
from changedetectionio.processors.restock_diff.processor import perform_site_check
|
||||
|
||||
|
||||
class TestCustomStringNormalization(unittest.TestCase):
|
||||
"""Test the text normalization logic for custom out-of-stock strings"""
|
||||
|
||||
def setUp(self):
|
||||
# Create a processor instance for testing
|
||||
self.processor = perform_site_check(datastore=None, watch_uuid='test')
|
||||
|
||||
def test_normalize_text_for_matching(self):
|
||||
"""Test the _normalize_text_for_matching method"""
|
||||
|
||||
test_cases = [
|
||||
# (input, expected_output)
|
||||
("Agotado", "agotado"),
|
||||
("AGOTADO", "agotado"), # Lowercase
|
||||
("Sin stock!", "sin stock!"), # Normalize whitespace
|
||||
("Pronto\t\nestarán\nen stock", "pronto estaran en stock"), # Multiple whitespace types + accents
|
||||
("¡Temporalmente AGOTADO!", "¡temporalmente agotado!"), # Complex case
|
||||
("", ""), # Empty string
|
||||
("café", "cafe"), # French accent
|
||||
("naïve", "naive"), # Multiple accents
|
||||
]
|
||||
|
||||
for input_text, expected in test_cases:
|
||||
with self.subTest(input_text=input_text):
|
||||
result = self.processor._normalize_text_for_matching(input_text)
|
||||
self.assertEqual(result, expected,
|
||||
f"Failed to normalize '{input_text}' -> expected '{expected}', got '{result}'")
|
||||
|
||||
def test_check_custom_strings_normalization(self):
|
||||
"""Test that custom string matching works with normalization"""
|
||||
|
||||
test_cases = [
|
||||
# (page_text, custom_strings, should_match, description)
|
||||
("AGOTADO", "agotado", True, "uppercase to lowercase"),
|
||||
("Agotado", "agotado", True, "single uppercase to lowercase"),
|
||||
("Sin stock!", "sin stock", True, "multiple spaces normalized"),
|
||||
("¡Pronto estarán en stock!", "pronto estaran en stock", True, "accents + spaces"),
|
||||
("TEMPORALMENTE AGOTADO", "temporalmente agotado", True, "multi-word uppercase"),
|
||||
("Available now", "agotado", False, "no match case"),
|
||||
("", "agotado", False, "empty text"),
|
||||
("agotado", "", False, "empty custom strings"),
|
||||
]
|
||||
|
||||
for page_text, custom_strings, should_match, description in test_cases:
|
||||
with self.subTest(description=description):
|
||||
result = self.processor._check_custom_strings(page_text, custom_strings, "out-of-stock")
|
||||
|
||||
if should_match:
|
||||
self.assertIsNotNone(result,
|
||||
f"Expected match for '{description}': '{page_text}' should match '{custom_strings}'")
|
||||
else:
|
||||
self.assertIsNone(result,
|
||||
f"Expected no match for '{description}': '{page_text}' should not match '{custom_strings}'")
|
||||
|
||||
def test_check_custom_strings_multiline(self):
|
||||
"""Test that multi-line custom strings work properly"""
|
||||
|
||||
page_text = "Product status: TEMPORALMENTE AGOTADO"
|
||||
custom_strings = """
|
||||
sin stock
|
||||
agotado
|
||||
temporalmente agotado
|
||||
"""
|
||||
|
||||
result = self.processor._check_custom_strings(page_text, custom_strings, "out-of-stock")
|
||||
self.assertIsNotNone(result)
|
||||
self.assertEqual(result.strip(), "temporalmente agotado")
|
||||
|
||||
def test_get_combined_instock_strings_normalization(self):
|
||||
"""Test that custom in-stock strings are normalized properly"""
|
||||
|
||||
restock_settings = {
|
||||
'custom_instock_strings': 'Disponible AHORA\nEn Stock\nDISPONÍBLE'
|
||||
}
|
||||
|
||||
result = self.processor._get_combined_instock_strings(restock_settings)
|
||||
|
||||
# Check that built-in strings are included
|
||||
self.assertIn('instock', result)
|
||||
self.assertIn('presale', result)
|
||||
|
||||
# Check that custom strings are normalized and included
|
||||
self.assertIn('disponible ahora', result)
|
||||
self.assertIn('en stock', result)
|
||||
self.assertIn('disponible', result) # accent removed
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 200 KiB After Width: | Height: | Size: 171 KiB |
@@ -117,9 +117,6 @@ price-parser
|
||||
|
||||
# flask_socket_io - incorrect package name, already have flask-socketio above
|
||||
|
||||
# So far for detecting correct favicon type, but for other things in the future
|
||||
python-magic
|
||||
|
||||
# Scheduler - Windows seemed to miss a lot of default timezone info (even "UTC" !)
|
||||
tzdata
|
||||
|
||||
|
||||
Reference in New Issue
Block a user