Compare commits

..

10 Commits

17 changed files with 126 additions and 81 deletions

View File

@@ -118,7 +118,8 @@ jobs:
sleep 3
# invert the check (it should be not 0/not running)
docker ps
docker logs sig-test
# check signal catch(STDOUT) log
docker logs sig-test | grep 'Shutdown: Got Signal - SIGINT' || exit 1
test -z "`docker ps|grep sig-test`"
if [ $? -ne 0 ]
then
@@ -138,7 +139,7 @@ jobs:
sleep 3
# invert the check (it should be not 0/not running)
docker ps
docker logs sig-test
docker logs sig-test | grep 'Shutdown: Got Signal - SIGTERM' || exit 1
test -z "`docker ps|grep sig-test`"
if [ $? -ne 0 ]
then

View File

@@ -2,7 +2,7 @@
# Read more https://github.com/dgtlmoon/changedetection.io/wiki
__version__ = '0.45.8.1'
__version__ = '0.45.9'
from distutils.util import strtobool
from json.decoder import JSONDecodeError

View File

@@ -3,7 +3,7 @@
{% from '_helpers.jinja' import render_field, render_checkbox_field, render_button %}
{% from '_common_fields.jinja' import render_common_settings_form %}
<script>
const notification_base_url="{{url_for('ajax_callback_send_notification_test')}}";
const notification_base_url="{{url_for('ajax_callback_send_notification_test', watch_uuid=uuid)}}";
</script>
<script src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>

View File

@@ -43,9 +43,11 @@ class JSActionExceptions(Exception):
return
class BrowserStepsStepTimout(Exception):
def __init__(self, step_n):
class BrowserStepsStepException(Exception):
def __init__(self, step_n, original_e):
self.step_n = step_n
self.original_e = original_e
print(f"Browser Steps exception at step {self.step_n}", str(original_e))
return
@@ -91,19 +93,20 @@ class ReplyWithContentButNoText(Exception):
class Fetcher():
browser_connection_is_custom = None
browser_connection_url = None
browser_steps = None
browser_steps_screenshot_path = None
content = None
error = None
fetcher_description = "No description"
browser_connection_url = None
headers = {}
instock_data = None
instock_data_js = ""
status_code = None
webdriver_js_execute_code = None
xpath_data = None
xpath_element_js = ""
instock_data = None
instock_data_js = ""
# Will be needed in the future by the VisualSelector, always get this where possible.
screenshot = False
@@ -172,7 +175,7 @@ class Fetcher():
def iterate_browser_steps(self):
from changedetectionio.blueprint.browser_steps.browser_steps import steppable_browser_interface
from playwright._impl._errors import TimeoutError
from playwright._impl._errors import TimeoutError, Error
from jinja2 import Environment
jinja2_env = Environment(extensions=['jinja2_time.TimeExtension'])
@@ -202,10 +205,10 @@ class Fetcher():
optional_value=optional_value)
self.screenshot_step(step_n)
self.save_step_html(step_n)
except TimeoutError as e:
print(str(e))
except (Error, TimeoutError) as e:
# Stop processing here
raise BrowserStepsStepTimout(step_n=step_n)
raise BrowserStepsStepException(step_n=step_n, original_e=e)
# It's always good to reset these
def delete_browser_steps_screenshots(self):
@@ -252,16 +255,19 @@ class base_html_playwright(Fetcher):
proxy = None
def __init__(self, proxy_override=None, browser_connection_url=None):
def __init__(self, proxy_override=None, custom_browser_connection_url=None):
super().__init__()
self.browser_type = os.getenv("PLAYWRIGHT_BROWSER_TYPE", 'chromium').strip('"')
# .strip('"') is going to save someone a lot of time when they accidently wrap the env value
if not browser_connection_url:
self.browser_connection_url = os.getenv("PLAYWRIGHT_DRIVER_URL", 'ws://playwright-chrome:3000').strip('"')
if custom_browser_connection_url:
self.browser_connection_is_custom = True
self.browser_connection_url = custom_browser_connection_url
else:
self.browser_connection_url = browser_connection_url
# Fallback to fetching from system
# .strip('"') is going to save someone a lot of time when they accidently wrap the env value
self.browser_connection_url = os.getenv("PLAYWRIGHT_DRIVER_URL", 'ws://playwright-chrome:3000').strip('"')
# If any proxy settings are enabled, then we should setup the proxy object
proxy_args = {}
@@ -311,7 +317,6 @@ class base_html_playwright(Fetcher):
is_binary=False):
from pkg_resources import resource_string
from urllib.parse import urlparse
extra_wait_ms = (int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)) + self.render_extract_delay) * 1000
@@ -324,19 +329,13 @@ class base_html_playwright(Fetcher):
from requests.exceptions import ConnectTimeout, ReadTimeout
wait_browserless_seconds = 240
# Could be set from the browser-override
browserless_function_url = self.browser_connection_url
browserless_function_url = os.getenv('BROWSERLESS_FUNCTION_URL')
from urllib.parse import urlparse
if not browserless_function_url:
# Convert/try to guess from PLAYWRIGHT_DRIVER_URL
o = urlparse(os.getenv('PLAYWRIGHT_DRIVER_URL'))
browserless_function_url = o._replace(scheme="http")._replace(path="function").geturl()
# Or use the system config defaults
if not self.browser_connection_url:
browserless_function_url = os.getenv('BROWSERLESS_FUNCTION_URL')
if not self.browser_connection_url:
browserless_function_url = os.getenv('PLAYWRIGHT_DRIVER_URL')
# Convert always to http:// and function/ for browserless
o = urlparse(browserless_function_url)
browserless_function_url = o._replace(scheme="http")._replace(path="function").geturl()
# Append proxy connect string
if self.proxy:
@@ -428,8 +427,10 @@ class base_html_playwright(Fetcher):
current_include_filters=None,
is_binary=False):
# For now, USE_EXPERIMENTAL_PUPPETEER_FETCH is not supported by watches with BrowserSteps (for now!)
if not self.browser_steps and os.getenv('USE_EXPERIMENTAL_PUPPETEER_FETCH'):
# browser_connection_is_custom doesnt work with puppeteer style fetch (use playwright native too in this case)
if not self.browser_connection_is_custom and not self.browser_steps and os.getenv('USE_EXPERIMENTAL_PUPPETEER_FETCH'):
if strtobool(os.getenv('USE_EXPERIMENTAL_PUPPETEER_FETCH')):
# Temporary backup solution until we rewrite the playwright code
return self.run_fetch_browserless_puppeteer(
@@ -576,15 +577,16 @@ class base_html_webdriver(Fetcher):
'socksProxy', 'socksVersion', 'socksUsername', 'socksPassword']
proxy = None
def __init__(self, proxy_override=None, browser_connection_url=None):
def __init__(self, proxy_override=None, custom_browser_connection_url=None):
super().__init__()
from selenium.webdriver.common.proxy import Proxy as SeleniumProxy
# .strip('"') is going to save someone a lot of time when they accidently wrap the env value
if not browser_connection_url:
if not custom_browser_connection_url:
self.browser_connection_url = os.getenv("WEBDRIVER_URL", 'http://browser-chrome:4444/wd/hub').strip('"')
else:
self.browser_connection_url = browser_connection_url
self.browser_connection_is_custom = True
self.browser_connection_url = custom_browser_connection_url
# If any proxy settings are enabled, then we should setup the proxy object
proxy_args = {}
@@ -681,7 +683,7 @@ class base_html_webdriver(Fetcher):
class html_requests(Fetcher):
fetcher_description = "Basic fast Plaintext/HTTP Client"
def __init__(self, proxy_override=None, browser_connection_url=None):
def __init__(self, proxy_override=None, custom_browser_connection_url=None):
super().__init__()
self.proxy_override = proxy_override
# browser_connection_url is none because its always 'launched locally'

View File

@@ -485,14 +485,18 @@ def changedetection_app(config=None, datastore_o=None):
# AJAX endpoint for sending a test
@app.route("/notification/send-test/<string:watch_uuid>", methods=['POST'])
@app.route("/notification/send-test", methods=['POST'])
@app.route("/notification/send-test/", methods=['POST'])
@login_optionally_required
def ajax_callback_send_notification_test():
def ajax_callback_send_notification_test(watch_uuid=None):
# Watch_uuid could be unsuet in the case its used in tag editor, global setings
import apprise
from .apprise_asset import asset
apobj = apprise.Apprise(asset=asset)
watch = datastore.data['watching'].get(watch_uuid) if watch_uuid else None
# validate URLS
if not len(request.form['notification_urls'].strip()):
@@ -505,9 +509,11 @@ def changedetection_app(config=None, datastore_o=None):
return make_response({'error': message}, 400)
try:
n_object = {'watch_url': request.form['window_url'],
'notification_urls': request.form['notification_urls'].splitlines()
}
# use the same as when it is triggered, but then override it with the form test values
n_object = {
'watch_url': request.form['window_url'],
'notification_urls': request.form['notification_urls'].splitlines()
}
# Only use if present, if not set in n_object it should use the default system value
if 'notification_format' in request.form and request.form['notification_format'].strip():
@@ -519,7 +525,9 @@ def changedetection_app(config=None, datastore_o=None):
if 'notification_body' in request.form and request.form['notification_body'].strip():
n_object['notification_body'] = request.form.get('notification_body', '').strip()
notification_q.put(n_object)
from . import update_worker
new_worker = update_worker.update_worker(update_q, notification_q, app, datastore)
new_worker.queue_notification_for_watch(notification_q=notification_q, n_object=n_object, watch=watch)
except Exception as e:
return make_response({'error': str(e)}, 400)

View File

@@ -38,6 +38,7 @@ base_config = {
'track_ldjson_price_data': None,
'headers': {}, # Extra headers to send
'ignore_text': [], # List of text to ignore when calculating the comparison checksum
'in_stock' : None,
'in_stock_only' : True, # Only trigger change on going to instock from out-of-stock
'include_filters': [],
'last_checked': 0,

View File

@@ -221,13 +221,14 @@ def process_notification(n_object, datastore):
# Notification title + body content parameters get created here.
# ( Where we prepare the tokens in the notification to be replaced with actual values )
def create_notification_parameters(n_object, datastore):
from copy import deepcopy
# in the case we send a test notification from the main settings, there is no UUID.
uuid = n_object['uuid'] if 'uuid' in n_object else ''
if uuid != '':
if uuid:
watch_title = datastore.data['watching'][uuid].get('title', '')
tag_list = []
tags = datastore.get_all_tags_for_watch(uuid)
@@ -255,7 +256,7 @@ def create_notification_parameters(n_object, datastore):
tokens.update(
{
'base_url': base_url,
'current_snapshot': n_object['current_snapshot'] if 'current_snapshot' in n_object else '',
'current_snapshot': n_object.get('current_snapshot', ''),
'diff': n_object.get('diff', ''), # Null default in the case we use a test
'diff_added': n_object.get('diff_added', ''), # Null default in the case we use a test
'diff_full': n_object.get('diff_full', ''), # Null default in the case we use a test

View File

@@ -43,14 +43,14 @@ class difference_detection_processor():
# In the case that the preferred fetcher was a browser config with custom connection URL..
# @todo - on save watch, if its extra_browser_ then it should be obvious it will use playwright (like if its requests now..)
browser_connection_url = None
custom_browser_connection_url = None
if prefer_fetch_backend.startswith('extra_browser_'):
(t, key) = prefer_fetch_backend.split('extra_browser_')
connection = list(
filter(lambda s: (s['browser_name'] == key), self.datastore.data['settings']['requests'].get('extra_browsers', [])))
if connection:
prefer_fetch_backend = 'base_html_playwright'
browser_connection_url = connection[0].get('browser_connection_url')
custom_browser_connection_url = connection[0].get('browser_connection_url')
# PDF should be html_requests because playwright will serve it up (so far) in a embedded page
# @todo https://github.com/dgtlmoon/changedetection.io/issues/2019
@@ -74,7 +74,7 @@ class difference_detection_processor():
# Now call the fetcher (playwright/requests/etc) with arguments that only a fetcher would need.
# When browser_connection_url is None, it method should default to working out whats the best defaults (os env vars etc)
self.fetcher = fetcher_obj(proxy_override=proxy_url,
browser_connection_url=browser_connection_url
custom_browser_connection_url=custom_browser_connection_url
)
if self.watch.has_browser_steps:

View File

@@ -4,6 +4,7 @@ function isItemInStock() {
' أخبرني عندما يتوفر',
'0 in stock',
'agotado',
'article épuisé',
'artikel zurzeit vergriffen',
'as soon as stock is available',
'ausverkauft', // sold out
@@ -17,10 +18,8 @@ function isItemInStock() {
'currently have any tickets for this',
'currently unavailable',
'dostępne wkrótce',
'dostępne wkrótce',
'en rupture de stock',
'ist derzeit nicht auf lager',
'ist derzeit nicht auf lager',
'item is no longer available',
'let me know when it\'s available',
'message if back in stock',
@@ -28,6 +27,8 @@ function isItemInStock() {
'nicht auf lager',
'nicht lieferbar',
'nicht zur verfügung',
'niet beschikbaar',
'niet leverbaar',
'no disponible temporalmente',
'no longer in stock',
'no tickets available',
@@ -44,6 +45,7 @@ function isItemInStock() {
'temporarily out of stock',
'temporarily unavailable',
'tickets unavailable',
'tijdelijk uitverkocht',
'unavailable tickets',
'we do not currently have an estimate of when this product will be back in stock.',
'zur zeit nicht an lager',

View File

@@ -24,14 +24,17 @@ $(document).ready(function() {
})
data = {
window_url : window.location.href,
notification_urls : $('.notification-urls').val(),
notification_body: $('#notification_body').val(),
notification_format: $('#notification_format').val(),
notification_title: $('#notification_title').val(),
notification_urls: $('.notification-urls').val(),
window_url: window.location.href,
}
for (key in data) {
if (!data[key].length) {
alert(key+" is empty, cannot send test.")
return;
}
if (!data['notification_urls'].length) {
alert("Notification URL list is empty, cannot send test.")
return;
}
$.ajax({

View File

@@ -248,6 +248,7 @@ class ChangeDetectionStore:
'check_count': 0,
'fetch_time' : 0.0,
'has_ldjson_price_data': None,
'in_stock': None,
'last_checked': 0,
'last_error': False,
'last_notification_error': False,

View File

@@ -14,7 +14,7 @@
{% if emailprefix %}
const email_notification_prefix=JSON.parse('{{ emailprefix|tojson }}');
{% endif %}
const notification_base_url="{{url_for('ajax_callback_send_notification_test')}}";
const notification_base_url="{{url_for('ajax_callback_send_notification_test', watch_uuid=uuid)}}";
const playwright_enabled={% if playwright_enabled %} true {% else %} false {% endif %};
const recheck_proxy_start_url="{{url_for('check_proxies.start_check', uuid=uuid)}}";
const proxy_recheck_status_url="{{url_for('check_proxies.get_recheck_status', uuid=uuid)}}";

View File

@@ -4,7 +4,7 @@
{% from '_helpers.jinja' import render_field, render_checkbox_field, render_button %}
{% from '_common_fields.jinja' import render_common_settings_form %}
<script>
const notification_base_url="{{url_for('ajax_callback_send_notification_test')}}";
const notification_base_url="{{url_for('ajax_callback_send_notification_test', watch_uuid=uuid)}}";
{% if emailprefix %}
const email_notification_prefix=JSON.parse('{{emailprefix|tojson}}');
{% endif %}

View File

@@ -141,7 +141,7 @@
{% if watch['processor'] == 'restock_diff' %}
<span class="restock-label {{'in-stock' if watch['in_stock'] else 'not-in-stock' }}" title="detecting restock conditions">
<!-- maybe some object watch['processor'][restock_diff] or.. -->
{% if watch['last_checked'] %}
{% if watch['last_checked'] and watch['in_stock'] != None %}
{% if watch['in_stock'] %} In stock {% else %} Not in stock {% endif %}
{% else %}
Not yet checked

View File

@@ -26,47 +26,61 @@ class update_worker(threading.Thread):
self.datastore = datastore
super().__init__(*args, **kwargs)
def queue_notification_for_watch(self, n_object, watch):
def queue_notification_for_watch(self, notification_q, n_object, watch):
from changedetectionio import diff
dates = []
trigger_text = ''
if watch:
watch_history = watch.history
dates = list(watch_history.keys())
trigger_text = watch.get('trigger_text', [])
watch_history = watch.history
dates = list(watch_history.keys())
# Add text that was triggered
snapshot_contents = watch.get_history_snapshot(dates[-1])
if len(dates):
snapshot_contents = watch.get_history_snapshot(dates[-1])
else:
snapshot_contents = "No snapshot/history available, the watch should fetch atleast once."
# HTML needs linebreak, but MarkDown and Text can use a linefeed
if n_object['notification_format'] == 'HTML':
if n_object.get('notification_format') == 'HTML':
line_feed_sep = "<br>"
# Snapshot will be plaintext on the disk, convert to some kind of HTML
snapshot_contents = snapshot_contents.replace('\n', line_feed_sep)
else:
line_feed_sep = "\n"
trigger_text = watch.get('trigger_text', [])
triggered_text = ''
if len(trigger_text):
from . import html_tools
triggered_text = html_tools.get_triggered_text(content=snapshot_contents, trigger_text=trigger_text)
if triggered_text:
triggered_text = line_feed_sep.join(triggered_text)
# Could be called as a 'test notification' with only 1 snapshot available
prev_snapshot = "Example text: example test\nExample text: change detection is cool\nExample text: some more examples\n"
current_snapshot = "Example text: example test\nExample text: change detection is fantastic\nExample text: even more examples\nExample text: a lot more examples"
if len(dates) > 1:
prev_snapshot = watch.get_history_snapshot(dates[-2])
current_snapshot = watch.get_history_snapshot(dates[-1])
n_object.update({
'current_snapshot': snapshot_contents,
'diff': diff.render_diff(watch.get_history_snapshot(dates[-2]), watch.get_history_snapshot(dates[-1]), line_feed_sep=line_feed_sep),
'diff_added': diff.render_diff(watch.get_history_snapshot(dates[-2]), watch.get_history_snapshot(dates[-1]), include_removed=False, line_feed_sep=line_feed_sep),
'diff_full': diff.render_diff(watch.get_history_snapshot(dates[-2]), watch.get_history_snapshot(dates[-1]), include_equal=True, line_feed_sep=line_feed_sep),
'diff_patch': diff.render_diff(watch.get_history_snapshot(dates[-2]), watch.get_history_snapshot(dates[-1]), line_feed_sep=line_feed_sep, patch_format=True),
'diff_removed': diff.render_diff(watch.get_history_snapshot(dates[-2]), watch.get_history_snapshot(dates[-1]), include_added=False, line_feed_sep=line_feed_sep),
'screenshot': watch.get_screenshot() if watch.get('notification_screenshot') else None,
'diff': diff.render_diff(prev_snapshot, current_snapshot, line_feed_sep=line_feed_sep),
'diff_added': diff.render_diff(prev_snapshot, current_snapshot, include_removed=False, line_feed_sep=line_feed_sep),
'diff_full': diff.render_diff(prev_snapshot, current_snapshot, include_equal=True, line_feed_sep=line_feed_sep),
'diff_patch': diff.render_diff(prev_snapshot, current_snapshot, line_feed_sep=line_feed_sep, patch_format=True),
'diff_removed': diff.render_diff(prev_snapshot, current_snapshot, include_added=False, line_feed_sep=line_feed_sep),
'screenshot': watch.get_screenshot() if watch and watch.get('notification_screenshot') else None,
'triggered_text': triggered_text,
'uuid': watch.get('uuid'),
'watch_url': watch.get('url'),
'uuid': watch.get('uuid') if watch else None,
'watch_url': watch.get('url') if watch else None,
})
logging.info (">> SENDING NOTIFICATION")
self.notification_q.put(n_object)
logging.info(">> SENDING NOTIFICATION")
notification_q.put(n_object)
# Prefer - Individual watch settings > Tag settings > Global settings (in that order)
def _check_cascading_vars(self, var_name, watch):
@@ -134,7 +148,7 @@ class update_worker(threading.Thread):
queued = False
if n_object and n_object.get('notification_urls'):
queued = True
self.queue_notification_for_watch(n_object, watch)
self.queue_notification_for_watch(notification_q=self.notification_q, n_object=n_object, watch=watch)
return queued
@@ -340,20 +354,32 @@ class update_worker(threading.Thread):
changed_detected = False
self.datastore.update_watch(uuid=uuid, update_obj={'last_error': False})
except content_fetcher.BrowserStepsStepTimout as e:
except content_fetcher.BrowserStepsStepException as e:
if not self.datastore.data['watching'].get(uuid):
continue
error_step = e.step_n + 1
err_text = f"Warning, browser step at position {error_step} could not run, target not found, check the watch, add a delay if necessary, view Browser Steps to see screenshot at that step"
from playwright._impl._errors import TimeoutError, Error
# Generally enough info for TimeoutError (couldnt locate the element after default seconds)
err_text = f"Browser step at position {error_step} could not run, check the watch, add a delay if necessary, view Browser Steps to see screenshot at that step."
if e.original_e.name == "TimeoutError":
# Just the first line is enough, the rest is the stack trace
err_text += " Could not find the target."
else:
# Other Error, more info is good.
err_text += " " + str(e.original_e).splitlines()[0]
print(f"BrowserSteps exception at step {error_step}", str(e.original_e))
self.datastore.update_watch(uuid=uuid,
update_obj={'last_error': err_text,
'browser_steps_last_error_step': error_step
}
)
if self.datastore.data['watching'][uuid].get('filter_failure_notification_send', False):
c = self.datastore.data['watching'][uuid].get('consecutive_filter_failures', 5)
c += 1

View File

@@ -90,7 +90,7 @@ services:
#
# Used for fetching pages via Playwright+Chrome where you need Javascript support.
# Note: works well but is deprecated, doesnt fetch full page screenshots and other issues
# Note: works well but is deprecated, does not fetch full page screenshots (doesnt work with Visual Selector) and other issues
# browser-chrome:
# hostname: browser-chrome
# image: selenium/standalone-chrome:4

View File

@@ -30,7 +30,7 @@ dnspython~=2.4 # related to eventlet fixes
# jq not available on Windows so must be installed manually
# Notification library
apprise~=1.6.0
apprise~=1.7.1
# apprise mqtt https://github.com/dgtlmoon/changedetection.io/issues/315
paho-mqtt