lee/changedetection.io
1
0
Fork 0
mirror of https://github.com/dgtlmoon/changedetection.io.git synced 2025-12-04 23:25:32 +00:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

UI - BrowserSteps - Show the screenshot of an error if it happened on a step, highlight which step had the error to make it easier to find out why the step didnt work, minor fixes to timeouts(#1883)

Browse Source
This commit is contained in:
dgtlmoon
2023-10-21 09:41:51 +02:00
committed by GitHub
parent b16d65741c
commit 1709e8f936
10 changed files with 96 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
changedetectionio/__init__.py
View File

@@ -622,7 +622,6 @@ def changedetection_app(config=None, datastore_o=None):
if request.args.get('unpause_on_save'):
extra_update_obj['paused'] = False
# Re #110, if they submit the same as the default value, set it to None, so we continue to follow the default
# Assume we use the default value, unless something relevant is different, then use the form value
# values could be None, 0 etc.
@@ -708,7 +707,6 @@ def changedetection_app(config=None, datastore_o=None):
# Only works reliably with Playwright
visualselector_enabled = os.getenv('PLAYWRIGHT_DRIVER_URL', False) and is_html_webdriver
output = render_template("edit.html",
available_processors=processors.available_processors(),
browser_steps_config=browser_step_ui_config,

41
changedetectionio/blueprint/browser_steps/__init__.py
View File

@@ -23,8 +23,10 @@
from distutils.util import strtobool
from flask import Blueprint, request, make_response
import os
import logging
import os
import re
from changedetectionio.store import ChangeDetectionStore
from changedetectionio import login_optionally_required
@@ -44,7 +46,7 @@ def construct_blueprint(datastore: ChangeDetectionStore):
# We keep the playwright session open for many minutes
seconds_keepalive = int(os.getenv('BROWSERSTEPS_MINUTES_KEEPALIVE', 10)) * 60
keepalive_seconds = int(os.getenv('BROWSERSTEPS_MINUTES_KEEPALIVE', 10)) * 60
browsersteps_start_session = {'start_time': time.time()}
@@ -56,16 +58,18 @@ def construct_blueprint(datastore: ChangeDetectionStore):
# Start the Playwright context, which is actually a nodejs sub-process and communicates over STDIN/STDOUT pipes
io_interface_context = io_interface_context.start()
keepalive_ms = ((keepalive_seconds + 3) * 1000)
base_url = os.getenv('PLAYWRIGHT_DRIVER_URL', '')
a = "?" if not '?' in base_url else '&'
base_url += a + f"timeout={keepalive_ms}"
# keep it alive for 10 seconds more than we advertise, sometimes it helps to keep it shutting down cleanly
keepalive = "&timeout={}".format(((seconds_keepalive + 3) * 1000))
try:
browsersteps_start_session['browser'] = io_interface_context.chromium.connect_over_cdp(
os.getenv('PLAYWRIGHT_DRIVER_URL', '') + keepalive)
browsersteps_start_session['browser'] = io_interface_context.chromium.connect_over_cdp(base_url)
except Exception as e:
if 'ECONNREFUSED' in str(e):
return make_response('Unable to start the Playwright Browser session, is it running?', 401)
else:
# Other errors, bad URL syntax, bad reply etc
return make_response(str(e), 401)
proxy_id = datastore.get_preferred_proxy_for_watch(uuid=watch_uuid)
@@ -118,6 +122,31 @@ def construct_blueprint(datastore: ChangeDetectionStore):
print("Starting connection with playwright - done")
return {'browsersteps_session_id': browsersteps_session_id}
@login_optionally_required
@browser_steps_blueprint.route("/browsersteps_image", methods=['GET'])
def browser_steps_fetch_screenshot_image():
from flask import (
make_response,
request,
send_from_directory,
)
uuid = request.args.get('uuid')
step_n = int(request.args.get('step_n'))
watch = datastore.data['watching'].get(uuid)
filename = f"step_before-{step_n}.jpeg" if request.args.get('type', '') == 'before' else f"step_{step_n}.jpeg"
if step_n and watch and os.path.isfile(os.path.join(watch.watch_data_dir, filename)):
response = make_response(send_from_directory(directory=watch.watch_data_dir, path=filename))
response.headers['Content-type'] = 'image/jpeg'
response.headers['Cache-Control'] = 'no-cache, no-store, must-revalidate'
response.headers['Pragma'] = 'no-cache'
response.headers['Expires'] = 0
return response
else:
return make_response('Unable to fetch image, is the URL correct? does the watch exist? does the step_type-n.jpeg exist?', 401)
# A request for an action was received
@login_optionally_required
@browser_steps_blueprint.route("/browsersteps_update", methods=['POST'])

4
changedetectionio/blueprint/browser_steps/browser_steps.py
View File

@@ -138,13 +138,13 @@ class steppable_browser_interface():
def action_wait_for_text(self, selector, value):
import json
v = json.dumps(value)
self.page.wait_for_function(f'document.querySelector("body").innerText.includes({v});', timeout=90000)
self.page.wait_for_function(f'document.querySelector("body").innerText.includes({v});', timeout=30000)
def action_wait_for_text_in_element(self, selector, value):
import json
s = json.dumps(selector)
v = json.dumps(value)
self.page.wait_for_function(f'document.querySelector({s}).innerText.includes({v});', timeout=90000)
self.page.wait_for_function(f'document.querySelector({s}).innerText.includes({v});', timeout=30000)
# @todo - in the future make some popout interface to capture what needs to be set
# https://playwright.dev/python/docs/api/class-keyboard

14
changedetectionio/model/Watch.py
View File

@@ -4,6 +4,7 @@ import os
import re
import time
import uuid
from pathlib import Path
# Allowable protocols, protects against javascript: etc
# file:// is further checked by ALLOW_FILE_URI
@@ -18,6 +19,7 @@ from changedetectionio.notification import (
base_config = {
'body': None,
'browser_steps_last_error_step': None,
'check_unique_lines': False, # On change-detected, compare against all history if its something new
'check_count': 0,
'date_created': None,
@@ -25,8 +27,8 @@ base_config = {
'extract_text': [], # Extract text by regex after filters
'extract_title_as_title': False,
'fetch_backend': 'system', # plaintext, playwright etc
'processor': 'text_json_diff', # could be restock_diff or others from .processors
'fetch_time': 0.0,
'processor': 'text_json_diff', # could be restock_diff or others from .processors
'filter_failure_notification_send': strtobool(os.getenv('FILTER_FAILURE_NOTIFICATION_SEND_DEFAULT', 'True')),
'filter_text_added': True,
'filter_text_replaced': True,
@@ -490,3 +492,13 @@ class model(dict):
filepath = os.path.join(self.watch_data_dir, 'last-fetched.br')
with open(filepath, 'wb') as f:
f.write(brotli.compress(contents, mode=brotli.MODE_TEXT))
@property
def get_browsersteps_available_screenshots(self):
"For knowing which screenshots are available to show the user in BrowserSteps UI"
available = []
for f in Path(self.watch_data_dir).glob('step_before-*.jpeg'):
step_n=re.search(r'step_before-(\d+)', f.name)
if step_n:
available.append(step_n.group(1))
return available

28
changedetectionio/static/js/browser-steps.js
View File

@@ -321,8 +321,14 @@ $(document).ready(function () {
var s = '<div class="control">' + '<a data-step-index=' + i + ' class="pure-button button-secondary button-green button-xsmall apply" >Apply</a>&nbsp;';
if (i > 0) {
// The first step never gets these (Goto-site)
s += '<a data-step-index=' + i + ' class="pure-button button-secondary button-xsmall clear" >Clear</a>&nbsp;' +
'<a data-step-index=' + i + ' class="pure-button button-secondary button-red button-xsmall remove" >Remove</a>';
s += `<a data-step-index="${i}" class="pure-button button-secondary button-xsmall clear" >Clear</a>&nbsp;` +
`<a data-step-index="${i}" class="pure-button button-secondary button-red button-xsmall remove" >Remove</a>`;
// if a screenshot is available
if (browser_steps_available_screenshots.includes(i.toString())) {
var d = (browser_steps_last_error_step === i+1) ? 'before' : 'after';
s += `&nbsp;<a data-step-index="${i}" class="pure-button button-secondary button-xsmall show-screenshot" title="Show screenshot from last run" data-type="${d}">Pic</a>&nbsp;`;
}
}
s += '</div>';
$(this).append(s)
@@ -437,6 +443,24 @@ $(document).ready(function () {
});
$('ul#browser_steps li .control .show-screenshot').click(function (element) {
var step_n = $(event.currentTarget).data('step-index');
w = window.open(this.href, "_blank", "width=640,height=480");
const t = $(event.currentTarget).data('type');
const url = browser_steps_fetch_screenshot_image_url + `&step_n=${step_n}&type=${t}`;
w.document.body.innerHTML = `<!DOCTYPE html>
<html lang="en">
<body>
<img src="${url}" style="width: 100%" alt="Browser Step at step ${step_n} from last run." title="Browser Step at step ${step_n} from last run."/>
</body>
</html>`;
w.document.title = `Browser Step at step ${step_n} from last run.`;
});
if (browser_steps_last_error_step) {
$("ul#browser_steps>li:nth-child("+browser_steps_last_error_step+")").addClass("browser-step-with-error");
}
$("ul#browser_steps select").change(function () {
set_greyed_state();

4
changedetectionio/static/styles/scss/parts/_browser-steps.scss
View File

@@ -6,6 +6,10 @@
}
li {
&.browser-step-with-error {
background-color: #ffd6d6;
border-radius: 4px;
}
&:not(:first-child) {
&:hover {
opacity: 1.0;

3
changedetectionio/static/styles/styles.css
View File

@@ -26,6 +26,9 @@
#browser_steps li {
list-style: decimal;
padding: 5px; }
#browser_steps li.browser-step-with-error {
background-color: #ffd6d6;
border-radius: 4px; }
#browser_steps li:not(:first-child):hover {
opacity: 1.0; }
#browser_steps li .control {

1
changedetectionio/store.py
View File

@@ -244,6 +244,7 @@ class ChangeDetectionStore:
import pathlib
self.__data['watching'][uuid].update({
'browser_steps_last_error_step' : None,
'check_count': 0,
'fetch_time' : 0.0,
'has_ldjson_price_data': None,

4
changedetectionio/templates/edit.html
View File

@@ -4,8 +4,10 @@
{% from '_common_fields.jinja' import render_common_settings_form %}
<script src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
<script>
const browser_steps_available_screenshots=JSON.parse('{{ watch.get_browsersteps_available_screenshots|tojson }}');
const browser_steps_config=JSON.parse('{{ browser_steps_config|tojson }}');
const browser_steps_fetch_screenshot_image_url="{{url_for('browser_steps.browser_steps_fetch_screenshot_image', uuid=uuid)}}";
const browser_steps_last_error_step={{ watch.browser_steps_last_error_step|tojson }};
const browser_steps_start_url="{{url_for('browser_steps.browsersteps_start_session', uuid=uuid)}}";
const browser_steps_sync_url="{{url_for('browser_steps.browsersteps_ui_update', uuid=uuid)}}";
{% if emailprefix %}

11
changedetectionio/update_worker.py
View File

@@ -238,7 +238,9 @@ class update_worker(threading.Thread):
# Used as a default and also by some tests
update_handler = text_json_diff.perform_site_check(datastore=self.datastore)
self.datastore.data['watching'][uuid]['browser_steps_last_error_step'] = None
changed_detected, update_obj, contents = update_handler.run(uuid, skip_when_checksum_same=queued_item_data.item.get('skip_when_checksum_same'))
# Re #342
# In Python 3, all strings are sequences of Unicode characters. There is a bytes type that holds raw bytes.
# We then convert/.decode('utf-8') for the notification etc
@@ -324,8 +326,13 @@ class update_worker(threading.Thread):
if not self.datastore.data['watching'].get(uuid):
continue
err_text = "Warning, browser step at position {} could not run, target not found, check the watch, add a delay if necessary.".format(e.step_n+1)
self.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text})
error_step = e.step_n + 1
err_text = f"Warning, browser step at position {error_step} could not run, target not found, check the watch, add a delay if necessary, view Browser Steps to see screenshot at that step"
self.datastore.update_watch(uuid=uuid,
update_obj={'last_error': err_text,
'browser_steps_last_error_step': error_step
}
)
if self.datastore.data['watching'][uuid].get('filter_failure_notification_send', False):