lee/changedetection.io
1
0
Fork 0
mirror of https://github.com/dgtlmoon/changedetection.io.git synced 2026-05-03 00:00:54 +00:00
Code Issues Actions 4 Packages Projects Releases Wiki Activity

WIP

Browse Source
This commit is contained in:
dgtlmoon
2022-05-22 19:03:20 +02:00
parent 49139e779a
commit 1594853ce5
2 changed files with 48 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files
changedetectionio/content_fetcher.py
+22 -26
View File
@@ -32,32 +32,26 @@ class Fetcher():
xpath_element_js = """
// Include the getXpath script directly, easier than fetching
!function(e,n){"object"==typeof exports&&"undefined"!=typeof module?module.exports=n():"function"==typeof define&&define.amd?define(n):(e=e||self).getXPath=n()}(this,function(){return function(e){var n=e;if(n&&n.id)return'//*[@id="'+n.id+'"]';for(var o=[];n&&Node.ELEMENT_NODE===n.nodeType;){for(var i=0,r=!1,d=n.previousSibling;d;)d.nodeType!==Node.DOCUMENT_TYPE_NODE&&d.nodeName===n.nodeName&&i++,d=d.previousSibling;for(d=n.nextSibling;d;){if(d.nodeName===n.nodeName){r=!0;break}d=d.nextSibling}o.push((n.prefix?n.prefix+":":"")+n.localName+(i||r?"["+(i+1)+"]":"")),n=n.parentNode}return o.length?"/"+o.reverse().join("/"):""}});
//# sourceMappingURL=index.umd.js.map
const findUpTag = (el) => {
let r = el
chained_css = [];
return null;
let r = el
chained_css = [el.tagName];
// Keep going up until we hit an ID tag, image it's like #list-widget div h4
while (r.parentNode) {
if(r.classList.length >0) {
// limit to just using 2 class names of each, stops from getting really huge selector strings
current_css='.'+Array.from(r.classList).slice(0, 2).join('.');
chained_css.unshift(current_css);
var f=chained_css.join(' ');
var q=document.querySelectorAll(f);
if(q.length==1) return current_css;
if(f.length >120) return null;
}
r = r.parentNode;
if('' !==r.id) {
chained_css.unshift("#"+r.id);
return chained_css.join(' ');
} else {
chained_css.unshift(r.tagName);
}
}
return null;
}
var elements = window.document.querySelectorAll(".hnname");
// @todo - if it's SVG or IMG, go into image diff mode
var elements = window.document.querySelectorAll("div,span,form,table,tbody,tr,td,a,p,ul,li,h1,h2,h3,h4");
var size_pos=[];
// after page fetch, inject this JS
// build a map of all elements and their positions (maybe that only include text?)
@@ -76,7 +70,7 @@ class Fetcher():
// 1st primitive - if it has class, try joining it all and select, if theres only one.. well thats us.
xpath_result=false;
/*
try {
var d= findUpTag(elements[i]);
if (d) {
@@ -85,8 +79,8 @@ class Fetcher():
} catch (e) {
var x=1;
}
*/
// You could swap it and default to getXpath and then try the smarter one
// default back to the less intelligent one
if (!xpath_result) {
xpath_result = getXPath(elements[i]);
@@ -127,8 +121,8 @@ class Fetcher():
});
}
}
return size_pos;
// https://stackoverflow.com/questions/1145850/how-to-get-height-of-entire-document-with-javascript
return {'size_pos':size_pos, 'browser_width': window.innerWidth, 'browser_height':document.body.scrollHeight};
"""
xpath_data = None
@@ -255,7 +249,6 @@ class base_html_playwright(Fetcher):
proxy=self.proxy
)
page = context.new_page()
page.set_viewport_size({"width": 1280, "height": 1024})
try:
response = page.goto(url, timeout=timeout * 1000, wait_until='commit')
# Wait_until = commit
@@ -273,6 +266,9 @@ class base_html_playwright(Fetcher):
if len(page.content().strip()) == 0:
raise EmptyReply(url=url, status_code=None)
# Bug 1(?) Set the viewport size AFTER loading the page
page.set_viewport_size({"width": 1280, "height": 1024})
self.status_code = response.status
self.content = page.content()
self.headers = response.all_headers()
@@ -283,13 +279,13 @@ class base_html_playwright(Fetcher):
page.evaluate("var css_filter=''")
self.xpath_data = page.evaluate("async () => {" + self.xpath_element_js + "}")
# Bug 1 in Playwright screenshot handling
# Bug 2 in Playwright screenshot handling
# Some bug where it gives the wrong screenshot size, but making a request with the clip set first seems to solve it
# JPEG is better here because the screenshots can be very very large
page.screenshot(type='jpeg', clip={'x': 1.0, 'y': 1.0, 'width': 1280, 'height': 1024})
self.screenshot = page.screenshot(type='jpeg', full_page=True, quality=92)
# Bug 2 - screenshot size is not the real size (but reported elements and everything else is fine)
# Bug 3 - screenshot size is not the real size (but reported elements and everything else is fine)
width = page.evaluate('async () => {return Math.max(document.documentElement.clientWidth || 0, window.innerWidth || 0)}')
context.close()
changedetectionio/static/js/visual-selector.js
+26 -18
View File
@@ -12,8 +12,8 @@ $(document).ready(function() {
var ctx = c.getContext("2d");
var current_default_xpath=$("#css_filter").val();
var x_scale;
var y_scale;
var x_scale=1;
var y_scale=1;
var selector_image = document.getElementById("selector-background");
var selector_image_rect;
var vh;
@@ -37,6 +37,8 @@ $(document).ready(function() {
}).done(function (data) {
$('.fetching-update-notice').html("Rendering..");
selector_data = data;
console.log("Reported browser width from backend: "+data['browser_width']);
set_scale();
reflow_selector();
$('.fetching-update-notice').fadeOut();
@@ -57,17 +59,21 @@ $(document).ready(function() {
// some things to check if the scaling doesnt work
// - that the widths/sizes really are about the actual screen size cat elements.json |grep -o width......|sort|uniq
selector_image_rect = selector_image.getBoundingClientRect();
console.log(selector_image.getBoundingClientRect());
// make the canvas the same size as the image
$('#selector-canvas').attr('height', selector_image_rect.height);
$('#selector-canvas').attr('width', selector_image_rect.width);
x_scale = selector_image_rect.width / selector_image.naturalWidth;
ctx.strokeStyle = 'rgb(255,0,0, 0.8)';
ctx.lineWidth = 2;
$('#selector-wrapper').attr('width', selector_image_rect.width);
x_scale = selector_image_rect.width / selector_data['browser_width'];
y_scale = selector_image_rect.height / selector_image.naturalHeight;
console.log(selector_image_rect.width +" "+ selector_image.naturalWidth);
ctx.strokeStyle = 'rgba(255,0,0, 0.8)';
ctx.fillStyle = 'rgba(255,0,0, 0.1)';
ctx.lineWidth = 3;
console.log("scaling set x: "+x_scale+" by y:"+y_scale);
}
function reflow_selector() {
@@ -76,13 +82,13 @@ $(document).ready(function() {
set_scale();
console.log(selector_data.length + " selectors found");
console.log(selector_data['size_pos'].length + " selectors found");
// highlight the default one if we can find it in the xPath list
// or the xpath matches the default one
for (var i = selector_data.length; i!==0; i--) {
var sel = selector_data[i-1];
if(selector_data[i - 1].xpath == current_default_xpath) {
for (var i = selector_data['size_pos'].length; i!==0; i--) {
var sel = selector_data['size_pos'][i-1];
if(selector_data['size_pos'][i - 1].xpath == current_default_xpath) {
ctx.strokeRect(sel.left * x_scale, sel.top * y_scale, sel.width * x_scale, sel.height * y_scale);
current_selected_i=i-1;
highlight_current_selected_i();
@@ -101,9 +107,10 @@ $(document).ready(function() {
// Reverse order - the most specific one should be deeper/"laster"
// Basically, find the most 'deepest'
for (var i = selector_data.length; i!==0; i--) {
var found=0;
for (var i = selector_data['size_pos'].length; i!==0; i--) {
// draw all of them? let them choose somehow?
var sel = selector_data[i-1];
var sel = selector_data['size_pos'][i-1];
// If we are in a bounding-box
if (e.offsetY > sel.top * y_scale && e.offsetY < sel.top * y_scale + sel.height * y_scale
&&
@@ -113,17 +120,18 @@ $(document).ready(function() {
// FOUND ONE
set_current_selected_text(sel.xpath);
ctx.strokeRect(sel.left * x_scale, sel.top * y_scale, sel.width * x_scale, sel.height * y_scale);
ctx.fillRect(sel.left * x_scale, sel.top * y_scale, sel.width * x_scale, sel.height * y_scale);
// no need to keep digging
// @todo or, O to go out/up, I to go in
// or double click to go up/out the selector?
current_selected_i=i-1;
console.log(sel);
found+=1;
break;
}
}
console.log("Found elements depth "+found);
}.debounce(5));
function set_current_selected_text(s) {
@@ -137,7 +145,7 @@ $(document).ready(function() {
return;
}
var sel = selector_data[current_selected_i];
var sel = selector_data['size_pos'][current_selected_i];
$("#css_filter").val('xpath:'+sel.xpath);
xctx.fillStyle = 'rgba(225,225,225,0.8)';
xctx.fillRect(0,0,c.width, c.height);