mirror of
https://github.com/dgtlmoon/changedetection.io.git
synced 2026-04-17 08:28:02 +00:00
Compare commits
1 Commits
default-fo
...
elementpat
| Author | SHA1 | Date | |
|---|---|---|---|
|
b72ecfa852 |
@@ -38,7 +38,7 @@ from flask_paginate import Pagination, get_page_parameter
|
||||
from changedetectionio import html_tools
|
||||
from changedetectionio.api import api_v1
|
||||
|
||||
__version__ = '0.43.1'
|
||||
__version__ = '0.43'
|
||||
|
||||
datastore = None
|
||||
|
||||
|
||||
@@ -219,15 +219,13 @@ class CreateWatch(Resource):
|
||||
|
||||
extras = copy.deepcopy(json_data)
|
||||
|
||||
# Because we renamed 'tag' to 'tags' but don't want to change the API (can do this in v2 of the API)
|
||||
tags = None
|
||||
# Because we renamed 'tag' to 'tags' but dont want to change the API (can do this in v2 of the API)
|
||||
if extras.get('tag'):
|
||||
tags = extras.get('tag')
|
||||
del extras['tag']
|
||||
extras['tags'] = extras.get('tag')
|
||||
|
||||
del extras['url']
|
||||
|
||||
new_uuid = self.datastore.add_watch(url=url, extras=extras, tag=tags)
|
||||
new_uuid = self.datastore.add_watch(url=url, extras=extras)
|
||||
if new_uuid:
|
||||
self.update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': new_uuid, 'skip_when_checksum_same': True}))
|
||||
return {'uuid': new_uuid}, 201
|
||||
|
||||
@@ -76,16 +76,6 @@ def construct_blueprint(datastore: ChangeDetectionStore):
|
||||
flash(f"Tag unlinked removed from {unlinked} watches")
|
||||
return redirect(url_for('tags.tags_overview_page'))
|
||||
|
||||
@tags_blueprint.route("/delete_all", methods=['GET'])
|
||||
@login_optionally_required
|
||||
def delete_all():
|
||||
for watch_uuid, watch in datastore.data['watching'].items():
|
||||
watch['tags'] = []
|
||||
datastore.data['settings']['application']['tags'] = {}
|
||||
|
||||
flash(f"All tags deleted")
|
||||
return redirect(url_for('tags.tags_overview_page'))
|
||||
|
||||
@tags_blueprint.route("/edit/<string:uuid>", methods=['GET'])
|
||||
@login_optionally_required
|
||||
def form_tag_edit(uuid):
|
||||
|
||||
@@ -316,10 +316,11 @@ class ValidateCSSJSONXPATHInput(object):
|
||||
if not self.allow_xpath:
|
||||
raise ValidationError("XPath not permitted in this field!")
|
||||
from lxml import etree, html
|
||||
import elementpath
|
||||
tree = html.fromstring("<html></html>")
|
||||
|
||||
try:
|
||||
tree.xpath(line.strip())
|
||||
elementpath.select(tree, line)
|
||||
except etree.XPathEvalError as e:
|
||||
message = field.gettext('\'%s\' is not a valid XPath expression. (%s)')
|
||||
raise ValidationError(message % (line, str(e)))
|
||||
|
||||
@@ -51,12 +51,13 @@ def element_removal(selectors: List[str], html_content):
|
||||
|
||||
# Return str Utf-8 of matched rules
|
||||
def xpath_filter(xpath_filter, html_content, append_pretty_line_formatting=False):
|
||||
import elementpath
|
||||
from lxml import etree, html
|
||||
|
||||
tree = html.fromstring(bytes(html_content, encoding='utf-8'))
|
||||
html_block = ""
|
||||
|
||||
r = tree.xpath(xpath_filter.strip(), namespaces={'re': 'http://exslt.org/regular-expressions'})
|
||||
r = elementpath.select(tree, xpath_filter.strip())
|
||||
#@note: //title/text() wont work where <title>CDATA..
|
||||
|
||||
for element in r:
|
||||
|
||||
@@ -85,8 +85,7 @@ class import_distill_io_json(Importer):
|
||||
now = time.time()
|
||||
self.new_uuids=[]
|
||||
|
||||
# @todo Use JSONSchema like in the API to validate here.
|
||||
|
||||
|
||||
try:
|
||||
data = json.loads(data.strip())
|
||||
except json.decoder.JSONDecodeError:
|
||||
@@ -121,8 +120,11 @@ class import_distill_io_json(Importer):
|
||||
except IndexError:
|
||||
pass
|
||||
|
||||
# Does this need to be here anymore?
|
||||
if d.get('tags', False):
|
||||
extras['tags'] = ", ".join(d['tags'])
|
||||
|
||||
new_uuid = datastore.add_watch(url=d['uri'].strip(),
|
||||
tag=",".join(d.get('tags', [])),
|
||||
extras=extras,
|
||||
write_to_disk_now=False)
|
||||
|
||||
|
||||
@@ -92,12 +92,6 @@ def process_notification(n_object, datastore):
|
||||
n_object.get('notification_format', default_notification_format),
|
||||
valid_notification_formats[default_notification_format],
|
||||
)
|
||||
|
||||
# If we arrived with 'System default' then look it up
|
||||
if n_format == default_notification_format_for_watch and datastore.data['settings']['application'].get('notification_format') != default_notification_format_for_watch:
|
||||
# Initially text or whatever
|
||||
n_format = datastore.data['settings']['application'].get('notification_format', valid_notification_formats[default_notification_format])
|
||||
|
||||
|
||||
# https://github.com/caronc/apprise/wiki/Development_LogCapture
|
||||
# Anything higher than or equal to WARNING (which covers things like Connection errors)
|
||||
|
||||
@@ -205,9 +205,10 @@ class ChangeDetectionStore:
|
||||
|
||||
# Clone a watch by UUID
|
||||
def clone(self, uuid):
|
||||
url = self.data['watching'][uuid].get('url')
|
||||
url = self.data['watching'][uuid]['url']
|
||||
tag = self.data['watching'][uuid].get('tags',[])
|
||||
extras = self.data['watching'][uuid]
|
||||
new_uuid = self.add_watch(url=url, extras=extras)
|
||||
new_uuid = self.add_watch(url=url, tag_uuids=tag, extras=extras)
|
||||
return new_uuid
|
||||
|
||||
def url_exists(self, url):
|
||||
@@ -247,9 +248,12 @@ class ChangeDetectionStore:
|
||||
if extras is None:
|
||||
extras = {}
|
||||
|
||||
# should always be str
|
||||
if tag is None or not tag:
|
||||
tag = ''
|
||||
|
||||
# Incase these are copied across, assume it's a reference and deepcopy()
|
||||
apply_extras = deepcopy(extras)
|
||||
apply_extras['tags'] = [] if not apply_extras.get('tags') else apply_extras.get('tags')
|
||||
|
||||
# Was it a share link? try to fetch the data
|
||||
if (url.startswith("https://changedetection.io/share/")):
|
||||
@@ -299,22 +303,20 @@ class ChangeDetectionStore:
|
||||
flash('Watch protocol is not permitted by SAFE_PROTOCOL_REGEX', 'error')
|
||||
return None
|
||||
|
||||
if tag and type(tag) == str:
|
||||
# Then it's probably a string of the actual tag by name, split and add it
|
||||
for t in tag.split(','):
|
||||
|
||||
# #Re 569
|
||||
# Could be in 'tags', var or extras, smash them together and strip
|
||||
apply_extras['tags'] = []
|
||||
if tag or extras.get('tags'):
|
||||
tags = list(filter(None, list(set().union(tag.split(','), extras.get('tags', '').split(',')))))
|
||||
for t in list(map(str.strip, tags)):
|
||||
# for each stripped tag, add tag as UUID
|
||||
for a_t in t.split(','):
|
||||
tag_uuid = self.add_tag(a_t)
|
||||
apply_extras['tags'].append(tag_uuid)
|
||||
apply_extras['tags'].append(self.add_tag(t))
|
||||
|
||||
# Or if UUIDs given directly
|
||||
if tag_uuids:
|
||||
apply_extras['tags'] = list(set(apply_extras['tags'] + tag_uuids))
|
||||
|
||||
# Make any uuids unique
|
||||
if apply_extras.get('tags'):
|
||||
apply_extras['tags'] = list(set(apply_extras.get('tags')))
|
||||
|
||||
new_watch = Watch.model(datastore_path=self.datastore_path, url=url)
|
||||
|
||||
new_uuid = new_watch.get('uuid')
|
||||
@@ -566,7 +568,7 @@ class ChangeDetectionStore:
|
||||
def add_tag(self, name):
|
||||
# If name exists, return that
|
||||
n = name.strip().lower()
|
||||
print (f">>> Adding new tag - '{n}'")
|
||||
print (f">>> Adding new tag - '{n}")
|
||||
if not n:
|
||||
return False
|
||||
|
||||
|
||||
@@ -267,7 +267,7 @@ def test_api_watch_PUT_update(client, live_server):
|
||||
|
||||
#live_server_setup(live_server)
|
||||
api_key = extract_api_key_from_UI(client)
|
||||
|
||||
time.sleep(1)
|
||||
# Create a watch
|
||||
set_original_response()
|
||||
test_url = url_for('test_endpoint', _external=True,
|
||||
@@ -283,6 +283,7 @@ def test_api_watch_PUT_update(client, live_server):
|
||||
|
||||
assert res.status_code == 201
|
||||
|
||||
time.sleep(1)
|
||||
|
||||
# Get a listing, it will be the first one
|
||||
res = client.get(
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
import time
|
||||
from flask import url_for
|
||||
from .util import live_server_setup, wait_for_all_checks, extract_rss_token_from_UI, get_UUID_for_tag_name, extract_UUID_from_client
|
||||
from .util import live_server_setup, wait_for_all_checks, extract_rss_token_from_UI, get_UUID_for_tag_name
|
||||
import os
|
||||
|
||||
|
||||
@@ -154,10 +154,6 @@ def test_tag_add_in_ui(client, live_server):
|
||||
)
|
||||
assert b"Tag added" in res.data
|
||||
assert b"new-test-tag" in res.data
|
||||
|
||||
res = client.get(url_for("tags.delete_all"), follow_redirects=True)
|
||||
assert b'All tags deleted' in res.data
|
||||
|
||||
res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
|
||||
assert b'Deleted' in res.data
|
||||
|
||||
@@ -223,11 +219,13 @@ def test_group_tag_notification(client, live_server):
|
||||
assert "test-tag" in notification_submission
|
||||
assert "other-tag" in notification_submission
|
||||
|
||||
#@todo Test that multiple notifications fired
|
||||
#@todo Test that each of multiple notifications with different settings
|
||||
res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
|
||||
assert b'Deleted' in res.data
|
||||
|
||||
#@todo Test that multiple notifications fired
|
||||
#@todo Test that each of multiple notifications with different settings
|
||||
|
||||
|
||||
def test_limit_tag_ui(client, live_server):
|
||||
#live_server_setup(live_server)
|
||||
|
||||
@@ -262,61 +260,3 @@ def test_limit_tag_ui(client, live_server):
|
||||
assert b'test-tag' in res.data
|
||||
assert res.data.count(b'processor-text_json_diff') == 20
|
||||
assert b"object at" not in res.data
|
||||
res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
|
||||
assert b'Deleted' in res.data
|
||||
res = client.get(url_for("tags.delete_all"), follow_redirects=True)
|
||||
assert b'All tags deleted' in res.data
|
||||
def test_clone_tag_on_import(client, live_server):
|
||||
#live_server_setup(live_server)
|
||||
test_url = url_for('test_endpoint', _external=True)
|
||||
res = client.post(
|
||||
url_for("import_page"),
|
||||
data={"urls": test_url + " test-tag, another-tag\r\n"},
|
||||
follow_redirects=True
|
||||
)
|
||||
|
||||
assert b"1 Imported" in res.data
|
||||
|
||||
res = client.get(url_for("index"))
|
||||
assert b'test-tag' in res.data
|
||||
assert b'another-tag' in res.data
|
||||
|
||||
watch_uuid = extract_UUID_from_client(client)
|
||||
res = client.get(url_for("form_clone", uuid=watch_uuid), follow_redirects=True)
|
||||
|
||||
assert b'Cloned' in res.data
|
||||
# 2 times plus the top link to tag
|
||||
assert res.data.count(b'test-tag') == 3
|
||||
assert res.data.count(b'another-tag') == 3
|
||||
res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
|
||||
assert b'Deleted' in res.data
|
||||
|
||||
def test_clone_tag_on_quickwatchform_add(client, live_server):
|
||||
#live_server_setup(live_server)
|
||||
|
||||
test_url = url_for('test_endpoint', _external=True)
|
||||
|
||||
res = client.post(
|
||||
url_for("form_quick_watch_add"),
|
||||
data={"url": test_url, "tags": ' test-tag, another-tag '},
|
||||
follow_redirects=True
|
||||
)
|
||||
|
||||
assert b"Watch added" in res.data
|
||||
|
||||
res = client.get(url_for("index"))
|
||||
assert b'test-tag' in res.data
|
||||
assert b'another-tag' in res.data
|
||||
|
||||
watch_uuid = extract_UUID_from_client(client)
|
||||
res = client.get(url_for("form_clone", uuid=watch_uuid), follow_redirects=True)
|
||||
|
||||
assert b'Cloned' in res.data
|
||||
# 2 times plus the top link to tag
|
||||
assert res.data.count(b'test-tag') == 3
|
||||
assert res.data.count(b'another-tag') == 3
|
||||
res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
|
||||
assert b'Deleted' in res.data
|
||||
|
||||
res = client.get(url_for("tags.delete_all"), follow_redirects=True)
|
||||
assert b'All tags deleted' in res.data
|
||||
@@ -112,7 +112,6 @@ def test_import_distillio(client, live_server):
|
||||
# did the tags work?
|
||||
res = client.get( url_for("index"))
|
||||
|
||||
# check tags
|
||||
assert b"nice stuff" in res.data
|
||||
assert b"nerd-news" in res.data
|
||||
|
||||
|
||||
@@ -24,6 +24,9 @@ def test_check_notification(client, live_server):
|
||||
#live_server_setup(live_server)
|
||||
set_original_response()
|
||||
|
||||
# Give the endpoint time to spin up
|
||||
time.sleep(1)
|
||||
|
||||
# Re 360 - new install should have defaults set
|
||||
res = client.get(url_for("settings_page"))
|
||||
notification_url = url_for('test_notification_endpoint', _external=True).replace('http', 'json')
|
||||
@@ -139,7 +142,8 @@ def test_check_notification(client, live_server):
|
||||
|
||||
# Did we see the URL that had a change, in the notification?
|
||||
# Diff was correctly executed
|
||||
|
||||
assert test_url in notification_submission
|
||||
assert ':-)' in notification_submission
|
||||
assert "Diff Full: Some initial text" in notification_submission
|
||||
assert "Diff: (changed) Which is across multiple lines" in notification_submission
|
||||
assert "(into) which has this one new line" in notification_submission
|
||||
@@ -152,8 +156,7 @@ def test_check_notification(client, live_server):
|
||||
assert "preview/" in notification_submission
|
||||
assert ":-)" in notification_submission
|
||||
assert "New ChangeDetection.io Notification - {}".format(test_url) in notification_submission
|
||||
assert test_url in notification_submission
|
||||
assert ':-)' in notification_submission
|
||||
|
||||
# Check the attachment was added, and that it is a JPEG from the original PNG
|
||||
notification_submission_object = json.loads(notification_submission)
|
||||
# We keep PNG screenshots for now
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
import time
|
||||
from flask import url_for
|
||||
from . util import live_server_setup
|
||||
from .util import live_server_setup, wait_for_all_checks
|
||||
|
||||
from ..html_tools import *
|
||||
|
||||
@@ -164,6 +164,7 @@ def test_check_xpath_text_function_utf8(client, live_server):
|
||||
assert b'Deleted' in res.data
|
||||
|
||||
def test_check_markup_xpath_filter_restriction(client, live_server):
|
||||
live_server_setup(live_server)
|
||||
sleep_time_for_fetch_thread = 3
|
||||
|
||||
xpath_filter = "//*[contains(@class, 'sametext')]"
|
||||
@@ -183,7 +184,7 @@ def test_check_markup_xpath_filter_restriction(client, live_server):
|
||||
assert b"1 Imported" in res.data
|
||||
|
||||
# Give the thread time to pick it up
|
||||
time.sleep(sleep_time_for_fetch_thread)
|
||||
wait_for_all_checks(client)
|
||||
|
||||
# Goto the edit page, add our ignore text
|
||||
# Add our URL to the import page
|
||||
@@ -195,7 +196,7 @@ def test_check_markup_xpath_filter_restriction(client, live_server):
|
||||
assert b"Updated watch." in res.data
|
||||
|
||||
# Give the thread time to pick it up
|
||||
time.sleep(sleep_time_for_fetch_thread)
|
||||
wait_for_all_checks(client)
|
||||
|
||||
# view it/reset state back to viewed
|
||||
client.get(url_for("diff_history_page", uuid="first"), follow_redirects=True)
|
||||
@@ -206,7 +207,7 @@ def test_check_markup_xpath_filter_restriction(client, live_server):
|
||||
# Trigger a check
|
||||
client.get(url_for("form_watch_checknow"), follow_redirects=True)
|
||||
# Give the thread time to pick it up
|
||||
time.sleep(sleep_time_for_fetch_thread)
|
||||
wait_for_all_checks(client)
|
||||
|
||||
res = client.get(url_for("index"))
|
||||
assert b'unviewed' not in res.data
|
||||
|
||||
@@ -65,45 +65,15 @@ class update_worker(threading.Thread):
|
||||
logging.info (">> SENDING NOTIFICATION")
|
||||
self.notification_q.put(n_object)
|
||||
|
||||
# Prefer - Individual watch settings > Tag settings > Global settings (in that order)
|
||||
def _check_cascading_vars(self, var_name, watch):
|
||||
|
||||
from changedetectionio.notification import (
|
||||
default_notification_format_for_watch,
|
||||
default_notification_body,
|
||||
default_notification_title
|
||||
)
|
||||
|
||||
|
||||
# Would be better if this was some kind of Object where Watch can reference the parent datastore etc
|
||||
v = watch.get(var_name)
|
||||
if v and not watch.get('notification_muted'):
|
||||
return v
|
||||
|
||||
tags = self.datastore.get_all_tags_for_watch(uuid=watch.get('uuid'))
|
||||
if tags:
|
||||
for tag_uuid, tag in tags.items():
|
||||
v = tag.get(var_name)
|
||||
if v and not tag.get('notification_muted'):
|
||||
return v
|
||||
|
||||
if self.datastore.data['settings']['application'].get(var_name):
|
||||
return self.datastore.data['settings']['application'].get(var_name)
|
||||
|
||||
# Otherwise could be defaults
|
||||
if var_name == 'notification_format':
|
||||
return default_notification_format_for_watch
|
||||
if var_name == 'notification_body':
|
||||
return default_notification_body
|
||||
if var_name == 'notification_title':
|
||||
return default_notification_title
|
||||
|
||||
return None
|
||||
|
||||
def send_content_changed_notification(self, watch_uuid):
|
||||
|
||||
from changedetectionio.notification import (
|
||||
default_notification_format_for_watch
|
||||
)
|
||||
|
||||
n_object = {}
|
||||
watch = self.datastore.data['watching'].get(watch_uuid)
|
||||
watch = self.datastore.data['watching'].get(watch_uuid, False)
|
||||
if not watch:
|
||||
return
|
||||
|
||||
@@ -117,20 +87,57 @@ class update_worker(threading.Thread):
|
||||
)
|
||||
|
||||
# Should be a better parent getter in the model object
|
||||
|
||||
# Prefer - Individual watch settings > Tag settings > Global settings (in that order)
|
||||
n_object['notification_urls'] = self._check_cascading_vars('notification_urls', watch)
|
||||
n_object['notification_title'] = self._check_cascading_vars('notification_title', watch)
|
||||
n_object['notification_body'] = self._check_cascading_vars('notification_body', watch)
|
||||
n_object['notification_format'] = self._check_cascading_vars('notification_format', watch)
|
||||
n_object['notification_urls'] = watch.get('notification_urls')
|
||||
|
||||
n_object['notification_title'] = watch['notification_title'] if watch['notification_title'] else \
|
||||
self.datastore.data['settings']['application']['notification_title']
|
||||
|
||||
n_object['notification_body'] = watch['notification_body'] if watch['notification_body'] else \
|
||||
self.datastore.data['settings']['application']['notification_body']
|
||||
|
||||
n_object['notification_format'] = watch['notification_format'] if watch['notification_format'] != default_notification_format_for_watch else \
|
||||
self.datastore.data['settings']['application']['notification_format']
|
||||
|
||||
# (Individual watch) Only prepare to notify if the rules above matched
|
||||
queued = False
|
||||
if n_object and n_object.get('notification_urls'):
|
||||
queued = True
|
||||
sent = False
|
||||
if 'notification_urls' in n_object and n_object['notification_urls']:
|
||||
sent = True
|
||||
self.queue_notification_for_watch(n_object, watch)
|
||||
|
||||
return queued
|
||||
# (Group tags) try by group tag
|
||||
if not sent:
|
||||
# Else, Try by tag, and use system default vars for format, body etc as fallback
|
||||
tags = self.datastore.get_all_tags_for_watch(uuid=watch_uuid)
|
||||
for tag_uuid, tag in tags.items():
|
||||
n_object = {}
|
||||
n_object['notification_urls'] = tag.get('notification_urls')
|
||||
|
||||
n_object['notification_title'] = tag.get('notification_title') if tag.get('notification_title') else \
|
||||
self.datastore.data['settings']['application']['notification_title']
|
||||
|
||||
n_object['notification_body'] = tag.get('notification_body') if tag.get('notification_body') else \
|
||||
self.datastore.data['settings']['application']['notification_body']
|
||||
|
||||
n_object['notification_format'] = tag.get('notification_format') if tag.get('notification_format') != default_notification_format_for_watch else \
|
||||
self.datastore.data['settings']['application']['notification_format']
|
||||
|
||||
if 'notification_urls' in n_object and n_object.get('notification_urls') and not tag.get('notification_muted'):
|
||||
sent = True
|
||||
self.queue_notification_for_watch(n_object, watch)
|
||||
|
||||
# (Group tags) try by global
|
||||
if not sent:
|
||||
# leave this as is, but repeat in a loop for each tag also
|
||||
n_object['notification_urls'] = self.datastore.data['settings']['application'].get('notification_urls')
|
||||
n_object['notification_title'] = self.datastore.data['settings']['application'].get('notification_title')
|
||||
n_object['notification_body'] = self.datastore.data['settings']['application'].get('notification_body')
|
||||
n_object['notification_format'] = self.datastore.data['settings']['application'].get('notification_format')
|
||||
if n_object.get('notification_urls') and n_object.get('notification_body') and n_object.get('notification_title'):
|
||||
sent = True
|
||||
self.queue_notification_for_watch(n_object, watch)
|
||||
|
||||
return sent
|
||||
|
||||
|
||||
def send_filter_failure_notification(self, watch_uuid):
|
||||
|
||||
@@ -42,6 +42,8 @@ paho-mqtt
|
||||
# (introduced once apprise became a dep)
|
||||
cryptography~=3.4
|
||||
|
||||
elementpath
|
||||
|
||||
# Used for CSS filtering
|
||||
beautifulsoup4
|
||||
|
||||
|
||||
Reference in New Issue
Block a user