lee/changedetection.io
1
0
Fork 0
mirror of https://github.com/dgtlmoon/changedetection.io.git synced 2026-09-25 06:46:11 +00:00
Code Issues Actions 4 Packages Projects Releases Wiki Activity

Merge branch 'master' into 2039-restock-use-itemprop

Browse Source
This commit is contained in:
dgtlmoon
2024-01-14 18:26:24 +01:00
committed by GitHub
parent 61185b5514 c3d825f38c
commit b71b457c24
30 changed files with 455 additions and 331 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/pypi-release.yml
+72
View File
@@ -0,0 +1,72 @@
name: Publish Python 🐍distribution 📦 to PyPI and TestPyPI
on: push
jobs:
build:
name: Build distribution 📦
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: "3.x"
- name: Install pypa/build
run: >-
python3 -m
pip install
build
--user
- name: Build a binary wheel and a source tarball
run: python3 -m build
- name: Store the distribution packages
uses: actions/upload-artifact@v3
with:
name: python-package-distributions
path: dist/
test-pypi-package:
name: Test the built 📦 package works basically.
runs-on: ubuntu-latest
needs:
- build
steps:
- name: Download all the dists
uses: actions/download-artifact@v3
with:
name: python-package-distributions
path: dist/
- name: Test that the basic pip built package runs without error
run: |
set -e
pip3 install dist/changedetection.io*.whl
changedetection.io -d /tmp -p 10000 &
sleep 3
curl http://127.0.0.1:10000/static/styles/pure-min.css >/dev/null
curl http://127.0.0.1:10000/ >/dev/null
killall changedetection.io
publish-to-pypi:
name: >-
Publish Python 🐍 distribution 📦 to PyPI
if: startsWith(github.ref, 'refs/tags/') # only publish to PyPI on tag pushes
needs:
- test-pypi-package
runs-on: ubuntu-latest
environment:
name: release
url: https://pypi.org/p/changedetection.io
permissions:
id-token: write # IMPORTANT: mandatory for trusted publishing
steps:
- name: Download all the dists
uses: actions/download-artifact@v3
with:
name: python-package-distributions
path: dist/
- name: Publish distribution 📦 to PyPI
uses: pypa/gh-action-pypi-publish@release/v1
.github/workflows/test-only.yml
+3 -2
View File
@@ -118,7 +118,8 @@ jobs:
sleep 3
# invert the check (it should be not 0/not running)
docker ps
docker logs sig-test
# check signal catch(STDOUT) log
docker logs sig-test | grep 'Shutdown: Got Signal - SIGINT' || exit 1
test -z "`docker ps|grep sig-test`"
if [ $? -ne 0 ]
then
@@ -138,7 +139,7 @@ jobs:
sleep 3
# invert the check (it should be not 0/not running)
docker ps
docker logs sig-test
docker logs sig-test | grep 'Shutdown: Got Signal - SIGTERM' || exit 1
test -z "`docker ps|grep sig-test`"
if [ $? -ne 0 ]
then
.github/workflows/test-pip-build.yml
-36
View File
@@ -1,36 +0,0 @@
name: ChangeDetection.io PIP package test
# Triggers the workflow on push or pull request events
# This line doesnt work, even tho it is the documented one
on: [push, pull_request]
# Changes to requirements.txt packages and Dockerfile may or may not always be compatible with arm etc, so worth testing
# @todo: some kind of path filter for requirements.txt and Dockerfile
jobs:
test-pip-build-basics:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Set up Python 3.11
uses: actions/setup-python@v4
with:
python-version: 3.11
- name: Test that the basic pip built package runs without error
run: |
set -e
mkdir dist
pip3 install wheel
python3 setup.py bdist_wheel
pip3 install -r requirements.txt
rm ./changedetection.py
rm -rf changedetectio
pip3 install dist/changedetection.io*.whl
changedetection.io -d /tmp -p 10000 &
sleep 3
curl http://127.0.0.1:10000/static/styles/pure-min.css >/dev/null
killall -9 changedetection.io
MANIFEST.in
+2
View File
@@ -10,6 +10,8 @@ prune changedetectionio/static/package-lock.json
prune changedetectionio/static/styles/node_modules
prune changedetectionio/static/styles/package-lock.json
include changedetection.py
include requirements.txt
include README-pip.md
global-exclude *.pyc
global-exclude node_modules
global-exclude venv
Procfile
-1
View File
@@ -1 +0,0 @@
web: python3 ./changedetection.py -C -d ./datastore -p $PORT
README.md
+5 -5
View File
@@ -17,7 +17,7 @@ _Live your data-life pro-actively._
- Nothing to install, access via browser login after signup.
- Super fast, no registration needed setup.
- Get started watching and receiving website change notifications straight away.
- See our [tutorials and how-to page for more inspiration](https://changedetection.io/tutorials)
### Target specific parts of the webpage using the Visual Selector tool.
@@ -98,7 +98,7 @@ Please :star: star :star: this project and help it grow! https://github.com/dgtl
With Docker composer, just clone this repository and..
```bash
$ docker-compose up -d
$ docker compose up -d
```
Docker standalone
@@ -137,10 +137,10 @@ docker rm $(docker ps -a -f name=changedetection.io -q)
docker run -d --restart always -p "127.0.0.1:5000:5000" -v datastore-volume:/datastore --name changedetection.io dgtlmoon/changedetection.io
```
### docker-compose
### docker compose
```bash
docker-compose pull && docker-compose up -d
docker compose pull && docker compose up -d
```
See the wiki for more information https://github.com/dgtlmoon/changedetection.io/wiki
@@ -249,7 +249,7 @@ Supports managing the website watch list [via our API](https://changedetection.i
Do you use changedetection.io to make money? does it save you time or money? Does it make your life easier? less stressful? Remember, we write this software when we should be doing actual paid work, we have to buy food and pay rent just like you.
Firstly, consider taking out a [change detection monthly subscription - unlimited checks and watches](https://changedetection.io?src=github) , even if you don't use it, you still get the warm fuzzy feeling of helping out the project. (And who knows, you might just use it!)
Firstly, consider taking out an officially supported [website change detection subscription](https://changedetection.io?src=github) , even if you don't use it, you still get the warm fuzzy feeling of helping out the project. (And who knows, you might just use it!)
Or directly donate an amount PayPal [![Donate](https://img.shields.io/badge/Donate-PayPal-green.svg)](https://www.paypal.com/donate/?hosted_button_id=7CP6HR9ZCNDYJ)
app.json
-21
View File
@@ -1,21 +0,0 @@
{
"name": "ChangeDetection.io",
"description": "The best and simplest self-hosted open source website change detection monitoring and notification service.",
"keywords": [
"changedetection",
"website monitoring"
],
"repository": "https://github.com/dgtlmoon/changedetection.io",
"success_url": "/",
"scripts": {
},
"env": {
},
"formation": {
"web": {
"quantity": 1,
"size": "free"
}
},
"image": "heroku/python"
}
changedetectionio/__init__.py
+1 -1
View File
@@ -2,7 +2,7 @@
# Read more https://github.com/dgtlmoon/changedetection.io/wiki
__version__ = '0.45.8.1'
__version__ = '0.45.12'
from distutils.util import strtobool
from json.decoder import JSONDecodeError
changedetectionio/api/api_v1.py
+12 -12
View File
@@ -30,7 +30,7 @@ class Watch(Resource):
self.update_q = kwargs['update_q']
# Get information about a single watch, excluding the history list (can be large)
# curl http://localhost:4000/api/v1/watch/<string:uuid>
# curl http://localhost:5000/api/v1/watch/<string:uuid>
# @todo - version2 - ?muted and ?paused should be able to be called together, return the watch struct not "OK"
# ?recheck=true
@auth.check_token
@@ -39,9 +39,9 @@ class Watch(Resource):
@api {get} /api/v1/watch/:uuid Single watch - get data, recheck, pause, mute.
@apiDescription Retrieve watch information and set muted/paused status
@apiExample {curl} Example usage:
curl http://localhost:4000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091 -H"x-api-key:813031b16330fe25e3780cf0325daa45"
curl "http://localhost:4000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091?muted=unmuted" -H"x-api-key:813031b16330fe25e3780cf0325daa45"
curl "http://localhost:4000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091?paused=unpaused" -H"x-api-key:813031b16330fe25e3780cf0325daa45"
curl http://localhost:5000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091 -H"x-api-key:813031b16330fe25e3780cf0325daa45"
curl "http://localhost:5000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091?muted=unmuted" -H"x-api-key:813031b16330fe25e3780cf0325daa45"
curl "http://localhost:5000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091?paused=unpaused" -H"x-api-key:813031b16330fe25e3780cf0325daa45"
@apiName Watch
@apiGroup Watch
@apiParam {uuid} uuid Watch unique ID.
@@ -84,7 +84,7 @@ class Watch(Resource):
"""
@api {delete} /api/v1/watch/:uuid Delete a watch and related history
@apiExample {curl} Example usage:
curl http://localhost:4000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091 -X DELETE -H"x-api-key:813031b16330fe25e3780cf0325daa45"
curl http://localhost:5000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091 -X DELETE -H"x-api-key:813031b16330fe25e3780cf0325daa45"
@apiParam {uuid} uuid Watch unique ID.
@apiName Delete
@apiGroup Watch
@@ -103,7 +103,7 @@ class Watch(Resource):
@api {put} /api/v1/watch/:uuid Update watch information
@apiExample {curl} Example usage:
Update (PUT)
curl http://localhost:4000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091 -X PUT -H"x-api-key:813031b16330fe25e3780cf0325daa45" -H "Content-Type: application/json" -d '{"url": "https://my-nice.com" , "tag": "new list"}'
curl http://localhost:5000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091 -X PUT -H"x-api-key:813031b16330fe25e3780cf0325daa45" -H "Content-Type: application/json" -d '{"url": "https://my-nice.com" , "tag": "new list"}'
@apiDescription Updates an existing watch using JSON, accepts the same structure as returned in <a href="#api-Watch-Watch">get single watch information</a>
@apiParam {uuid} uuid Watch unique ID.
@@ -132,13 +132,13 @@ class WatchHistory(Resource):
self.datastore = kwargs['datastore']
# Get a list of available history for a watch by UUID
# curl http://localhost:4000/api/v1/watch/<string:uuid>/history
# curl http://localhost:5000/api/v1/watch/<string:uuid>/history
def get(self, uuid):
"""
@api {get} /api/v1/watch/<string:uuid>/history Get a list of all historical snapshots available for a watch
@apiDescription Requires `uuid`, returns list
@apiExample {curl} Example usage:
curl http://localhost:4000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091/history -H"x-api-key:813031b16330fe25e3780cf0325daa45" -H "Content-Type: application/json"
curl http://localhost:5000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091/history -H"x-api-key:813031b16330fe25e3780cf0325daa45" -H "Content-Type: application/json"
{
"1676649279": "/tmp/data/6a4b7d5c-fee4-4616-9f43-4ac97046b595/cb7e9be8258368262246910e6a2a4c30.txt",
"1677092785": "/tmp/data/6a4b7d5c-fee4-4616-9f43-4ac97046b595/e20db368d6fc633e34f559ff67bb4044.txt",
@@ -166,7 +166,7 @@ class WatchSingleHistory(Resource):
@api {get} /api/v1/watch/<string:uuid>/history/<int:timestamp> Get single snapshot from watch
@apiDescription Requires watch `uuid` and `timestamp`. `timestamp` of "`latest`" for latest available snapshot, or <a href="#api-Watch_History-Get_list_of_available_stored_snapshots_for_watch">use the list returned here</a>
@apiExample {curl} Example usage:
curl http://localhost:4000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091/history/1677092977 -H"x-api-key:813031b16330fe25e3780cf0325daa45" -H "Content-Type: application/json"
curl http://localhost:5000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091/history/1677092977 -H"x-api-key:813031b16330fe25e3780cf0325daa45" -H "Content-Type: application/json"
@apiName Get single snapshot content
@apiGroup Watch History
@apiSuccess (200) {String} OK
@@ -202,7 +202,7 @@ class CreateWatch(Resource):
@api {post} /api/v1/watch Create a single watch
@apiDescription Requires atleast `url` set, can accept the same structure as <a href="#api-Watch-Watch">get single watch information</a> to create.
@apiExample {curl} Example usage:
curl http://localhost:4000/api/v1/watch -H"x-api-key:813031b16330fe25e3780cf0325daa45" -H "Content-Type: application/json" -d '{"url": "https://my-nice.com" , "tag": "nice list"}'
curl http://localhost:5000/api/v1/watch -H"x-api-key:813031b16330fe25e3780cf0325daa45" -H "Content-Type: application/json" -d '{"url": "https://my-nice.com" , "tag": "nice list"}'
@apiName Create
@apiGroup Watch
@apiSuccess (200) {String} OK Was created
@@ -245,7 +245,7 @@ class CreateWatch(Resource):
@api {get} /api/v1/watch List watches
@apiDescription Return concise list of available watches and some very basic info
@apiExample {curl} Example usage:
curl http://localhost:4000/api/v1/watch -H"x-api-key:813031b16330fe25e3780cf0325daa45"
curl http://localhost:5000/api/v1/watch -H"x-api-key:813031b16330fe25e3780cf0325daa45"
{
"6a4b7d5c-fee4-4616-9f43-4ac97046b595": {
"last_changed": 1677103794,
@@ -363,7 +363,7 @@ class SystemInfo(Resource):
@api {get} /api/v1/systeminfo Return system info
@apiDescription Return some info about the current system state
@apiExample {curl} Example usage:
curl http://localhost:4000/api/v1/systeminfo -H"x-api-key:813031b16330fe25e3780cf0325daa45"
curl http://localhost:5000/api/v1/systeminfo -H"x-api-key:813031b16330fe25e3780cf0325daa45"
HTTP/1.0 200
{
'queue_size': 10 ,
changedetectionio/blueprint/browser_steps/__init__.py
+1 -1
View File
@@ -58,7 +58,7 @@ def construct_blueprint(datastore: ChangeDetectionStore):
io_interface_context = io_interface_context.start()
keepalive_ms = ((keepalive_seconds + 3) * 1000)
base_url = os.getenv('PLAYWRIGHT_DRIVER_URL', '')
base_url = os.getenv('PLAYWRIGHT_DRIVER_URL', '').strip('"')
a = "?" if not '?' in base_url else '&'
base_url += a + f"timeout={keepalive_ms}"
changedetectionio/blueprint/tags/templates/edit-tag.html
+1 -1
View File
@@ -3,7 +3,7 @@
{% from '_helpers.jinja' import render_field, render_checkbox_field, render_button %}
{% from '_common_fields.jinja' import render_common_settings_form %}
<script>
const notification_base_url="{{url_for('ajax_callback_send_notification_test')}}";
const notification_base_url="{{url_for('ajax_callback_send_notification_test', watch_uuid=uuid)}}";
</script>
<script src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
changedetectionio/content_fetcher.py
+28 -19
View File
@@ -43,9 +43,11 @@ class JSActionExceptions(Exception):
return
class BrowserStepsStepTimout(Exception):
def __init__(self, step_n):
class BrowserStepsStepException(Exception):
def __init__(self, step_n, original_e):
self.step_n = step_n
self.original_e = original_e
print(f"Browser Steps exception at step {self.step_n}", str(original_e))
return
@@ -91,19 +93,20 @@ class ReplyWithContentButNoText(Exception):
class Fetcher():
browser_connection_is_custom = None
browser_connection_url = None
browser_steps = None
browser_steps_screenshot_path = None
content = None
error = None
fetcher_description = "No description"
browser_connection_url = None
headers = {}
instock_data = None
instock_data_js = ""
status_code = None
webdriver_js_execute_code = None
xpath_data = None
xpath_element_js = ""
instock_data = None
instock_data_js = ""
# Will be needed in the future by the VisualSelector, always get this where possible.
screenshot = False
@@ -172,7 +175,7 @@ class Fetcher():
def iterate_browser_steps(self):
from changedetectionio.blueprint.browser_steps.browser_steps import steppable_browser_interface
from playwright._impl._errors import TimeoutError
from playwright._impl._errors import TimeoutError, Error
from jinja2 import Environment
jinja2_env = Environment(extensions=['jinja2_time.TimeExtension'])
@@ -202,10 +205,10 @@ class Fetcher():
optional_value=optional_value)
self.screenshot_step(step_n)
self.save_step_html(step_n)
except TimeoutError as e:
print(str(e))
except (Error, TimeoutError) as e:
# Stop processing here
raise BrowserStepsStepTimout(step_n=step_n)
raise BrowserStepsStepException(step_n=step_n, original_e=e)
# It's always good to reset these
def delete_browser_steps_screenshots(self):
@@ -252,16 +255,19 @@ class base_html_playwright(Fetcher):
proxy = None
def __init__(self, proxy_override=None, browser_connection_url=None):
def __init__(self, proxy_override=None, custom_browser_connection_url=None):
super().__init__()
self.browser_type = os.getenv("PLAYWRIGHT_BROWSER_TYPE", 'chromium').strip('"')
# .strip('"') is going to save someone a lot of time when they accidently wrap the env value
if not browser_connection_url:
self.browser_connection_url = os.getenv("PLAYWRIGHT_DRIVER_URL", 'ws://playwright-chrome:3000').strip('"')
if custom_browser_connection_url:
self.browser_connection_is_custom = True
self.browser_connection_url = custom_browser_connection_url
else:
self.browser_connection_url = browser_connection_url
# Fallback to fetching from system
# .strip('"') is going to save someone a lot of time when they accidently wrap the env value
self.browser_connection_url = os.getenv("PLAYWRIGHT_DRIVER_URL", 'ws://playwright-chrome:3000').strip('"')
# If any proxy settings are enabled, then we should setup the proxy object
proxy_args = {}
@@ -421,8 +427,10 @@ class base_html_playwright(Fetcher):
current_include_filters=None,
is_binary=False):
# For now, USE_EXPERIMENTAL_PUPPETEER_FETCH is not supported by watches with BrowserSteps (for now!)
if not self.browser_steps and os.getenv('USE_EXPERIMENTAL_PUPPETEER_FETCH'):
# browser_connection_is_custom doesnt work with puppeteer style fetch (use playwright native too in this case)
if not self.browser_connection_is_custom and not self.browser_steps and os.getenv('USE_EXPERIMENTAL_PUPPETEER_FETCH'):
if strtobool(os.getenv('USE_EXPERIMENTAL_PUPPETEER_FETCH')):
# Temporary backup solution until we rewrite the playwright code
return self.run_fetch_browserless_puppeteer(
@@ -569,15 +577,16 @@ class base_html_webdriver(Fetcher):
'socksProxy', 'socksVersion', 'socksUsername', 'socksPassword']
proxy = None
def __init__(self, proxy_override=None, browser_connection_url=None):
def __init__(self, proxy_override=None, custom_browser_connection_url=None):
super().__init__()
from selenium.webdriver.common.proxy import Proxy as SeleniumProxy
# .strip('"') is going to save someone a lot of time when they accidently wrap the env value
if not browser_connection_url:
if not custom_browser_connection_url:
self.browser_connection_url = os.getenv("WEBDRIVER_URL", 'http://browser-chrome:4444/wd/hub').strip('"')
else:
self.browser_connection_url = browser_connection_url
self.browser_connection_is_custom = True
self.browser_connection_url = custom_browser_connection_url
# If any proxy settings are enabled, then we should setup the proxy object
proxy_args = {}
@@ -674,7 +683,7 @@ class base_html_webdriver(Fetcher):
class html_requests(Fetcher):
fetcher_description = "Basic fast Plaintext/HTTP Client"
def __init__(self, proxy_override=None, browser_connection_url=None):
def __init__(self, proxy_override=None, custom_browser_connection_url=None):
super().__init__()
self.proxy_override = proxy_override
# browser_connection_url is none because its always 'launched locally'
changedetectionio/flask_app.py
+22 -5
View File
@@ -485,14 +485,18 @@ def changedetection_app(config=None, datastore_o=None):
# AJAX endpoint for sending a test
@app.route("/notification/send-test/<string:watch_uuid>", methods=['POST'])
@app.route("/notification/send-test", methods=['POST'])
@app.route("/notification/send-test/", methods=['POST'])
@login_optionally_required
def ajax_callback_send_notification_test():
def ajax_callback_send_notification_test(watch_uuid=None):
# Watch_uuid could be unsuet in the case its used in tag editor, global setings
import apprise
from .apprise_asset import asset
apobj = apprise.Apprise(asset=asset)
watch = datastore.data['watching'].get(watch_uuid) if watch_uuid else None
# validate URLS
if not len(request.form['notification_urls'].strip()):
@@ -505,9 +509,11 @@ def changedetection_app(config=None, datastore_o=None):
return make_response({'error': message}, 400)
try:
n_object = {'watch_url': request.form['window_url'],
'notification_urls': request.form['notification_urls'].splitlines()
}
# use the same as when it is triggered, but then override it with the form test values
n_object = {
'watch_url': request.form['window_url'],
'notification_urls': request.form['notification_urls'].splitlines()
}
# Only use if present, if not set in n_object it should use the default system value
if 'notification_format' in request.form and request.form['notification_format'].strip():
@@ -519,7 +525,9 @@ def changedetection_app(config=None, datastore_o=None):
if 'notification_body' in request.form and request.form['notification_body'].strip():
n_object['notification_body'] = request.form.get('notification_body', '').strip()
notification_q.put(n_object)
from . import update_worker
new_worker = update_worker.update_worker(update_q, notification_q, app, datastore)
new_worker.queue_notification_for_watch(notification_q=notification_q, n_object=n_object, watch=watch)
except Exception as e:
return make_response({'error': str(e)}, 400)
@@ -1584,6 +1592,15 @@ def notification_runner():
try:
from changedetectionio import notification
# Fallback to system config if not set
if not n_object.get('notification_body') and datastore.data['settings']['application'].get('notification_body'):
n_object['notification_body'] = datastore.data['settings']['application'].get('notification_body')
if not n_object.get('notification_title') and datastore.data['settings']['application'].get('notification_title'):
n_object['notification_title'] = datastore.data['settings']['application'].get('notification_title')
if not n_object.get('notification_format') and datastore.data['settings']['application'].get('notification_format'):
n_object['notification_title'] = datastore.data['settings']['application'].get('notification_format')
sent_obj = notification.process_notification(n_object, datastore)
changedetectionio/forms.py
+1
View File
@@ -43,6 +43,7 @@ valid_method = {
'PUT',
'PATCH',
'DELETE',
'OPTIONS',
}
default_method = 'GET'
changedetectionio/model/Watch.py
+1
View File
@@ -38,6 +38,7 @@ base_config = {
'track_ldjson_price_data': None,
'headers': {}, # Extra headers to send
'ignore_text': [], # List of text to ignore when calculating the comparison checksum
'in_stock' : None,
'in_stock_only' : True, # Only trigger change on going to instock from out-of-stock
'include_filters': [],
'last_checked': 0,
changedetectionio/notification.py
+79 -73
View File
@@ -1,4 +1,5 @@
import apprise
import time
from jinja2 import Environment, BaseLoader
from apprise import NotifyFormat
import json
@@ -119,8 +120,8 @@ def process_notification(n_object, datastore):
# Get the notification body from datastore
jinja2_env = Environment(loader=BaseLoader)
n_body = jinja2_env.from_string(n_object.get('notification_body', default_notification_body)).render(**notification_parameters)
n_title = jinja2_env.from_string(n_object.get('notification_title', default_notification_title)).render(**notification_parameters)
n_body = jinja2_env.from_string(n_object.get('notification_body', '')).render(**notification_parameters)
n_title = jinja2_env.from_string(n_object.get('notification_title', '')).render(**notification_parameters)
n_format = valid_notification_formats.get(
n_object.get('notification_format', default_notification_format),
valid_notification_formats[default_notification_format],
@@ -131,103 +132,108 @@ def process_notification(n_object, datastore):
# Initially text or whatever
n_format = datastore.data['settings']['application'].get('notification_format', valid_notification_formats[default_notification_format])
# https://github.com/caronc/apprise/wiki/Development_LogCapture
# Anything higher than or equal to WARNING (which covers things like Connection errors)
# raise it as an exception
apobjs=[]
sent_objs=[]
sent_objs = []
from .apprise_asset import asset
for url in n_object['notification_urls']:
url = jinja2_env.from_string(url).render(**notification_parameters)
apobj = apprise.Apprise(debug=True, asset=asset)
url = url.strip()
if len(url):
apobj = apprise.Apprise(debug=True, asset=asset)
if not n_object.get('notification_urls'):
return None
with apprise.LogCapture(level=apprise.logging.DEBUG) as logs:
for url in n_object['notification_urls']:
url = url.strip()
print(">> Process Notification: AppRise notifying {}".format(url))
with apprise.LogCapture(level=apprise.logging.DEBUG) as logs:
# Re 323 - Limit discord length to their 2000 char limit total or it wont send.
# Because different notifications may require different pre-processing, run each sequentially :(
# 2000 bytes minus -
# 200 bytes for the overhead of the _entire_ json payload, 200 bytes for {tts, wait, content} etc headers
# Length of URL - Incase they specify a longer custom avatar_url
url = jinja2_env.from_string(url).render(**notification_parameters)
# So if no avatar_url is specified, add one so it can be correctly calculated into the total payload
k = '?' if not '?' in url else '&'
if not 'avatar_url' in url \
and not url.startswith('mail') \
and not url.startswith('post') \
and not url.startswith('get') \
and not url.startswith('delete') \
and not url.startswith('put'):
url += k + 'avatar_url=https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/changedetectionio/static/images/avatar-256x256.png'
# Re 323 - Limit discord length to their 2000 char limit total or it wont send.
# Because different notifications may require different pre-processing, run each sequentially :(
# 2000 bytes minus -
# 200 bytes for the overhead of the _entire_ json payload, 200 bytes for {tts, wait, content} etc headers
# Length of URL - Incase they specify a longer custom avatar_url
if url.startswith('tgram://'):
# Telegram only supports a limit subset of HTML, remove the '<br>' we place in.
# re https://github.com/dgtlmoon/changedetection.io/issues/555
# @todo re-use an existing library we have already imported to strip all non-allowed tags
n_body = n_body.replace('<br>', '\n')
n_body = n_body.replace('</br>', '\n')
# real limit is 4096, but minus some for extra metadata
payload_max_size = 3600
body_limit = max(0, payload_max_size - len(n_title))
n_title = n_title[0:payload_max_size]
n_body = n_body[0:body_limit]
# So if no avatar_url is specified, add one so it can be correctly calculated into the total payload
k = '?' if not '?' in url else '&'
if not 'avatar_url' in url \
and not url.startswith('mail') \
and not url.startswith('post') \
and not url.startswith('get') \
and not url.startswith('delete') \
and not url.startswith('put'):
url += k + 'avatar_url=https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/changedetectionio/static/images/avatar-256x256.png'
elif url.startswith('discord://') or url.startswith('https://discordapp.com/api/webhooks') or url.startswith('https://discord.com/api'):
# real limit is 2000, but minus some for extra metadata
payload_max_size = 1700
body_limit = max(0, payload_max_size - len(n_title))
n_title = n_title[0:payload_max_size]
n_body = n_body[0:body_limit]
if url.startswith('tgram://'):
# Telegram only supports a limit subset of HTML, remove the '<br>' we place in.
# re https://github.com/dgtlmoon/changedetection.io/issues/555
# @todo re-use an existing library we have already imported to strip all non-allowed tags
n_body = n_body.replace('<br>', '\n')
n_body = n_body.replace('</br>', '\n')
# real limit is 4096, but minus some for extra metadata
payload_max_size = 3600
body_limit = max(0, payload_max_size - len(n_title))
n_title = n_title[0:payload_max_size]
n_body = n_body[0:body_limit]
elif url.startswith('mailto'):
# Apprise will default to HTML, so we need to override it
# So that whats' generated in n_body is in line with what is going to be sent.
# https://github.com/caronc/apprise/issues/633#issuecomment-1191449321
if not 'format=' in url and (n_format == 'Text' or n_format == 'Markdown'):
prefix = '?' if not '?' in url else '&'
# Apprise format is lowercase text https://github.com/caronc/apprise/issues/633
n_format = n_format.tolower()
url = "{}{}format={}".format(url, prefix, n_format)
# If n_format == HTML, then apprise email should default to text/html and we should be sending HTML only
elif url.startswith('discord://') or url.startswith('https://discordapp.com/api/webhooks') or url.startswith(
'https://discord.com/api'):
# real limit is 2000, but minus some for extra metadata
payload_max_size = 1700
body_limit = max(0, payload_max_size - len(n_title))
n_title = n_title[0:payload_max_size]
n_body = n_body[0:body_limit]
apobj.add(url)
elif url.startswith('mailto'):
# Apprise will default to HTML, so we need to override it
# So that whats' generated in n_body is in line with what is going to be sent.
# https://github.com/caronc/apprise/issues/633#issuecomment-1191449321
if not 'format=' in url and (n_format == 'Text' or n_format == 'Markdown'):
prefix = '?' if not '?' in url else '&'
# Apprise format is lowercase text https://github.com/caronc/apprise/issues/633
n_format = n_format.lower()
url = f"{url}{prefix}format={n_format}"
# If n_format == HTML, then apprise email should default to text/html and we should be sending HTML only
apobj.notify(
title=n_title,
body=n_body,
body_format=n_format,
# False is not an option for AppRise, must be type None
attach=n_object.get('screenshot', None)
)
apobj.add(url)
apobj.clear()
sent_objs.append({'title': n_title,
'body': n_body,
'url': url,
'body_format': n_format})
# Incase it needs to exist in memory for a while after to process(?)
apobjs.append(apobj)
# Blast off the notifications tht are set in .add()
apobj.notify(
title=n_title,
body=n_body,
body_format=n_format,
# False is not an option for AppRise, must be type None
attach=n_object.get('screenshot', None)
)
# Returns empty string if nothing found, multi-line string otherwise
log_value = logs.getvalue()
if log_value and 'WARNING' in log_value or 'ERROR' in log_value:
raise Exception(log_value)
# Give apprise time to register an error
time.sleep(3)
sent_objs.append({'title': n_title,
'body': n_body,
'url' : url,
'body_format': n_format})
# Returns empty string if nothing found, multi-line string otherwise
log_value = logs.getvalue()
if log_value and 'WARNING' in log_value or 'ERROR' in log_value:
raise Exception(log_value)
# Return what was sent for better logging - after the for loop
return sent_objs
# Notification title + body content parameters get created here.
# ( Where we prepare the tokens in the notification to be replaced with actual values )
def create_notification_parameters(n_object, datastore):
from copy import deepcopy
# in the case we send a test notification from the main settings, there is no UUID.
uuid = n_object['uuid'] if 'uuid' in n_object else ''
if uuid != '':
if uuid:
watch_title = datastore.data['watching'][uuid].get('title', '')
tag_list = []
tags = datastore.get_all_tags_for_watch(uuid)
@@ -255,7 +261,7 @@ def create_notification_parameters(n_object, datastore):
tokens.update(
{
'base_url': base_url,
'current_snapshot': n_object['current_snapshot'] if 'current_snapshot' in n_object else '',
'current_snapshot': n_object.get('current_snapshot', ''),
'diff': n_object.get('diff', ''), # Null default in the case we use a test
'diff_added': n_object.get('diff_added', ''), # Null default in the case we use a test
'diff_full': n_object.get('diff_full', ''), # Null default in the case we use a test
changedetectionio/processors/__init__.py
+3 -3
View File
@@ -43,14 +43,14 @@ class difference_detection_processor():
# In the case that the preferred fetcher was a browser config with custom connection URL..
# @todo - on save watch, if its extra_browser_ then it should be obvious it will use playwright (like if its requests now..)
browser_connection_url = None
custom_browser_connection_url = None
if prefer_fetch_backend.startswith('extra_browser_'):
(t, key) = prefer_fetch_backend.split('extra_browser_')
connection = list(
filter(lambda s: (s['browser_name'] == key), self.datastore.data['settings']['requests'].get('extra_browsers', [])))
if connection:
prefer_fetch_backend = 'base_html_playwright'
browser_connection_url = connection[0].get('browser_connection_url')
custom_browser_connection_url = connection[0].get('browser_connection_url')
# PDF should be html_requests because playwright will serve it up (so far) in a embedded page
# @todo https://github.com/dgtlmoon/changedetection.io/issues/2019
@@ -74,7 +74,7 @@ class difference_detection_processor():
# Now call the fetcher (playwright/requests/etc) with arguments that only a fetcher would need.
# When browser_connection_url is None, it method should default to working out whats the best defaults (os env vars etc)
self.fetcher = fetcher_obj(proxy_override=proxy_url,
browser_connection_url=browser_connection_url
custom_browser_connection_url=custom_browser_connection_url
)
if self.watch.has_browser_steps:
changedetectionio/processors/restock_diff.py
+5 -1
View File
@@ -109,4 +109,8 @@ class perform_site_check(difference_detection_processor):
# All cases
changed_detected = True
return changed_detected, update_obj, self.fetcher.instock_data.encode('utf-8')
# Always record the new checksum
update_obj["previous_md5"] = fetched_md5
return changed_detected, update_obj, self.fetcher.instock_data.encode('utf-8').strip()
changedetectionio/res/stock-not-in-stock.js
+122 -106
View File
@@ -1,115 +1,131 @@
function isItemInStock() {
// @todo Pass these in so the same list can be used in non-JS fetchers
const outOfStockTexts = [
' أخبرني عندما يتوفر',
'0 in stock',
'agotado',
'artikel zurzeit vergriffen',
'as soon as stock is available',
'ausverkauft', // sold out
'available for back order',
'back-order or out of stock',
'backordered',
'benachrichtigt mich', // notify me
'brak na stanie',
'brak w magazynie',
'coming soon',
'currently have any tickets for this',
'currently unavailable',
'dostępne wkrótce',
'dostępne wkrótce',
'en rupture de stock',
'ist derzeit nicht auf lager',
'ist derzeit nicht auf lager',
'item is no longer available',
'let me know when it\'s available',
'message if back in stock',
'nachricht bei',
'nicht auf lager',
'nicht lieferbar',
'nicht zur verfügung',
'no disponible temporalmente',
'no longer in stock',
'no tickets available',
'not available',
'not currently available',
'not in stock',
'notify me when available',
'não estamos a aceitar encomendas',
'out of stock',
'out-of-stock',
'produkt niedostępny',
'sold out',
'sold-out',
'temporarily out of stock',
'temporarily unavailable',
'tickets unavailable',
'unavailable tickets',
'we do not currently have an estimate of when this product will be back in stock.',
'zur zeit nicht an lager',
'品切れ',
'已售完',
'품절'
];
// @todo Pass these in so the same list can be used in non-JS fetchers
const outOfStockTexts = [
' أخبرني عندما يتوفر',
'0 in stock',
'agotado',
'article épuisé',
'artikel zurzeit vergriffen',
'as soon as stock is available',
'ausverkauft', // sold out
'available for back order',
'back-order or out of stock',
'backordered',
'benachrichtigt mich', // notify me
'brak na stanie',
'brak w magazynie',
'coming soon',
'currently have any tickets for this',
'currently unavailable',
'dostępne wkrótce',
'en rupture de stock',
'ist derzeit nicht auf lager',
'item is no longer available',
'let me know when it\'s available',
'message if back in stock',
'nachricht bei',
'nicht auf lager',
'nicht lieferbar',
'nicht zur verfügung',
'niet beschikbaar',
'niet leverbaar',
'no disponible temporalmente',
'no longer in stock',
'no tickets available',
'not available',
'not currently available',
'not in stock',
'notify me when available',
'não estamos a aceitar encomendas',
'out of stock',
'out-of-stock',
'produkt niedostępny',
'sold out',
'sold-out',
'temporarily out of stock',
'temporarily unavailable',
'tickets unavailable',
'tijdelijk uitverkocht',
'unavailable tickets',
'we do not currently have an estimate of when this product will be back in stock.',
'we don\'t know when or if this item will be back in stock.',
'zur zeit nicht an lager',
'品切れ',
'已售完',
'품절'
];
const negateOutOfStockRegexs = [
'[0-9] in stock'
]
var negateOutOfStockRegexs_r = [];
for (let i = 0; i < negateOutOfStockRegexs.length; i++) {
negateOutOfStockRegexs_r.push(new RegExp(negateOutOfStockRegexs[0], 'g'));
}
const elementsWithZeroChildren = Array.from(document.getElementsByTagName('*')).filter(element => element.children.length === 0);
// REGEXS THAT REALLY MEAN IT'S IN STOCK
for (let i = elementsWithZeroChildren.length - 1; i >= 0; i--) {
const element = elementsWithZeroChildren[i];
if (element.offsetWidth > 0 || element.offsetHeight > 0 || element.getClientRects().length > 0) {
var elementText="";
if (element.tagName.toLowerCase() === "input") {
elementText = element.value.toLowerCase();
} else {
elementText = element.textContent.toLowerCase();
}
if (elementText.length) {
// try which ones could mean its in stock
for (let i = 0; i < negateOutOfStockRegexs.length; i++) {
if (negateOutOfStockRegexs_r[i].test(elementText)) {
return 'Possibly in stock';
}
}
}
function getElementBaseText(element) {
// .textContent can include text from children which may give the wrong results
// scan only immediate TEXT_NODEs, which will be a child of the element
var text = "";
for (var i = 0; i < element.childNodes.length; ++i)
if (element.childNodes[i].nodeType === Node.TEXT_NODE)
text += element.childNodes[i].textContent;
return text.toLowerCase().trim();
}
}
// OTHER STUFF THAT COULD BE THAT IT'S OUT OF STOCK
for (let i = elementsWithZeroChildren.length - 1; i >= 0; i--) {
const element = elementsWithZeroChildren[i];
if (element.offsetWidth > 0 || element.offsetHeight > 0 || element.getClientRects().length > 0) {
var elementText="";
if (element.tagName.toLowerCase() === "input") {
elementText = element.value.toLowerCase();
} else {
elementText = element.textContent.toLowerCase();
}
if (elementText.length) {
// and these mean its out of stock
for (const outOfStockText of outOfStockTexts) {
if (elementText.includes(outOfStockText)) {
return elementText; // item is out of stock
}
}
}
const negateOutOfStockRegexs = [
'[0-9] in stock'
]
var negateOutOfStockRegexs_r = [];
for (let i = 0; i < negateOutOfStockRegexs.length; i++) {
negateOutOfStockRegexs_r.push(new RegExp(negateOutOfStockRegexs[0], 'g'));
}
}
return 'Possibly in stock'; // possibly in stock, cant decide otherwise.
// The out-of-stock or in-stock-text is generally always above-the-fold
// and often below-the-fold is a list of related products that may or may not contain trigger text
// so it's good to filter to just the 'above the fold' elements
// and it should be atleast 100px from the top to ignore items in the toolbar, sometimes menu items like "Coming soon" exist
const elementsToScan = Array.from(document.getElementsByTagName('*')).filter(element => element.getBoundingClientRect().top + window.scrollY <= window.innerHeight && element.getBoundingClientRect().top + window.scrollY >= 100);
var elementText = "";
// REGEXS THAT REALLY MEAN IT'S IN STOCK
for (let i = elementsToScan.length - 1; i >= 0; i--) {
const element = elementsToScan[i];
elementText = "";
if (element.tagName.toLowerCase() === "input") {
elementText = element.value.toLowerCase();
} else {
elementText = getElementBaseText(element);
}
if (elementText.length) {
// try which ones could mean its in stock
for (let i = 0; i < negateOutOfStockRegexs.length; i++) {
if (negateOutOfStockRegexs_r[i].test(elementText)) {
return 'Possibly in stock';
}
}
}
}
// OTHER STUFF THAT COULD BE THAT IT'S OUT OF STOCK
for (let i = elementsToScan.length - 1; i >= 0; i--) {
const element = elementsToScan[i];
if (element.offsetWidth > 0 || element.offsetHeight > 0 || element.getClientRects().length > 0) {
elementText = "";
if (element.tagName.toLowerCase() === "input") {
elementText = element.value.toLowerCase();
} else {
elementText = getElementBaseText(element);
}
if (elementText.length) {
// and these mean its out of stock
for (const outOfStockText of outOfStockTexts) {
if (elementText.includes(outOfStockText)) {
return outOfStockText; // item is out of stock
}
}
}
}
}
return 'Possibly in stock'; // possibly in stock, cant decide otherwise.
}
// returns the element text that makes it think it's out of stock
return isItemInStock();
return isItemInStock().trim()
changedetectionio/static/js/notifications.js
+10 -7
View File
@@ -24,14 +24,17 @@ $(document).ready(function() {
})
data = {
window_url : window.location.href,
notification_urls : $('.notification-urls').val(),
notification_body: $('#notification_body').val(),
notification_format: $('#notification_format').val(),
notification_title: $('#notification_title').val(),
notification_urls: $('.notification-urls').val(),
window_url: window.location.href,
}
for (key in data) {
if (!data[key].length) {
alert(key+" is empty, cannot send test.")
return;
}
if (!data['notification_urls'].length) {
alert("Notification URL list is empty, cannot send test.")
return;
}
$.ajax({
changedetectionio/store.py
+1
View File
@@ -248,6 +248,7 @@ class ChangeDetectionStore:
'check_count': 0,
'fetch_time' : 0.0,
'has_ldjson_price_data': None,
'in_stock': None,
'last_checked': 0,
'last_error': False,
'last_notification_error': False,
changedetectionio/templates/edit.html
+1 -1
View File
@@ -14,7 +14,7 @@
{% if emailprefix %}
const email_notification_prefix=JSON.parse('{{ emailprefix|tojson }}');
{% endif %}
const notification_base_url="{{url_for('ajax_callback_send_notification_test')}}";
const notification_base_url="{{url_for('ajax_callback_send_notification_test', watch_uuid=uuid)}}";
const playwright_enabled={% if playwright_enabled %} true {% else %} false {% endif %};
const recheck_proxy_start_url="{{url_for('check_proxies.start_check', uuid=uuid)}}";
const proxy_recheck_status_url="{{url_for('check_proxies.get_recheck_status', uuid=uuid)}}";
changedetectionio/templates/settings.html
+1 -1
View File
@@ -4,7 +4,7 @@
{% from '_helpers.jinja' import render_field, render_checkbox_field, render_button %}
{% from '_common_fields.jinja' import render_common_settings_form %}
<script>
const notification_base_url="{{url_for('ajax_callback_send_notification_test')}}";
const notification_base_url="{{url_for('ajax_callback_send_notification_test', watch_uuid=uuid)}}";
{% if emailprefix %}
const email_notification_prefix=JSON.parse('{{emailprefix|tojson}}');
{% endif %}
changedetectionio/tests/smtp/test_notification_smtp.py
+17 -1
View File
@@ -97,6 +97,17 @@ def test_check_notification_email_formats_default_Text_override_HTML(client, liv
set_original_response()
global smtp_test_server
notification_url = f'mailto://changedetection@{smtp_test_server}:11025/?to=fff@home.com'
notification_body = f"""<!DOCTYPE html>
<html lang="en">
<head>
<title>My Webpage</title>
</head>
<body>
<h1>Test</h1>
{default_notification_body}
</body>
</html>
"""
#####################
# Set this up for when we remove the notification from the watch, it should fallback with these details
@@ -104,7 +115,7 @@ def test_check_notification_email_formats_default_Text_override_HTML(client, liv
url_for("settings_page"),
data={"application-notification_urls": notification_url,
"application-notification_title": "fallback-title " + default_notification_title,
"application-notification_body": default_notification_body,
"application-notification_body": notification_body,
"application-notification_format": 'Text',
"requests-time_between_check-minutes": 180,
'application-fetch_backend': "html_requests"},
@@ -161,5 +172,10 @@ def test_check_notification_email_formats_default_Text_override_HTML(client, liv
assert 'Content-Type: text/html' in msg
assert '(removed) So let\'s see what happens.<br>' in msg # the html part
# https://github.com/dgtlmoon/changedetection.io/issues/2103
assert '<h1>Test</h1>' in msg
assert '&lt;' not in msg
assert 'Content-Type: text/html' in msg
res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
assert b'Deleted' in res.data
changedetectionio/tests/test_notification_errors.py
+14 -5
View File
@@ -1,8 +1,7 @@
import os
import time
import re
from flask import url_for
from . util import set_original_response, set_modified_response, live_server_setup
from .util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks
import logging
def test_check_notification_error_handling(client, live_server):
@@ -11,7 +10,7 @@ def test_check_notification_error_handling(client, live_server):
set_original_response()
# Give the endpoint time to spin up
time.sleep(2)
time.sleep(1)
# Set a URL and fetch it, then set a notification URL which is going to give errors
test_url = url_for('test_endpoint', _external=True)
@@ -22,12 +21,16 @@ def test_check_notification_error_handling(client, live_server):
)
assert b"Watch added" in res.data
time.sleep(2)
wait_for_all_checks(client)
set_modified_response()
working_notification_url = url_for('test_notification_endpoint', _external=True).replace('http', 'json')
broken_notification_url = "jsons://broken-url-xxxxxxxx123/test"
res = client.post(
url_for("edit_page", uuid="first"),
data={"notification_urls": "jsons://broken-url-xxxxxxxx123/test",
# A URL with errors should not block the one that is working
data={"notification_urls": f"{broken_notification_url}\r\n{working_notification_url}",
"notification_title": "xxx",
"notification_body": "xxxxx",
"notification_format": "Text",
@@ -63,4 +66,10 @@ def test_check_notification_error_handling(client, live_server):
found_name_resolution_error = b"Temporary failure in name resolution" in res.data or b"Name or service not known" in res.data
assert found_name_resolution_error
# And the working one, which is after the 'broken' one should still have fired
with open("test-datastore/notification.txt", "r") as f:
notification_submission = f.read()
os.unlink("test-datastore/notification.txt")
assert 'xxxxx' in notification_submission
client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
changedetectionio/update_worker.py
+49 -23
View File
@@ -26,47 +26,61 @@ class update_worker(threading.Thread):
self.datastore = datastore
super().__init__(*args, **kwargs)
def queue_notification_for_watch(self, n_object, watch):
def queue_notification_for_watch(self, notification_q, n_object, watch):
from changedetectionio import diff
dates = []
trigger_text = ''
if watch:
watch_history = watch.history
dates = list(watch_history.keys())
trigger_text = watch.get('trigger_text', [])
watch_history = watch.history
dates = list(watch_history.keys())
# Add text that was triggered
snapshot_contents = watch.get_history_snapshot(dates[-1])
if len(dates):
snapshot_contents = watch.get_history_snapshot(dates[-1])
else:
snapshot_contents = "No snapshot/history available, the watch should fetch atleast once."
# HTML needs linebreak, but MarkDown and Text can use a linefeed
if n_object['notification_format'] == 'HTML':
if n_object.get('notification_format') == 'HTML':
line_feed_sep = "<br>"
# Snapshot will be plaintext on the disk, convert to some kind of HTML
snapshot_contents = snapshot_contents.replace('\n', line_feed_sep)
else:
line_feed_sep = "\n"
trigger_text = watch.get('trigger_text', [])
triggered_text = ''
if len(trigger_text):
from . import html_tools
triggered_text = html_tools.get_triggered_text(content=snapshot_contents, trigger_text=trigger_text)
if triggered_text:
triggered_text = line_feed_sep.join(triggered_text)
# Could be called as a 'test notification' with only 1 snapshot available
prev_snapshot = "Example text: example test\nExample text: change detection is cool\nExample text: some more examples\n"
current_snapshot = "Example text: example test\nExample text: change detection is fantastic\nExample text: even more examples\nExample text: a lot more examples"
if len(dates) > 1:
prev_snapshot = watch.get_history_snapshot(dates[-2])
current_snapshot = watch.get_history_snapshot(dates[-1])
n_object.update({
'current_snapshot': snapshot_contents,
'diff': diff.render_diff(watch.get_history_snapshot(dates[-2]), watch.get_history_snapshot(dates[-1]), line_feed_sep=line_feed_sep),
'diff_added': diff.render_diff(watch.get_history_snapshot(dates[-2]), watch.get_history_snapshot(dates[-1]), include_removed=False, line_feed_sep=line_feed_sep),
'diff_full': diff.render_diff(watch.get_history_snapshot(dates[-2]), watch.get_history_snapshot(dates[-1]), include_equal=True, line_feed_sep=line_feed_sep),
'diff_patch': diff.render_diff(watch.get_history_snapshot(dates[-2]), watch.get_history_snapshot(dates[-1]), line_feed_sep=line_feed_sep, patch_format=True),
'diff_removed': diff.render_diff(watch.get_history_snapshot(dates[-2]), watch.get_history_snapshot(dates[-1]), include_added=False, line_feed_sep=line_feed_sep),
'screenshot': watch.get_screenshot() if watch.get('notification_screenshot') else None,
'diff': diff.render_diff(prev_snapshot, current_snapshot, line_feed_sep=line_feed_sep),
'diff_added': diff.render_diff(prev_snapshot, current_snapshot, include_removed=False, line_feed_sep=line_feed_sep),
'diff_full': diff.render_diff(prev_snapshot, current_snapshot, include_equal=True, line_feed_sep=line_feed_sep),
'diff_patch': diff.render_diff(prev_snapshot, current_snapshot, line_feed_sep=line_feed_sep, patch_format=True),
'diff_removed': diff.render_diff(prev_snapshot, current_snapshot, include_added=False, line_feed_sep=line_feed_sep),
'screenshot': watch.get_screenshot() if watch and watch.get('notification_screenshot') else None,
'triggered_text': triggered_text,
'uuid': watch.get('uuid'),
'watch_url': watch.get('url'),
'uuid': watch.get('uuid') if watch else None,
'watch_url': watch.get('url') if watch else None,
})
logging.info (">> SENDING NOTIFICATION")
self.notification_q.put(n_object)
logging.info(">> SENDING NOTIFICATION")
notification_q.put(n_object)
# Prefer - Individual watch settings > Tag settings > Global settings (in that order)
def _check_cascading_vars(self, var_name, watch):
@@ -134,7 +148,7 @@ class update_worker(threading.Thread):
queued = False
if n_object and n_object.get('notification_urls'):
queued = True
self.queue_notification_for_watch(n_object, watch)
self.queue_notification_for_watch(notification_q=self.notification_q, n_object=n_object, watch=watch)
return queued
@@ -174,9 +188,9 @@ class update_worker(threading.Thread):
return
threshold = self.datastore.data['settings']['application'].get('filter_failure_notification_threshold_attempts')
n_object = {'notification_title': "Changedetection.io - Alert - Browser step at position {} could not be run".format(step_n+1),
'notification_body': "Your configured browser step at position {} for {{watch['url']}} "
'notification_body': "Your configured browser step at position {} for {{{{watch_url}}}} "
"did not appear on the page after {} attempts, did the page change layout? "
"Does it need a delay added?\n\nLink: {{base_url}}/edit/{{watch_uuid}}\n\n"
"Does it need a delay added?\n\nLink: {{{{base_url}}}}/edit/{{{{watch_uuid}}}}\n\n"
"Thanks - Your omniscient changedetection.io installation :)\n".format(step_n+1, threshold),
'notification_format': 'text'}
@@ -340,20 +354,32 @@ class update_worker(threading.Thread):
changed_detected = False
self.datastore.update_watch(uuid=uuid, update_obj={'last_error': False})
except content_fetcher.BrowserStepsStepTimout as e:
except content_fetcher.BrowserStepsStepException as e:
if not self.datastore.data['watching'].get(uuid):
continue
error_step = e.step_n + 1
err_text = f"Warning, browser step at position {error_step} could not run, target not found, check the watch, add a delay if necessary, view Browser Steps to see screenshot at that step"
from playwright._impl._errors import TimeoutError, Error
# Generally enough info for TimeoutError (couldnt locate the element after default seconds)
err_text = f"Browser step at position {error_step} could not run, check the watch, add a delay if necessary, view Browser Steps to see screenshot at that step."
if e.original_e.name == "TimeoutError":
# Just the first line is enough, the rest is the stack trace
err_text += " Could not find the target."
else:
# Other Error, more info is good.
err_text += " " + str(e.original_e).splitlines()[0]
print(f"BrowserSteps exception at step {error_step}", str(e.original_e))
self.datastore.update_watch(uuid=uuid,
update_obj={'last_error': err_text,
'browser_steps_last_error_step': error_step
}
)
if self.datastore.data['watching'][uuid].get('filter_failure_notification_send', False):
c = self.datastore.data['watching'][uuid].get('consecutive_filter_failures', 5)
c += 1
docker-compose.yml
+1 -1
View File
@@ -90,7 +90,7 @@ services:
#
# Used for fetching pages via Playwright+Chrome where you need Javascript support.
# Note: works well but is deprecated, doesnt fetch full page screenshots and other issues
# Note: works well but is deprecated, does not fetch full page screenshots (doesnt work with Visual Selector) and other issues
# browser-chrome:
# hostname: browser-chrome
# image: selenium/standalone-chrome:4
heroku.yml
-2
View File
@@ -1,2 +0,0 @@
run:
changedetection: python3 ./changedetection.py -C -d ./datastore -p $PORT
requirements.txt
+1 -1
View File
@@ -30,7 +30,7 @@ dnspython~=2.4 # related to eventlet fixes
# jq not available on Windows so must be installed manually
# Notification library
apprise~=1.6.0
apprise~=1.7.1
# apprise mqtt https://github.com/dgtlmoon/changedetection.io/issues/315
paho-mqtt
setup.py
+2 -2
View File
@@ -27,7 +27,7 @@ install_requires = open('requirements.txt').readlines()
setup(
name='changedetection.io',
version=find_version("changedetectionio", "__init__.py"),
description='Website change detection and monitoring service',
description='Website change detection and monitoring service, detect changes to web pages and send alerts/notifications.',
long_description=open('README-pip.md').read(),
long_description_content_type='text/markdown',
keywords='website change monitor for changes notification change detection '
@@ -41,7 +41,7 @@ setup(
include_package_data=True,
install_requires=install_requires,
license="Apache License 2.0",
python_requires=">= 3.7",
python_requires=">= 3.10",
classifiers=['Intended Audience :: Customer Service',
'Intended Audience :: Developers',
'Intended Audience :: Education',