From 8ec17a4c83a0a39ce3730ebe428c2fae0a8dfba5 Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Tue, 28 Dec 2021 17:07:41 +0100 Subject: [PATCH 01/10] Re #267 - Pass settings for the proxy setup for webdriver (#326) * Re #267 - Pass HTTP_PROXY as the proxy setup for webdriver * Update README.md --- README.md | 19 ++----------------- changedetectionio/content_fetcher.py | 24 ++++++++++++++++++++++-- changedetectionio/forms.py | 4 +++- docker-compose.yml | 14 ++++++++++++-- 4 files changed, 39 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 0c1320103..19f194833 100644 --- a/README.md +++ b/README.md @@ -136,24 +136,9 @@ When you enable a `json:` filter, you can even automatically extract and parse e `json:$.price` would give `23.50`, or you can extract the whole structure -### Proxy +### Proxy configuration -A proxy for ChangeDetection.io can be configured by setting environment the -`HTTP_PROXY`, `HTTPS_PROXY` variables, examples are also in the `docker-compose.yml` - -`NO_PROXY` exclude list can be specified by following `"localhost,192.168.0.0/24"` - -as `docker run` with `-e` - -``` -docker run -d --restart always -e HTTPS_PROXY="socks5h://10.10.1.10:1080" -p "127.0.0.1:5000:5000" -v datastore-volume:/datastore --name changedetection.io dgtlmoon/changedetection.io -``` - -With `docker-compose`, see the `Proxy support example` in docker-compose.yml. - -For more information see https://docs.python-requests.org/en/master/user/advanced/#proxies - -This proxy support also extends to the notifications https://github.com/caronc/apprise/issues/387#issuecomment-841718867 +See the wiki https://github.com/dgtlmoon/changedetection.io/wiki/Proxy-configuration ### RaspberriPi support? diff --git a/changedetectionio/content_fetcher.py b/changedetectionio/content_fetcher.py index 1f6ef14ae..d1eb41b9f 100644 --- a/changedetectionio/content_fetcher.py +++ b/changedetectionio/content_fetcher.py @@ -3,6 +3,7 @@ import time from abc import ABC, abstractmethod from selenium import webdriver from selenium.webdriver.common.desired_capabilities import DesiredCapabilities +from selenium.webdriver.common.proxy import Proxy as SeleniumProxy from selenium.common.exceptions import WebDriverException import urllib3.exceptions @@ -65,15 +66,34 @@ class html_webdriver(Fetcher): command_executor = '' + # Configs for Proxy setup + # In the ENV vars, is prefixed with "webdriver_", so it is for example "webdriver_sslProxy" + selenium_proxy_settings_mappings = ['ftpProxy', 'httpProxy', 'noProxy', + 'proxyAutoconfigUrl', 'sslProxy', 'autodetect', + 'socksProxy', 'socksUsername', 'socksPassword'] + proxy=None + def __init__(self): - self.command_executor = os.getenv("WEBDRIVER_URL", 'http://browser-chrome:4444/wd/hub') + # .strip('"') is going to save someone a lot of time when they accidently wrap the env value + self.command_executor = os.getenv("WEBDRIVER_URL", 'http://browser-chrome:4444/wd/hub').strip('"') + + # If any proxy settings are enabled, then we should setup the proxy object + proxy_args = {} + for k in self.selenium_proxy_settings_mappings: + v = os.getenv('webdriver_' + k, False) + if v: + proxy_args[k] = v.strip('"') + + if proxy_args: + self.proxy = SeleniumProxy(raw=proxy_args) def run(self, url, timeout, request_headers): # check env for WEBDRIVER_URL driver = webdriver.Remote( command_executor=self.command_executor, - desired_capabilities=DesiredCapabilities.CHROME) + desired_capabilities=DesiredCapabilities.CHROME, + proxy=self.proxy) try: driver.get(url) diff --git a/changedetectionio/forms.py b/changedetectionio/forms.py index bc6501656..1028c213b 100644 --- a/changedetectionio/forms.py +++ b/changedetectionio/forms.py @@ -106,10 +106,12 @@ class ValidateContentFetcherIsReady(object): except urllib3.exceptions.MaxRetryError as e: driver_url = some_object.command_executor message = field.gettext('Content fetcher \'%s\' did not respond.' % (field.data)) - message += '
'+field.gettext('Be sure that the selenium/webdriver runner is running and accessible via network from this container/host.') + message += '
' + field.gettext( + 'Be sure that the selenium/webdriver runner is running and accessible via network from this container/host.') message += '
' + field.gettext('Did you follow the instructions in the wiki?') message += '

' + field.gettext('WebDriver Host: %s' % (driver_url)) message += '
Go here for more information' + message += '
'+field.gettext('Content fetcher did not respond properly, unable to use it.\n %s' % (str(e))) raise ValidationError(message) diff --git a/docker-compose.yml b/docker-compose.yml index d06de4025..1060ed581 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -13,13 +13,23 @@ services: # - PUID=1000 # - PGID=1000 - # # Alternative WebDriver/selenium URL, do not use "'s or 's! + # + # Alternative WebDriver/selenium URL, do not use "'s or 's! # - WEBDRIVER_URL=http://browser-chrome:4444/wd/hub - # Proxy support example. + # + # WebDriver proxy settings webdriver_ftpProxy, webdriver_httpProxy, webdriver_noProxy, + # webdriver_proxyAutoconfigUrl, webdriver_sslProxy, webdriver_autodetect, + # webdriver_socksProxy, webdriver_socksUsername, webdriver_socksPassword + # + # https://selenium-python.readthedocs.io/api.html#module-selenium.webdriver.common.proxy + # + # Plain requsts - proxy support example. # - HTTP_PROXY=socks5h://10.10.1.10:1080 # - HTTPS_PROXY=socks5h://10.10.1.10:1080 + # # An exclude list (useful for notification URLs above) can be specified by with # - NO_PROXY="localhost,192.168.0.0/24" + # # Base URL of your changedetection.io install (Added to the notification alert) # - BASE_URL=https://mysite.com From 98461d813e478402420465f9183dbd61f3019846 Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Tue, 28 Dec 2021 18:57:39 +0100 Subject: [PATCH 02/10] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 19f194833..8ea37588b 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,6 @@ Open source web page monitoring, notification and change detection. #### Example use cases -Know when ... - Products and services have a change in pricing - Governmental department updates (changes are often only on their websites) - New software releases, security advisories when you're not on their mailing list. @@ -30,7 +29,8 @@ Know when ... - Trigger API calls via notifications when text appears on a website - Glue together APIs using the JSON filter and JSON notifications - Create RSS feeds based on changes in web content - +- You have a very sensitive list of URLs to watch and you do _not_ want to use the paid alternatives. (Remember, _you_ are the product) + _Need an actual Chrome runner with Javascript support? We support fetching via WebDriver!_ **Get monitoring now! super simple.** From c4990fa3f986955aa4a7da5387eb9d8c3aa5c7de Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Tue, 28 Dec 2021 18:59:43 +0100 Subject: [PATCH 03/10] Create CONTRIBUTING.md --- CONTRIBUTING.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 CONTRIBUTING.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 000000000..ef12c87a7 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,5 @@ +Contributing is always welcome! + +I am no professional flask developer, if you know a better way that something can be done, please let me know! + +Otherwise, it's always best to PR into the `dev` branch. From e3bcd8c9bfd6a680d74f4224f31e41cb72d77f05 Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Wed, 29 Dec 2021 08:55:37 +0100 Subject: [PATCH 04/10] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 8ea37588b..22cf50de7 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,7 @@ Open source web page monitoring, notification and change detection. - Festivals with changes - Realestate listing changes - COVID related news from government websites +- University/organisation news from their website - Detect and monitor changes in JSON API responses - API monitoring and alerting - Trigger API calls via notifications when text appears on a website From dfcae4ee643f4991816421ca47011baef8168e04 Mon Sep 17 00:00:00 2001 From: Simon Caron Date: Wed, 29 Dec 2021 17:18:29 -0500 Subject: [PATCH 05/10] Extend Request Parameters to add Body & Method (#325) --- changedetectionio/__init__.py | 2 + changedetectionio/content_fetcher.py | 6 +- changedetectionio/fetch_site_status.py | 4 +- changedetectionio/forms.py | 24 +++ changedetectionio/store.py | 2 + changedetectionio/templates/edit.html | 29 +++- changedetectionio/tests/test_headers.py | 80 --------- changedetectionio/tests/test_request.py | 211 ++++++++++++++++++++++++ changedetectionio/tests/util.py | 15 ++ 9 files changed, 282 insertions(+), 91 deletions(-) delete mode 100644 changedetectionio/tests/test_headers.py create mode 100644 changedetectionio/tests/test_request.py diff --git a/changedetectionio/__init__.py b/changedetectionio/__init__.py index 0af94a489..4b8127589 100644 --- a/changedetectionio/__init__.py +++ b/changedetectionio/__init__.py @@ -445,6 +445,8 @@ def changedetection_app(config=None, datastore_o=None): 'tag': form.tag.data.strip(), 'title': form.title.data.strip(), 'headers': form.headers.data, + 'body': form.body.data, + 'method': form.method.data, 'fetch_backend': form.fetch_backend.data, 'trigger_text': form.trigger_text.data, 'notification_title': form.notification_title.data, diff --git a/changedetectionio/content_fetcher.py b/changedetectionio/content_fetcher.py index d1eb41b9f..b2a2d7948 100644 --- a/changedetectionio/content_fetcher.py +++ b/changedetectionio/content_fetcher.py @@ -131,10 +131,12 @@ class html_webdriver(Fetcher): class html_requests(Fetcher): fetcher_description = "Basic fast Plaintext/HTTP Client" - def run(self, url, timeout, request_headers): + def run(self, url, timeout, request_headers, request_body, request_method): import requests - r = requests.get(url, + r = requests.request(method=request_method, + data=request_body, + url=url, headers=request_headers, timeout=timeout, verify=False) diff --git a/changedetectionio/fetch_site_status.py b/changedetectionio/fetch_site_status.py index fc8c1e6e1..69ff7de02 100644 --- a/changedetectionio/fetch_site_status.py +++ b/changedetectionio/fetch_site_status.py @@ -80,6 +80,8 @@ class perform_site_check(): else: timeout = self.datastore.data['settings']['requests']['timeout'] url = self.datastore.get_val(uuid, 'url') + request_body = self.datastore.get_val(uuid, 'body') + request_method = self.datastore.get_val(uuid, 'method') # Pluggable content fetcher prefer_backend = watch['fetch_backend'] @@ -91,7 +93,7 @@ class perform_site_check(): fetcher = klass() - fetcher.run(url, timeout, request_headers) + fetcher.run(url, timeout, request_headers, request_body, request_method) # Fetching complete, now filters # @todo move to class / maybe inside of fetcher abstract base? diff --git a/changedetectionio/forms.py b/changedetectionio/forms.py index 1028c213b..dc06c67a0 100644 --- a/changedetectionio/forms.py +++ b/changedetectionio/forms.py @@ -8,6 +8,16 @@ import re from changedetectionio.notification import default_notification_format, valid_notification_formats, default_notification_body, default_notification_title +valid_method = { + 'GET', + 'POST', + 'PUT', + 'PATCH', + 'DELETE', +} + +default_method = 'GET' + class StringListField(StringField): widget = widgets.TextArea() @@ -224,8 +234,22 @@ class watchForm(commonSettingsForm): ignore_text = StringListField('Ignore Text', [ValidateListRegex()]) headers = StringDictKeyValue('Request Headers') + body = TextAreaField('Request Body', [validators.Optional()]) + method = SelectField('Request Method', choices=valid_method, default=default_method) trigger_text = StringListField('Trigger/wait for text', [validators.Optional(), ValidateListRegex()]) + def validate(self, **kwargs): + if not super().validate(): + return False + + result = True + + # Fail form validation when a body is set for a GET + if self.method.data == 'GET' and self.body.data: + self.body.errors.append('Body must be empty when Request Method is set to GET') + result = False + + return result class globalSettingsForm(commonSettingsForm): diff --git a/changedetectionio/store.py b/changedetectionio/store.py index 5728cbb5f..60f3d8261 100644 --- a/changedetectionio/store.py +++ b/changedetectionio/store.py @@ -70,6 +70,8 @@ class ChangeDetectionStore: 'previous_md5': "", 'uuid': str(uuid_builder.uuid4()), 'headers': {}, # Extra headers to send + 'body': None, + 'method': 'GET', 'history': {}, # Dict of timestamp and output stripped filename 'ignore_text': [], # List of text to ignore when calculating the comparison checksum # Custom notification content diff --git a/changedetectionio/templates/edit.html b/changedetectionio/templates/edit.html index c4f6ee623..4b62bcd96 100644 --- a/changedetectionio/templates/edit.html +++ b/changedetectionio/templates/edit.html @@ -9,6 +9,7 @@
-
- {{ render_field(form.headers, rows=5, placeholder="Example -Cookie: foobar -User-Agent: wonderbra 1.0") }} - - Note: ONLY used by Basic fast Plaintext/HTTP Client - -
{{ render_field(form.fetch_backend) }} @@ -62,6 +55,26 @@ User-Agent: wonderbra 1.0") }}
+
+ Note: These settings are ONLY used by Basic fast Plaintext/HTTP Client. +
+ {{ render_field(form.headers, rows=5, placeholder="Example +Cookie: foobar +User-Agent: wonderbra 1.0") }} +
+
+ {{ render_field(form.body, rows=5, placeholder="Example +{ + \"name\":\"John\", + \"age\":30, + \"car\":null +}") }} +
+
+ {{ render_field(form.method) }} +
+
+
Note: These settings override the global settings.
diff --git a/changedetectionio/tests/test_headers.py b/changedetectionio/tests/test_headers.py deleted file mode 100644 index 389439783..000000000 --- a/changedetectionio/tests/test_headers.py +++ /dev/null @@ -1,80 +0,0 @@ -import json -import time -from flask import url_for -from . util import set_original_response, set_modified_response, live_server_setup - -# Hard to just add more live server URLs when one test is already running (I think) -# So we add our test here (was in a different file) -def test_headers_in_request(client, live_server): - live_server_setup(live_server) - - # Add our URL to the import page - test_url = url_for('test_headers', _external=True) - - # Add the test URL twice, we will check - res = client.post( - url_for("import_page"), - data={"urls": test_url}, - follow_redirects=True - ) - assert b"1 Imported" in res.data - - res = client.post( - url_for("import_page"), - data={"urls": test_url}, - follow_redirects=True - ) - assert b"1 Imported" in res.data - - cookie_header = '_ga=GA1.2.1022228332; cookie-preferences=analytics:accepted;' - - - # Add some headers to a request - res = client.post( - url_for("edit_page", uuid="first"), - data={ - "url": test_url, - "tag": "", - "fetch_backend": "html_requests", - "headers": "xxx:ooo\ncool:yeah\r\ncookie:"+cookie_header}, - follow_redirects=True - ) - assert b"Updated watch." in res.data - - - # Give the thread time to pick up the first version - time.sleep(5) - - # The service should echo back the request headers - res = client.get( - url_for("preview_page", uuid="first"), - follow_redirects=True - ) - - # Flask will convert the header key to uppercase - assert b"Xxx:ooo" in res.data - assert b"Cool:yeah" in res.data - - # The test call service will return the headers as the body - from html import escape - assert escape(cookie_header).encode('utf-8') in res.data - - time.sleep(5) - - # Re #137 - Examine the JSON index file, it should have only one set of headers entered - watches_with_headers = 0 - with open('test-datastore/url-watches.json') as f: - app_struct = json.load(f) - for uuid in app_struct['watching']: - if (len(app_struct['watching'][uuid]['headers'])): - watches_with_headers += 1 - - # Should be only one with headers set - assert watches_with_headers==1 - - - - - - - diff --git a/changedetectionio/tests/test_request.py b/changedetectionio/tests/test_request.py new file mode 100644 index 000000000..ab613d8fa --- /dev/null +++ b/changedetectionio/tests/test_request.py @@ -0,0 +1,211 @@ +import json +import time +from flask import url_for +from . util import set_original_response, set_modified_response, live_server_setup + +def test_setup(live_server): + live_server_setup(live_server) + +# Hard to just add more live server URLs when one test is already running (I think) +# So we add our test here (was in a different file) +def test_headers_in_request(client, live_server): + # Add our URL to the import page + test_url = url_for('test_headers', _external=True) + + # Add the test URL twice, we will check + res = client.post( + url_for("import_page"), + data={"urls": test_url}, + follow_redirects=True + ) + assert b"1 Imported" in res.data + + res = client.post( + url_for("import_page"), + data={"urls": test_url}, + follow_redirects=True + ) + assert b"1 Imported" in res.data + + cookie_header = '_ga=GA1.2.1022228332; cookie-preferences=analytics:accepted;' + + + # Add some headers to a request + res = client.post( + url_for("edit_page", uuid="first"), + data={ + "url": test_url, + "tag": "", + "fetch_backend": "html_requests", + "headers": "xxx:ooo\ncool:yeah\r\ncookie:"+cookie_header}, + follow_redirects=True + ) + assert b"Updated watch." in res.data + + + # Give the thread time to pick up the first version + time.sleep(5) + + # The service should echo back the request headers + res = client.get( + url_for("preview_page", uuid="first"), + follow_redirects=True + ) + + # Flask will convert the header key to uppercase + assert b"Xxx:ooo" in res.data + assert b"Cool:yeah" in res.data + + # The test call service will return the headers as the body + from html import escape + assert escape(cookie_header).encode('utf-8') in res.data + + time.sleep(5) + + # Re #137 - Examine the JSON index file, it should have only one set of headers entered + watches_with_headers = 0 + with open('test-datastore/url-watches.json') as f: + app_struct = json.load(f) + for uuid in app_struct['watching']: + if (len(app_struct['watching'][uuid]['headers'])): + watches_with_headers += 1 + + # Should be only one with headers set + assert watches_with_headers==1 + +def test_body_in_request(client, live_server): + # Add our URL to the import page + test_url = url_for('test_body', _external=True) + + # Add the test URL twice, we will check + res = client.post( + url_for("import_page"), + data={"urls": test_url}, + follow_redirects=True + ) + assert b"1 Imported" in res.data + + res = client.post( + url_for("import_page"), + data={"urls": test_url}, + follow_redirects=True + ) + assert b"1 Imported" in res.data + + body_value = 'Test Body Value' + + # Attempt to add a body with a GET method + res = client.post( + url_for("edit_page", uuid="first"), + data={ + "url": test_url, + "tag": "", + "method": "GET", + "fetch_backend": "html_requests", + "body": "invalid"}, + follow_redirects=True + ) + assert b"Body must be empty when Request Method is set to GET" in res.data + + # Add a properly formatted body with a proper method + res = client.post( + url_for("edit_page", uuid="first"), + data={ + "url": test_url, + "tag": "", + "method": "POST", + "fetch_backend": "html_requests", + "body": body_value}, + follow_redirects=True + ) + assert b"Updated watch." in res.data + + # Give the thread time to pick up the first version + time.sleep(5) + + # The service should echo back the body + res = client.get( + url_for("preview_page", uuid="first"), + follow_redirects=True + ) + + # Check if body returned contains the specified data + assert str.encode(body_value) in res.data + + watches_with_body = 0 + with open('test-datastore/url-watches.json') as f: + app_struct = json.load(f) + for uuid in app_struct['watching']: + if app_struct['watching'][uuid]['body']==body_value: + watches_with_body += 1 + + # Should be only one with body set + assert watches_with_body==1 + +def test_method_in_request(client, live_server): + # Add our URL to the import page + test_url = url_for('test_method', _external=True) + + # Add the test URL twice, we will check + res = client.post( + url_for("import_page"), + data={"urls": test_url}, + follow_redirects=True + ) + assert b"1 Imported" in res.data + + res = client.post( + url_for("import_page"), + data={"urls": test_url}, + follow_redirects=True + ) + assert b"1 Imported" in res.data + + # Attempt to add a method which is not valid + res = client.post( + url_for("edit_page", uuid="first"), + data={ + "url": test_url, + "tag": "", + "fetch_backend": "html_requests", + "method": "invalid"}, + follow_redirects=True + ) + assert b"Not a valid choice" in res.data + + # Add a properly formatted body + res = client.post( + url_for("edit_page", uuid="first"), + data={ + "url": test_url, + "tag": "", + "fetch_backend": "html_requests", + "method": "PATCH"}, + follow_redirects=True + ) + assert b"Updated watch." in res.data + + # Give the thread time to pick up the first version + time.sleep(5) + + # The service should echo back the request verb + res = client.get( + url_for("preview_page", uuid="first"), + follow_redirects=True + ) + + # The test call service will return the verb as the body + assert b"PATCH" in res.data + + time.sleep(5) + + watches_with_method = 0 + with open('test-datastore/url-watches.json') as f: + app_struct = json.load(f) + for uuid in app_struct['watching']: + if app_struct['watching'][uuid]['method'] == 'PATCH': + watches_with_method += 1 + + # Should be only one with method set to PATCH + assert watches_with_method == 1 + diff --git a/changedetectionio/tests/util.py b/changedetectionio/tests/util.py index a24e6ae37..80eb98207 100644 --- a/changedetectionio/tests/util.py +++ b/changedetectionio/tests/util.py @@ -56,6 +56,21 @@ def live_server_setup(live_server): return "\n".join(output) + # Just return the body in the request + @live_server.app.route('/test-body', methods=['POST', 'GET']) + def test_body(): + + from flask import request + + return request.data + + # Just return the verb in the request + @live_server.app.route('/test-method', methods=['POST', 'GET', 'PATCH']) + def test_method(): + + from flask import request + + return request.method # Where we POST to as a notification @live_server.app.route('/test_notification_endpoint', methods=['POST', 'GET']) From d602cf46467d7c877186772804edb5bae1bbb087 Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Wed, 29 Dec 2021 23:28:34 +0100 Subject: [PATCH 06/10] Aligning call signatures #325 --- changedetectionio/content_fetcher.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/changedetectionio/content_fetcher.py b/changedetectionio/content_fetcher.py index b2a2d7948..69713b2cd 100644 --- a/changedetectionio/content_fetcher.py +++ b/changedetectionio/content_fetcher.py @@ -23,7 +23,7 @@ class Fetcher(): return self.error @abstractmethod - def run(self, url, timeout, request_headers): + def run(self, url, timeout, request_headers, request_body, request_method): # Should set self.error, self.status_code and self.content pass @@ -87,7 +87,9 @@ class html_webdriver(Fetcher): if proxy_args: self.proxy = SeleniumProxy(raw=proxy_args) - def run(self, url, timeout, request_headers): + def run(self, url, timeout, request_headers, request_body, request_method): + + # request_body, request_method unused for now, until some magic in the future happens. # check env for WEBDRIVER_URL driver = webdriver.Remote( From cf9554b169011853e2a005ab3b60f89010555f88 Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Wed, 29 Dec 2021 23:31:53 +0100 Subject: [PATCH 07/10] Move 'request type' field to the new 'Requests' tab --- changedetectionio/templates/edit.html | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/changedetectionio/templates/edit.html b/changedetectionio/templates/edit.html index 4b62bcd96..aeb8c6148 100644 --- a/changedetectionio/templates/edit.html +++ b/changedetectionio/templates/edit.html @@ -42,13 +42,6 @@ href="{{ url_for('settings_page', uuid=uuid) }}">default global settings. {% endif %}
-
- {{ render_field(form.fetch_backend) }} - -

Use the Basic method (default) where your watched sites don't need Javascript to render.

-

The Chrome/Javascript method requires a network connection to a running WebDriver+Chrome server, set by the ENV var 'WEBDRIVER_URL'.

- -
{{ render_field(form.extract_title_as_title) }}
@@ -56,8 +49,16 @@
- Note: These settings are ONLY used by Basic fast Plaintext/HTTP Client. +
+ {{ render_field(form.fetch_backend) }} + +

Use the Basic method (default) where your watched sites don't need Javascript to render.

+

The Chrome/Javascript method requires a network connection to a running WebDriver+Chrome server, set by the ENV var 'WEBDRIVER_URL'.

+ +
+
+ Note: Request Headers and Body settings are ONLY used by Basic fast Plaintext/HTTP Client. {{ render_field(form.headers, rows=5, placeholder="Example Cookie: foobar User-Agent: wonderbra 1.0") }} From d6a6ef2c1deba8d531c4174d458152696035bf5e Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Wed, 29 Dec 2021 23:37:04 +0100 Subject: [PATCH 08/10] Unify Filters and Triggers tabs into a single tab --- changedetectionio/templates/edit.html | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/changedetectionio/templates/edit.html b/changedetectionio/templates/edit.html index aeb8c6148..f30c07059 100644 --- a/changedetectionio/templates/edit.html +++ b/changedetectionio/templates/edit.html @@ -10,9 +10,8 @@
@@ -52,13 +51,16 @@
{{ render_field(form.fetch_backend) }} -

Use the Basic method (default) where your watched sites don't need Javascript to render.

+

Use the Basic method (default) where your watched site doesn't need Javascript to render.

The Chrome/Javascript method requires a network connection to a running WebDriver+Chrome server, set by the ENV var 'WEBDRIVER_URL'.

- Note: Request Headers and Body settings are ONLY used by Basic fast Plaintext/HTTP Client. +
+ {{ render_field(form.method) }} +
+ Note: Request Headers and Body settings are ONLY used by Basic fast Plaintext/HTTP Client fetch method. {{ render_field(form.headers, rows=5, placeholder="Example Cookie: foobar User-Agent: wonderbra 1.0") }} @@ -71,9 +73,7 @@ User-Agent: wonderbra 1.0") }} \"car\":null }") }} -
- {{ render_field(form.method) }} -
+
@@ -85,7 +85,7 @@ User-Agent: wonderbra 1.0") }}
-
+
{{ render_field(form.css_filter, placeholder=".class-name or #some-id, or other CSS selector rule.", @@ -112,9 +112,6 @@ User-Agent: wonderbra 1.0") }}
-
- -
{{ render_field(form.trigger_text, rows=5, placeholder="Some text to wait for in a line @@ -127,6 +124,7 @@ User-Agent: wonderbra 1.0") }}
+
From f1933b786c3f04682c8b6721f8454c0b2c217084 Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Wed, 29 Dec 2021 23:57:30 +0100 Subject: [PATCH 09/10] RSS Link links you back to the difference UI/JS page, RSS Description is the page you're watching, and RSS Title is the page you're watching --- changedetectionio/__init__.py | 15 +++++++++++++-- changedetectionio/templates/settings.html | 2 +- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/changedetectionio/__init__.py b/changedetectionio/__init__.py index 4b8127589..ea01e39a0 100644 --- a/changedetectionio/__init__.py +++ b/changedetectionio/__init__.py @@ -268,9 +268,20 @@ def changedetection_app(config=None, datastore_o=None): # @todo In the future make this a configurable link back (see work on BASE_URL https://github.com/dgtlmoon/changedetection.io/pull/228) guid = "{}/{}".format(watch['uuid'], watch['last_changed']) fe = fg.add_entry() + + + # Include a link to the diff page, they will have to login here to see if password protection is enabled. + # Description is the page you watch, link takes you to the diff JS UI page + base_url = datastore.data['settings']['application']['base_url'] + if base_url == '': + base_url = "" + diff_link = "{}{}".format(base_url, url_for('diff_history_page', uuid=watch['uuid'])) + + # @todo use title if it exists + fe.link(diff_link) fe.title(watch['url']) - fe.link(href=watch['url']) - fe.description(watch['url']) + fe.description(href=watch['url']) + fe.guid(guid, permalink=False) dt = datetime.datetime.fromtimestamp(int(watch['newest_history_key'])) dt = dt.replace(tzinfo=pytz.UTC) diff --git a/changedetectionio/templates/settings.html b/changedetectionio/templates/settings.html index b684c94fa..3a048cc45 100644 --- a/changedetectionio/templates/settings.html +++ b/changedetectionio/templates/settings.html @@ -36,7 +36,7 @@ {{ render_field(form.base_url, placeholder="http://yoursite.com:5000/", class="m-d") }} - Base URL used for the {base_url} token in notifications, default value is the ENV var 'BASE_URL' (Currently "{{current_base_url}}"), + Base URL used for the {base_url} token in notifications and RSS links.
Default value is the ENV var 'BASE_URL' (Currently "{{current_base_url}}"), read more here.
From a37a5038d8f93ef68034ab7d563d127dc84d1b7c Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Thu, 30 Dec 2021 00:04:38 +0100 Subject: [PATCH 10/10] Fix broken RSS link fields --- changedetectionio/__init__.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/changedetectionio/__init__.py b/changedetectionio/__init__.py index ea01e39a0..98f1a9548 100644 --- a/changedetectionio/__init__.py +++ b/changedetectionio/__init__.py @@ -275,12 +275,15 @@ def changedetection_app(config=None, datastore_o=None): base_url = datastore.data['settings']['application']['base_url'] if base_url == '': base_url = "" - diff_link = "{}{}".format(base_url, url_for('diff_history_page', uuid=watch['uuid'])) - + + diff_link = {'href': "{}{}".format(base_url, url_for('diff_history_page', uuid=watch['uuid']))} + # @todo use title if it exists - fe.link(diff_link) - fe.title(watch['url']) - fe.description(href=watch['url']) + fe.link(link=diff_link) + fe.title(title=watch['url']) + + # @todo in the future Any code html is valid.]]> + fe.description(description=watch['url']) fe.guid(guid, permalink=False) dt = datetime.datetime.fromtimestamp(int(watch['newest_history_key']))