diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 000000000..ef12c87a7 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,5 @@ +Contributing is always welcome! + +I am no professional flask developer, if you know a better way that something can be done, please let me know! + +Otherwise, it's always best to PR into the `dev` branch. diff --git a/README.md b/README.md index 0c1320103..22cf50de7 100644 --- a/README.md +++ b/README.md @@ -18,19 +18,20 @@ Open source web page monitoring, notification and change detection. #### Example use cases -Know when ... - Products and services have a change in pricing - Governmental department updates (changes are often only on their websites) - New software releases, security advisories when you're not on their mailing list. - Festivals with changes - Realestate listing changes - COVID related news from government websites +- University/organisation news from their website - Detect and monitor changes in JSON API responses - API monitoring and alerting - Trigger API calls via notifications when text appears on a website - Glue together APIs using the JSON filter and JSON notifications - Create RSS feeds based on changes in web content - +- You have a very sensitive list of URLs to watch and you do _not_ want to use the paid alternatives. (Remember, _you_ are the product) + _Need an actual Chrome runner with Javascript support? We support fetching via WebDriver!_ **Get monitoring now! super simple.** @@ -136,24 +137,9 @@ When you enable a `json:` filter, you can even automatically extract and parse e `json:$.price` would give `23.50`, or you can extract the whole structure -### Proxy +### Proxy configuration -A proxy for ChangeDetection.io can be configured by setting environment the -`HTTP_PROXY`, `HTTPS_PROXY` variables, examples are also in the `docker-compose.yml` - -`NO_PROXY` exclude list can be specified by following `"localhost,192.168.0.0/24"` - -as `docker run` with `-e` - -``` -docker run -d --restart always -e HTTPS_PROXY="socks5h://10.10.1.10:1080" -p "127.0.0.1:5000:5000" -v datastore-volume:/datastore --name changedetection.io dgtlmoon/changedetection.io -``` - -With `docker-compose`, see the `Proxy support example` in docker-compose.yml. - -For more information see https://docs.python-requests.org/en/master/user/advanced/#proxies - -This proxy support also extends to the notifications https://github.com/caronc/apprise/issues/387#issuecomment-841718867 +See the wiki https://github.com/dgtlmoon/changedetection.io/wiki/Proxy-configuration ### RaspberriPi support? diff --git a/changedetectionio/__init__.py b/changedetectionio/__init__.py index 3be55b5f2..ac2d205a6 100644 --- a/changedetectionio/__init__.py +++ b/changedetectionio/__init__.py @@ -268,9 +268,23 @@ def changedetection_app(config=None, datastore_o=None): # @todo In the future make this a configurable link back (see work on BASE_URL https://github.com/dgtlmoon/changedetection.io/pull/228) guid = "{}/{}".format(watch['uuid'], watch['last_changed']) fe = fg.add_entry() - fe.title(watch['url']) - fe.link(href=watch['url']) - fe.description(watch['url']) + + + # Include a link to the diff page, they will have to login here to see if password protection is enabled. + # Description is the page you watch, link takes you to the diff JS UI page + base_url = datastore.data['settings']['application']['base_url'] + if base_url == '': + base_url = "" + + diff_link = {'href': "{}{}".format(base_url, url_for('diff_history_page', uuid=watch['uuid']))} + + # @todo use title if it exists + fe.link(link=diff_link) + fe.title(title=watch['url']) + + # @todo in the future Any code html is valid.]]> + fe.description(description=watch['url']) + fe.guid(guid, permalink=False) dt = datetime.datetime.fromtimestamp(int(watch['newest_history_key'])) dt = dt.replace(tzinfo=pytz.UTC) @@ -454,6 +468,8 @@ def changedetection_app(config=None, datastore_o=None): 'tag': form.tag.data.strip(), 'title': form.title.data.strip(), 'headers': form.headers.data, + 'body': form.body.data, + 'method': form.method.data, 'fetch_backend': form.fetch_backend.data, 'trigger_text': form.trigger_text.data, 'notification_title': form.notification_title.data, diff --git a/changedetectionio/content_fetcher.py b/changedetectionio/content_fetcher.py index 1f6ef14ae..69713b2cd 100644 --- a/changedetectionio/content_fetcher.py +++ b/changedetectionio/content_fetcher.py @@ -3,6 +3,7 @@ import time from abc import ABC, abstractmethod from selenium import webdriver from selenium.webdriver.common.desired_capabilities import DesiredCapabilities +from selenium.webdriver.common.proxy import Proxy as SeleniumProxy from selenium.common.exceptions import WebDriverException import urllib3.exceptions @@ -22,7 +23,7 @@ class Fetcher(): return self.error @abstractmethod - def run(self, url, timeout, request_headers): + def run(self, url, timeout, request_headers, request_body, request_method): # Should set self.error, self.status_code and self.content pass @@ -65,15 +66,36 @@ class html_webdriver(Fetcher): command_executor = '' - def __init__(self): - self.command_executor = os.getenv("WEBDRIVER_URL", 'http://browser-chrome:4444/wd/hub') + # Configs for Proxy setup + # In the ENV vars, is prefixed with "webdriver_", so it is for example "webdriver_sslProxy" + selenium_proxy_settings_mappings = ['ftpProxy', 'httpProxy', 'noProxy', + 'proxyAutoconfigUrl', 'sslProxy', 'autodetect', + 'socksProxy', 'socksUsername', 'socksPassword'] + proxy=None - def run(self, url, timeout, request_headers): + def __init__(self): + # .strip('"') is going to save someone a lot of time when they accidently wrap the env value + self.command_executor = os.getenv("WEBDRIVER_URL", 'http://browser-chrome:4444/wd/hub').strip('"') + + # If any proxy settings are enabled, then we should setup the proxy object + proxy_args = {} + for k in self.selenium_proxy_settings_mappings: + v = os.getenv('webdriver_' + k, False) + if v: + proxy_args[k] = v.strip('"') + + if proxy_args: + self.proxy = SeleniumProxy(raw=proxy_args) + + def run(self, url, timeout, request_headers, request_body, request_method): + + # request_body, request_method unused for now, until some magic in the future happens. # check env for WEBDRIVER_URL driver = webdriver.Remote( command_executor=self.command_executor, - desired_capabilities=DesiredCapabilities.CHROME) + desired_capabilities=DesiredCapabilities.CHROME, + proxy=self.proxy) try: driver.get(url) @@ -111,10 +133,12 @@ class html_webdriver(Fetcher): class html_requests(Fetcher): fetcher_description = "Basic fast Plaintext/HTTP Client" - def run(self, url, timeout, request_headers): + def run(self, url, timeout, request_headers, request_body, request_method): import requests - r = requests.get(url, + r = requests.request(method=request_method, + data=request_body, + url=url, headers=request_headers, timeout=timeout, verify=False) diff --git a/changedetectionio/fetch_site_status.py b/changedetectionio/fetch_site_status.py index fc8c1e6e1..69ff7de02 100644 --- a/changedetectionio/fetch_site_status.py +++ b/changedetectionio/fetch_site_status.py @@ -80,6 +80,8 @@ class perform_site_check(): else: timeout = self.datastore.data['settings']['requests']['timeout'] url = self.datastore.get_val(uuid, 'url') + request_body = self.datastore.get_val(uuid, 'body') + request_method = self.datastore.get_val(uuid, 'method') # Pluggable content fetcher prefer_backend = watch['fetch_backend'] @@ -91,7 +93,7 @@ class perform_site_check(): fetcher = klass() - fetcher.run(url, timeout, request_headers) + fetcher.run(url, timeout, request_headers, request_body, request_method) # Fetching complete, now filters # @todo move to class / maybe inside of fetcher abstract base? diff --git a/changedetectionio/forms.py b/changedetectionio/forms.py index bc6501656..dc06c67a0 100644 --- a/changedetectionio/forms.py +++ b/changedetectionio/forms.py @@ -8,6 +8,16 @@ import re from changedetectionio.notification import default_notification_format, valid_notification_formats, default_notification_body, default_notification_title +valid_method = { + 'GET', + 'POST', + 'PUT', + 'PATCH', + 'DELETE', +} + +default_method = 'GET' + class StringListField(StringField): widget = widgets.TextArea() @@ -106,10 +116,12 @@ class ValidateContentFetcherIsReady(object): except urllib3.exceptions.MaxRetryError as e: driver_url = some_object.command_executor message = field.gettext('Content fetcher \'%s\' did not respond.' % (field.data)) - message += '
'+field.gettext('Be sure that the selenium/webdriver runner is running and accessible via network from this container/host.') + message += '
' + field.gettext( + 'Be sure that the selenium/webdriver runner is running and accessible via network from this container/host.') message += '
' + field.gettext('Did you follow the instructions in the wiki?') message += '

' + field.gettext('WebDriver Host: %s' % (driver_url)) message += '
Go here for more information' + message += '
'+field.gettext('Content fetcher did not respond properly, unable to use it.\n %s' % (str(e))) raise ValidationError(message) @@ -222,8 +234,22 @@ class watchForm(commonSettingsForm): ignore_text = StringListField('Ignore Text', [ValidateListRegex()]) headers = StringDictKeyValue('Request Headers') + body = TextAreaField('Request Body', [validators.Optional()]) + method = SelectField('Request Method', choices=valid_method, default=default_method) trigger_text = StringListField('Trigger/wait for text', [validators.Optional(), ValidateListRegex()]) + def validate(self, **kwargs): + if not super().validate(): + return False + + result = True + + # Fail form validation when a body is set for a GET + if self.method.data == 'GET' and self.body.data: + self.body.errors.append('Body must be empty when Request Method is set to GET') + result = False + + return result class globalSettingsForm(commonSettingsForm): diff --git a/changedetectionio/store.py b/changedetectionio/store.py index 31fc689c6..23795a6f0 100644 --- a/changedetectionio/store.py +++ b/changedetectionio/store.py @@ -70,6 +70,8 @@ class ChangeDetectionStore: 'previous_md5': "", 'uuid': str(uuid_builder.uuid4()), 'headers': {}, # Extra headers to send + 'body': None, + 'method': 'GET', 'history': {}, # Dict of timestamp and output stripped filename 'ignore_text': [], # List of text to ignore when calculating the comparison checksum # Custom notification content diff --git a/changedetectionio/templates/edit.html b/changedetectionio/templates/edit.html index c4f6ee623..f30c07059 100644 --- a/changedetectionio/templates/edit.html +++ b/changedetectionio/templates/edit.html @@ -9,9 +9,9 @@
@@ -41,27 +41,41 @@ href="{{ url_for('settings_page', uuid=uuid) }}">default global settings. {% endif %} -
- {{ render_field(form.headers, rows=5, placeholder="Example -Cookie: foobar -User-Agent: wonderbra 1.0") }} - - Note: ONLY used by Basic fast Plaintext/HTTP Client - -
-
- {{ render_field(form.fetch_backend) }} - -

Use the Basic method (default) where your watched sites don't need Javascript to render.

-

The Chrome/Javascript method requires a network connection to a running WebDriver+Chrome server, set by the ENV var 'WEBDRIVER_URL'.

- -
{{ render_field(form.extract_title_as_title) }}
+
+
+ {{ render_field(form.fetch_backend) }} + +

Use the Basic method (default) where your watched site doesn't need Javascript to render.

+

The Chrome/Javascript method requires a network connection to a running WebDriver+Chrome server, set by the ENV var 'WEBDRIVER_URL'.

+ +
+ +
+
+ {{ render_field(form.method) }} +
+ Note: Request Headers and Body settings are ONLY used by Basic fast Plaintext/HTTP Client fetch method. + {{ render_field(form.headers, rows=5, placeholder="Example +Cookie: foobar +User-Agent: wonderbra 1.0") }} +
+
+ {{ render_field(form.body, rows=5, placeholder="Example +{ + \"name\":\"John\", + \"age\":30, + \"car\":null +}") }} +
+ +
+
Note: These settings override the global settings.
@@ -71,7 +85,7 @@ User-Agent: wonderbra 1.0") }}
-
+
{{ render_field(form.css_filter, placeholder=".class-name or #some-id, or other CSS selector rule.", @@ -98,9 +112,6 @@ User-Agent: wonderbra 1.0") }}
-
- -
{{ render_field(form.trigger_text, rows=5, placeholder="Some text to wait for in a line @@ -113,6 +124,7 @@ User-Agent: wonderbra 1.0") }}
+
diff --git a/changedetectionio/templates/settings.html b/changedetectionio/templates/settings.html index b684c94fa..3a048cc45 100644 --- a/changedetectionio/templates/settings.html +++ b/changedetectionio/templates/settings.html @@ -36,7 +36,7 @@ {{ render_field(form.base_url, placeholder="http://yoursite.com:5000/", class="m-d") }} - Base URL used for the {base_url} token in notifications, default value is the ENV var 'BASE_URL' (Currently "{{current_base_url}}"), + Base URL used for the {base_url} token in notifications and RSS links.
Default value is the ENV var 'BASE_URL' (Currently "{{current_base_url}}"), read more here.
diff --git a/changedetectionio/tests/test_headers.py b/changedetectionio/tests/test_headers.py deleted file mode 100644 index 389439783..000000000 --- a/changedetectionio/tests/test_headers.py +++ /dev/null @@ -1,80 +0,0 @@ -import json -import time -from flask import url_for -from . util import set_original_response, set_modified_response, live_server_setup - -# Hard to just add more live server URLs when one test is already running (I think) -# So we add our test here (was in a different file) -def test_headers_in_request(client, live_server): - live_server_setup(live_server) - - # Add our URL to the import page - test_url = url_for('test_headers', _external=True) - - # Add the test URL twice, we will check - res = client.post( - url_for("import_page"), - data={"urls": test_url}, - follow_redirects=True - ) - assert b"1 Imported" in res.data - - res = client.post( - url_for("import_page"), - data={"urls": test_url}, - follow_redirects=True - ) - assert b"1 Imported" in res.data - - cookie_header = '_ga=GA1.2.1022228332; cookie-preferences=analytics:accepted;' - - - # Add some headers to a request - res = client.post( - url_for("edit_page", uuid="first"), - data={ - "url": test_url, - "tag": "", - "fetch_backend": "html_requests", - "headers": "xxx:ooo\ncool:yeah\r\ncookie:"+cookie_header}, - follow_redirects=True - ) - assert b"Updated watch." in res.data - - - # Give the thread time to pick up the first version - time.sleep(5) - - # The service should echo back the request headers - res = client.get( - url_for("preview_page", uuid="first"), - follow_redirects=True - ) - - # Flask will convert the header key to uppercase - assert b"Xxx:ooo" in res.data - assert b"Cool:yeah" in res.data - - # The test call service will return the headers as the body - from html import escape - assert escape(cookie_header).encode('utf-8') in res.data - - time.sleep(5) - - # Re #137 - Examine the JSON index file, it should have only one set of headers entered - watches_with_headers = 0 - with open('test-datastore/url-watches.json') as f: - app_struct = json.load(f) - for uuid in app_struct['watching']: - if (len(app_struct['watching'][uuid]['headers'])): - watches_with_headers += 1 - - # Should be only one with headers set - assert watches_with_headers==1 - - - - - - - diff --git a/changedetectionio/tests/test_request.py b/changedetectionio/tests/test_request.py new file mode 100644 index 000000000..ab613d8fa --- /dev/null +++ b/changedetectionio/tests/test_request.py @@ -0,0 +1,211 @@ +import json +import time +from flask import url_for +from . util import set_original_response, set_modified_response, live_server_setup + +def test_setup(live_server): + live_server_setup(live_server) + +# Hard to just add more live server URLs when one test is already running (I think) +# So we add our test here (was in a different file) +def test_headers_in_request(client, live_server): + # Add our URL to the import page + test_url = url_for('test_headers', _external=True) + + # Add the test URL twice, we will check + res = client.post( + url_for("import_page"), + data={"urls": test_url}, + follow_redirects=True + ) + assert b"1 Imported" in res.data + + res = client.post( + url_for("import_page"), + data={"urls": test_url}, + follow_redirects=True + ) + assert b"1 Imported" in res.data + + cookie_header = '_ga=GA1.2.1022228332; cookie-preferences=analytics:accepted;' + + + # Add some headers to a request + res = client.post( + url_for("edit_page", uuid="first"), + data={ + "url": test_url, + "tag": "", + "fetch_backend": "html_requests", + "headers": "xxx:ooo\ncool:yeah\r\ncookie:"+cookie_header}, + follow_redirects=True + ) + assert b"Updated watch." in res.data + + + # Give the thread time to pick up the first version + time.sleep(5) + + # The service should echo back the request headers + res = client.get( + url_for("preview_page", uuid="first"), + follow_redirects=True + ) + + # Flask will convert the header key to uppercase + assert b"Xxx:ooo" in res.data + assert b"Cool:yeah" in res.data + + # The test call service will return the headers as the body + from html import escape + assert escape(cookie_header).encode('utf-8') in res.data + + time.sleep(5) + + # Re #137 - Examine the JSON index file, it should have only one set of headers entered + watches_with_headers = 0 + with open('test-datastore/url-watches.json') as f: + app_struct = json.load(f) + for uuid in app_struct['watching']: + if (len(app_struct['watching'][uuid]['headers'])): + watches_with_headers += 1 + + # Should be only one with headers set + assert watches_with_headers==1 + +def test_body_in_request(client, live_server): + # Add our URL to the import page + test_url = url_for('test_body', _external=True) + + # Add the test URL twice, we will check + res = client.post( + url_for("import_page"), + data={"urls": test_url}, + follow_redirects=True + ) + assert b"1 Imported" in res.data + + res = client.post( + url_for("import_page"), + data={"urls": test_url}, + follow_redirects=True + ) + assert b"1 Imported" in res.data + + body_value = 'Test Body Value' + + # Attempt to add a body with a GET method + res = client.post( + url_for("edit_page", uuid="first"), + data={ + "url": test_url, + "tag": "", + "method": "GET", + "fetch_backend": "html_requests", + "body": "invalid"}, + follow_redirects=True + ) + assert b"Body must be empty when Request Method is set to GET" in res.data + + # Add a properly formatted body with a proper method + res = client.post( + url_for("edit_page", uuid="first"), + data={ + "url": test_url, + "tag": "", + "method": "POST", + "fetch_backend": "html_requests", + "body": body_value}, + follow_redirects=True + ) + assert b"Updated watch." in res.data + + # Give the thread time to pick up the first version + time.sleep(5) + + # The service should echo back the body + res = client.get( + url_for("preview_page", uuid="first"), + follow_redirects=True + ) + + # Check if body returned contains the specified data + assert str.encode(body_value) in res.data + + watches_with_body = 0 + with open('test-datastore/url-watches.json') as f: + app_struct = json.load(f) + for uuid in app_struct['watching']: + if app_struct['watching'][uuid]['body']==body_value: + watches_with_body += 1 + + # Should be only one with body set + assert watches_with_body==1 + +def test_method_in_request(client, live_server): + # Add our URL to the import page + test_url = url_for('test_method', _external=True) + + # Add the test URL twice, we will check + res = client.post( + url_for("import_page"), + data={"urls": test_url}, + follow_redirects=True + ) + assert b"1 Imported" in res.data + + res = client.post( + url_for("import_page"), + data={"urls": test_url}, + follow_redirects=True + ) + assert b"1 Imported" in res.data + + # Attempt to add a method which is not valid + res = client.post( + url_for("edit_page", uuid="first"), + data={ + "url": test_url, + "tag": "", + "fetch_backend": "html_requests", + "method": "invalid"}, + follow_redirects=True + ) + assert b"Not a valid choice" in res.data + + # Add a properly formatted body + res = client.post( + url_for("edit_page", uuid="first"), + data={ + "url": test_url, + "tag": "", + "fetch_backend": "html_requests", + "method": "PATCH"}, + follow_redirects=True + ) + assert b"Updated watch." in res.data + + # Give the thread time to pick up the first version + time.sleep(5) + + # The service should echo back the request verb + res = client.get( + url_for("preview_page", uuid="first"), + follow_redirects=True + ) + + # The test call service will return the verb as the body + assert b"PATCH" in res.data + + time.sleep(5) + + watches_with_method = 0 + with open('test-datastore/url-watches.json') as f: + app_struct = json.load(f) + for uuid in app_struct['watching']: + if app_struct['watching'][uuid]['method'] == 'PATCH': + watches_with_method += 1 + + # Should be only one with method set to PATCH + assert watches_with_method == 1 + diff --git a/changedetectionio/tests/util.py b/changedetectionio/tests/util.py index a24e6ae37..80eb98207 100644 --- a/changedetectionio/tests/util.py +++ b/changedetectionio/tests/util.py @@ -56,6 +56,21 @@ def live_server_setup(live_server): return "\n".join(output) + # Just return the body in the request + @live_server.app.route('/test-body', methods=['POST', 'GET']) + def test_body(): + + from flask import request + + return request.data + + # Just return the verb in the request + @live_server.app.route('/test-method', methods=['POST', 'GET', 'PATCH']) + def test_method(): + + from flask import request + + return request.method # Where we POST to as a notification @live_server.app.route('/test_notification_endpoint', methods=['POST', 'GET']) diff --git a/docker-compose.yml b/docker-compose.yml index d06de4025..1060ed581 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -13,13 +13,23 @@ services: # - PUID=1000 # - PGID=1000 - # # Alternative WebDriver/selenium URL, do not use "'s or 's! + # + # Alternative WebDriver/selenium URL, do not use "'s or 's! # - WEBDRIVER_URL=http://browser-chrome:4444/wd/hub - # Proxy support example. + # + # WebDriver proxy settings webdriver_ftpProxy, webdriver_httpProxy, webdriver_noProxy, + # webdriver_proxyAutoconfigUrl, webdriver_sslProxy, webdriver_autodetect, + # webdriver_socksProxy, webdriver_socksUsername, webdriver_socksPassword + # + # https://selenium-python.readthedocs.io/api.html#module-selenium.webdriver.common.proxy + # + # Plain requsts - proxy support example. # - HTTP_PROXY=socks5h://10.10.1.10:1080 # - HTTPS_PROXY=socks5h://10.10.1.10:1080 + # # An exclude list (useful for notification URLs above) can be specified by with # - NO_PROXY="localhost,192.168.0.0/24" + # # Base URL of your changedetection.io install (Added to the notification alert) # - BASE_URL=https://mysite.com