diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 1b8274141..a45f05976 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -30,11 +30,11 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@v4 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@v1 + uses: github/codeql-action/init@v2 with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. @@ -45,7 +45,7 @@ jobs: # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild - uses: github/codeql-action/autobuild@v1 + uses: github/codeql-action/autobuild@v2 # ℹ️ Command-line programs to run using the OS shell. # 📚 https://git.io/JvXDl @@ -59,4 +59,4 @@ jobs: # make release - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v1 + uses: github/codeql-action/analyze@v2 diff --git a/.github/workflows/containers.yml b/.github/workflows/containers.yml index 9c358d4fa..9a3136c09 100644 --- a/.github/workflows/containers.yml +++ b/.github/workflows/containers.yml @@ -39,9 +39,9 @@ jobs: # Or if we are in a tagged release scenario. if: ${{ github.event.workflow_run.conclusion == 'success' }} || ${{ github.event.release.tag_name }} != '' steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up Python 3.9 - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: 3.9 @@ -58,27 +58,27 @@ jobs: echo ${{ github.ref }} > changedetectionio/tag.txt - name: Set up QEMU - uses: docker/setup-qemu-action@v1 + uses: docker/setup-qemu-action@v3 with: image: tonistiigi/binfmt:latest platforms: all - name: Login to GitHub Container Registry - uses: docker/login-action@v1 + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Login to Docker Hub Container Registry - uses: docker/login-action@v1 + uses: docker/login-action@v3 with: username: ${{ secrets.DOCKER_HUB_USERNAME }} password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} - name: Set up Docker Buildx id: buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v3 with: install: true version: latest @@ -88,7 +88,7 @@ jobs: - name: Build and push :dev id: docker_build if: ${{ github.ref }} == "refs/heads/master" - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v5 with: context: ./ file: ./Dockerfile @@ -105,7 +105,7 @@ jobs: - name: Build and push :tag id: docker_build_tag_release if: github.event_name == 'release' && startsWith(github.event.release.tag_name, '0.') - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v5 with: context: ./ file: ./Dockerfile @@ -125,7 +125,7 @@ jobs: run: echo step SHA ${{ steps.vars.outputs.sha_short }} tag ${{steps.vars.outputs.tag}} branch ${{steps.vars.outputs.branch}} digest ${{ steps.docker_build.outputs.digest }} - name: Cache Docker layers - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: /tmp/.buildx-cache key: ${{ runner.os }}-buildx-${{ github.sha }} diff --git a/.github/workflows/test-container-build.yml b/.github/workflows/test-container-build.yml index 53c91aa79..0c0ce6b2d 100644 --- a/.github/workflows/test-container-build.yml +++ b/.github/workflows/test-container-build.yml @@ -24,22 +24,22 @@ jobs: test-container-build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up Python 3.9 - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: 3.9 # Just test that the build works, some libraries won't compile on ARM/rPi etc - name: Set up QEMU - uses: docker/setup-qemu-action@v1 + uses: docker/setup-qemu-action@v3 with: image: tonistiigi/binfmt:latest platforms: all - name: Set up Docker Buildx id: buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v3 with: install: true version: latest @@ -49,7 +49,7 @@ jobs: # Check we can still build under alpine/musl - name: Test that the docker containers can build (musl via alpine check) id: docker_build_musl - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v5 with: context: ./ file: ./.github/test/Dockerfile-alpine @@ -57,7 +57,7 @@ jobs: - name: Test that the docker containers can build id: docker_build - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v5 # https://github.com/docker/build-push-action#customizing with: context: ./ diff --git a/.github/workflows/test-only.yml b/.github/workflows/test-only.yml index 9acfb3f88..003cb1d22 100644 --- a/.github/workflows/test-only.yml +++ b/.github/workflows/test-only.yml @@ -7,11 +7,11 @@ jobs: test-application: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 # Mainly just for link/flake8 - name: Set up Python 3.10 - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: '3.10' diff --git a/.github/workflows/test-pip-build.yml b/.github/workflows/test-pip-build.yml index cf9d94189..61d94e13c 100644 --- a/.github/workflows/test-pip-build.yml +++ b/.github/workflows/test-pip-build.yml @@ -11,10 +11,10 @@ jobs: test-pip-build-basics: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up Python 3.9 - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: 3.9 diff --git a/README.md b/README.md index 700f4ba3c..68e113295 100644 --- a/README.md +++ b/README.md @@ -226,7 +226,7 @@ The application also supports notifying you that it can follow this information ## Proxy Configuration -See the wiki https://github.com/dgtlmoon/changedetection.io/wiki/Proxy-configuration , we also support using [BrightData proxy services where possible]( https://github.com/dgtlmoon/changedetection.io/wiki/Proxy-configuration#brightdata-proxy-support) +See the wiki https://github.com/dgtlmoon/changedetection.io/wiki/Proxy-configuration , we also support using [Bright Data proxy services where possible]( https://github.com/dgtlmoon/changedetection.io/wiki/Proxy-configuration#brightdata-proxy-support) ## Raspberry Pi support? diff --git a/changedetectionio/__init__.py b/changedetectionio/__init__.py index c9e12f07a..ea6baab87 100644 --- a/changedetectionio/__init__.py +++ b/changedetectionio/__init__.py @@ -38,7 +38,9 @@ from flask_paginate import Pagination, get_page_parameter from changedetectionio import html_tools from changedetectionio.api import api_v1 -__version__ = '0.45.1' +__version__ = '0.45.2' + +from changedetectionio.store import BASE_URL_NOT_SET_TEXT datastore = None @@ -356,12 +358,10 @@ def changedetection_app(config=None, datastore_o=None): # Include a link to the diff page, they will have to login here to see if password protection is enabled. # Description is the page you watch, link takes you to the diff JS UI page # Dict val base_url will get overriden with the env var if it is set. - ext_base_url = datastore.data['settings']['application'].get('base_url') - if ext_base_url: - # Go with overriden value - diff_link = {'href': "{}{}".format(ext_base_url, url_for('diff_history_page', uuid=watch['uuid'], _external=False))} - else: - diff_link = {'href': url_for('diff_history_page', uuid=watch['uuid'], _external=True)} + ext_base_url = datastore.data['settings']['application'].get('active_base_url') + + # Because we are called via whatever web server, flask should figure out the right path ( + diff_link = {'href': url_for('diff_history_page', uuid=watch['uuid'], _external=True)} fe.link(link=diff_link) @@ -714,7 +714,6 @@ def changedetection_app(config=None, datastore_o=None): output = render_template("edit.html", available_processors=processors.available_processors(), browser_steps_config=browser_step_ui_config, - current_base_url=datastore.data['settings']['application']['base_url'], emailprefix=os.getenv('NOTIFICATION_MAIL_BUTTON_PREFIX', False), form=form, has_default_notification_urls=True if len(datastore.data['settings']['application']['notification_urls']) else False, @@ -804,7 +803,6 @@ def changedetection_app(config=None, datastore_o=None): output = render_template("settings.html", form=form, - current_base_url = datastore.data['settings']['application']['base_url'], hide_remove_pass=os.getenv("SALTED_PASS", False), api_key=datastore.data['settings']['application'].get('api_access_token'), emailprefix=os.getenv('NOTIFICATION_MAIL_BUTTON_PREFIX', False), @@ -1270,10 +1268,10 @@ def changedetection_app(config=None, datastore_o=None): update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid, 'skip_when_checksum_same': False})) i = 1 - elif tag != None: + elif tag: # Items that have this current tag for watch_uuid, watch in datastore.data['watching'].items(): - if (tag != None and tag in watch.get('tags', {})): + if tag in watch.get('tags', {}): if watch_uuid not in running_uuids and not datastore.data['watching'][watch_uuid]['paused']: update_q.put( queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': watch_uuid, 'skip_when_checksum_same': False}) diff --git a/changedetectionio/blueprint/check_proxies/__init__.py b/changedetectionio/blueprint/check_proxies/__init__.py index bb5ed9b21..60c98436c 100644 --- a/changedetectionio/blueprint/check_proxies/__init__.py +++ b/changedetectionio/blueprint/check_proxies/__init__.py @@ -57,9 +57,11 @@ def construct_blueprint(datastore: ChangeDetectionStore): status.update({'status': 'ERROR OTHER', 'length': len(contents), 'text': f"Got empty reply with code {e.status_code} - Access denied"}) else: status.update({'status': 'ERROR OTHER', 'length': len(contents) if contents else 0, 'text': f"Empty reply with code {e.status_code}, needs chrome?"}) - + except content_fetcher.ReplyWithContentButNoText as e: + txt = f"Got reply but with no content - Status code {e.status_code} - It's possible that the filters were found, but contained no usable text (or contained only an image)." + status.update({'status': 'ERROR', 'text': txt}) except Exception as e: - status.update({'status': 'ERROR OTHER', 'length': len(contents) if contents else 0, 'text': 'Error: '+str(e)}) + status.update({'status': 'ERROR OTHER', 'length': len(contents) if contents else 0, 'text': 'Error: '+type(e).__name__+str(e)}) else: status.update({'status': 'OK', 'length': len(contents), 'text': ''}) diff --git a/changedetectionio/content_fetcher.py b/changedetectionio/content_fetcher.py index db9e23dc5..dab956a5a 100644 --- a/changedetectionio/content_fetcher.py +++ b/changedetectionio/content_fetcher.py @@ -77,11 +77,13 @@ class ScreenshotUnavailable(Exception): class ReplyWithContentButNoText(Exception): - def __init__(self, status_code, url, screenshot=None): + def __init__(self, status_code, url, screenshot=None, has_filters=False, html_content=''): # Set this so we can use it in other parts of the app self.status_code = status_code self.url = url self.screenshot = screenshot + self.has_filters = has_filters + self.html_content = html_content return @@ -343,8 +345,8 @@ class base_html_playwright(Fetcher): 'req_headers': request_headers, 'screenshot_quality': int(os.getenv("PLAYWRIGHT_SCREENSHOT_QUALITY", 72)), 'url': url, - 'user_agent': request_headers.get('User-Agent', 'Mozilla/5.0'), - 'proxy_username': self.proxy.get('username','') if self.proxy else False, + 'user_agent': {k.lower(): v for k, v in request_headers.items()}.get('user-agent', None), + 'proxy_username': self.proxy.get('username', '') if self.proxy else False, 'proxy_password': self.proxy.get('password', '') if self.proxy else False, 'no_cache_list': [ 'twitter', @@ -443,7 +445,7 @@ class base_html_playwright(Fetcher): # Set user agent to prevent Cloudflare from blocking the browser # Use the default one configured in the App.py model that's passed from fetch_site_status.py context = browser.new_context( - user_agent=request_headers.get('User-Agent', 'Mozilla/5.0'), + user_agent={k.lower(): v for k, v in request_headers.items()}.get('user-agent', None), proxy=self.proxy, # This is needed to enable JavaScript execution on GitHub and others bypass_csp=True, @@ -684,7 +686,7 @@ class html_requests(Fetcher): is_binary=False): # Make requests use a more modern looking user-agent - if not 'User-Agent' in request_headers: + if not {k.lower(): v for k, v in request_headers.items()}.get('user-agent', None): request_headers['User-Agent'] = os.getenv("DEFAULT_SETTINGS_HEADERS_USERAGENT", 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.66 Safari/537.36') diff --git a/changedetectionio/forms.py b/changedetectionio/forms.py index 4725b1811..33cea7a50 100644 --- a/changedetectionio/forms.py +++ b/changedetectionio/forms.py @@ -229,16 +229,19 @@ class ValidateJinja2Template(object): def __call__(self, form, field): from changedetectionio import notification - from jinja2 import Environment, BaseLoader, TemplateSyntaxError + from jinja2 import Environment, BaseLoader, TemplateSyntaxError, UndefinedError from jinja2.meta import find_undeclared_variables try: jinja2_env = Environment(loader=BaseLoader) jinja2_env.globals.update(notification.valid_tokens) + rendered = jinja2_env.from_string(field.data).render() except TemplateSyntaxError as e: raise ValidationError(f"This is not a valid Jinja2 template: {e}") from e + except UndefinedError as e: + raise ValidationError(f"A variable or function is not defined: {e}") from e ast = jinja2_env.parse(field.data) undefined = ", ".join(find_undeclared_variables(ast)) @@ -502,7 +505,10 @@ class globalSettingsRequestForm(Form): class globalSettingsApplicationForm(commonSettingsForm): api_access_token_enabled = BooleanField('API access token security check enabled', default=True, validators=[validators.Optional()]) - base_url = StringField('Base URL', validators=[validators.Optional()]) + base_url = StringField('Notification base URL override', + validators=[validators.Optional()], + render_kw={"placeholder": os.getenv('BASE_URL', 'Not set')} + ) empty_pages_are_a_change = BooleanField('Treat empty pages as a change?', default=False) fetch_backend = RadioField('Fetch Method', default="html_requests", choices=content_fetcher.available_fetchers(), validators=[ValidateContentFetcherIsReady()]) global_ignore_text = StringListField('Ignore Text', [ValidateListRegex()]) diff --git a/changedetectionio/notification.py b/changedetectionio/notification.py index 27c7afa71..6c0f53f65 100644 --- a/changedetectionio/notification.py +++ b/changedetectionio/notification.py @@ -208,15 +208,11 @@ def create_notification_parameters(n_object, datastore): watch_tag = '' # Create URLs to customise the notification with - base_url = datastore.data['settings']['application']['base_url'] + # active_base_url - set in store.py data property + base_url = datastore.data['settings']['application'].get('active_base_url') watch_url = n_object['watch_url'] - # Re #148 - Some people have just {{ base_url }} in the body or title, but this may break some notification services - # like 'Join', so it's always best to atleast set something obvious so that they are not broken. - if base_url == '': - base_url = "" - diff_url = "{}/diff/{}".format(base_url, uuid) preview_url = "{}/preview/{}".format(base_url, uuid) @@ -226,7 +222,7 @@ def create_notification_parameters(n_object, datastore): # Valid_tokens also used as a field validator tokens.update( { - 'base_url': base_url if base_url is not None else '', + 'base_url': base_url, 'current_snapshot': n_object['current_snapshot'] if 'current_snapshot' in n_object else '', 'diff': n_object.get('diff', ''), # Null default in the case we use a test 'diff_added': n_object.get('diff_added', ''), # Null default in the case we use a test diff --git a/changedetectionio/processors/text_json_diff.py b/changedetectionio/processors/text_json_diff.py index 0ae4aca0c..1bf31fa4f 100644 --- a/changedetectionio/processors/text_json_diff.py +++ b/changedetectionio/processors/text_json_diff.py @@ -314,7 +314,12 @@ class perform_site_check(difference_detection_processor): # Treat pages with no renderable text content as a change? No by default empty_pages_are_a_change = self.datastore.data['settings']['application'].get('empty_pages_are_a_change', False) if not is_json and not empty_pages_are_a_change and len(stripped_text_from_html.strip()) == 0: - raise content_fetcher.ReplyWithContentButNoText(url=url, status_code=fetcher.get_last_status_code(), screenshot=screenshot) + raise content_fetcher.ReplyWithContentButNoText(url=url, + status_code=fetcher.get_last_status_code(), + screenshot=screenshot, + has_filters=has_filter_rule, + html_content=html_content + ) # We rely on the actual text in the html output.. many sites have random script vars etc, # in the future we'll implement other mechanisms. diff --git a/changedetectionio/res/puppeteer_fetch.js b/changedetectionio/res/puppeteer_fetch.js index d376729f9..6015fbe81 100644 --- a/changedetectionio/res/puppeteer_fetch.js +++ b/changedetectionio/res/puppeteer_fetch.js @@ -18,7 +18,9 @@ module.exports = async ({page, context}) => { await page.setBypassCSP(true) await page.setExtraHTTPHeaders(req_headers); - await page.setUserAgent(user_agent); + if (user_agent) { + await page.setUserAgent(user_agent); + } // https://ourcodeworld.com/articles/read/1106/how-to-solve-puppeteer-timeouterror-navigation-timeout-of-30000-ms-exceeded await page.setDefaultNavigationTimeout(0); diff --git a/changedetectionio/res/stock-not-in-stock.js b/changedetectionio/res/stock-not-in-stock.js index c0ca9b2f6..7023fb9b2 100644 --- a/changedetectionio/res/stock-not-in-stock.js +++ b/changedetectionio/res/stock-not-in-stock.js @@ -5,14 +5,19 @@ function isItemInStock() { 'agotado', 'artikel zurzeit vergriffen', 'as soon as stock is available', + 'ausverkauft', // sold out 'available for back order', + 'back-order or out of stock', 'backordered', + 'benachrichtigt mich', // notify me 'brak na stanie', 'brak w magazynie', 'coming soon', 'currently have any tickets for this', 'currently unavailable', + 'dostępne wkrótce', 'en rupture de stock', + 'ist derzeit nicht auf lager', 'item is no longer available', 'message if back in stock', 'nachricht bei', @@ -37,6 +42,7 @@ function isItemInStock() { 'unavailable tickets', 'we do not currently have an estimate of when this product will be back in stock.', 'zur zeit nicht an lager', + '已售完', ]; diff --git a/changedetectionio/static/js/browser-steps.js b/changedetectionio/static/js/browser-steps.js index f2214e4a8..5540ef5c8 100644 --- a/changedetectionio/static/js/browser-steps.js +++ b/changedetectionio/static/js/browser-steps.js @@ -208,7 +208,7 @@ $(document).ready(function () { console.log(x); if (x && first_available.length) { // @todo will it let you click shit that has a layer ontop? probably not. - if (x['tagtype'] === 'text' || x['tagtype'] === 'email' || x['tagName'] === 'textarea' || x['tagtype'] === 'password' || x['tagtype'] === 'search') { + if (x['tagtype'] === 'text' || x['tagtype'] === 'number' || x['tagtype'] === 'email' || x['tagName'] === 'textarea' || x['tagtype'] === 'password' || x['tagtype'] === 'search') { $('select', first_available).val('Enter text in field').change(); $('input[type=text]', first_available).first().val(x['xpath']); $('input[placeholder="Value"]', first_available).addClass('ok').click().focus(); diff --git a/changedetectionio/static/js/global-settings.js b/changedetectionio/static/js/global-settings.js index dc7818bb5..16ba33e1a 100644 --- a/changedetectionio/static/js/global-settings.js +++ b/changedetectionio/static/js/global-settings.js @@ -32,5 +32,10 @@ $(document).ready(function () { window.getSelection().removeAllRanges(); }); + + $("#notification-token-toggle").click(function (e) { + e.preventDefault(); + $('#notification-tokens-info').toggle(); + }); }); diff --git a/changedetectionio/static/js/watch-settings.js b/changedetectionio/static/js/watch-settings.js index 353b23a33..0b419959b 100644 --- a/changedetectionio/static/js/watch-settings.js +++ b/changedetectionio/static/js/watch-settings.js @@ -42,4 +42,8 @@ $(document).ready(function () { $('#notification_urls').val(''); e.preventDefault(); }); + $("#notification-token-toggle").click(function (e) { + e.preventDefault(); + $('#notification-tokens-info').toggle(); + }); }); diff --git a/changedetectionio/static/styles/scss/parts/_browser-steps.scss b/changedetectionio/static/styles/scss/parts/_browser-steps.scss index d43720cc4..4d64ce38c 100644 --- a/changedetectionio/static/styles/scss/parts/_browser-steps.scss +++ b/changedetectionio/static/styles/scss/parts/_browser-steps.scss @@ -44,7 +44,7 @@ #browser-steps .flex-wrapper { display: flex; flex-flow: row; - height: 600px; /*@todo make this dynamic */ + height: 70vh; } /* this is duplicate :( */ diff --git a/changedetectionio/static/styles/styles.css b/changedetectionio/static/styles/styles.css index c3e5620d2..709ab6368 100644 --- a/changedetectionio/static/styles/styles.css +++ b/changedetectionio/static/styles/styles.css @@ -50,8 +50,7 @@ #browser-steps .flex-wrapper { display: flex; flex-flow: row; - height: 600px; - /*@todo make this dynamic */ } + height: 70vh; } /* this is duplicate :( */ #browsersteps-selector-wrapper { diff --git a/changedetectionio/store.py b/changedetectionio/store.py index 388756d7b..9ac2218b9 100644 --- a/changedetectionio/store.py +++ b/changedetectionio/store.py @@ -18,6 +18,9 @@ import threading import time import uuid as uuid_builder +# Because the server will run as a daemon and wont know the URL for notification links when firing off a notification +BASE_URL_NOT_SET_TEXT = '("Base URL" not set - see settings - notifications)' + dictfilt = lambda x, y: dict([ (i,x[i]) for i in x if i in set(y) ]) # Is there an existing library to ensure some data store (JSON etc) is in sync with CRUD methods? @@ -175,12 +178,21 @@ class ChangeDetectionStore: @property def data(self): - # Re #152, Return env base_url if not overriden, @todo also prefer the proxy pass url - env_base_url = os.getenv('BASE_URL','') - if not self.__data['settings']['application']['base_url']: - self.__data['settings']['application']['base_url'] = env_base_url.strip('" ') + # Re #152, Return env base_url if not overriden + # Re #148 - Some people have just {{ base_url }} in the body or title, but this may break some notification services + # like 'Join', so it's always best to atleast set something obvious so that they are not broken. - return self.__data + active_base_url = BASE_URL_NOT_SET_TEXT + if self.__data['settings']['application'].get('base_url'): + active_base_url = self.__data['settings']['application'].get('base_url') + elif os.getenv('BASE_URL'): + active_base_url = os.getenv('BASE_URL') + + # I looked at various ways todo the following, but in the end just copying the dict seemed simplest/most reliable + # even given the memory tradeoff - if you know a better way.. maybe return d|self.__data.. or something + d = self.__data + d['settings']['application']['active_base_url'] = active_base_url.strip('" ') + return d # Delete a single watch by UUID def delete(self, uuid): @@ -327,6 +339,9 @@ class ChangeDetectionStore: if k in apply_extras: del apply_extras[k] + if not apply_extras.get('date_created'): + apply_extras['date_created'] = int(time.time()) + new_watch.update(apply_extras) new_watch.ensure_data_dir_exists() self.__data['watching'][new_uuid] = new_watch diff --git a/changedetectionio/templates/_common_fields.jinja b/changedetectionio/templates/_common_fields.jinja index 6b6f6e05a..84a7b0678 100644 --- a/changedetectionio/templates/_common_fields.jinja +++ b/changedetectionio/templates/_common_fields.jinja @@ -13,9 +13,9 @@
@@ -35,18 +35,14 @@
{{ render_field(form.notification_body , rows=5, class="notification-body", placeholder=settings_application['notification_body']) }} - Body for all notifications -
-
- - {{ render_field(form.notification_format , class="notification-format") }} - Format for all notifications + Body for all notifications ‐ You can use Jinja2 templating in the notification title, body and URL, and tokens from below. + +
-

- You can use Jinja2 templating in the notification title, body and URL. -

- +
Show token/placeholders
+
+ +
+ {{ render_field(form.notification_format , class="notification-format") }} + Format for all notifications +
{% endmacro %} diff --git a/changedetectionio/templates/settings.html b/changedetectionio/templates/settings.html index 1d09faa5f..7bca51fed 100644 --- a/changedetectionio/templates/settings.html +++ b/changedetectionio/templates/settings.html @@ -62,14 +62,6 @@ Allow access to view watch diff page when password is enabled (Good for sharing the diff page) -
- {{ render_field(form.application.form.base_url, placeholder="http://yoursite.com:5000/", - class="m-d") }} - - Base URL used for the {{ '{{ base_url }}' }} token in notifications and RSS links.
Default value is the ENV var 'BASE_URL' (Currently "{{settings_application['current_base_url']}}"), - read more here. - -
{{ render_field(form.application.form.pager_size) }} Number of items per page in the watch overview list, 0 to disable. @@ -100,6 +92,13 @@ {{ render_common_settings_form(form.application.form, emailprefix, settings_application) }}
+
+ {{ render_field(form.application.form.base_url, class="m-d") }} + + Base URL used for the {{ '{{ base_url }}' }} token in notification links.
+ Default value is the system environment variable 'BASE_URL' - read more here. + +
diff --git a/changedetectionio/templates/watch-overview.html b/changedetectionio/templates/watch-overview.html index 968bfc800..687012c72 100644 --- a/changedetectionio/templates/watch-overview.html +++ b/changedetectionio/templates/watch-overview.html @@ -119,6 +119,9 @@ Try adding external proxies/locations {% endif %} + {% if 'empty result or contain only an image' in watch.last_error %} + more help here. + {% endif %}
{% endif %} {% if watch.last_notification_error is defined and watch.last_notification_error != False %} diff --git a/changedetectionio/tests/test_backup.py b/changedetectionio/tests/test_backup.py index 89fd66a5d..f03fcf374 100644 --- a/changedetectionio/tests/test_backup.py +++ b/changedetectionio/tests/test_backup.py @@ -1,6 +1,6 @@ #!/usr/bin/python3 -from .util import set_original_response, set_modified_response, live_server_setup +from .util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks from flask import url_for from urllib.request import urlopen from zipfile import ZipFile @@ -19,12 +19,12 @@ def test_backup(client, live_server): # Add our URL to the import page res = client.post( url_for("import_page"), - data={"urls": url_for('test_endpoint', _external=True)}, + data={"urls": url_for('test_endpoint', _external=True)+"?somechar=őőőőőőőő"}, follow_redirects=True ) assert b"1 Imported" in res.data - time.sleep(3) + wait_for_all_checks(client) res = client.get( url_for("get_backup"), diff --git a/changedetectionio/tests/test_css_selector.py b/changedetectionio/tests/test_css_selector.py index 0dfe2af78..dcc103318 100644 --- a/changedetectionio/tests/test_css_selector.py +++ b/changedetectionio/tests/test_css_selector.py @@ -2,7 +2,7 @@ import time from flask import url_for -from . util import live_server_setup +from .util import live_server_setup, wait_for_all_checks from ..html_tools import * @@ -176,3 +176,77 @@ def test_check_multiple_filters(client, live_server): assert b"Blob A" in res.data # CSS was ok assert b"Blob B" in res.data # xPath was ok assert b"Blob C" not in res.data # Should not be included + +# The filter exists, but did not contain anything useful +# Mainly used when the filter contains just an IMG, this can happen when someone selects an image in the visual-selector +# Tests fetcher can throw a "ReplyWithContentButNoText" exception after applying filter and extracting text +def test_filter_is_empty_help_suggestion(client, live_server): + #live_server_setup(live_server) + + include_filters = "#blob-a" + + with open("test-datastore/endpoint-content.txt", "w") as f: + f.write(""" +
+ +
+ + + """) + + + # Add our URL to the import page + test_url = url_for('test_endpoint', _external=True) + res = client.post( + url_for("import_page"), + data={"urls": test_url}, + follow_redirects=True + ) + assert b"1 Imported" in res.data + wait_for_all_checks(client) + + # Goto the edit page, add our ignore text + # Add our URL to the import page + res = client.post( + url_for("edit_page", uuid="first"), + data={"include_filters": include_filters, + "url": test_url, + "tags": "", + "headers": "", + 'fetch_backend': "html_requests"}, + follow_redirects=True + ) + assert b"Updated watch." in res.data + + wait_for_all_checks(client) + + + res = client.get( + url_for("index"), + follow_redirects=True + ) + + assert b'empty result or contain only an image' in res.data + + + ### Just an empty selector, no image + + with open("test-datastore/endpoint-content.txt", "w") as f: + f.write(""" +
+ +
+ + + """) + + res = client.get(url_for("form_watch_checknow"), follow_redirects=True) + wait_for_all_checks(client) + + res = client.get( + url_for("index"), + follow_redirects=True + ) + + assert b'empty result or contain only an image' not in res.data + assert b'but contained no usable text' in res.data diff --git a/changedetectionio/update_worker.py b/changedetectionio/update_worker.py index 63a0aab42..1cfc0a84b 100644 --- a/changedetectionio/update_worker.py +++ b/changedetectionio/update_worker.py @@ -3,7 +3,7 @@ import threading import queue import time -from changedetectionio import content_fetcher +from changedetectionio import content_fetcher, html_tools from .processors.text_json_diff import FilterNotFoundInResponse from .processors.restock_diff import UnableToExtractRestockData @@ -251,7 +251,20 @@ class update_worker(threading.Thread): # Totally fine, it's by choice - just continue on, nothing more to care about # Page had elements/content but no renderable text # Backend (not filters) gave zero output - self.datastore.update_watch(uuid=uuid, update_obj={'last_error': "Got HTML content but no text found (With {} reply code).".format(e.status_code)}) + extra_help = "" + if e.has_filters: + # Maybe it contains an image? offer a more helpful link + has_img = html_tools.include_filters(include_filters='img', + html_content=e.html_content) + if has_img: + extra_help = ", it's possible that the filters you have give an empty result or contain only an image." + else: + extra_help = ", it's possible that the filters were found, but contained no usable text." + + self.datastore.update_watch(uuid=uuid, update_obj={ + 'last_error': f"Got HTML content but no text found (With {e.status_code} reply code){extra_help}" + }) + if e.screenshot: self.datastore.save_screenshot(watch_uuid=uuid, screenshot=e.screenshot) process_changedetection_results = False