diff --git a/changedetectionio/__init__.py b/changedetectionio/__init__.py index 8ec6bb8da..074928516 100644 --- a/changedetectionio/__init__.py +++ b/changedetectionio/__init__.py @@ -175,6 +175,7 @@ def main(): # proxy_set_header Host "localhost"; # proxy_set_header X-Forwarded-Prefix /app; + if os.getenv('USE_X_SETTINGS'): logger.info("USE_X_SETTINGS is ENABLED") from werkzeug.middleware.proxy_fix import ProxyFix diff --git a/changedetectionio/content_fetchers/puppeteer.py b/changedetectionio/content_fetchers/puppeteer.py index cad1b6b85..725be3b35 100644 --- a/changedetectionio/content_fetchers/puppeteer.py +++ b/changedetectionio/content_fetchers/puppeteer.py @@ -9,7 +9,6 @@ from loguru import logger from changedetectionio.content_fetchers.base import Fetcher, manage_user_agent from changedetectionio.content_fetchers.exceptions import PageUnloadable, Non200ErrorCodeReceived, EmptyReply, BrowserFetchTimedOut, BrowserConnectError - class fetcher(Fetcher): fetcher_description = "Puppeteer/direct {}/Javascript".format( os.getenv("PLAYWRIGHT_BROWSER_TYPE", 'chromium').capitalize() @@ -93,15 +92,39 @@ class fetcher(Fetcher): ignoreHTTPSErrors=True ) except websockets.exceptions.InvalidStatusCode as e: - raise BrowserConnectError(msg=f"Error while trying to connect the browser, Code {e.status_code} (check your access)") + raise BrowserConnectError(msg=f"Error while trying to connect the browser, Code {e.status_code} (check your access, whitelist IP, password etc)") except websockets.exceptions.InvalidURI: raise BrowserConnectError(msg=f"Error connecting to the browser, check your browser connection address (should be ws:// or wss://") except Exception as e: raise BrowserConnectError(msg=f"Error connecting to the browser {str(e)}") - else: - self.page = await browser.newPage() - await self.page.setUserAgent(manage_user_agent(headers=request_headers, current_ua=await self.page.evaluate('navigator.userAgent'))) + # Better is to launch chrome with the URL as arg + # non-headless - newPage() will launch an extra tab/window, .browser should already contain 1 page/tab + # headless - ask a new page + self.page = (pages := await browser.pages) and len(pages) or await browser.newPage() + + try: + from pyppeteerstealth import inject_evasions_into_page + except ImportError: + logger.debug("pyppeteerstealth module not available, skipping") + pass + else: + # I tried hooking events via self.page.on(Events.Page.DOMContentLoaded, inject_evasions_requiring_obj_to_page) + # But I could never get it to fire reliably, so we just inject it straight after + await inject_evasions_into_page(self.page) + + # This user agent is similar to what was used when tweaking the evasions in inject_evasions_into_page(..) + user_agent = None + if request_headers: + user_agent = next((value for key, value in request_headers.items() if key.lower().strip() == 'user-agent'), None) + if user_agent: + await self.page.setUserAgent(user_agent) + # Remove it so it's not sent again with headers after + [request_headers.pop(key) for key in list(request_headers) if key.lower().strip() == 'user-agent'.lower().strip()] + + if not user_agent: + # Attempt to strip 'HeadlessChrome' etc + await self.page.setUserAgent(manage_user_agent(headers=request_headers, current_ua=await self.page.evaluate('navigator.userAgent'))) await self.page.setBypassCSP(True) if request_headers: diff --git a/changedetectionio/content_fetchers/requests.py b/changedetectionio/content_fetchers/requests.py index b743dbcec..2c28cda7c 100644 --- a/changedetectionio/content_fetchers/requests.py +++ b/changedetectionio/content_fetchers/requests.py @@ -30,11 +30,6 @@ class fetcher(Fetcher): if self.browser_steps_get_valid_steps(): raise BrowserStepsInUnsupportedFetcher(url=url) - # Make requests use a more modern looking user-agent - if not {k.lower(): v for k, v in request_headers.items()}.get('user-agent', None): - request_headers['User-Agent'] = os.getenv("DEFAULT_SETTINGS_HEADERS_USERAGENT", - 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.66 Safari/537.36') - proxies = {} # Allows override the proxy on a per-request basis diff --git a/changedetectionio/flask_app.py b/changedetectionio/flask_app.py index 49880b5df..41f80a773 100644 --- a/changedetectionio/flask_app.py +++ b/changedetectionio/flask_app.py @@ -338,8 +338,11 @@ def changedetection_app(config=None, datastore_o=None): # @todo needs a .itemsWithTag() or something - then we can use that in Jinaj2 and throw this away for uuid, watch in datastore.data['watching'].items(): + # @todo tag notification_muted skip also (improve Watch model) + if watch.get('notification_muted'): + continue if limit_tag and not limit_tag in watch['tags']: - continue + continue watch['uuid'] = uuid sorted_watches.append(watch) @@ -768,7 +771,7 @@ def changedetection_app(config=None, datastore_o=None): jq_support=jq_support, playwright_enabled=os.getenv('PLAYWRIGHT_DRIVER_URL', False), settings_application=datastore.data['settings']['application'], - using_global_webdriver_wait=default['webdriver_delay'] is None, + using_global_webdriver_wait=not default['webdriver_delay'], uuid=uuid, visualselector_enabled=visualselector_enabled, watch=watch @@ -1063,6 +1066,8 @@ def changedetection_app(config=None, datastore_o=None): content = [] ignored_line_numbers = [] trigger_line_numbers = [] + versions = [] + timestamp = None # More for testing, possible to return the first/only if uuid == 'first': @@ -1082,57 +1087,53 @@ def changedetection_app(config=None, datastore_o=None): if (watch.get('fetch_backend') == 'system' and system_uses_webdriver) or watch.get('fetch_backend') == 'html_webdriver' or watch.get('fetch_backend', '').startswith('extra_browser_'): is_html_webdriver = True - # Never requested successfully, but we detected a fetch error if datastore.data['watching'][uuid].history_n == 0 and (watch.get_error_text() or watch.get_error_snapshot()): flash("Preview unavailable - No fetch/check completed or triggers not reached", "error") - output = render_template("preview.html", - content=content, - history_n=watch.history_n, - extra_stylesheets=extra_stylesheets, -# current_diff_url=watch['url'], - watch=watch, - uuid=uuid, - is_html_webdriver=is_html_webdriver, - last_error=watch['last_error'], - last_error_text=watch.get_error_text(), - last_error_screenshot=watch.get_error_snapshot()) - return output + else: + # So prepare the latest preview or not + preferred_version = request.args.get('version') + versions = list(watch.history.keys()) + timestamp = versions[-1] + if preferred_version and preferred_version in versions: + timestamp = preferred_version - timestamp = list(watch.history.keys())[-1] - try: - tmp = watch.get_history_snapshot(timestamp).splitlines() + try: + versions = list(watch.history.keys()) + tmp = watch.get_history_snapshot(timestamp).splitlines() - # Get what needs to be highlighted - ignore_rules = watch.get('ignore_text', []) + datastore.data['settings']['application']['global_ignore_text'] + # Get what needs to be highlighted + ignore_rules = watch.get('ignore_text', []) + datastore.data['settings']['application']['global_ignore_text'] - # .readlines will keep the \n, but we will parse it here again, in the future tidy this up - ignored_line_numbers = html_tools.strip_ignore_text(content="\n".join(tmp), - wordlist=ignore_rules, - mode='line numbers' - ) + # .readlines will keep the \n, but we will parse it here again, in the future tidy this up + ignored_line_numbers = html_tools.strip_ignore_text(content="\n".join(tmp), + wordlist=ignore_rules, + mode='line numbers' + ) - trigger_line_numbers = html_tools.strip_ignore_text(content="\n".join(tmp), - wordlist=watch['trigger_text'], - mode='line numbers' - ) - # Prepare the classes and lines used in the template - i=0 - for l in tmp: - classes=[] - i+=1 - if i in ignored_line_numbers: - classes.append('ignored') - if i in trigger_line_numbers: - classes.append('triggered') - content.append({'line': l, 'classes': ' '.join(classes)}) + trigger_line_numbers = html_tools.strip_ignore_text(content="\n".join(tmp), + wordlist=watch['trigger_text'], + mode='line numbers' + ) + # Prepare the classes and lines used in the template + i=0 + for l in tmp: + classes=[] + i+=1 + if i in ignored_line_numbers: + classes.append('ignored') + if i in trigger_line_numbers: + classes.append('triggered') + content.append({'line': l, 'classes': ' '.join(classes)}) - except Exception as e: - content.append({'line': f"File doesnt exist or unable to read timestamp {timestamp}", 'classes': ''}) + except Exception as e: + content.append({'line': f"File doesnt exist or unable to read timestamp {timestamp}", 'classes': ''}) output = render_template("preview.html", content=content, + current_version=timestamp, history_n=watch.history_n, extra_stylesheets=extra_stylesheets, + extra_title=f" - Diff - {watch.label} @ {timestamp}", ignored_line_numbers=ignored_line_numbers, triggered_line_numbers=trigger_line_numbers, current_diff_url=watch['url'], @@ -1142,7 +1143,10 @@ def changedetection_app(config=None, datastore_o=None): is_html_webdriver=is_html_webdriver, last_error=watch['last_error'], last_error_text=watch.get_error_text(), - last_error_screenshot=watch.get_error_snapshot()) + last_error_screenshot=watch.get_error_snapshot(), + versions=versions + ) + return output diff --git a/changedetectionio/forms.py b/changedetectionio/forms.py index 2d64a2273..673be9caa 100644 --- a/changedetectionio/forms.py +++ b/changedetectionio/forms.py @@ -526,6 +526,10 @@ class SingleExtraBrowser(Form): browser_connection_url = StringField('Browser connection URL', [validators.Optional()], render_kw={"placeholder": "wss://brightdata... wss://oxylabs etc", "size":50}) # @todo do the validation here instead +class DefaultUAInputForm(Form): + html_requests = StringField('Plaintext requests', validators=[validators.Optional()], render_kw={"placeholder": ""}) + if os.getenv("PLAYWRIGHT_DRIVER_URL") or os.getenv("WEBDRIVER_URL"): + html_webdriver = StringField('Chrome requests', validators=[validators.Optional()], render_kw={"placeholder": ""}) # datastore.data['settings']['requests'].. class globalSettingsRequestForm(Form): @@ -537,6 +541,8 @@ class globalSettingsRequestForm(Form): extra_proxies = FieldList(FormField(SingleExtraProxy), min_entries=5) extra_browsers = FieldList(FormField(SingleExtraBrowser), min_entries=5) + default_ua = FormField(DefaultUAInputForm, label="Default User-Agent overrides") + def validate_extra_proxies(self, extra_validators=None): for e in self.data['extra_proxies']: if e.get('proxy_name') or e.get('proxy_url'): diff --git a/changedetectionio/model/App.py b/changedetectionio/model/App.py index 1202d5db1..75384f170 100644 --- a/changedetectionio/model/App.py +++ b/changedetectionio/model/App.py @@ -6,6 +6,7 @@ from changedetectionio.notification import ( ) _FILTER_FAILURE_THRESHOLD_ATTEMPTS_DEFAULT = 6 +DEFAULT_SETTINGS_HEADERS_USERAGENT='Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.66 Safari/537.36' class model(dict): base_config = { @@ -22,6 +23,10 @@ class model(dict): 'time_between_check': {'weeks': None, 'days': None, 'hours': 3, 'minutes': None, 'seconds': None}, 'timeout': int(getenv("DEFAULT_SETTINGS_REQUESTS_TIMEOUT", "45")), # Default 45 seconds 'workers': int(getenv("DEFAULT_SETTINGS_REQUESTS_WORKERS", "10")), # Number of threads, lower is better for slow connections + 'default_ua': { + 'html_requests': getenv("DEFAULT_SETTINGS_HEADERS_USERAGENT", DEFAULT_SETTINGS_HEADERS_USERAGENT), + 'html_webdriver': None, + } }, 'application': { # Custom notification content diff --git a/changedetectionio/notification.py b/changedetectionio/notification.py index 4fa357384..41285ce45 100644 --- a/changedetectionio/notification.py +++ b/changedetectionio/notification.py @@ -122,10 +122,6 @@ def process_notification(n_object, datastore): # Insert variables into the notification content notification_parameters = create_notification_parameters(n_object, datastore) - # Get the notification body from datastore - n_body = jinja_render(template_str=n_object.get('notification_body', ''), **notification_parameters) - n_title = jinja_render(template_str=n_object.get('notification_title', ''), **notification_parameters) - n_format = valid_notification_formats.get( n_object.get('notification_format', default_notification_format), valid_notification_formats[default_notification_format], @@ -151,6 +147,11 @@ def process_notification(n_object, datastore): with apprise.LogCapture(level=apprise.logging.DEBUG) as logs: for url in n_object['notification_urls']: + + # Get the notification body from datastore + n_body = jinja_render(template_str=n_object.get('notification_body', ''), **notification_parameters) + n_title = jinja_render(template_str=n_object.get('notification_title', ''), **notification_parameters) + url = url.strip() if not url: logger.warning(f"Process Notification: skipping empty notification URL.") diff --git a/changedetectionio/processors/__init__.py b/changedetectionio/processors/__init__.py index e2b544811..8702ee5d1 100644 --- a/changedetectionio/processors/__init__.py +++ b/changedetectionio/processors/__init__.py @@ -97,6 +97,10 @@ class difference_detection_processor(): request_headers.update(self.datastore.get_all_base_headers()) request_headers.update(self.datastore.get_all_headers_in_textfile_for_watch(uuid=self.watch.get('uuid'))) + ua = self.datastore.data['settings']['requests'].get('default_ua') + if ua and ua.get(prefer_fetch_backend): + request_headers.update({'User-Agent': ua.get(prefer_fetch_backend)}) + # https://github.com/psf/requests/issues/4525 # Requests doesnt yet support brotli encoding, so don't put 'br' here, be totally sure that the user cannot # do this by accident. diff --git a/changedetectionio/static/images/gradient-border.png b/changedetectionio/static/images/gradient-border.png deleted file mode 100644 index 4c7705f8c..000000000 Binary files a/changedetectionio/static/images/gradient-border.png and /dev/null differ diff --git a/changedetectionio/static/js/diff-overview.js b/changedetectionio/static/js/diff-overview.js index 767cf6e1f..95e6dd7a0 100644 --- a/changedetectionio/static/js/diff-overview.js +++ b/changedetectionio/static/js/diff-overview.js @@ -8,6 +8,13 @@ $(document).ready(function () { } }) + $('.needs-localtime').each(function () { + for (var option of this.options) { + var dateObject = new Date(option.value * 1000); + option.label = dateObject.toLocaleString(undefined, {dateStyle: "full", timeStyle: "medium"}); + } + }); + // Load it when the #screenshot tab is in use, so we dont give a slow experience when waiting for the text diff to load window.addEventListener('hashchange', function (e) { toggle(location.hash); diff --git a/changedetectionio/static/js/diff-render.js b/changedetectionio/static/js/diff-render.js index 53f1d68f5..ea69d364f 100644 --- a/changedetectionio/static/js/diff-render.js +++ b/changedetectionio/static/js/diff-render.js @@ -79,12 +79,7 @@ $(document).ready(function () { $('#jump-next-diff').click(); } - $('.needs-localtime').each(function () { - for (var option of this.options) { - var dateObject = new Date(option.value * 1000); - option.label = dateObject.toLocaleString(undefined, {dateStyle: "full", timeStyle: "medium"}); - } - }) + onDiffTypeChange( document.querySelector('#settings [name="diff_type"]:checked'), ); diff --git a/changedetectionio/static/js/preview.js b/changedetectionio/static/js/preview.js new file mode 100644 index 000000000..a9895cb21 --- /dev/null +++ b/changedetectionio/static/js/preview.js @@ -0,0 +1,49 @@ +function redirect_to_version(version) { + var currentUrl = window.location.href; + var baseUrl = currentUrl.split('?')[0]; // Base URL without query parameters + var anchor = ''; + + // Check if there is an anchor + if (baseUrl.indexOf('#') !== -1) { + anchor = baseUrl.substring(baseUrl.indexOf('#')); + baseUrl = baseUrl.substring(0, baseUrl.indexOf('#')); + } + window.location.href = baseUrl + '?version=' + version + anchor; +} + +document.addEventListener('keydown', function (event) { + var selectElement = document.getElementById('preview-version'); + if (selectElement) { + var selectedOption = selectElement.querySelector('option:checked'); + if (selectedOption) { + if (event.key === 'ArrowLeft') { + if (selectedOption.previousElementSibling) { + redirect_to_version(selectedOption.previousElementSibling.value); + } + } else if (event.key === 'ArrowRight') { + if (selectedOption.nextElementSibling) { + redirect_to_version(selectedOption.nextElementSibling.value); + } + } + } + } +}); + + +document.getElementById('preview-version').addEventListener('change', function () { + redirect_to_version(this.value); +}); + +var selectElement = document.getElementById('preview-version'); +if (selectElement) { + var selectedOption = selectElement.querySelector('option:checked'); + if (selectedOption) { + if (selectedOption.previousElementSibling) { + document.getElementById('btn-previous').href = "?version=" + selectedOption.previousElementSibling.value; + } + if (selectedOption.nextElementSibling) { + document.getElementById('btn-next').href = "?version=" + selectedOption.nextElementSibling.value; + } + + } +} diff --git a/changedetectionio/static/styles/scss/styles.scss b/changedetectionio/static/styles/scss/styles.scss index 55bcc4cbf..1c1e8b5b4 100644 --- a/changedetectionio/static/styles/scss/styles.scss +++ b/changedetectionio/static/styles/scss/styles.scss @@ -243,7 +243,6 @@ body::after { body::before { // background-image set in base.html so it works with reverse proxies etc content: ""; - background-size: cover } body:after, @@ -1083,6 +1082,9 @@ ul { li { list-style: none; font-size: 0.8rem; + > * { + display: inline-block; + } } } diff --git a/changedetectionio/static/styles/styles.css b/changedetectionio/static/styles/styles.css index c18d8d96e..b09d55990 100644 --- a/changedetectionio/static/styles/styles.css +++ b/changedetectionio/static/styles/styles.css @@ -574,8 +574,7 @@ body::after { opacity: 0.91; } body::before { - content: ""; - background-size: cover; } + content: ""; } body:after, body:before { @@ -1173,6 +1172,8 @@ ul { #quick-watch-processor-type ul li { list-style: none; font-size: 0.8rem; } + #quick-watch-processor-type ul li > * { + display: inline-block; } .restock-label { padding: 3px; diff --git a/changedetectionio/store.py b/changedetectionio/store.py index 884c617a1..afa6b2ae6 100644 --- a/changedetectionio/store.py +++ b/changedetectionio/store.py @@ -554,7 +554,6 @@ class ChangeDetectionStore: return os.path.isfile(filepath) def get_all_base_headers(self): - from .model.App import parse_headers_from_text_file headers = {} # Global app settings headers.update(self.data['settings'].get('headers', {})) diff --git a/changedetectionio/templates/base.html b/changedetectionio/templates/base.html index c4c664cb2..bbfe8634d 100644 --- a/changedetectionio/templates/base.html +++ b/changedetectionio/templates/base.html @@ -6,7 +6,7 @@ Change Detection{{extra_title}} - + {% if extra_stylesheets %} @@ -24,12 +24,6 @@ - - @@ -89,8 +83,8 @@
  • - - + + diff --git a/changedetectionio/templates/edit.html b/changedetectionio/templates/edit.html index f8c0eba4c..6d6d19fcd 100644 --- a/changedetectionio/templates/edit.html +++ b/changedetectionio/templates/edit.html @@ -433,7 +433,8 @@ Unavailable") }}
    {% if visualselector_enabled %} - The Visual Selector tool lets you select the text elements that will be used for the change detection ‐ after the Browser Steps has completed, this tool is a helper to manage filters in the "CSS/JSONPath/JQ/XPath Filters" box of the Filters & Triggers tab. + The Visual Selector tool lets you select the text elements that will be used for the change detection ‐ after the Browser Steps has completed.
    + This tool is a helper to manage filters in the "CSS/JSONPath/JQ/XPath Filters" box of the Filters & Triggers tab.
    diff --git a/changedetectionio/templates/preview.html b/changedetectionio/templates/preview.html index 5cc61bedc..8bc231e16 100644 --- a/changedetectionio/templates/preview.html +++ b/changedetectionio/templates/preview.html @@ -1,72 +1,103 @@ {% extends 'base.html' %} {% block content %} - - - - -
    -
      - {% if last_error_text %}
    • Error Text
      • {% endif %} - {% if last_error_screenshot %}
    • Error Screenshot
      • {% endif %} - {% if history_n > 0 %} -
    • Text
      • -
    • Screenshot
      • + + + + + {% if versions|length >= 2 %} +
      + +
      + + + +
      + +
      + Keyboard: + ← Previous   + → Next +
      + {% endif %} + +
      + +
      + + +
      +
      +
      {{ watch.error_text_ctime|format_seconds_ago }} seconds ago
      + 
                   {{ last_error_text }}
      +
      + +
      +
      {{ watch.snapshot_error_screenshot_ctime|format_seconds_ago }} seconds ago +
      + Current erroring screenshot from most recent request +
      + +
      +
      {{ watch.snapshot_text_ctime|format_timestamp_timeago }}
      + Grey lines are ignored Blue lines are triggers + Pro-tip: Highlight text to add to ignore filters + + + + + + + +
      + {% for row in content %} +
      {{ row.line }}
      + {% endfor %} +
      +
      + +
      +
      + For now, Differences are performed on text, not graphically, only the latest screenshot is available. +
      +
      + {% if is_html_webdriver %} + {% if screenshot %} +
      {{ watch.snapshot_screenshot_ctime|format_timestamp_timeago }}
      + Current screenshot from most recent request + {% else %} + No screenshot available just yet! Try rechecking the page. + {% endif %} + {% else %} + Screenshot requires Playwright/WebDriver enabled + {% endif %} +
      - -
      -
      {{watch.snapshot_error_screenshot_ctime|format_seconds_ago}} seconds ago
      - Current erroring screenshot from most recent request -
      - -
      -
      {{watch.snapshot_text_ctime|format_timestamp_timeago}}
      - Grey lines are ignored Blue lines are triggers Pro-tip: Highlight text to add to ignore filters - - - - - - - -
      - {% for row in content %} -
      {{row.line}}
      - {% endfor %} -
      -
      - -
      -
      - For now, Differences are performed on text, not graphically, only the latest screenshot is available. -
      -
      - {% if is_html_webdriver %} - {% if screenshot %} -
      {{watch.snapshot_screenshot_ctime|format_timestamp_timeago}}
      - Current screenshot from most recent request - {% else %} - No screenshot available just yet! Try rechecking the page. - {% endif %} - {% else %} - Screenshot requires Playwright/WebDriver enabled - {% endif %} -
      -
    {% endblock %} diff --git a/changedetectionio/templates/settings.html b/changedetectionio/templates/settings.html index e72c7818a..0e3cea344 100644 --- a/changedetectionio/templates/settings.html +++ b/changedetectionio/templates/settings.html @@ -108,8 +108,6 @@

    Use the Basic method (default) where your watched sites don't need Javascript to render.

    The Chrome/Javascript method requires a network connection to a running WebDriver+Chrome server, set by the ENV var 'WEBDRIVER_URL'.

    -
    - Tip: Connect using Bright Data and Oxylabs Proxies, find out more here.
    @@ -121,6 +119,18 @@ {{ render_field(form.application.form.webdriver_delay) }}
    +
    + {{ render_field(form.requests.form.default_ua) }} + + Applied to all requests.

    + Note: Simply changing the User-Agent often does not defeat anti-robot technologies, it's important to consider all of the ways that the browser is detected. +     +
    +
    +
    + Tip: Connect using Bright Data and Oxylabs Proxies, find out more here. + +
    @@ -190,7 +200,7 @@ nav - + Chrome Chrome Webstore

    diff --git a/changedetectionio/templates/watch-overview.html b/changedetectionio/templates/watch-overview.html index 6a25208ea..15f538fb8 100644 --- a/changedetectionio/templates/watch-overview.html +++ b/changedetectionio/templates/watch-overview.html @@ -13,7 +13,7 @@
    {{ render_nolabel_field(form.url, placeholder="https://...", required=true) }} - {{ render_nolabel_field(form.tags, value=active_tag.title if active_tag else '', placeholder="watch label / tag") }} + {{ render_nolabel_field(form.tags, value=active_tag.title if active_tag_uuid else '', placeholder="watch label / tag") }} {{ render_nolabel_field(form.watch_submit_button, title="Watch this URL!" ) }} {{ render_nolabel_field(form.edit_and_watch_submit_button, title="Edit first then Watch") }}
    @@ -46,7 +46,7 @@ {% endif %} {% if search_q %}
    Searching "{{search_q}}"
    {% endif %}
    - All + All {% for uuid, tag in tags %} @@ -67,11 +67,11 @@ {% set link_order = "desc" if sort_order == 'asc' else "asc" %} {% set arrow_span = "" %} - # + # - Website - Last Checked - Last Changed + Website + Last Checked + Last Changed @@ -95,11 +95,11 @@ {{ loop.index+pagination.skip }} {% if not watch.paused %} - Pause checks + Pause checks {% else %} - UnPause checks + UnPause checks {% endif %} - Mute notifications + Mute notifications {{watch.title if watch.title is not none and watch.title|length > 0 else watch.url}} @@ -204,7 +204,7 @@ all {% if active_tag_uuid %} in "{{active_tag.title}}"{%endif%}
  • - RSS Feed + RSS Feed
    • {{ pagination.links }} diff --git a/changedetectionio/tests/test_jsonpath_jq_selector.py b/changedetectionio/tests/test_jsonpath_jq_selector.py index 5dfdfef2b..1202849f6 100644 --- a/changedetectionio/tests/test_jsonpath_jq_selector.py +++ b/changedetectionio/tests/test_jsonpath_jq_selector.py @@ -479,8 +479,9 @@ def test_correct_header_detect(client, live_server): url_for("preview_page", uuid="first"), follow_redirects=True ) - assert b'"world":' in res.data - assert res.data.count(b'{') >= 2 + + assert b'"hello": 123,' in res.data + assert b'"world": 123' in res.data res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True) assert b'Deleted' in res.data diff --git a/changedetectionio/tests/test_request.py b/changedetectionio/tests/test_request.py index 869ea3496..cfbc7825a 100644 --- a/changedetectionio/tests/test_request.py +++ b/changedetectionio/tests/test_request.py @@ -256,12 +256,40 @@ def test_method_in_request(client, live_server): def test_headers_textfile_in_request(client, live_server): #live_server_setup(live_server) # Add our URL to the import page + + webdriver_ua = "Hello fancy webdriver UA 1.0" + requests_ua = "Hello basic requests UA 1.1" + test_url = url_for('test_headers', _external=True) if os.getenv('PLAYWRIGHT_DRIVER_URL'): # Because its no longer calling back to localhost but from the browser container, set in test-only.yml test_url = test_url.replace('localhost', 'cdio') - print ("TEST URL IS ",test_url) + form_data = { + "application-fetch_backend": "html_requests", + "application-minutes_between_check": 180, + "requests-default_ua-html_requests": requests_ua + } + + if os.getenv('PLAYWRIGHT_DRIVER_URL'): + form_data["requests-default_ua-html_webdriver"] = webdriver_ua + + res = client.post( + url_for("settings_page"), + data=form_data, + follow_redirects=True + ) + assert b'Settings updated' in res.data + + res = client.get(url_for("settings_page")) + + # Only when some kind of real browser is setup + if os.getenv('PLAYWRIGHT_DRIVER_URL'): + assert b'requests-default_ua-html_webdriver' in res.data + + # Field should always be there + assert b"requests-default_ua-html_requests" in res.data + # Add the test URL twice, we will check res = client.post( url_for("import_page"), @@ -272,15 +300,14 @@ def test_headers_textfile_in_request(client, live_server): wait_for_all_checks(client) - # Add some headers to a request res = client.post( url_for("edit_page", uuid="first"), data={ - "url": test_url, - "tags": "testtag", - "fetch_backend": 'html_webdriver' if os.getenv('PLAYWRIGHT_DRIVER_URL') else 'html_requests', - "headers": "xxx:ooo\ncool:yeah\r\n"}, + "url": test_url, + "tags": "testtag", + "fetch_backend": 'html_webdriver' if os.getenv('PLAYWRIGHT_DRIVER_URL') else 'html_requests', + "headers": "xxx:ooo\ncool:yeah\r\n"}, follow_redirects=True ) assert b"Updated watch." in res.data @@ -292,7 +319,7 @@ def test_headers_textfile_in_request(client, live_server): with open('test-datastore/headers.txt', 'w') as f: f.write("global-header: nice\r\nnext-global-header: nice") - with open('test-datastore/'+extract_UUID_from_client(client)+'/headers.txt', 'w') as f: + with open('test-datastore/' + extract_UUID_from_client(client) + '/headers.txt', 'w') as f: f.write("watch-header: nice") client.get(url_for("form_watch_checknow"), follow_redirects=True) @@ -306,7 +333,7 @@ def test_headers_textfile_in_request(client, live_server): # Not needed anymore os.unlink('test-datastore/headers.txt') os.unlink('test-datastore/headers-testtag.txt') - os.unlink('test-datastore/'+extract_UUID_from_client(client)+'/headers.txt') + os.unlink('test-datastore/' + extract_UUID_from_client(client) + '/headers.txt') # The service should echo back the request verb res = client.get( url_for("preview_page", uuid="first"), @@ -319,7 +346,12 @@ def test_headers_textfile_in_request(client, live_server): assert b"Watch-Header:nice" in res.data assert b"Tag-Header:test" in res.data + # Check the custom UA from system settings page made it through + if os.getenv('PLAYWRIGHT_DRIVER_URL'): + assert "User-Agent:".encode('utf-8') + webdriver_ua.encode('utf-8') in res.data + else: + assert "User-Agent:".encode('utf-8') + requests_ua.encode('utf-8') in res.data - #unlink headers.txt on start/stop + # unlink headers.txt on start/stop res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True) - assert b'Deleted' in res.data \ No newline at end of file + assert b'Deleted' in res.data diff --git a/requirements.txt b/requirements.txt index 049163abf..371e5ff94 100644 --- a/requirements.txt +++ b/requirements.txt @@ -78,8 +78,8 @@ jq~=1.3; python_version >= "3.8" and sys_platform == "linux" pillow # playwright is installed at Dockerfile build time because it's not available on all platforms -# experimental release pyppeteer-ng==2.0.0rc5 +pyppeteerstealth>=0.0.4 # Include pytest, so if theres a support issue we can ask them to run these tests on their setup pytest ~=7.2