diff --git a/changedetectionio/__init__.py b/changedetectionio/__init__.py index 8ec6bb8da..6fb76ce43 100644 --- a/changedetectionio/__init__.py +++ b/changedetectionio/__init__.py @@ -2,7 +2,7 @@ # Read more https://github.com/dgtlmoon/changedetection.io/wiki -__version__ = '0.45.22' +__version__ = '0.45.23' from changedetectionio.strtobool import strtobool from json.decoder import JSONDecodeError @@ -175,6 +175,7 @@ def main(): # proxy_set_header Host "localhost"; # proxy_set_header X-Forwarded-Prefix /app; + if os.getenv('USE_X_SETTINGS'): logger.info("USE_X_SETTINGS is ENABLED") from werkzeug.middleware.proxy_fix import ProxyFix diff --git a/changedetectionio/blueprint/browser_steps/__init__.py b/changedetectionio/blueprint/browser_steps/__init__.py index 24440908e..307970997 100644 --- a/changedetectionio/blueprint/browser_steps/__init__.py +++ b/changedetectionio/blueprint/browser_steps/__init__.py @@ -84,7 +84,9 @@ def construct_blueprint(datastore: ChangeDetectionStore): # Tell Playwright to connect to Chrome and setup a new session via our stepper interface browsersteps_start_session['browserstepper'] = browser_steps.browsersteps_live_ui( playwright_browser=browsersteps_start_session['browser'], - proxy=proxy) + proxy=proxy, + start_url=datastore.data['watching'][watch_uuid].get('url') + ) # For test #browsersteps_start_session['browserstepper'].action_goto_url(value="http://example.com?time="+str(time.time())) @@ -167,11 +169,6 @@ def construct_blueprint(datastore: ChangeDetectionStore): step_n = int(request.form.get('step_n')) is_last_step = strtobool(request.form.get('is_last_step')) - if step_operation == 'Goto site': - step_operation = 'goto_url' - step_optional_value = datastore.data['watching'][uuid].get('url') - step_selector = None - # @todo try.. accept.. nice errors not popups.. try: diff --git a/changedetectionio/blueprint/browser_steps/browser_steps.py b/changedetectionio/blueprint/browser_steps/browser_steps.py index 6aac24466..76f3d7561 100644 --- a/changedetectionio/blueprint/browser_steps/browser_steps.py +++ b/changedetectionio/blueprint/browser_steps/browser_steps.py @@ -49,6 +49,10 @@ browser_step_ui_config = {'Choose one': '0 0', # ONLY Works in Playwright because we need the fullscreen screenshot class steppable_browser_interface(): page = None + start_url = None + + def __init__(self, start_url): + self.start_url = start_url # Convert and perform "Click Button" for example def call_action(self, action_name, selector=None, optional_value=None): @@ -87,6 +91,10 @@ class steppable_browser_interface(): logger.debug(f"Time to goto URL {time.time()-now:.2f}s") return response + # Incase they request to go back to the start + def action_goto_site(self, selector=None, value=None): + return self.action_goto_url(value=self.start_url) + def action_click_element_containing_text(self, selector=None, value=''): if not len(value.strip()): return @@ -194,10 +202,11 @@ class browsersteps_live_ui(steppable_browser_interface): browser_type = os.getenv("PLAYWRIGHT_BROWSER_TYPE", 'chromium').strip('"') - def __init__(self, playwright_browser, proxy=None, headers=None): + def __init__(self, playwright_browser, proxy=None, headers=None, start_url=None): self.headers = headers or {} self.age_start = time.time() self.playwright_browser = playwright_browser + self.start_url = start_url if self.context is None: self.connect(proxy=proxy) diff --git a/changedetectionio/content_fetchers/base.py b/changedetectionio/content_fetchers/base.py index ca2fd0191..f817341d3 100644 --- a/changedetectionio/content_fetchers/base.py +++ b/changedetectionio/content_fetchers/base.py @@ -112,23 +112,26 @@ class Fetcher(): def browser_steps_get_valid_steps(self): if self.browser_steps is not None and len(self.browser_steps): - valid_steps = filter( - lambda s: (s['operation'] and len(s['operation']) and s['operation'] != 'Choose one' and s['operation'] != 'Goto site'), - self.browser_steps) + valid_steps = list(filter( + lambda s: (s['operation'] and len(s['operation']) and s['operation'] != 'Choose one'), + self.browser_steps)) + + # Just incase they selected Goto site by accident with older JS + if valid_steps and valid_steps[0]['operation'] == 'Goto site': + del(valid_steps[0]) return valid_steps return None - def iterate_browser_steps(self): + def iterate_browser_steps(self, start_url=None): from changedetectionio.blueprint.browser_steps.browser_steps import steppable_browser_interface from playwright._impl._errors import TimeoutError, Error from changedetectionio.safe_jinja import render as jinja_render - step_n = 0 if self.browser_steps is not None and len(self.browser_steps): - interface = steppable_browser_interface() + interface = steppable_browser_interface(start_url=start_url) interface.page = self.page valid_steps = self.browser_steps_get_valid_steps() diff --git a/changedetectionio/content_fetchers/playwright.py b/changedetectionio/content_fetchers/playwright.py index 7950e0334..04ab2759f 100644 --- a/changedetectionio/content_fetchers/playwright.py +++ b/changedetectionio/content_fetchers/playwright.py @@ -119,7 +119,7 @@ class fetcher(Fetcher): # Re-use as much code from browser steps as possible so its the same from changedetectionio.blueprint.browser_steps.browser_steps import steppable_browser_interface - browsersteps_interface = steppable_browser_interface() + browsersteps_interface = steppable_browser_interface(start_url=url) browsersteps_interface.page = self.page response = browsersteps_interface.action_goto_url(value=url) @@ -172,7 +172,7 @@ class fetcher(Fetcher): # Run Browser Steps here if self.browser_steps_get_valid_steps(): - self.iterate_browser_steps() + self.iterate_browser_steps(start_url=url) self.page.wait_for_timeout(extra_wait * 1000) diff --git a/changedetectionio/content_fetchers/puppeteer.py b/changedetectionio/content_fetchers/puppeteer.py index cad1b6b85..725be3b35 100644 --- a/changedetectionio/content_fetchers/puppeteer.py +++ b/changedetectionio/content_fetchers/puppeteer.py @@ -9,7 +9,6 @@ from loguru import logger from changedetectionio.content_fetchers.base import Fetcher, manage_user_agent from changedetectionio.content_fetchers.exceptions import PageUnloadable, Non200ErrorCodeReceived, EmptyReply, BrowserFetchTimedOut, BrowserConnectError - class fetcher(Fetcher): fetcher_description = "Puppeteer/direct {}/Javascript".format( os.getenv("PLAYWRIGHT_BROWSER_TYPE", 'chromium').capitalize() @@ -93,15 +92,39 @@ class fetcher(Fetcher): ignoreHTTPSErrors=True ) except websockets.exceptions.InvalidStatusCode as e: - raise BrowserConnectError(msg=f"Error while trying to connect the browser, Code {e.status_code} (check your access)") + raise BrowserConnectError(msg=f"Error while trying to connect the browser, Code {e.status_code} (check your access, whitelist IP, password etc)") except websockets.exceptions.InvalidURI: raise BrowserConnectError(msg=f"Error connecting to the browser, check your browser connection address (should be ws:// or wss://") except Exception as e: raise BrowserConnectError(msg=f"Error connecting to the browser {str(e)}") - else: - self.page = await browser.newPage() - await self.page.setUserAgent(manage_user_agent(headers=request_headers, current_ua=await self.page.evaluate('navigator.userAgent'))) + # Better is to launch chrome with the URL as arg + # non-headless - newPage() will launch an extra tab/window, .browser should already contain 1 page/tab + # headless - ask a new page + self.page = (pages := await browser.pages) and len(pages) or await browser.newPage() + + try: + from pyppeteerstealth import inject_evasions_into_page + except ImportError: + logger.debug("pyppeteerstealth module not available, skipping") + pass + else: + # I tried hooking events via self.page.on(Events.Page.DOMContentLoaded, inject_evasions_requiring_obj_to_page) + # But I could never get it to fire reliably, so we just inject it straight after + await inject_evasions_into_page(self.page) + + # This user agent is similar to what was used when tweaking the evasions in inject_evasions_into_page(..) + user_agent = None + if request_headers: + user_agent = next((value for key, value in request_headers.items() if key.lower().strip() == 'user-agent'), None) + if user_agent: + await self.page.setUserAgent(user_agent) + # Remove it so it's not sent again with headers after + [request_headers.pop(key) for key in list(request_headers) if key.lower().strip() == 'user-agent'.lower().strip()] + + if not user_agent: + # Attempt to strip 'HeadlessChrome' etc + await self.page.setUserAgent(manage_user_agent(headers=request_headers, current_ua=await self.page.evaluate('navigator.userAgent'))) await self.page.setBypassCSP(True) if request_headers: diff --git a/changedetectionio/content_fetchers/requests.py b/changedetectionio/content_fetchers/requests.py index b743dbcec..2c28cda7c 100644 --- a/changedetectionio/content_fetchers/requests.py +++ b/changedetectionio/content_fetchers/requests.py @@ -30,11 +30,6 @@ class fetcher(Fetcher): if self.browser_steps_get_valid_steps(): raise BrowserStepsInUnsupportedFetcher(url=url) - # Make requests use a more modern looking user-agent - if not {k.lower(): v for k, v in request_headers.items()}.get('user-agent', None): - request_headers['User-Agent'] = os.getenv("DEFAULT_SETTINGS_HEADERS_USERAGENT", - 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.66 Safari/537.36') - proxies = {} # Allows override the proxy on a per-request basis diff --git a/changedetectionio/flask_app.py b/changedetectionio/flask_app.py index ee1324240..41f80a773 100644 --- a/changedetectionio/flask_app.py +++ b/changedetectionio/flask_app.py @@ -338,8 +338,11 @@ def changedetection_app(config=None, datastore_o=None): # @todo needs a .itemsWithTag() or something - then we can use that in Jinaj2 and throw this away for uuid, watch in datastore.data['watching'].items(): + # @todo tag notification_muted skip also (improve Watch model) + if watch.get('notification_muted'): + continue if limit_tag and not limit_tag in watch['tags']: - continue + continue watch['uuid'] = uuid sorted_watches.append(watch) @@ -619,7 +622,6 @@ def changedetection_app(config=None, datastore_o=None): from .blueprint.browser_steps.browser_steps import browser_step_ui_config from . import processors - using_default_check_time = True # More for testing, possible to return the first/only if not datastore.data['watching'].keys(): flash("No watches to edit", "error") @@ -644,10 +646,6 @@ def changedetection_app(config=None, datastore_o=None): # be sure we update with a copy instead of accidently editing the live object by reference default = deepcopy(datastore.data['watching'][uuid]) - # Show system wide default if nothing configured - if all(value == 0 or value == None for value in datastore.data['watching'][uuid]['time_between_check'].values()): - default['time_between_check'] = deepcopy(datastore.data['settings']['requests']['time_between_check']) - # Defaults for proxy choice if datastore.proxy_list is not None: # When enabled # @todo @@ -685,18 +683,8 @@ def changedetection_app(config=None, datastore_o=None): if request.args.get('unpause_on_save'): extra_update_obj['paused'] = False - # Re #110, if they submit the same as the default value, set it to None, so we continue to follow the default - # Assume we use the default value, unless something relevant is different, then use the form value - # values could be None, 0 etc. - # Set to None unless the next for: says that something is different - extra_update_obj['time_between_check'] = dict.fromkeys(form.time_between_check.data) - for k, v in form.time_between_check.data.items(): - if v and v != datastore.data['settings']['requests']['time_between_check'][k]: - extra_update_obj['time_between_check'] = form.time_between_check.data - using_default_check_time = False - break - + extra_update_obj['time_between_check'] = form.time_between_check.data # Ignore text form_ignore_text = form.ignore_text.data @@ -777,14 +765,13 @@ def changedetection_app(config=None, datastore_o=None): extra_title=f" - Edit - {watch.label}", form=form, has_default_notification_urls=True if len(datastore.data['settings']['application']['notification_urls']) else False, - has_empty_checktime=using_default_check_time, has_extra_headers_file=len(datastore.get_all_headers_in_textfile_for_watch(uuid=uuid)) > 0, has_special_tag_options=_watch_has_tag_options_set(watch=watch), is_html_webdriver=is_html_webdriver, jq_support=jq_support, playwright_enabled=os.getenv('PLAYWRIGHT_DRIVER_URL', False), settings_application=datastore.data['settings']['application'], - using_global_webdriver_wait=default['webdriver_delay'] is None, + using_global_webdriver_wait=not default['webdriver_delay'], uuid=uuid, visualselector_enabled=visualselector_enabled, watch=watch @@ -863,11 +850,13 @@ def changedetection_app(config=None, datastore_o=None): flash("An error occurred, please see below.", "error") output = render_template("settings.html", - form=form, - hide_remove_pass=os.getenv("SALTED_PASS", False), api_key=datastore.data['settings']['application'].get('api_access_token'), emailprefix=os.getenv('NOTIFICATION_MAIL_BUTTON_PREFIX', False), - settings_application=datastore.data['settings']['application']) + form=form, + hide_remove_pass=os.getenv("SALTED_PASS", False), + min_system_recheck_seconds=int(os.getenv('MINIMUM_SECONDS_RECHECK_TIME', 3)), + settings_application=datastore.data['settings']['application'] + ) return output @@ -1077,6 +1066,8 @@ def changedetection_app(config=None, datastore_o=None): content = [] ignored_line_numbers = [] trigger_line_numbers = [] + versions = [] + timestamp = None # More for testing, possible to return the first/only if uuid == 'first': @@ -1096,57 +1087,53 @@ def changedetection_app(config=None, datastore_o=None): if (watch.get('fetch_backend') == 'system' and system_uses_webdriver) or watch.get('fetch_backend') == 'html_webdriver' or watch.get('fetch_backend', '').startswith('extra_browser_'): is_html_webdriver = True - # Never requested successfully, but we detected a fetch error if datastore.data['watching'][uuid].history_n == 0 and (watch.get_error_text() or watch.get_error_snapshot()): flash("Preview unavailable - No fetch/check completed or triggers not reached", "error") - output = render_template("preview.html", - content=content, - history_n=watch.history_n, - extra_stylesheets=extra_stylesheets, -# current_diff_url=watch['url'], - watch=watch, - uuid=uuid, - is_html_webdriver=is_html_webdriver, - last_error=watch['last_error'], - last_error_text=watch.get_error_text(), - last_error_screenshot=watch.get_error_snapshot()) - return output + else: + # So prepare the latest preview or not + preferred_version = request.args.get('version') + versions = list(watch.history.keys()) + timestamp = versions[-1] + if preferred_version and preferred_version in versions: + timestamp = preferred_version - timestamp = list(watch.history.keys())[-1] - try: - tmp = watch.get_history_snapshot(timestamp).splitlines() + try: + versions = list(watch.history.keys()) + tmp = watch.get_history_snapshot(timestamp).splitlines() - # Get what needs to be highlighted - ignore_rules = watch.get('ignore_text', []) + datastore.data['settings']['application']['global_ignore_text'] + # Get what needs to be highlighted + ignore_rules = watch.get('ignore_text', []) + datastore.data['settings']['application']['global_ignore_text'] - # .readlines will keep the \n, but we will parse it here again, in the future tidy this up - ignored_line_numbers = html_tools.strip_ignore_text(content="\n".join(tmp), - wordlist=ignore_rules, - mode='line numbers' - ) + # .readlines will keep the \n, but we will parse it here again, in the future tidy this up + ignored_line_numbers = html_tools.strip_ignore_text(content="\n".join(tmp), + wordlist=ignore_rules, + mode='line numbers' + ) - trigger_line_numbers = html_tools.strip_ignore_text(content="\n".join(tmp), - wordlist=watch['trigger_text'], - mode='line numbers' - ) - # Prepare the classes and lines used in the template - i=0 - for l in tmp: - classes=[] - i+=1 - if i in ignored_line_numbers: - classes.append('ignored') - if i in trigger_line_numbers: - classes.append('triggered') - content.append({'line': l, 'classes': ' '.join(classes)}) + trigger_line_numbers = html_tools.strip_ignore_text(content="\n".join(tmp), + wordlist=watch['trigger_text'], + mode='line numbers' + ) + # Prepare the classes and lines used in the template + i=0 + for l in tmp: + classes=[] + i+=1 + if i in ignored_line_numbers: + classes.append('ignored') + if i in trigger_line_numbers: + classes.append('triggered') + content.append({'line': l, 'classes': ' '.join(classes)}) - except Exception as e: - content.append({'line': f"File doesnt exist or unable to read timestamp {timestamp}", 'classes': ''}) + except Exception as e: + content.append({'line': f"File doesnt exist or unable to read timestamp {timestamp}", 'classes': ''}) output = render_template("preview.html", content=content, + current_version=timestamp, history_n=watch.history_n, extra_stylesheets=extra_stylesheets, + extra_title=f" - Diff - {watch.label} @ {timestamp}", ignored_line_numbers=ignored_line_numbers, triggered_line_numbers=trigger_line_numbers, current_diff_url=watch['url'], @@ -1156,7 +1143,10 @@ def changedetection_app(config=None, datastore_o=None): is_html_webdriver=is_html_webdriver, last_error=watch['last_error'], last_error_text=watch.get_error_text(), - last_error_screenshot=watch.get_error_snapshot()) + last_error_screenshot=watch.get_error_snapshot(), + versions=versions + ) + return output @@ -1668,14 +1658,14 @@ def notification_runner(): # Trim the log length notification_debug_log = notification_debug_log[-100:] -# Thread runner to check every minute, look for new watches to feed into the Queue. +# Threaded runner, look for new watches to feed into the Queue. def ticker_thread_check_time_launch_checks(): import random from changedetectionio import update_worker proxy_last_called_time = {} - recheck_time_minimum_seconds = int(os.getenv('MINIMUM_SECONDS_RECHECK_TIME', 20)) + recheck_time_minimum_seconds = int(os.getenv('MINIMUM_SECONDS_RECHECK_TIME', 3)) logger.debug(f"System env MINIMUM_SECONDS_RECHECK_TIME {recheck_time_minimum_seconds}") # Spin up Workers that do the fetching @@ -1729,9 +1719,7 @@ def ticker_thread_check_time_launch_checks(): continue # If they supplied an individual entry minutes to threshold. - - watch_threshold_seconds = watch.threshold_seconds() - threshold = watch_threshold_seconds if watch_threshold_seconds > 0 else recheck_time_system_seconds + threshold = recheck_time_system_seconds if watch.get('time_between_check_use_default') else watch.threshold_seconds() # #580 - Jitter plus/minus amount of time to make the check seem more random to the server jitter = datastore.data['settings']['requests'].get('jitter_seconds', 0) diff --git a/changedetectionio/forms.py b/changedetectionio/forms.py index 4f74f9783..673be9caa 100644 --- a/changedetectionio/forms.py +++ b/changedetectionio/forms.py @@ -453,6 +453,7 @@ class watchForm(commonSettingsForm): tags = StringTagUUID('Group tag', [validators.Optional()], default='') time_between_check = FormField(TimeBetweenCheckForm) + time_between_check_use_default = BooleanField('Use global settings for time between check', default=False) include_filters = StringListField('CSS/JSONPath/JQ/XPath Filters', [ValidateCSSJSONXPATHInput()], default='') @@ -525,6 +526,10 @@ class SingleExtraBrowser(Form): browser_connection_url = StringField('Browser connection URL', [validators.Optional()], render_kw={"placeholder": "wss://brightdata... wss://oxylabs etc", "size":50}) # @todo do the validation here instead +class DefaultUAInputForm(Form): + html_requests = StringField('Plaintext requests', validators=[validators.Optional()], render_kw={"placeholder": ""}) + if os.getenv("PLAYWRIGHT_DRIVER_URL") or os.getenv("WEBDRIVER_URL"): + html_webdriver = StringField('Chrome requests', validators=[validators.Optional()], render_kw={"placeholder": ""}) # datastore.data['settings']['requests'].. class globalSettingsRequestForm(Form): @@ -536,6 +541,8 @@ class globalSettingsRequestForm(Form): extra_proxies = FieldList(FormField(SingleExtraProxy), min_entries=5) extra_browsers = FieldList(FormField(SingleExtraBrowser), min_entries=5) + default_ua = FormField(DefaultUAInputForm, label="Default User-Agent overrides") + def validate_extra_proxies(self, extra_validators=None): for e in self.data['extra_proxies']: if e.get('proxy_name') or e.get('proxy_url'): diff --git a/changedetectionio/model/App.py b/changedetectionio/model/App.py index 1202d5db1..75384f170 100644 --- a/changedetectionio/model/App.py +++ b/changedetectionio/model/App.py @@ -6,6 +6,7 @@ from changedetectionio.notification import ( ) _FILTER_FAILURE_THRESHOLD_ATTEMPTS_DEFAULT = 6 +DEFAULT_SETTINGS_HEADERS_USERAGENT='Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.66 Safari/537.36' class model(dict): base_config = { @@ -22,6 +23,10 @@ class model(dict): 'time_between_check': {'weeks': None, 'days': None, 'hours': 3, 'minutes': None, 'seconds': None}, 'timeout': int(getenv("DEFAULT_SETTINGS_REQUESTS_TIMEOUT", "45")), # Default 45 seconds 'workers': int(getenv("DEFAULT_SETTINGS_REQUESTS_WORKERS", "10")), # Number of threads, lower is better for slow connections + 'default_ua': { + 'html_requests': getenv("DEFAULT_SETTINGS_HEADERS_USERAGENT", DEFAULT_SETTINGS_HEADERS_USERAGENT), + 'html_webdriver': None, + } }, 'application': { # Custom notification content diff --git a/changedetectionio/model/Watch.py b/changedetectionio/model/Watch.py index 6bd41e9ef..a88a220dd 100644 --- a/changedetectionio/model/Watch.py +++ b/changedetectionio/model/Watch.py @@ -12,9 +12,10 @@ from loguru import logger # file:// is further checked by ALLOW_FILE_URI SAFE_PROTOCOL_REGEX='^(http|https|ftp|file):' -minimum_seconds_recheck_time = int(os.getenv('MINIMUM_SECONDS_RECHECK_TIME', 60)) +minimum_seconds_recheck_time = int(os.getenv('MINIMUM_SECONDS_RECHECK_TIME', 3)) mtable = {'seconds': 1, 'minutes': 60, 'hours': 3600, 'days': 86400, 'weeks': 86400 * 7} + def is_safe_url(test_url): # See https://github.com/dgtlmoon/changedetection.io/issues/1358 diff --git a/changedetectionio/model/__init__.py b/changedetectionio/model/__init__.py index dac2aabfa..333bcd995 100644 --- a/changedetectionio/model/__init__.py +++ b/changedetectionio/model/__init__.py @@ -12,7 +12,6 @@ class Restock(dict): default_values.update(dict(*args, **kwargs)) super().__init__(default_values.copy()) - class watch_base(dict): def __init__(self, *arg, **kw): @@ -37,6 +36,7 @@ class watch_base(dict): 'track_ldjson_price_data': None, 'headers': {}, # Extra headers to send 'ignore_text': [], # List of text to ignore when calculating the comparison checksum + 'in_stock': None, 'in_stock_only': True, # Only trigger change on going to instock from out-of-stock 'include_filters': [], 'last_checked': 0, @@ -55,7 +55,6 @@ class watch_base(dict): 'previous_md5': False, 'previous_md5_before_filters': False, # Used for skipping changedetection entirely 'proxy': None, # Preferred proxy connection - 'restock': {}, # Restock/price storage 'remote_server_reply': None, # From 'server' reply header 'sort_text_alphabetically': False, 'subtractive_selectors': [], @@ -66,6 +65,7 @@ class watch_base(dict): # Requires setting to None on submit if it's the same as the default # Should be all None by default, so we use the system default in this case. 'time_between_check': {'weeks': None, 'days': None, 'hours': None, 'minutes': None, 'seconds': None}, + 'time_between_check_use_default': True, 'title': None, 'trigger_text': [], # List of text or regex to wait for until a change is detected 'url': '', diff --git a/changedetectionio/notification.py b/changedetectionio/notification.py index f634a1afa..41285ce45 100644 --- a/changedetectionio/notification.py +++ b/changedetectionio/notification.py @@ -48,7 +48,7 @@ from apprise.decorators import notify def apprise_custom_api_call_wrapper(body, title, notify_type, *args, **kwargs): import requests from apprise.utils import parse_url as apprise_parse_url - from apprise.URLBase import URLBase + from apprise import URLBase url = kwargs['meta'].get('url') @@ -122,10 +122,6 @@ def process_notification(n_object, datastore): # Insert variables into the notification content notification_parameters = create_notification_parameters(n_object, datastore) - # Get the notification body from datastore - n_body = jinja_render(template_str=n_object.get('notification_body', ''), **notification_parameters) - n_title = jinja_render(template_str=n_object.get('notification_title', ''), **notification_parameters) - n_format = valid_notification_formats.get( n_object.get('notification_format', default_notification_format), valid_notification_formats[default_notification_format], @@ -151,6 +147,11 @@ def process_notification(n_object, datastore): with apprise.LogCapture(level=apprise.logging.DEBUG) as logs: for url in n_object['notification_urls']: + + # Get the notification body from datastore + n_body = jinja_render(template_str=n_object.get('notification_body', ''), **notification_parameters) + n_title = jinja_render(template_str=n_object.get('notification_title', ''), **notification_parameters) + url = url.strip() if not url: logger.warning(f"Process Notification: skipping empty notification URL.") diff --git a/changedetectionio/processors/__init__.py b/changedetectionio/processors/__init__.py index e2b544811..8702ee5d1 100644 --- a/changedetectionio/processors/__init__.py +++ b/changedetectionio/processors/__init__.py @@ -97,6 +97,10 @@ class difference_detection_processor(): request_headers.update(self.datastore.get_all_base_headers()) request_headers.update(self.datastore.get_all_headers_in_textfile_for_watch(uuid=self.watch.get('uuid'))) + ua = self.datastore.data['settings']['requests'].get('default_ua') + if ua and ua.get(prefer_fetch_backend): + request_headers.update({'User-Agent': ua.get(prefer_fetch_backend)}) + # https://github.com/psf/requests/issues/4525 # Requests doesnt yet support brotli encoding, so don't put 'br' here, be totally sure that the user cannot # do this by accident. diff --git a/changedetectionio/static/images/gradient-border.png b/changedetectionio/static/images/gradient-border.png deleted file mode 100644 index 4c7705f8c..000000000 Binary files a/changedetectionio/static/images/gradient-border.png and /dev/null differ diff --git a/changedetectionio/static/js/browser-steps.js b/changedetectionio/static/js/browser-steps.js index 7c9c38d8c..4e576bd45 100644 --- a/changedetectionio/static/js/browser-steps.js +++ b/changedetectionio/static/js/browser-steps.js @@ -26,7 +26,8 @@ $(document).ready(function () { set_scale(); }); // Should always be disabled - $('#browser_steps >li:first-child select').val('Goto site').attr('disabled', 'disabled'); + $('#browser_steps-0-operation option[value="Goto site"]').prop("selected", "selected"); + $('#browser_steps-0-operation').attr('disabled', 'disabled'); $('#browsersteps-click-start').click(function () { $("#browsersteps-click-start").fadeOut(); diff --git a/changedetectionio/static/js/diff-overview.js b/changedetectionio/static/js/diff-overview.js index 767cf6e1f..95e6dd7a0 100644 --- a/changedetectionio/static/js/diff-overview.js +++ b/changedetectionio/static/js/diff-overview.js @@ -8,6 +8,13 @@ $(document).ready(function () { } }) + $('.needs-localtime').each(function () { + for (var option of this.options) { + var dateObject = new Date(option.value * 1000); + option.label = dateObject.toLocaleString(undefined, {dateStyle: "full", timeStyle: "medium"}); + } + }); + // Load it when the #screenshot tab is in use, so we dont give a slow experience when waiting for the text diff to load window.addEventListener('hashchange', function (e) { toggle(location.hash); diff --git a/changedetectionio/static/js/diff-render.js b/changedetectionio/static/js/diff-render.js index 53f1d68f5..ea69d364f 100644 --- a/changedetectionio/static/js/diff-render.js +++ b/changedetectionio/static/js/diff-render.js @@ -79,12 +79,7 @@ $(document).ready(function () { $('#jump-next-diff').click(); } - $('.needs-localtime').each(function () { - for (var option of this.options) { - var dateObject = new Date(option.value * 1000); - option.label = dateObject.toLocaleString(undefined, {dateStyle: "full", timeStyle: "medium"}); - } - }) + onDiffTypeChange( document.querySelector('#settings [name="diff_type"]:checked'), ); diff --git a/changedetectionio/static/js/preview.js b/changedetectionio/static/js/preview.js new file mode 100644 index 000000000..a9895cb21 --- /dev/null +++ b/changedetectionio/static/js/preview.js @@ -0,0 +1,49 @@ +function redirect_to_version(version) { + var currentUrl = window.location.href; + var baseUrl = currentUrl.split('?')[0]; // Base URL without query parameters + var anchor = ''; + + // Check if there is an anchor + if (baseUrl.indexOf('#') !== -1) { + anchor = baseUrl.substring(baseUrl.indexOf('#')); + baseUrl = baseUrl.substring(0, baseUrl.indexOf('#')); + } + window.location.href = baseUrl + '?version=' + version + anchor; +} + +document.addEventListener('keydown', function (event) { + var selectElement = document.getElementById('preview-version'); + if (selectElement) { + var selectedOption = selectElement.querySelector('option:checked'); + if (selectedOption) { + if (event.key === 'ArrowLeft') { + if (selectedOption.previousElementSibling) { + redirect_to_version(selectedOption.previousElementSibling.value); + } + } else if (event.key === 'ArrowRight') { + if (selectedOption.nextElementSibling) { + redirect_to_version(selectedOption.nextElementSibling.value); + } + } + } + } +}); + + +document.getElementById('preview-version').addEventListener('change', function () { + redirect_to_version(this.value); +}); + +var selectElement = document.getElementById('preview-version'); +if (selectElement) { + var selectedOption = selectElement.querySelector('option:checked'); + if (selectedOption) { + if (selectedOption.previousElementSibling) { + document.getElementById('btn-previous').href = "?version=" + selectedOption.previousElementSibling.value; + } + if (selectedOption.nextElementSibling) { + document.getElementById('btn-next').href = "?version=" + selectedOption.nextElementSibling.value; + } + + } +} diff --git a/changedetectionio/static/js/watch-settings.js b/changedetectionio/static/js/watch-settings.js index 22bf48ed2..73c66191c 100644 --- a/changedetectionio/static/js/watch-settings.js +++ b/changedetectionio/static/js/watch-settings.js @@ -1,3 +1,17 @@ +function toggleOpacity(checkboxSelector, fieldSelector) { + const checkbox = document.querySelector(checkboxSelector); + const fields = document.querySelectorAll(fieldSelector); + function updateOpacity() { + const opacityValue = checkbox.checked ? 0.6 : 1; + fields.forEach(field => { + field.style.opacity = opacityValue; + }); + } + // Initial setup + updateOpacity(); + checkbox.addEventListener('change', updateOpacity); +} + $(document).ready(function () { $('#notification-setting-reset-to-default').click(function (e) { $('#notification_title').val(''); @@ -10,4 +24,7 @@ $(document).ready(function () { e.preventDefault(); $('#notification-tokens-info').toggle(); }); + + toggleOpacity('#time_between_check_use_default', '#time_between_check'); }); + diff --git a/changedetectionio/static/styles/scss/styles.scss b/changedetectionio/static/styles/scss/styles.scss index 727f2c8aa..7d00d7390 100644 --- a/changedetectionio/static/styles/scss/styles.scss +++ b/changedetectionio/static/styles/scss/styles.scss @@ -248,7 +248,6 @@ body::after { body::before { // background-image set in base.html so it works with reverse proxies etc content: ""; - background-size: cover } body:after, @@ -933,23 +932,26 @@ body.full-width { font-size: .875em; } } - .text-filtering { - h3 { - margin-top: 0; - } - border: 1px solid #ccc; - padding: 1rem; - border-radius: 5px; - margin-bottom: 1rem; - fieldset:last-of-type { +} + +.border-fieldset { + h3 { + margin-top: 0; + } + border: 1px solid #ccc; + padding: 1rem; + border-radius: 5px; + margin-bottom: 1rem; + fieldset:last-of-type { + padding-bottom: 0; + .pure-control-group { padding-bottom: 0; - .pure-control-group { - padding-bottom: 0; - } } } } + + ul { padding-left: 1em; padding-top: 0px; @@ -1084,6 +1086,9 @@ ul { li { list-style: none; font-size: 0.8rem; + > * { + display: inline-block; + } } } diff --git a/changedetectionio/static/styles/styles.css b/changedetectionio/static/styles/styles.css index 990390128..4870a5126 100644 --- a/changedetectionio/static/styles/styles.css +++ b/changedetectionio/static/styles/styles.css @@ -577,8 +577,7 @@ body::after { opacity: 0.91; } body::before { - content: ""; - background-size: cover; } + content: ""; } body:after, body:before { @@ -1044,17 +1043,18 @@ body.full-width .edit-form { color: var(--color-text-input-description); } .edit-form .pure-form-message-inline code { font-size: .875em; } - .edit-form .text-filtering { - border: 1px solid #ccc; - padding: 1rem; - border-radius: 5px; - margin-bottom: 1rem; } - .edit-form .text-filtering h3 { - margin-top: 0; } - .edit-form .text-filtering fieldset:last-of-type { + +.border-fieldset { + border: 1px solid #ccc; + padding: 1rem; + border-radius: 5px; + margin-bottom: 1rem; } + .border-fieldset h3 { + margin-top: 0; } + .border-fieldset fieldset:last-of-type { + padding-bottom: 0; } + .border-fieldset fieldset:last-of-type .pure-control-group { padding-bottom: 0; } - .edit-form .text-filtering fieldset:last-of-type .pure-control-group { - padding-bottom: 0; } ul { padding-left: 1em; @@ -1173,6 +1173,8 @@ ul { #quick-watch-processor-type ul li { list-style: none; font-size: 0.8rem; } + #quick-watch-processor-type ul li > * { + display: inline-block; } .restock-label.in-stock { background-color: var(--color-background-button-green); diff --git a/changedetectionio/store.py b/changedetectionio/store.py index 796498dda..a9f99eaf7 100644 --- a/changedetectionio/store.py +++ b/changedetectionio/store.py @@ -555,7 +555,6 @@ class ChangeDetectionStore: return os.path.isfile(filepath) def get_all_base_headers(self): - from .model.App import parse_headers_from_text_file headers = {} # Global app settings headers.update(self.data['settings'].get('headers', {})) @@ -880,3 +879,16 @@ class ChangeDetectionStore: self.__data["watching"][awatch]['include_filters'][num] = 'xpath1:' + selector if selector.startswith('xpath:'): self.__data["watching"][awatch]['include_filters'][num] = selector.replace('xpath:', 'xpath1:', 1) + + # Use more obvious default time setting + def update_15(self): + for uuid in self.__data["watching"]: + if self.__data["watching"][uuid]['time_between_check'] == self.__data['settings']['requests']['time_between_check']: + # What the old logic was, which was pretty confusing + self.__data["watching"][uuid]['time_between_check_use_default'] = True + elif all(value is None or value == 0 for value in self.__data["watching"][uuid]['time_between_check'].values()): + self.__data["watching"][uuid]['time_between_check_use_default'] = True + else: + # Something custom here + self.__data["watching"][uuid]['time_between_check_use_default'] = False + diff --git a/changedetectionio/templates/base.html b/changedetectionio/templates/base.html index c4c664cb2..87018a7de 100644 --- a/changedetectionio/templates/base.html +++ b/changedetectionio/templates/base.html @@ -6,7 +6,9 @@ Change Detection{{extra_title}} - + {% if app_rss_token %} + + {% endif %} {% if extra_stylesheets %} @@ -24,12 +26,6 @@ - - @@ -89,8 +85,8 @@
  • - - + + diff --git a/changedetectionio/templates/edit.html b/changedetectionio/templates/edit.html index 21e371049..583e9055a 100644 --- a/changedetectionio/templates/edit.html +++ b/changedetectionio/templates/edit.html @@ -85,15 +85,9 @@ {{ render_field(form.tags) }} Organisational tag/group name used in the main listing page -
    +
    {{ render_field(form.time_between_check, class="time-check-widget") }} - {% if has_empty_checktime %} - Currently using the default global settings, change to another value if you want to be specific. - {% else %} - Set to blank to use the default global settings. - {% endif %} + {{ render_checkbox_field(form.time_between_check_use_default, class="use-default-timecheck") }}
    {{ render_checkbox_field(form.extract_title_as_title) }} @@ -328,7 +322,7 @@ nav -
    +

    Text filtering

    Limit trigger/ignore/block/extract to;
    @@ -437,7 +431,8 @@ Unavailable") }}
    {% if visualselector_enabled %} - The Visual Selector tool lets you select the text elements that will be used for the change detection ‐ after the Browser Steps has completed, this tool is a helper to manage filters in the "CSS/JSONPath/JQ/XPath Filters" box of the Filters & Triggers tab. + The Visual Selector tool lets you select the text elements that will be used for the change detection ‐ after the Browser Steps has completed.
    + This tool is a helper to manage filters in the "CSS/JSONPath/JQ/XPath Filters" box of the Filters & Triggers tab.
    diff --git a/changedetectionio/templates/preview.html b/changedetectionio/templates/preview.html index 5cc61bedc..8bc231e16 100644 --- a/changedetectionio/templates/preview.html +++ b/changedetectionio/templates/preview.html @@ -1,72 +1,103 @@ {% extends 'base.html' %} {% block content %} - - - - -
    -
      - {% if last_error_text %}
    • Error Text
      • {% endif %} - {% if last_error_screenshot %}
    • Error Screenshot
      • {% endif %} - {% if history_n > 0 %} -
    • Text
      • -
    • Screenshot
      • + + + + + {% if versions|length >= 2 %} +
      + +
      + + + +
      + +
      + Keyboard: + ← Previous   + → Next +
      + {% endif %} + +
      + +
      + + +
      +
      +
      {{ watch.error_text_ctime|format_seconds_ago }} seconds ago
      + 
                   {{ last_error_text }}
      +
      + +
      +
      {{ watch.snapshot_error_screenshot_ctime|format_seconds_ago }} seconds ago +
      + Current erroring screenshot from most recent request +
      + +
      +
      {{ watch.snapshot_text_ctime|format_timestamp_timeago }}
      + Grey lines are ignored Blue lines are triggers + Pro-tip: Highlight text to add to ignore filters + + + + + + + +
      + {% for row in content %} +
      {{ row.line }}
      + {% endfor %} +
      +
      + +
      +
      + For now, Differences are performed on text, not graphically, only the latest screenshot is available. +
      +
      + {% if is_html_webdriver %} + {% if screenshot %} +
      {{ watch.snapshot_screenshot_ctime|format_timestamp_timeago }}
      + Current screenshot from most recent request + {% else %} + No screenshot available just yet! Try rechecking the page. + {% endif %} + {% else %} + Screenshot requires Playwright/WebDriver enabled + {% endif %} +
      - -
      -
      {{watch.snapshot_error_screenshot_ctime|format_seconds_ago}} seconds ago
      - Current erroring screenshot from most recent request -
      - -
      -
      {{watch.snapshot_text_ctime|format_timestamp_timeago}}
      - Grey lines are ignored Blue lines are triggers Pro-tip: Highlight text to add to ignore filters - - - - - - - -
      - {% for row in content %} -
      {{row.line}}
      - {% endfor %} -
      -
      - -
      -
      - For now, Differences are performed on text, not graphically, only the latest screenshot is available. -
      -
      - {% if is_html_webdriver %} - {% if screenshot %} -
      {{watch.snapshot_screenshot_ctime|format_timestamp_timeago}}
      - Current screenshot from most recent request - {% else %} - No screenshot available just yet! Try rechecking the page. - {% endif %} - {% else %} - Screenshot requires Playwright/WebDriver enabled - {% endif %} -
      -
    {% endblock %} diff --git a/changedetectionio/templates/settings.html b/changedetectionio/templates/settings.html index 78387a48c..0e3cea344 100644 --- a/changedetectionio/templates/settings.html +++ b/changedetectionio/templates/settings.html @@ -31,7 +31,7 @@
    {{ render_field(form.requests.form.time_between_check, class="time-check-widget") }} - Default time for all watches, when the watch does not have a specific time setting. + Default recheck time for all watches, current system minimum is {{min_system_recheck_seconds}} seconds (more info).
    {{ render_field(form.requests.form.jitter_seconds, class="jitter_seconds") }} @@ -108,8 +108,6 @@

    Use the Basic method (default) where your watched sites don't need Javascript to render.

    The Chrome/Javascript method requires a network connection to a running WebDriver+Chrome server, set by the ENV var 'WEBDRIVER_URL'.

    -
    - Tip: Connect using Bright Data and Oxylabs Proxies, find out more here.
    @@ -121,6 +119,18 @@ {{ render_field(form.application.form.webdriver_delay) }}
    +
    + {{ render_field(form.requests.form.default_ua) }} + + Applied to all requests.

    + Note: Simply changing the User-Agent often does not defeat anti-robot technologies, it's important to consider all of the ways that the browser is detected. +     +
    +
    +
    + Tip: Connect using Bright Data and Oxylabs Proxies, find out more here. + +
    @@ -190,7 +200,7 @@ nav - + Chrome Chrome Webstore

    diff --git a/changedetectionio/templates/watch-overview.html b/changedetectionio/templates/watch-overview.html index 91d30adbe..40147d38c 100644 --- a/changedetectionio/templates/watch-overview.html +++ b/changedetectionio/templates/watch-overview.html @@ -13,7 +13,7 @@
    {{ render_nolabel_field(form.url, placeholder="https://...", required=true) }} - {{ render_nolabel_field(form.tags, value=active_tag.title if active_tag else '', placeholder="watch label / tag") }} + {{ render_nolabel_field(form.tags, value=active_tag.title if active_tag_uuid else '', placeholder="watch label / tag") }} {{ render_nolabel_field(form.watch_submit_button, title="Watch this URL!" ) }} {{ render_nolabel_field(form.edit_and_watch_submit_button, title="Edit first then Watch") }}
    @@ -46,7 +46,7 @@ {% endif %} {% if search_q %}
    Searching "{{search_q}}"
    {% endif %}
    - All + All {% for uuid, tag in tags %} @@ -72,14 +72,14 @@ {% set link_order = "desc" if sort_order == 'asc' else "asc" %} {% set arrow_span = "" %} - # + # - Website + Website {% if any_has_restock_price_processor %} Restock & Price {% endif %} - Last Checked - Last Changed + Last Checked + Last Changed @@ -104,11 +104,11 @@ {{ loop.index+pagination.skip }} {% if not watch.paused %} - Pause checks + Pause checks {% else %} - UnPause checks + UnPause checks {% endif %} - Mute notifications + Mute notifications {{watch.title if watch.title is not none and watch.title|length > 0 else watch.url}} @@ -223,7 +223,7 @@ all {% if active_tag_uuid %} in "{{active_tag.title}}"{%endif%}
  • - RSS Feed + RSS Feed
    • {{ pagination.links }} diff --git a/changedetectionio/tests/test_jsonpath_jq_selector.py b/changedetectionio/tests/test_jsonpath_jq_selector.py index 5dfdfef2b..1202849f6 100644 --- a/changedetectionio/tests/test_jsonpath_jq_selector.py +++ b/changedetectionio/tests/test_jsonpath_jq_selector.py @@ -479,8 +479,9 @@ def test_correct_header_detect(client, live_server): url_for("preview_page", uuid="first"), follow_redirects=True ) - assert b'"world":' in res.data - assert res.data.count(b'{') >= 2 + + assert b'"hello": 123,' in res.data + assert b'"world": 123' in res.data res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True) assert b'Deleted' in res.data diff --git a/changedetectionio/tests/test_request.py b/changedetectionio/tests/test_request.py index 869ea3496..cfbc7825a 100644 --- a/changedetectionio/tests/test_request.py +++ b/changedetectionio/tests/test_request.py @@ -256,12 +256,40 @@ def test_method_in_request(client, live_server): def test_headers_textfile_in_request(client, live_server): #live_server_setup(live_server) # Add our URL to the import page + + webdriver_ua = "Hello fancy webdriver UA 1.0" + requests_ua = "Hello basic requests UA 1.1" + test_url = url_for('test_headers', _external=True) if os.getenv('PLAYWRIGHT_DRIVER_URL'): # Because its no longer calling back to localhost but from the browser container, set in test-only.yml test_url = test_url.replace('localhost', 'cdio') - print ("TEST URL IS ",test_url) + form_data = { + "application-fetch_backend": "html_requests", + "application-minutes_between_check": 180, + "requests-default_ua-html_requests": requests_ua + } + + if os.getenv('PLAYWRIGHT_DRIVER_URL'): + form_data["requests-default_ua-html_webdriver"] = webdriver_ua + + res = client.post( + url_for("settings_page"), + data=form_data, + follow_redirects=True + ) + assert b'Settings updated' in res.data + + res = client.get(url_for("settings_page")) + + # Only when some kind of real browser is setup + if os.getenv('PLAYWRIGHT_DRIVER_URL'): + assert b'requests-default_ua-html_webdriver' in res.data + + # Field should always be there + assert b"requests-default_ua-html_requests" in res.data + # Add the test URL twice, we will check res = client.post( url_for("import_page"), @@ -272,15 +300,14 @@ def test_headers_textfile_in_request(client, live_server): wait_for_all_checks(client) - # Add some headers to a request res = client.post( url_for("edit_page", uuid="first"), data={ - "url": test_url, - "tags": "testtag", - "fetch_backend": 'html_webdriver' if os.getenv('PLAYWRIGHT_DRIVER_URL') else 'html_requests', - "headers": "xxx:ooo\ncool:yeah\r\n"}, + "url": test_url, + "tags": "testtag", + "fetch_backend": 'html_webdriver' if os.getenv('PLAYWRIGHT_DRIVER_URL') else 'html_requests', + "headers": "xxx:ooo\ncool:yeah\r\n"}, follow_redirects=True ) assert b"Updated watch." in res.data @@ -292,7 +319,7 @@ def test_headers_textfile_in_request(client, live_server): with open('test-datastore/headers.txt', 'w') as f: f.write("global-header: nice\r\nnext-global-header: nice") - with open('test-datastore/'+extract_UUID_from_client(client)+'/headers.txt', 'w') as f: + with open('test-datastore/' + extract_UUID_from_client(client) + '/headers.txt', 'w') as f: f.write("watch-header: nice") client.get(url_for("form_watch_checknow"), follow_redirects=True) @@ -306,7 +333,7 @@ def test_headers_textfile_in_request(client, live_server): # Not needed anymore os.unlink('test-datastore/headers.txt') os.unlink('test-datastore/headers-testtag.txt') - os.unlink('test-datastore/'+extract_UUID_from_client(client)+'/headers.txt') + os.unlink('test-datastore/' + extract_UUID_from_client(client) + '/headers.txt') # The service should echo back the request verb res = client.get( url_for("preview_page", uuid="first"), @@ -319,7 +346,12 @@ def test_headers_textfile_in_request(client, live_server): assert b"Watch-Header:nice" in res.data assert b"Tag-Header:test" in res.data + # Check the custom UA from system settings page made it through + if os.getenv('PLAYWRIGHT_DRIVER_URL'): + assert "User-Agent:".encode('utf-8') + webdriver_ua.encode('utf-8') in res.data + else: + assert "User-Agent:".encode('utf-8') + requests_ua.encode('utf-8') in res.data - #unlink headers.txt on start/stop + # unlink headers.txt on start/stop res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True) - assert b'Deleted' in res.data \ No newline at end of file + assert b'Deleted' in res.data diff --git a/changedetectionio/tests/test_watch_fields_storage.py b/changedetectionio/tests/test_watch_fields_storage.py index 5044598ce..7dc3f7480 100644 --- a/changedetectionio/tests/test_watch_fields_storage.py +++ b/changedetectionio/tests/test_watch_fields_storage.py @@ -54,102 +54,3 @@ def test_check_watch_field_storage(client, live_server): assert b"woohoo" in res.data assert b"curl: foo" in res.data - - -# Re https://github.com/dgtlmoon/changedetection.io/issues/110 -def test_check_recheck_global_setting(client, live_server): - - res = client.post( - url_for("settings_page"), - data={ - "requests-time_between_check-minutes": 1566, - 'application-fetch_backend': "html_requests" - }, - follow_redirects=True - ) - assert b"Settings updated." in res.data - - # Now add a record - - test_url = "http://somerandomsitewewatch.com" - - res = client.post( - url_for("import_page"), - data={"urls": test_url}, - follow_redirects=True - ) - assert b"1 Imported" in res.data - - # Now visit the edit page, it should have the default minutes - - res = client.get( - url_for("edit_page", uuid="first"), - follow_redirects=True - ) - - # Should show the default minutes - assert b"change to another value if you want to be specific" in res.data - assert b"1566" in res.data - - res = client.post( - url_for("settings_page"), - data={ - "requests-time_between_check-minutes": 222, - 'application-fetch_backend': "html_requests" - }, - follow_redirects=True - ) - assert b"Settings updated." in res.data - - res = client.get( - url_for("edit_page", uuid="first"), - follow_redirects=True - ) - - # Should show the default minutes - assert b"change to another value if you want to be specific" in res.data - assert b"222" in res.data - - # Now change it specifically, it should show the new minutes - res = client.post( - url_for("edit_page", uuid="first"), - data={"url": test_url, - "time_between_check-minutes": 55, - 'fetch_backend': "html_requests" - }, - follow_redirects=True - ) - - res = client.get( - url_for("edit_page", uuid="first"), - follow_redirects=True - ) - assert b"55" in res.data - - # Now submit an empty field, it should give back the default global minutes - res = client.post( - url_for("settings_page"), - data={ - "requests-time_between_check-minutes": 666, - "application-fetch_backend": "html_requests" - }, - follow_redirects=True - ) - assert b"Settings updated." in res.data - - res = client.post( - url_for("edit_page", uuid="first"), - data={"url": test_url, - "time_between_check-minutes": "", - 'fetch_backend': "html_requests" - }, - follow_redirects=True - ) - - assert b"Updated watch." in res.data - - res = client.get( - url_for("edit_page", uuid="first"), - follow_redirects=True - ) - assert b"666" in res.data diff --git a/changedetectionio/tests/visualselector/test_fetch_data.py b/changedetectionio/tests/visualselector/test_fetch_data.py index 2f460d7c7..15677f31f 100644 --- a/changedetectionio/tests/visualselector/test_fetch_data.py +++ b/changedetectionio/tests/visualselector/test_fetch_data.py @@ -102,10 +102,9 @@ def test_basic_browserstep(client, live_server): "url": test_url, "tags": "", 'fetch_backend': "html_webdriver", - 'browser_steps-0-operation': 'Goto site', - 'browser_steps-1-operation': 'Click element', - 'browser_steps-1-selector': 'button[name=test-button]', - 'browser_steps-1-optional_value': '', + 'browser_steps-0-operation': 'Click element', + 'browser_steps-0-selector': 'button[name=test-button]', + 'browser_steps-0-optional_value': '', # For now, cookies doesnt work in headers because it must be a full cookiejar object 'headers': "testheader: yes\buser-agent: MyCustomAgent", }, @@ -141,10 +140,9 @@ def test_basic_browserstep(client, live_server): "url": four_o_four_url, "tags": "", 'fetch_backend': "html_webdriver", - 'browser_steps-0-operation': 'Goto site', - 'browser_steps-1-operation': 'Click element', - 'browser_steps-1-selector': 'button[name=test-button]', - 'browser_steps-1-optional_value': '' + 'browser_steps-0-operation': 'Click element', + 'browser_steps-0-selector': 'button[name=test-button]', + 'browser_steps-0-optional_value': '' }, follow_redirects=True ) diff --git a/docker-compose.yml b/docker-compose.yml index 1b5bd9afc..4cf176058 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -54,7 +54,9 @@ services: # # Default number of parallel/concurrent fetchers # - FETCH_WORKERS=10 - + # + # Absolute minimum seconds to recheck, overrides any watch minimum, change to 0 to disable + # - MINIMUM_SECONDS_RECHECK_TIME=3 # Comment out ports: when using behind a reverse proxy , enable networks: etc. ports: - 5000:5000 diff --git a/requirements.txt b/requirements.txt index 553bfe602..64575b3ae 100644 --- a/requirements.txt +++ b/requirements.txt @@ -36,7 +36,7 @@ dnspython==2.3.0 # related to eventlet fixes # jq not available on Windows so must be installed manually # Notification library -apprise~=1.7.4 +apprise~=1.8.0 # apprise mqtt https://github.com/dgtlmoon/changedetection.io/issues/315 # and 2.0.0 https://github.com/dgtlmoon/changedetection.io/issues/2241 not yet compatible @@ -52,7 +52,10 @@ cryptography~=3.4 beautifulsoup4 # XPath filtering, lxml is required by bs4 anyway, but put it here to be safe. -lxml >=4.8.0,<6 +# #2328 - 5.2.0 and 5.2.1 had extra CPU flag CFLAGS set which was not compatible on older hardware +# It could be advantageous to run its own pypi package here with those performance flags set +# https://bugs.launchpad.net/lxml/+bug/2059910/comments/16 +lxml >=4.8.0,<6,!=5.2.0,!=5.2.1 # XPath 2.0-3.1 support - 4.2.0 broke something? elementpath==4.1.5 @@ -74,8 +77,8 @@ jq~=1.3; python_version >= "3.8" and sys_platform == "linux" pillow # playwright is installed at Dockerfile build time because it's not available on all platforms -# experimental release pyppeteer-ng==2.0.0rc5 +pyppeteerstealth>=0.0.4 # Include pytest, so if theres a support issue we can ask them to run these tests on their setup pytest ~=7.2