Re #1774 - adding test for self closing tag that could break xpath

.github/test/Dockerfile-alpine

@@ -2,7 +2,7 @@
 # Test that we can still build on Alpine (musl modified libc https://musl.libc.org/)
 # Some packages wont install via pypi because they dont have a wheel available under this architecture.
 
-FROM ghcr.io/linuxserver/baseimage-alpine:3.16
+FROM ghcr.io/linuxserver/baseimage-alpine:3.18
 ENV PYTHONUNBUFFERED=1
 
 COPY requirements.txt /requirements.txt
@@ -26,6 +26,6 @@ RUN \
     py3-pip && \
   echo "**** pip3 install test of changedetection.io ****" && \
   pip3 install -U pip wheel setuptools && \
-  pip3 install -U --no-cache-dir --find-links https://wheel-index.linuxserver.io/alpine-3.16/ -r /requirements.txt && \
+  pip3 install -U --no-cache-dir --find-links https://wheel-index.linuxserver.io/alpine-3.18/ -r /requirements.txt && \
   apk del --purge \
     build-dependencies

.github/workflows/containers.yml

@@ -40,10 +40,10 @@ jobs:
     if: ${{ github.event.workflow_run.conclusion == 'success' }} || ${{ github.event.release.tag_name }} != ''
     steps:
       - uses: actions/checkout@v4
-      - name: Set up Python 3.9
+      - name: Set up Python 3.11
         uses: actions/setup-python@v4
         with:
-          python-version: 3.9
+          python-version: 3.11
 
       - name: Install dependencies
         run: |

.github/workflows/test-container-build.yml

@@ -25,10 +25,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
         - uses: actions/checkout@v4
-        - name: Set up Python 3.9
+        - name: Set up Python 3.11
           uses: actions/setup-python@v4
           with:
-            python-version: 3.9
+            python-version: 3.11
 
         # Just test that the build works, some libraries won't compile on ARM/rPi etc
         - name: Set up QEMU

.github/workflows/test-only.yml

@@ -10,10 +10,10 @@ jobs:
       - uses: actions/checkout@v4
 
       # Mainly just for link/flake8
-      - name: Set up Python 3.10
+      - name: Set up Python 3.11
         uses: actions/setup-python@v4
         with:
-          python-version: '3.10'
+          python-version: '3.11'
 
       - name: Lint with flake8
         run: |
@@ -83,6 +83,7 @@ jobs:
         run: |
           cd changedetectionio
           ./run_proxy_tests.sh
+          # And again with PLAYWRIGHT_DRIVER_URL=..
           cd ..
 
       - name: Test changedetection.io container starts+runs basically without error
@@ -98,4 +99,4 @@ jobs:
 
 #export WEBDRIVER_URL=http://localhost:4444/wd/hub
 #pytest tests/fetchers/test_content.py
-#pytest tests/test_errorhandling.py
+#pytest tests/test_errorhandling.py

.github/workflows/test-pip-build.yml

@@ -13,10 +13,10 @@ jobs:
     steps:
         - uses: actions/checkout@v4
 
-        - name: Set up Python 3.9
+        - name: Set up Python 3.11
           uses: actions/setup-python@v4
           with:
-            python-version: 3.9
+            python-version: 3.11
 
 
         - name: Test that the basic pip built package runs without error

Dockerfile

@@ -1,5 +1,5 @@
 # pip dependencies install stage
-FROM python:3.10-slim-bullseye as builder
+FROM python:3.11-slim-bullseye as builder
 
 # See `cryptography` pin comment in requirements.txt
 ARG CRYPTOGRAPHY_DONT_BUILD_RUST=1
@@ -29,7 +29,7 @@ RUN pip install --target=/dependencies playwright~=1.27.1 \
     || echo "WARN: Failed to install Playwright. The application can still run, but the Playwright option will be disabled."
 
 # Final image stage
-FROM python:3.10-slim-bullseye
+FROM python:3.11-slim-bullseye
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
     libssl1.1 \

README-pip.md

@@ -2,19 +2,44 @@
 
 Live your data-life pro-actively, track website content changes and receive notifications via Discord, Email, Slack, Telegram and 70+ more
 
-[<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/docs/screenshot.png" style="max-width:100%;" alt="Self-hosted web page change monitoring"  title="Self-hosted web page change monitoring"  />](https://changedetection.io)
+[<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/docs/screenshot.png" style="max-width:100%;" alt="Self-hosted web page change monitoring, list of websites with changes"  title="Self-hosted web page change monitoring, list of websites with changes"  />](https://changedetection.io)
 
 
 [**Don't have time? Let us host it for you! try our extremely affordable subscription use our proxies and support!**](https://changedetection.io) 
 
 
-#### Example use cases
+### Target specific parts of the webpage using the Visual Selector tool.
+
+Available when connected to a <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Playwright-content-fetcher">playwright content fetcher</a> (included as part of our subscription service)
+
+[<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/docs/visualselector-anim.gif" style="max-width:100%;" alt="Select parts and elements of a web page to monitor for changes"  title="Select parts and elements of a web page to monitor for changes" />](https://changedetection.io?src=pip)
+
+### Easily see what changed, examine by word, line, or individual character.
+
+[<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/docs/screenshot-diff.png" style="max-width:100%;" alt="Self-hosted web page change monitoring context difference "  title="Self-hosted web page change monitoring context difference " />](https://changedetection.io?src=pip)
+
+
+### Perform interactive browser steps
+
+Fill in text boxes, click buttons and more, setup your changedetection scenario. 
+
+Using the **Browser Steps** configuration, add basic steps before performing change detection, such as logging into websites, adding a product to a cart, accept cookie logins, entering dates and refining searches.
+
+[<img src="docs/browsersteps-anim.gif" style="max-width:100%;" alt="Website change detection with interactive browser steps, detect changes behind login and password, search queries and more"  title="Website change detection with interactive browser steps, detect changes behind login and password, search queries and more" />](https://changedetection.io?src=pip)
+
+After **Browser Steps** have been run, then visit the **Visual Selector** tab to refine the content you're interested in.
+Requires Playwright to be enabled.
+
+
+### Example use cases
 
 - Products and services have a change in pricing
 - _Out of stock notification_ and _Back In stock notification_
+- Monitor and track PDF file changes, know when a PDF file has text changes.
 - Governmental department updates (changes are often only on their websites)
 - New software releases, security advisories when you're not on their mailing list.
 - Festivals with changes
+- Discogs restock alerts and monitoring
 - Realestate listing changes
 - Know when your favourite whiskey is on sale, or other special deals are announced before anyone else
 - COVID related news from government websites
@@ -27,18 +52,34 @@ Live your data-life pro-actively, track website content changes and receive noti
 - Create RSS feeds based on changes in web content
 - Monitor HTML source code for unexpected changes, strengthen your PCI compliance
 - You have a very sensitive list of URLs to watch and you do _not_ want to use the paid alternatives. (Remember, _you_ are the product)
+- Get notified when certain keywords appear in Twitter search results
+- Proactively search for jobs, get notified when companies update their careers page, search job portals for keywords.
+- Get alerts when new job positions are open on Bamboo HR and other job platforms
+- Website defacement monitoring
+- Pokémon Card Restock Tracker / Pokémon TCG Tracker
+- RegTech - stay ahead of regulatory changes, regulatory compliance
 
 _Need an actual Chrome runner with Javascript support? We support fetching via WebDriver and Playwright!</a>_
 
 #### Key Features
 
 - Lots of trigger filters, such as "Trigger on text", "Remove text by selector", "Ignore text", "Extract text", also using regular-expressions!
-- Target elements with xPath and CSS Selectors, Easily monitor complex JSON with JSONPath or jq
+- Target elements with xPath(1.0) and CSS Selectors, Easily monitor complex JSON with JSONPath or jq
 - Switch between fast non-JS and Chrome JS based "fetchers"
+- Track changes in PDF files (Monitor text changed in the PDF, Also monitor PDF filesize and checksums)
 - Easily specify how often a site should be checked
 - Execute JS before extracting text (Good for logging in, see examples in the UI!)
 - Override Request Headers, Specify `POST` or `GET` and other methods
 - Use the "Visual Selector" to help target specific elements
+- Configurable [proxy per watch](https://github.com/dgtlmoon/changedetection.io/wiki/Proxy-configuration)
+- Send a screenshot with the notification when a change is detected in the web page
+
+We [recommend and use Bright Data](https://brightdata.grsm.io/n0r16zf7eivq) global proxy services, Bright Data will match any first deposit up to $100 using our signup link.
+
+[Oxylabs](https://oxylabs.go2cloud.org/SH2d) is also an excellent proxy provider and well worth using, they offer Residental, ISP, Rotating and many other proxy types to suit your project. 
+
+Please :star: star :star: this project and help it grow! https://github.com/dgtlmoon/changedetection.io/
+
 
 
 ```bash

README.md

@@ -5,7 +5,7 @@
 _Live your data-life pro-actively._ 
 
 
-[<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/docs/screenshot.png" style="max-width:100%;" alt="Self-hosted web page change monitoring"  title="Self-hosted web page change monitoring"  />](https://changedetection.io?src=github)
+[<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/docs/screenshot.png" style="max-width:100%;" alt="Self-hosted web site page change monitoring"  title="Self-hosted web site page change monitoring"  />](https://changedetection.io?src=github)
 
 [![Release Version][release-shield]][release-link] [![Docker Pulls][docker-pulls]][docker-link] [![License][license-shield]](LICENSE.md)
 
@@ -22,7 +22,7 @@ _Live your data-life pro-actively._
 
 Available when connected to a <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Playwright-content-fetcher">playwright content fetcher</a> (included as part of our subscription service)
 
-[<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/docs/visualselector-anim.gif" style="max-width:100%;" alt="Self-hosted web page change monitoring context difference "  title="Self-hosted web page change monitoring context difference " />](https://changedetection.io?src=github)
+[<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/docs/visualselector-anim.gif" style="max-width:100%;" alt="Select parts and elements of a web page to monitor for changes"  title="Select parts and elements of a web page to monitor for changes" />](https://changedetection.io?src=github)
 
 ### Easily see what changed, examine by word, line, or individual character.
 
@@ -35,7 +35,7 @@ Fill in text boxes, click buttons and more, setup your changedetection scenario.
 
 Using the **Browser Steps** configuration, add basic steps before performing change detection, such as logging into websites, adding a product to a cart, accept cookie logins, entering dates and refining searches.
 
-[<img src="docs/browsersteps-anim.gif" style="max-width:100%;" alt="Self-hosted web page change monitoring context difference "  title="Website change detection with interactive browser steps, login, cookies etc" />](https://changedetection.io?src=github)
+[<img src="docs/browsersteps-anim.gif" style="max-width:100%;" alt="Website change detection with interactive browser steps, detect changes behind login and password, search queries and more"  title="Website change detection with interactive browser steps, detect changes behind login and password, search queries and more" />](https://changedetection.io?src=github)
 
 After **Browser Steps** have been run, then visit the **Visual Selector** tab to refine the content you're interested in.
 Requires Playwright to be enabled.
@@ -226,7 +226,7 @@ The application also supports notifying you that it can follow this information
 
 ## Proxy Configuration
 
-See the wiki https://github.com/dgtlmoon/changedetection.io/wiki/Proxy-configuration , we also support using [Bright Data proxy services where possible]( https://github.com/dgtlmoon/changedetection.io/wiki/Proxy-configuration#brightdata-proxy-support)
+See the wiki https://github.com/dgtlmoon/changedetection.io/wiki/Proxy-configuration , we also support using [Bright Data proxy services where possible](https://github.com/dgtlmoon/changedetection.io/wiki/Proxy-configuration#brightdata-proxy-support) and [Oxylabs](https://oxylabs.go2cloud.org/SH2d) proxy services.
 
 ## Raspberry Pi support?
 

changedetectionio/__init__.py

@@ -38,7 +38,7 @@ from flask_paginate import Pagination, get_page_parameter
 from changedetectionio import html_tools
 from changedetectionio.api import api_v1
 
-__version__ = '0.45.2'
+__version__ = '0.45.3'
 
 from changedetectionio.store import BASE_URL_NOT_SET_TEXT
 
@@ -186,7 +186,6 @@ class User(flask_login.UserMixin):
 
     pass
 
-
 def login_optionally_required(func):
     @wraps(func)
     def decorated_view(*args, **kwargs):
@@ -199,7 +198,6 @@ def login_optionally_required(func):
         # Permitted
         elif request.endpoint == 'diff_history_page' and datastore.data['settings']['application'].get('shared_diff_access'):
             return func(*args, **kwargs)
-
         elif request.method in flask_login.config.EXEMPT_METHODS:
             return func(*args, **kwargs)
         elif app.config.get('LOGIN_DISABLED'):
@@ -715,6 +713,7 @@ def changedetection_app(config=None, datastore_o=None):
                                      available_processors=processors.available_processors(),
                                      browser_steps_config=browser_step_ui_config,
                                      emailprefix=os.getenv('NOTIFICATION_MAIL_BUTTON_PREFIX', False),
+                                     extra_title=f" - Edit - {watch.label}",
                                      form=form,
                                      has_default_notification_urls=True if len(datastore.data['settings']['application']['notification_urls']) else False,
                                      has_empty_checktime=using_default_check_time,
@@ -914,21 +913,29 @@ def changedetection_app(config=None, datastore_o=None):
 
         # Read as binary and force decode as UTF-8
         # Windows may fail decode in python if we just use 'r' mode (chardet decode exception)
-        try:
-            newest_version_file_contents = watch.get_history_snapshot(dates[-1])
-        except Exception as e:
-            newest_version_file_contents = "Unable to read {}.\n".format(dates[-1])
-
-        previous_version = request.args.get('previous_version')
-        previous_timestamp = dates[-2]
-        if previous_version:
-            previous_timestamp = previous_version
+        from_version = request.args.get('from_version')
+        from_version_index = -2 # second newest
+        if from_version and from_version in dates:
+            from_version_index = dates.index(from_version)
+        else:
+            from_version = dates[from_version_index]
 
         try:
-            previous_version_file_contents = watch.get_history_snapshot(previous_timestamp)
+            from_version_file_contents = watch.get_history_snapshot(dates[from_version_index])
         except Exception as e:
-            previous_version_file_contents = "Unable to read {}.\n".format(previous_timestamp)
+            from_version_file_contents = "Unable to read to-version at index{}.\n".format(dates[from_version_index])
 
+        to_version = request.args.get('to_version')
+        to_version_index = -1
+        if to_version and to_version in dates:
+            to_version_index = dates.index(to_version)
+        else:
+            to_version = dates[to_version_index]
+
+        try:
+            to_version_file_contents = watch.get_history_snapshot(dates[to_version_index])
+        except Exception as e:
+            to_version_file_contents = "Unable to read to-version at index{}.\n".format(dates[to_version_index])
 
         screenshot_url = watch.get_screenshot()
 
@@ -944,22 +951,24 @@ def changedetection_app(config=None, datastore_o=None):
 
         output = render_template("diff.html",
                                  current_diff_url=watch['url'],
-                                 current_previous_version=str(previous_version),
+                                 from_version=str(from_version),
+                                 to_version=str(to_version),
                                  extra_stylesheets=extra_stylesheets,
-                                 extra_title=" - Diff - {}".format(watch['title'] if watch['title'] else watch['url']),
+                                 extra_title=f" - Diff - {watch.label}",
                                  extract_form=extract_form,
                                  is_html_webdriver=is_html_webdriver,
                                  last_error=watch['last_error'],
                                  last_error_screenshot=watch.get_error_snapshot(),
                                  last_error_text=watch.get_error_text(),
                                  left_sticky=True,
-                                 newest=newest_version_file_contents,
+                                 newest=to_version_file_contents,
                                  newest_version_timestamp=dates[-1],
                                  password_enabled_and_share_is_off=password_enabled_and_share_is_off,
-                                 previous=previous_version_file_contents,
+                                 from_version_file_contents=from_version_file_contents,
+                                 to_version_file_contents=to_version_file_contents,
                                  screenshot=screenshot_url,
                                  uuid=uuid,
-                                 versions=dates[:-1], # All except current/last
+                                 versions=dates, # All except current/last
                                  watch_a=watch
                                  )
 
@@ -1431,6 +1440,7 @@ def changedetection_app(config=None, datastore_o=None):
         return redirect(url_for('index'))
 
     @app.route("/highlight_submit_ignore_url", methods=['POST'])
+    @login_optionally_required
     def highlight_submit_ignore_url():
         import re
         mode = request.form.get('mode')

changedetectionio/content_fetcher.py

@@ -1,12 +1,15 @@
-import hashlib
 from abc import abstractmethod
+from distutils.util import strtobool
+from urllib.parse import urlparse
 import chardet
+import hashlib
 import json
 import logging
 import os
 import requests
 import sys
 import time
+import urllib.parse
 
 visualselector_xpath_selectors = 'div,span,form,table,tbody,tr,td,a,p,ul,li,h1,h2,h3,h4, header, footer, section, article, aside, details, main, nav, section, summary'
 
@@ -266,7 +269,6 @@ class base_html_playwright(Fetcher):
 
         if self.proxy:
             # Playwright needs separate username and password values
-            from urllib.parse import urlparse
             parsed = urlparse(self.proxy.get('server'))
             if parsed.username:
                 self.proxy['username'] = parsed.username
@@ -321,14 +323,13 @@ class base_html_playwright(Fetcher):
 
         # Append proxy connect string
         if self.proxy:
-            import urllib.parse
             # Remove username/password if it exists in the URL or you will receive "ERR_NO_SUPPORTED_PROXIES" error
             # Actual authentication handled by Puppeteer/node
             o = urlparse(self.proxy.get('server'))
-            proxy_url = urllib.parse.quote(o._replace(netloc="{}:{}".format(o.hostname, o.port)).geturl())
+            # Remove scheme, socks5:// doesnt always work and it will autodetect anyway
+            proxy_url = urllib.parse.quote(o._replace(netloc="{}:{}".format(o.hostname, o.port)).geturl().replace(f"{o.scheme}://", '', 1))
             browserless_function_url = f"{browserless_function_url}&--proxy-server={proxy_url}&dumpio=true"
 
-
         try:
             amp = '&' if '?' in browserless_function_url else '?'
             response = requests.request(
@@ -347,7 +348,7 @@ class base_html_playwright(Fetcher):
                         'url': url,
                         'user_agent': {k.lower(): v for k, v in request_headers.items()}.get('user-agent', None),
                         'proxy_username': self.proxy.get('username', '') if self.proxy else False,
-                        'proxy_password': self.proxy.get('password', '') if self.proxy else False,
+                        'proxy_password': self.proxy.get('password', '') if self.proxy and self.proxy.get('username') else False,
                         'no_cache_list': [
                             'twitter',
                             '.pdf'
@@ -416,8 +417,8 @@ class base_html_playwright(Fetcher):
                 lambda s: (s['operation'] and len(s['operation']) and s['operation'] != 'Choose one' and s['operation'] != 'Goto site'),
                 self.browser_steps))
 
-        if not has_browser_steps:
-            if os.getenv('USE_EXPERIMENTAL_PUPPETEER_FETCH'):
+        if not has_browser_steps and os.getenv('USE_EXPERIMENTAL_PUPPETEER_FETCH'):
+            if strtobool(os.getenv('USE_EXPERIMENTAL_PUPPETEER_FETCH')):
                 # Temporary backup solution until we rewrite the playwright code
                 return self.run_fetch_browserless_puppeteer(
                     url,
@@ -434,6 +435,7 @@ class base_html_playwright(Fetcher):
 
         self.delete_browser_steps_screenshots()
         response = None
+
         with sync_playwright() as p:
             browser_type = getattr(p, self.browser_type)
 
@@ -442,6 +444,9 @@ class base_html_playwright(Fetcher):
             # 60,000 connection timeout only
             browser = browser_type.connect_over_cdp(self.command_executor, timeout=60000)
 
+            # SOCKS5 with authentication is not supported (yet)
+            # https://github.com/microsoft/playwright/issues/10567
+
             # Set user agent to prevent Cloudflare from blocking the browser
             # Use the default one configured in the App.py model that's passed from fetch_site_status.py
             context = browser.new_context(
@@ -478,7 +483,6 @@ class base_html_playwright(Fetcher):
                 print("Content Fetcher > retrying request got error - ", str(e))
                 time.sleep(1)
                 response = self.page.goto(url, wait_until='commit')
-
             except Exception as e:
                 print("Content Fetcher > Other exception when page.goto", str(e))
                 context.close()
@@ -614,7 +618,6 @@ class base_html_webdriver(Fetcher):
         from selenium.common.exceptions import WebDriverException
         # request_body, request_method unused for now, until some magic in the future happens.
 
-        # check env for WEBDRIVER_URL
         self.driver = webdriver.Remote(
             command_executor=self.command_executor,
             desired_capabilities=DesiredCapabilities.CHROME,
@@ -693,6 +696,10 @@ class html_requests(Fetcher):
         proxies = {}
 
         # Allows override the proxy on a per-request basis
+
+        # https://requests.readthedocs.io/en/latest/user/advanced/#socks
+        # Should also work with `socks5://user:pass@host:port` type syntax.
+
         if self.proxy_override:
             proxies = {'http': self.proxy_override, 'https': self.proxy_override, 'ftp': self.proxy_override}
         else:

changedetectionio/forms.py

@@ -481,7 +481,7 @@ class SingleExtraProxy(Form):
 
     # maybe better to set some <script>var..
     proxy_name = StringField('Name', [validators.Optional()], render_kw={"placeholder": "Name"})
-    proxy_url = StringField('Proxy URL', [validators.Optional()], render_kw={"placeholder": "http://user:pass@...:3128", "size":50})
+    proxy_url = StringField('Proxy URL', [validators.Optional()], render_kw={"placeholder": "socks5:// or regular proxy http://user:pass@...:3128", "size":50})
     # @todo do the validation here instead
 
 # datastore.data['settings']['requests']..

changedetectionio/html_tools.py

@@ -7,13 +7,14 @@ from typing import List
 import json
 import re
 
+
 # HTML added to be sure each result matching a filter (.example) gets converted to a new line by Inscriptis
 TEXT_FILTER_LIST_LINE_SUFFIX = "<br>"
 
 PERL_STYLE_REGEX = r'^/(.*?)/([a-z]*)?$'
 # 'price' , 'lowPrice', 'highPrice' are usually under here
-# all of those may or may not appear on different websites
-LD_JSON_PRODUCT_OFFER_SELECTOR = "json:$..offers"
+# All of those may or may not appear on different websites - I didnt find a way todo case-insensitive searching here
+LD_JSON_PRODUCT_OFFER_SELECTORS = ["json:$..offers", "json:$..Offers"]
 
 class JSONNotFound(ValueError):
     def __init__(self, msg):
@@ -161,7 +162,6 @@ def extract_json_as_string(content, json_filter, ensure_is_ldjson_info_type=None
 
         # Foreach <script json></script> blob.. just return the first that matches json_filter
         # As a last resort, try to parse the whole <body>
-        s = []
         soup = BeautifulSoup(content, 'html.parser')
 
         if ensure_is_ldjson_info_type:
@@ -187,13 +187,24 @@ def extract_json_as_string(content, json_filter, ensure_is_ldjson_info_type=None
         
         for json_data in bs_jsons:
             stripped_text_from_html = _parse_json(json_data, json_filter)
+
             if ensure_is_ldjson_info_type:
                 # Could sometimes be list, string or something else random
                 if isinstance(json_data, dict):
                     # If it has LD JSON 'key' @type, and @type is 'product', and something was found for the search
                     # (Some sites have multiple of the same ld+json @type='product', but some have the review part, some have the 'price' part)
-                    if json_data.get('@type', False) and json_data.get('@type','').lower() == ensure_is_ldjson_info_type.lower() and stripped_text_from_html:
-                        break
+                    # @type could also be a list (Product, SubType)
+                    # LD_JSON auto-extract also requires some content PLUS the ldjson to be present
+                    # 1833 - could be either str or dict, should not be anything else
+                    if json_data.get('@type') and stripped_text_from_html:
+                        try:
+                            if json_data.get('@type') == str or json_data.get('@type') == dict:
+                                types = [json_data.get('@type')] if isinstance(json_data.get('@type'), str) else json_data.get('@type')
+                                if ensure_is_ldjson_info_type.lower() in [x.lower().strip() for x in types]:
+                                    break
+                        except:
+                            continue
+
             elif stripped_text_from_html:
                 break
 
@@ -283,9 +294,18 @@ def html_to_text(html_content: str, render_anchor_tag_content=False) -> str:
 
 # Does LD+JSON exist with a @type=='product' and a .price set anywhere?
 def has_ldjson_product_info(content):
+    pricing_data = ''
+
     try:
-        pricing_data = extract_json_as_string(content=content, json_filter=LD_JSON_PRODUCT_OFFER_SELECTOR, ensure_is_ldjson_info_type="product")
-    except JSONNotFound as e:
+        if not 'application/ld+json' in content:
+            return False
+
+        for filter in LD_JSON_PRODUCT_OFFER_SELECTORS:
+            pricing_data += extract_json_as_string(content=content,
+                                                  json_filter=filter,
+                                                  ensure_is_ldjson_info_type="product")
+
+    except Exception as e:
         # Totally fine
         return False
     x=bool(pricing_data)

changedetectionio/model/Watch.py

@@ -167,9 +167,7 @@ class model(dict):
     @property
     def label(self):
         # Used for sorting
-        if self['title']:
-            return self['title']
-        return self['url']
+        return self.get('title') if self.get('title') else self.get('url')
 
     @property
     def last_changed(self):

changedetectionio/processors/text_json_diff.py

@@ -17,7 +17,7 @@ urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 
 name = 'Webpage Text/HTML, JSON and PDF changes'
 description = 'Detects all text changes where possible'
-
+json_filter_prefixes = ['json:', 'jq:']
 
 class FilterNotFoundInResponse(ValueError):
     def __init__(self, msg):
@@ -196,7 +196,7 @@ class perform_site_check(difference_detection_processor):
 
         # Inject a virtual LD+JSON price tracker rule
         if watch.get('track_ldjson_price_data', '') == PRICE_DATA_TRACK_ACCEPT:
-            include_filters_rule.append(html_tools.LD_JSON_PRODUCT_OFFER_SELECTOR)
+            include_filters_rule += html_tools.LD_JSON_PRODUCT_OFFER_SELECTORS
 
         has_filter_rule = len(include_filters_rule) and len(include_filters_rule[0].strip())
         has_subtractive_selectors = len(subtractive_selectors) and len(subtractive_selectors[0].strip())
@@ -214,7 +214,6 @@ class perform_site_check(difference_detection_processor):
                 pass
 
         if has_filter_rule:
-            json_filter_prefixes = ['json:', 'jq:']
             for filter in include_filters_rule:
                 if any(prefix in filter for prefix in json_filter_prefixes):
                     stripped_text_from_html += html_tools.extract_json_as_string(content=fetcher.content, json_filter=filter)

changedetectionio/res/puppeteer_fetch.js

@@ -18,6 +18,7 @@ module.exports = async ({page, context}) => {
 
     await page.setBypassCSP(true)
     await page.setExtraHTTPHeaders(req_headers);
+
     if (user_agent) {
         await page.setUserAgent(user_agent);
     }
@@ -26,6 +27,10 @@ module.exports = async ({page, context}) => {
     await page.setDefaultNavigationTimeout(0);
 
     if (proxy_username) {
+        // Setting Proxy-Authentication header is deprecated, and doing so can trigger header change errors from Puppeteer
+        // https://github.com/puppeteer/puppeteer/issues/676 ?
+        // https://help.brightdata.com/hc/en-us/articles/12632549957649-Proxy-Manager-How-to-Guides#h_01HAKWR4Q0AFS8RZTNYWRDFJC2
+        // https://cri.dev/posts/2020-03-30-How-to-solve-Puppeteer-Chrome-Error-ERR_INVALID_ARGUMENT/
         await page.authenticate({
             username: proxy_username,
             password: proxy_password

changedetectionio/run_proxy_tests.sh

@@ -10,6 +10,40 @@ set -x
 docker run --network changedet-network -d --name squid-one --hostname squid-one --rm -v `pwd`/tests/proxy_list/squid.conf:/etc/squid/conf.d/debian.conf ubuntu/squid:4.13-21.10_edge
 docker run --network changedet-network -d --name squid-two --hostname squid-two --rm -v `pwd`/tests/proxy_list/squid.conf:/etc/squid/conf.d/debian.conf ubuntu/squid:4.13-21.10_edge
 
+# SOCKS5 related - start simple Socks5 proxy server
+# SOCKSTEST=xyz should show in the logs of this service to confirm it fetched
+docker run --network changedet-network -d --hostname socks5proxy --name socks5proxy -p 1080:1080 -e PROXY_USER=proxy_user123 -e PROXY_PASSWORD=proxy_pass123 serjs/go-socks5-proxy
+docker run --network changedet-network -d --hostname socks5proxy-noauth -p 1081:1080 --name socks5proxy-noauth  serjs/go-socks5-proxy
+
+echo "---------------------------------- SOCKS5 -------------------"
+# SOCKS5 related - test from proxies.json
+docker run --network changedet-network \
+  -v `pwd`/tests/proxy_socks5/proxies.json-example:/app/changedetectionio/test-datastore/proxies.json \
+  --rm \
+  -e "SOCKSTEST=proxiesjson" \
+  test-changedetectionio \
+  bash -c 'cd changedetectionio && pytest tests/proxy_socks5/test_socks5_proxy_sources.py'
+
+# SOCKS5 related - by manually entering in UI
+docker run --network changedet-network \
+  --rm \
+  -e "SOCKSTEST=manual" \
+  test-changedetectionio \
+  bash -c 'cd changedetectionio && pytest tests/proxy_socks5/test_socks5_proxy.py'
+
+# SOCKS5 related - test from proxies.json via playwright - NOTE- PLAYWRIGHT DOESNT SUPPORT AUTHENTICATING PROXY
+docker run --network changedet-network \
+  -e "SOCKSTEST=manual-playwright" \
+  -v `pwd`/tests/proxy_socks5/proxies.json-example-noauth:/app/changedetectionio/test-datastore/proxies.json \
+  -e "PLAYWRIGHT_DRIVER_URL=ws://browserless:3000" \
+  --rm \
+  test-changedetectionio \
+  bash -c 'cd changedetectionio && pytest tests/proxy_socks5/test_socks5_proxy_sources.py'
+
+echo "socks5 server logs"
+docker logs socks5proxy
+echo "----------------------------------"
+
 # Used for configuring a custom proxy URL via the UI
 docker run --network changedet-network -d \
   --name squid-custom \

changedetectionio/static/js/diff-render.js

@@ -1,110 +1,120 @@
-var a = document.getElementById("a");
-var b = document.getElementById("b");
-var result = document.getElementById("result");
-
-function changed() {
-  // https://github.com/kpdecker/jsdiff/issues/389
-  // I would love to use `{ignoreWhitespace: true}` here but it breaks the formatting
-  options = {
-    ignoreWhitespace: document.getElementById("ignoreWhitespace").checked,
-  };
-
-  var diff = Diff[window.diffType](a.textContent, b.textContent, options);
-  var fragment = document.createDocumentFragment();
-  for (var i = 0; i < diff.length; i++) {
-    if (diff[i].added && diff[i + 1] && diff[i + 1].removed) {
-      var swap = diff[i];
-      diff[i] = diff[i + 1];
-      diff[i + 1] = swap;
-    }
-
-    var node;
-    if (diff[i].removed) {
-      node = document.createElement("del");
-      node.classList.add("change");
-      const wrapper = node.appendChild(document.createElement("span"));
-      wrapper.appendChild(document.createTextNode(diff[i].value));
-    } else if (diff[i].added) {
-      node = document.createElement("ins");
-      node.classList.add("change");
-      const wrapper = node.appendChild(document.createElement("span"));
-      wrapper.appendChild(document.createTextNode(diff[i].value));
-    } else {
-      node = document.createTextNode(diff[i].value);
-    }
-    fragment.appendChild(node);
-  }
-
-  result.textContent = "";
-  result.appendChild(fragment);
-
-  // Jump at start
-  inputs.current = 0;
-  next_diff();
-}
-
-window.onload = function () {
-  /* Convert what is options from UTC time.time() to local browser time */
-  var diffList = document.getElementById("diff-version");
-  if (typeof diffList != "undefined" && diffList != null) {
-    for (var option of diffList.options) {
-      var dateObject = new Date(option.value * 1000);
-      option.label = dateObject.toLocaleString();
-    }
-  }
-
-  /* Set current version date as local time in the browser also */
-  var current_v = document.getElementById("current-v-date");
-  var dateObject = new Date(newest_version_timestamp * 1000);
-  current_v.innerHTML = dateObject.toLocaleString();
-  onDiffTypeChange(
-    document.querySelector('#settings [name="diff_type"]:checked'),
-  );
-  changed();
-};
-
-a.onpaste = a.onchange = b.onpaste = b.onchange = changed;
-
-if ("oninput" in a) {
-  a.oninput = b.oninput = changed;
-} else {
-  a.onkeyup = b.onkeyup = changed;
-}
-
-function onDiffTypeChange(radio) {
-  window.diffType = radio.value;
-  // Not necessary
-  //	document.title = "Diff " + radio.value.slice(4);
-}
-
-var radio = document.getElementsByName("diff_type");
-for (var i = 0; i < radio.length; i++) {
-  radio[i].onchange = function (e) {
-    onDiffTypeChange(e.target);
-    changed();
-  };
-}
-
-document.getElementById("ignoreWhitespace").onchange = function (e) {
-  changed();
-};
-
-var inputs = document.getElementsByClassName("change");
-inputs.current = 0;
-
-function next_diff() {
-  var element = inputs[inputs.current];
-  var headerOffset = 80;
-  var elementPosition = element.getBoundingClientRect().top;
-  var offsetPosition = elementPosition - headerOffset + window.scrollY;
-
-  window.scrollTo({
-    top: offsetPosition,
-    behavior: "smooth",
-  });
-
-  inputs.current++;
-  if (inputs.current >= inputs.length) {
+$(document).ready(function () {
+    var a = document.getElementById("a");
+    var b = document.getElementById("b");
+    var result = document.getElementById("result");
+    var inputs = document.getElementsByClassName("change");
     inputs.current = 0;
-  }
-}
+
+    $('#jump-next-diff').click(function () {
+
+        var element = inputs[inputs.current];
+        var headerOffset = 80;
+        var elementPosition = element.getBoundingClientRect().top;
+        var offsetPosition = elementPosition - headerOffset + window.scrollY;
+
+        window.scrollTo({
+            top: offsetPosition,
+            behavior: "smooth",
+        });
+
+        inputs.current++;
+        if (inputs.current >= inputs.length) {
+            inputs.current = 0;
+        }
+    });
+
+    function changed() {
+        // https://github.com/kpdecker/jsdiff/issues/389
+        // I would love to use `{ignoreWhitespace: true}` here but it breaks the formatting
+        options = {
+            ignoreWhitespace: document.getElementById("ignoreWhitespace").checked,
+        };
+
+        var diff = Diff[window.diffType](a.textContent, b.textContent, options);
+        var fragment = document.createDocumentFragment();
+        for (var i = 0; i < diff.length; i++) {
+            if (diff[i].added && diff[i + 1] && diff[i + 1].removed) {
+                var swap = diff[i];
+                diff[i] = diff[i + 1];
+                diff[i + 1] = swap;
+            }
+
+            var node;
+            if (diff[i].removed) {
+                node = document.createElement("del");
+                node.classList.add("change");
+                const wrapper = node.appendChild(document.createElement("span"));
+                wrapper.appendChild(document.createTextNode(diff[i].value));
+            } else if (diff[i].added) {
+                node = document.createElement("ins");
+                node.classList.add("change");
+                const wrapper = node.appendChild(document.createElement("span"));
+                wrapper.appendChild(document.createTextNode(diff[i].value));
+            } else {
+                node = document.createTextNode(diff[i].value);
+            }
+            fragment.appendChild(node);
+        }
+
+        result.textContent = "";
+        result.appendChild(fragment);
+
+        // Jump at start
+        inputs.current = 0;
+
+        // For nice mouse-over hover/title information
+        const removed_current_option = $('#diff-version option:selected')
+        if (removed_current_option) {
+            $('del').each(function () {
+                $(this).prop('title', 'Removed '+removed_current_option[0].label);
+            });
+        }
+        const inserted_current_option = $('#current-version option:selected')
+        if (removed_current_option) {
+            $('ins').each(function () {
+                $(this).prop('title', 'Inserted '+inserted_current_option[0].label);
+            });
+        }
+
+        next_diff();
+    }
+
+    $('.needs-localtime').each(function () {
+        for (var option of this.options) {
+            var dateObject = new Date(option.value * 1000);
+            option.label = dateObject.toLocaleString(undefined, {dateStyle: "full", timeStyle: "medium"});
+        }
+    })
+    onDiffTypeChange(
+        document.querySelector('#settings [name="diff_type"]:checked'),
+    );
+    changed();
+
+    a.onpaste = a.onchange = b.onpaste = b.onchange = changed;
+
+    if ("oninput" in a) {
+        a.oninput = b.oninput = changed;
+    } else {
+        a.onkeyup = b.onkeyup = changed;
+    }
+
+    function onDiffTypeChange(radio) {
+        window.diffType = radio.value;
+        // Not necessary
+        //	document.title = "Diff " + radio.value.slice(4);
+    }
+
+    var radio = document.getElementsByName("diff_type");
+    for (var i = 0; i < radio.length; i++) {
+        radio[i].onchange = function (e) {
+            onDiffTypeChange(e.target);
+            changed();
+        };
+    }
+
+    document.getElementById("ignoreWhitespace").onchange = function (e) {
+        changed();
+    };
+
+});
+

changedetectionio/static/styles/diff.css

@@ -187,6 +187,10 @@ ins {
     padding: 0.5em; }
   #settings ins {
     padding: 0.5em; }
+  #settings option:checked {
+    font-weight: bold; }
+  #settings [type=radio], #settings [type=checkbox] {
+    vertical-align: middle; }
 
 .source {
   position: absolute;

changedetectionio/static/styles/scss/diff.scss

@@ -77,6 +77,13 @@ ins {
   ins {
     padding: 0.5em;
   }
+
+  option:checked {
+    font-weight: bold;
+  }
+  [type=radio],[type=checkbox] {
+    vertical-align: middle;
+  }
 }
 
 .source {

changedetectionio/static/styles/scss/styles.scss

@@ -471,7 +471,11 @@ footer {
   padding: 10px;
 
   &#left-sticky {
-    left: 0px;
+    left: 0;
+    position: fixed;
+    border-top-right-radius: 5px;
+    border-bottom-right-radius: 5px;
+    box-shadow: 1px 1px 4px var(--color-shadow-jump);
   }
 
   &#right-sticky {

changedetectionio/static/styles/styles.css

@@ -667,7 +667,11 @@ footer {
   background: var(--color-background);
   padding: 10px; }
   .sticky-tab#left-sticky {
-    left: 0px; }
+    left: 0;
+    position: fixed;
+    border-top-right-radius: 5px;
+    border-bottom-right-radius: 5px;
+    box-shadow: 1px 1px 4px var(--color-shadow-jump); }
   .sticky-tab#right-sticky {
     right: 0px; }
   .sticky-tab#hosted-sticky {

changedetectionio/store.py

@@ -42,6 +42,7 @@ class ChangeDetectionStore:
         self.__data = App.model()
         self.datastore_path = datastore_path
         self.json_store_path = "{}/url-watches.json".format(self.datastore_path)
+        print(">>> Datastore path is ", self.json_store_path)
         self.needs_write = False
         self.start_time = time.time()
         self.stop_thread = False
@@ -95,6 +96,14 @@ class ChangeDetectionStore:
                 self.add_watch(url='https://changedetection.io/CHANGELOG.txt',
                                tag='changedetection.io',
                                extras={'fetch_backend': 'html_requests'})
+
+            updates_available = self.get_updates_available()
+            self.__data['settings']['application']['schema_version'] = updates_available.pop()
+
+        else:
+            # Bump the update version by running updates
+            self.run_updates()
+
         self.__data['version_tag'] = version_tag
 
         # Just to test that proxies.json if it exists, doesnt throw a parsing error on startup
@@ -124,9 +133,6 @@ class ChangeDetectionStore:
             secret = secrets.token_hex(16)
             self.__data['settings']['application']['api_access_token'] = secret
 
-        # Bump the update version by running updates
-        self.run_updates()
-
         self.needs_write = True
 
         # Finally start the thread that will manage periodic data saves to JSON
@@ -624,14 +630,8 @@ class ChangeDetectionStore:
     def tag_exists_by_name(self, tag_name):
         return any(v.get('title', '').lower() == tag_name.lower() for k, v in self.__data['settings']['application']['tags'].items())
 
-    # Run all updates
-    # IMPORTANT - Each update could be run even when they have a new install and the schema is correct
-    #             So therefor - each `update_n` should be very careful about checking if it needs to actually run
-    #             Probably we should bump the current update schema version with each tag release version?
-    def run_updates(self):
+    def get_updates_available(self):
         import inspect
-        import shutil
-
         updates_available = []
         for i, o in inspect.getmembers(self, predicate=inspect.ismethod):
             m = re.search(r'update_(\d+)$', i)
@@ -639,6 +639,15 @@ class ChangeDetectionStore:
                 updates_available.append(int(m.group(1)))
         updates_available.sort()
 
+        return updates_available
+
+    # Run all updates
+    # IMPORTANT - Each update could be run even when they have a new install and the schema is correct
+    #             So therefor - each `update_n` should be very careful about checking if it needs to actually run
+    #             Probably we should bump the current update schema version with each tag release version?
+    def run_updates(self):
+        import shutil
+        updates_available = self.get_updates_available()
         for update_n in updates_available:
             if update_n > self.__data['settings']['application']['schema_version']:
                 print ("Applying update_{}".format((update_n)))

changedetectionio/templates/base.html

@@ -121,7 +121,8 @@
     {% endif %}
     {% if left_sticky %}
       <div class="sticky-tab" id="left-sticky">
-        <a href="{{url_for('preview_page', uuid=uuid)}}">Show current snapshot</a>
+        <a href="{{url_for('preview_page', uuid=uuid)}}">Show current snapshot</a><br>
+          Visualise <strong>triggers</strong> and <strong>ignored text</strong>
       </div>
     {% endif %}
     {% if right_sticky %}

changedetectionio/templates/diff.html

@@ -13,10 +13,31 @@
 <script src="{{url_for('static_content', group='js', filename='diff-overview.js')}}" defer></script>
 
 <div id="settings">
-    <h1>Differences</h1>
     <form class="pure-form " action="" method="GET">
         <fieldset>
-
+            {% if versions|length >= 1 %}
+                <strong>Compare</strong>
+                <del class="change"><span>from</span></del>
+                <select id="diff-version" name="from_version" class="needs-localtime">
+                    {% for version in versions|reverse %}
+                        <option value="{{ version }}" {% if version== from_version %} selected="" {% endif %}>
+                            {{ version }}
+                        </option>
+                    {% endfor %}
+                </select>
+                <ins class="change"><span>to</span></ins>
+                <select id="current-version" name="to_version" class="needs-localtime">
+                    {% for version in versions|reverse %}
+                        <option value="{{ version }}" {% if version== to_version %} selected="" {% endif %}>
+                            {{ version }}
+                        </option>
+                    {% endfor %}
+                </select>
+                <button type="submit" class="pure-button pure-button-primary">Go</button>
+            {% endif %}
+        </fieldset>
+        <fieldset>
+            <strong>Style</strong>
             <label for="diffWords" class="pure-checkbox">
                 <input type="radio" name="diff_type" id="diffWords" value="diffWords"> Words</label>
             <label for="diffLines" class="pure-checkbox">
@@ -26,32 +47,20 @@
                 <input type="radio" name="diff_type" id="diffChars" value="diffChars"> Chars</label>
             <!-- @todo - when mimetype is JSON, select this by default? -->
             <label for="diffJson" class="pure-checkbox">
-                <input type="radio" name="diff_type" id="diffJson" value="diffJson" > JSON</label>
+                <input type="radio" name="diff_type" id="diffJson" value="diffJson"> JSON</label>
 
-            {% if versions|length >= 1 %}
-            <label for="diff-version">Compare newest (<span id="current-v-date"></span>) with</label>
-            <select id="diff-version" name="previous_version">
-                {% for version in versions|reverse %}
-                <option value="{{version}}" {% if version== current_previous_version %} selected="" {% endif %}>
-                    {{version}}
-                </option>
-                {% endfor %}
-            </select>
-            <button type="submit" class="pure-button pure-button-primary">Go</button>
-            {% endif %}
-        </fieldset>
-    </form>
-    <del>Removed text</del>
-    <ins>Inserted Text</ins>
-    <span>
+            <span>
         <!-- https://github.com/kpdecker/jsdiff/issues/389 ? -->
         <label for="ignoreWhitespace" class="pure-checkbox" id="label-diff-ignorewhitespace">
-            <input type="checkbox" id="ignoreWhitespace" name="ignoreWhitespace" > Ignore Whitespace</label>
+            <input type="checkbox" id="ignoreWhitespace" name="ignoreWhitespace"> Ignore Whitespace</label>
     </span>
+        </fieldset>
+    </form>
+
 </div>
 
 <div id="diff-jump">
-    <a onclick="next_diff();">Jump</a>
+    <a id="jump-next-diff">Jump</a>
 </div>
 
 <script src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
@@ -79,8 +88,6 @@
     </div>
 
      <div class="tab-pane-inner" id="text">
-         <div class="tip">Pro-tip: Use <strong>show current snapshot</strong> tab to visualise what will be ignored, highlight text to add to ignore filters</div>
-
          {% if password_enabled_and_share_is_off %}
            <div class="tip">Pro-tip: You can enable <strong>"share access when password is enabled"</strong> from settings</div>
          {% endif %}
@@ -91,8 +98,8 @@
              <tbody>
              <tr>
                  <!-- just proof of concept copied straight from github.com/kpdecker/jsdiff -->
-                 <td id="a" style="display: none;">{{previous}}</td>
-                 <td id="b" style="display: none;">{{newest}}</td>
+                 <td id="a" style="display: none;">{{from_version_file_contents}}</td>
+                 <td id="b" style="display: none;">{{to_version_file_contents}}</td>
                  <td id="diff-col">
                      <span id="result" class="highlightable-filter"></span>
                  </td>

changedetectionio/templates/edit.html

@@ -109,7 +109,7 @@
                         <span class="pure-form-message-inline">
                             <p>Use the <strong>Basic</strong> method (default) where your watched site doesn't need Javascript to render.</p>
                             <p>The <strong>Chrome/Javascript</strong> method requires a network connection to a running WebDriver+Chrome server, set by the ENV var 'WEBDRIVER_URL'. </p>
-                            Tip: <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Proxy-configuration#brightdata-proxy-support">Connect using BrightData Proxies, find out more here.</a>
+                            Tip: <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Proxy-configuration#brightdata-proxy-support">Connect using Bright Data and Oxylabs Proxies, find out more here.</a>
                         </span>
                     </div>
                 {% if form.proxy %}

changedetectionio/templates/settings.html

@@ -109,7 +109,7 @@
                         <p>The <strong>Chrome/Javascript</strong> method requires a network connection to a running WebDriver+Chrome server, set by the ENV var 'WEBDRIVER_URL'. </p>
                     </span>
                     <br>
-                    Tip: <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Proxy-configuration#brightdata-proxy-support">Connect using BrightData Proxies, find out more here.</a>
+                    Tip: <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Proxy-configuration#brightdata-proxy-support">Connect using Bright Data and Oxylabs Proxies, find out more here.</a>
                 </div>
                 <fieldset class="pure-group" id="webdriver-override-options">
                     <div class="pure-form-message-inline">
@@ -229,7 +229,8 @@ nav
 
                 <div class="pure-control-group">
                 {{ render_field(form.requests.form.extra_proxies) }}
-                <span class="pure-form-message-inline">"Name" will be used for selecting the proxy in the Watch Edit settings</span>
+                <span class="pure-form-message-inline">"Name" will be used for selecting the proxy in the Watch Edit settings</span><br>
+                <span class="pure-form-message-inline">SOCKS5 proxies with authentication are only supported with 'plain requests' fetcher, for other fetchers you should whitelist the IP access instead</span>
                 </div>
             </div>
             <div id="actions">

changedetectionio/tests/fetchers/test_content.py

@@ -28,8 +28,6 @@ def test_fetch_webdriver_content(client, live_server):
     )
 
     assert b"1 Imported" in res.data
-    time.sleep(3)
-
     wait_for_all_checks(client)
 
 

changedetectionio/tests/proxy_list/test_multiple_proxy.py

@@ -2,12 +2,11 @@
 
 import time
 from flask import url_for
-from ..util import live_server_setup
+from ..util import live_server_setup, wait_for_all_checks
+
 
 def test_preferred_proxy(client, live_server):
-    time.sleep(1)
     live_server_setup(live_server)
-    time.sleep(1)
     url = "http://chosen.changedetection.io"
 
     res = client.post(
@@ -20,7 +19,7 @@ def test_preferred_proxy(client, live_server):
 
     assert b"1 Imported" in res.data
 
-    time.sleep(2)
+    wait_for_all_checks(client)
     res = client.post(
         url_for("edit_page", uuid="first"),
         data={
@@ -34,5 +33,5 @@ def test_preferred_proxy(client, live_server):
         follow_redirects=True
     )
     assert b"Updated watch." in res.data
-    time.sleep(2)
+    wait_for_all_checks(client)
     # Now the request should appear in the second-squid logs

changedetectionio/tests/proxy_socks5/proxies.json-example

@@ -0,0 +1,6 @@
+{
+  "socks5proxy": {
+    "label": "socks5proxy",
+    "url": "socks5://proxy_user123:proxy_pass123@socks5proxy:1080"
+  }
+}

changedetectionio/tests/proxy_socks5/proxies.json-example-noauth

@@ -0,0 +1,6 @@
+{
+  "socks5proxy": {
+    "label": "socks5proxy",
+    "url": "socks5://socks5proxy-noauth:1080"
+  }
+}

changedetectionio/tests/proxy_socks5/test_socks5_proxy.py

@@ -0,0 +1,63 @@
+#!/usr/bin/python3
+import os
+import time
+from flask import url_for
+from changedetectionio.tests.util import live_server_setup, wait_for_all_checks
+
+
+def test_socks5(client, live_server):
+    live_server_setup(live_server)
+
+    # Setup a proxy
+    res = client.post(
+        url_for("settings_page"),
+        data={
+            "requests-time_between_check-minutes": 180,
+            "application-ignore_whitespace": "y",
+            "application-fetch_backend": "html_requests",
+            # set in .github/workflows/test-only.yml
+            "requests-extra_proxies-0-proxy_url": "socks5://proxy_user123:proxy_pass123@socks5proxy:1080",
+            "requests-extra_proxies-0-proxy_name": "socks5proxy",
+        },
+        follow_redirects=True
+    )
+
+    assert b"Settings updated." in res.data
+
+    test_url = "https://changedetection.io/CHANGELOG.txt?socks-test-tag=" + os.getenv('SOCKSTEST', '')
+
+    res = client.post(
+        url_for("form_quick_watch_add"),
+        data={"url": test_url, "tags": '', 'edit_and_watch_submit_button': 'Edit > Watch'},
+        follow_redirects=True
+    )
+    assert b"Watch added in Paused state, saving will unpause" in res.data
+
+    res = client.get(
+        url_for("edit_page", uuid="first", unpause_on_save=1),
+    )
+    # check the proxy is offered as expected
+    assert b'ui-0socks5proxy' in res.data
+
+    res = client.post(
+        url_for("edit_page", uuid="first", unpause_on_save=1),
+        data={
+            "include_filters": "",
+            "fetch_backend": 'html_webdriver' if os.getenv('PLAYWRIGHT_DRIVER_URL') else 'html_requests',
+            "headers": "",
+            "proxy": "ui-0socks5proxy",
+            "tags": "",
+            "url": test_url,
+        },
+        follow_redirects=True
+    )
+    assert b"unpaused" in res.data
+    wait_for_all_checks(client)
+
+    res = client.get(
+        url_for("preview_page", uuid="first"),
+        follow_redirects=True
+    )
+
+    # Should see the proper string
+    assert "+0200:".encode('utf-8') in res.data

changedetectionio/tests/proxy_socks5/test_socks5_proxy_sources.py

@@ -0,0 +1,52 @@
+#!/usr/bin/python3
+import os
+import time
+from flask import url_for
+from changedetectionio.tests.util import live_server_setup, wait_for_all_checks
+
+
+# should be proxies.json mounted from run_proxy_tests.sh already
+# -v `pwd`/tests/proxy_socks5/proxies.json-example:/app/changedetectionio/test-datastore/proxies.json
+def test_socks5_from_proxiesjson_file(client, live_server):
+    live_server_setup(live_server)
+
+    test_url = "https://changedetection.io/CHANGELOG.txt?socks-test-tag=" + os.getenv('SOCKSTEST', '')
+
+    res = client.get(url_for("settings_page"))
+    assert b'name="requests-proxy" type="radio" value="socks5proxy"' in res.data
+
+    res = client.post(
+        url_for("form_quick_watch_add"),
+        data={"url": test_url, "tags": '', 'edit_and_watch_submit_button': 'Edit > Watch'},
+        follow_redirects=True
+    )
+    assert b"Watch added in Paused state, saving will unpause" in res.data
+
+    res = client.get(
+        url_for("edit_page", uuid="first", unpause_on_save=1),
+    )
+    # check the proxy is offered as expected
+    assert b'name="proxy" type="radio" value="socks5proxy"' in res.data
+
+    res = client.post(
+        url_for("edit_page", uuid="first", unpause_on_save=1),
+        data={
+            "include_filters": "",
+            "fetch_backend": 'html_webdriver' if os.getenv('PLAYWRIGHT_DRIVER_URL') else 'html_requests',
+            "headers": "",
+            "proxy": "socks5proxy",
+            "tags": "",
+            "url": test_url,
+        },
+        follow_redirects=True
+    )
+    assert b"unpaused" in res.data
+    wait_for_all_checks(client)
+
+    res = client.get(
+        url_for("preview_page", uuid="first"),
+        follow_redirects=True
+    )
+
+    # Should see the proper string
+    assert "+0200:".encode('utf-8') in res.data

changedetectionio/tests/test_automatic_follow_ldjson_price.py

@@ -2,7 +2,8 @@
 
 import time
 from flask import url_for
-from .util import live_server_setup, extract_UUID_from_client, extract_api_key_from_UI
+from .util import live_server_setup, extract_UUID_from_client, extract_api_key_from_UI, wait_for_all_checks
+
 
 def set_response_with_ldjson():
     test_return_data = """<html>
@@ -27,7 +28,7 @@ def set_response_with_ldjson():
            "description":"You dont need it",
            "mpn":"111111",
            "sku":"22222",
-           "offers":{
+           "Offers":{
               "@type":"AggregateOffer",
               "lowPrice":8097000,
               "highPrice":8099900,
@@ -75,12 +76,11 @@ def set_response_without_ldjson():
         f.write(test_return_data)
     return None
 
-# actually only really used by the distll.io importer, but could be handy too
-def test_check_ldjson_price_autodetect(client, live_server):
+def test_setup(client, live_server):
     live_server_setup(live_server)
 
-    # Give the endpoint time to spin up
-    time.sleep(1)
+# actually only really used by the distll.io importer, but could be handy too
+def test_check_ldjson_price_autodetect(client, live_server):
 
     set_response_with_ldjson()
 
@@ -92,7 +92,7 @@ def test_check_ldjson_price_autodetect(client, live_server):
         follow_redirects=True
     )
     assert b"1 Imported" in res.data
-    time.sleep(3)
+    wait_for_all_checks(client)
 
     # Should get a notice that it's available
     res = client.get(url_for("index"))
@@ -102,11 +102,11 @@ def test_check_ldjson_price_autodetect(client, live_server):
     uuid = extract_UUID_from_client(client)
 
     client.get(url_for('price_data_follower.accept', uuid=uuid, follow_redirects=True))
-    time.sleep(2)
+    wait_for_all_checks(client)
 
     # Trigger a check
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    time.sleep(2)
+    wait_for_all_checks(client)
     # Offer should be gone
     res = client.get(url_for("index"))
     assert b'Embedded price data' not in res.data
@@ -138,9 +138,97 @@ def test_check_ldjson_price_autodetect(client, live_server):
         follow_redirects=True
     )
     assert b"1 Imported" in res.data
-    time.sleep(3)
+    wait_for_all_checks(client)
     res = client.get(url_for("index"))
     assert b'ldjson-price-track-offer' not in res.data
     
     ##########################################################################################
     client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+
+
+def _test_runner_check_bad_format_ignored(live_server, client, has_ldjson_price_data):
+
+    test_url = url_for('test_endpoint', _external=True)
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+    wait_for_all_checks(client)
+
+    for k,v in client.application.config.get('DATASTORE').data['watching'].items():
+        assert v.get('last_error') == False
+        assert v.get('has_ldjson_price_data') == has_ldjson_price_data
+
+
+    ##########################################################################################
+    client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+
+
+def test_bad_ldjson_is_correctly_ignored(client, live_server):
+    #live_server_setup(live_server)
+    test_return_data = """
+            <html>
+            <head>
+                <script type="application/ld+json">
+                    {
+                        "@context": "http://schema.org",
+                        "@type": ["Product", "SubType"],
+                        "name": "My test product",
+                        "description": "",
+                        "offers": {
+                            "note" : "You can see the case-insensitive OffERS key, it should work",
+                            "@type": "Offer",
+                            "offeredBy": {
+                                "@type": "Organization",
+                                "name":"Person",
+                                "telephone":"+1 999 999 999"
+                            },
+                            "price": "1",
+                            "priceCurrency": "EUR",
+                            "url": "/some/url"
+                        }
+                    }
+                </script>
+            </head>
+            <body>
+            <div class="yes">Some extra stuff</div>
+            </body></html>
+     """
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write(test_return_data)
+
+    _test_runner_check_bad_format_ignored(live_server=live_server, client=client, has_ldjson_price_data=True)
+    test_return_data = """
+            <html>
+            <head>
+                <script type="application/ld+json">
+                    {
+                        "@context": "http://schema.org",
+                        "@type": ["Product", "SubType"],
+                        "name": "My test product",
+                        "description": "",
+                        "BrokenOffers": {
+                            "@type": "Offer",
+                            "offeredBy": {
+                                "@type": "Organization",
+                                "name":"Person",
+                                "telephone":"+1 999 999 999"
+                            },
+                            "price": "1",
+                            "priceCurrency": "EUR",
+                            "url": "/some/url"
+                        }
+                    }
+                </script>
+            </head>
+            <body>
+            <div class="yes">Some extra stuff</div>
+            </body></html>
+     """
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write(test_return_data)
+
+    _test_runner_check_bad_format_ignored(live_server=live_server, client=client, has_ldjson_price_data=False)
+

changedetectionio/tests/test_backend.py

@@ -89,7 +89,7 @@ def test_check_basic_change_detection_functionality(client, live_server):
 
     # Following the 'diff' link, it should no longer display as 'unviewed' even after we recheck it a few times
     res = client.get(url_for("diff_history_page", uuid="first"))
-    assert b'Compare newest' in res.data
+    assert b'selected=""' in res.data, "Confirm diff history page loaded"
 
     # Check the [preview] pulls the right one
     res = client.get(

changedetectionio/tests/test_xpath_selector.py

@@ -2,7 +2,7 @@
 
 import time
 from flask import url_for
-from . util import live_server_setup
+from .util import live_server_setup, wait_for_all_checks
 
 from ..html_tools import *
 
@@ -86,14 +86,14 @@ def test_check_xpath_filter_utf8(client, live_server):
         follow_redirects=True
     )
     assert b"1 Imported" in res.data
-    time.sleep(1)
+    wait_for_all_checks(client)
     res = client.post(
         url_for("edit_page", uuid="first"),
         data={"include_filters": filter, "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests"},
         follow_redirects=True
     )
     assert b"Updated watch." in res.data
-    time.sleep(3)
+    wait_for_all_checks(client)
     res = client.get(url_for("index"))
     assert b'Unicode strings with encoding declaration are not supported.' not in res.data
     res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
@@ -140,14 +140,14 @@ def test_check_xpath_text_function_utf8(client, live_server):
         follow_redirects=True
     )
     assert b"1 Imported" in res.data
-    time.sleep(1)
+    wait_for_all_checks(client)
     res = client.post(
         url_for("edit_page", uuid="first"),
         data={"include_filters": filter, "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests"},
         follow_redirects=True
     )
     assert b"Updated watch." in res.data
-    time.sleep(3)
+    wait_for_all_checks(client)
     res = client.get(url_for("index"))
     assert b'Unicode strings with encoding declaration are not supported.' not in res.data
 
@@ -164,7 +164,6 @@ def test_check_xpath_text_function_utf8(client, live_server):
     assert b'Deleted' in res.data
 
 def test_check_markup_xpath_filter_restriction(client, live_server):
-    sleep_time_for_fetch_thread = 3
 
     xpath_filter = "//*[contains(@class, 'sametext')]"
 
@@ -183,7 +182,7 @@ def test_check_markup_xpath_filter_restriction(client, live_server):
     assert b"1 Imported" in res.data
 
     # Give the thread time to pick it up
-    time.sleep(sleep_time_for_fetch_thread)
+    wait_for_all_checks(client)
 
     # Goto the edit page, add our ignore text
     # Add our URL to the import page
@@ -195,7 +194,7 @@ def test_check_markup_xpath_filter_restriction(client, live_server):
     assert b"Updated watch." in res.data
 
     # Give the thread time to pick it up
-    time.sleep(sleep_time_for_fetch_thread)
+    wait_for_all_checks(client)
 
     # view it/reset state back to viewed
     client.get(url_for("diff_history_page", uuid="first"), follow_redirects=True)
@@ -206,7 +205,7 @@ def test_check_markup_xpath_filter_restriction(client, live_server):
     # Trigger a check
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
     # Give the thread time to pick it up
-    time.sleep(sleep_time_for_fetch_thread)
+    wait_for_all_checks(client)
 
     res = client.get(url_for("index"))
     assert b'unviewed' not in res.data
@@ -216,9 +215,6 @@ def test_check_markup_xpath_filter_restriction(client, live_server):
 
 def test_xpath_validation(client, live_server):
 
-    # Give the endpoint time to spin up
-    time.sleep(1)
-
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
@@ -227,7 +223,7 @@ def test_xpath_validation(client, live_server):
         follow_redirects=True
     )
     assert b"1 Imported" in res.data
-    time.sleep(2)
+    wait_for_all_checks(client)
 
     res = client.post(
         url_for("edit_page", uuid="first"),
@@ -244,11 +240,8 @@ def test_check_with_prefix_include_filters(client, live_server):
     res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
 
-    # Give the endpoint time to spin up
-    time.sleep(1)
-
     set_original_response()
-
+    wait_for_all_checks(client)
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
@@ -257,7 +250,7 @@ def test_check_with_prefix_include_filters(client, live_server):
         follow_redirects=True
     )
     assert b"1 Imported" in res.data
-    time.sleep(3)
+    wait_for_all_checks(client)
 
     res = client.post(
         url_for("edit_page", uuid="first"),
@@ -266,7 +259,7 @@ def test_check_with_prefix_include_filters(client, live_server):
     )
 
     assert b"Updated watch." in res.data
-    time.sleep(3)
+    wait_for_all_checks(client)
 
     res = client.get(
         url_for("preview_page", uuid="first"),
@@ -277,3 +270,46 @@ def test_check_with_prefix_include_filters(client, live_server):
     assert b"Some text that will change" not in res.data #not in selector
 
     client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+def test_various_rules(client, live_server):
+    # Just check these don't error
+    #live_server_setup(live_server)
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write("""<html>
+       <body>
+     Some initial text<br>
+     <p>Which is across multiple lines</p>
+     <br>
+     So let's see what happens.  <br>
+     <div class="sametext">Some text thats the same</div>
+     <div class="changetext">Some text that will change</div>
+     <a href=''>some linky </a>
+     <a href=''>another some linky </a>
+     <!-- related to https://github.com/dgtlmoon/changedetection.io/pull/1774 -->
+     <input   type="email"   id="email" />
+     </body>
+     </html>
+    """)
+    test_url = url_for('test_endpoint', _external=True)
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+    wait_for_all_checks(client)
+
+
+    for r in ['//div', '//a', 'xpath://div', 'xpath://a']:
+        res = client.post(
+            url_for("edit_page", uuid="first"),
+            data={"include_filters": r,
+                  "url": test_url,
+                  "tags": "",
+                  "headers": "",
+                  'fetch_backend': "html_requests"},
+            follow_redirects=True
+        )
+        wait_for_all_checks(client)
+        assert b"Updated watch." in res.data
+        res = client.get(url_for("index"))
+        assert b'fetch-error' not in res.data, f"Should not see errors after '{r} filter"

docker-compose.yml

@@ -81,7 +81,7 @@ services:
 #        restart: unless-stopped
 
      # Used for fetching pages via Playwright+Chrome where you need Javascript support.
-
+     # Note: Playwright/browserless not supported on ARM type devices (rPi etc)
 #    playwright-chrome:
 #        hostname: playwright-chrome
 #        image: browserless/chrome

requirements.txt

@@ -16,7 +16,7 @@ validators~=0.21
 # Set these versions together to avoid a RequestsDependencyWarning
 # >= 2.26 also adds Brotli support if brotli is installed
 brotli~=1.0
-requests[socks] ~=2.28
+requests[socks]
 
 urllib3>1.26
 chardet>2.3.0

runtime.txt

@@ -1 +1 @@
-python-3.9.15
+python-3.11.5

setup.py

@@ -41,7 +41,7 @@ setup(
     include_package_data=True,
     install_requires=install_requires,
     license="Apache License 2.0",
-    python_requires=">= 3.6",
+    python_requires=">= 3.7",
     classifiers=['Intended Audience :: Customer Service',
                  'Intended Audience :: Developers',
                  'Intended Audience :: Education',