mirror of
https://github.com/dgtlmoon/changedetection.io.git
synced 2026-02-18 04:06:03 +00:00
Compare commits
10 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
7558ca5fda | ||
|
|
383c3b427f | ||
|
|
b01ba5d8a1 | ||
|
|
86e5184cef | ||
|
|
1dbf1f5db5 | ||
|
|
c5bd7da647 | ||
|
|
549e167746 | ||
|
|
9d38b45173 | ||
|
|
3558e9ee10 | ||
|
|
4b94de7e0c |
33
.github/nginx-reverse-proxy-test.conf
vendored
Normal file
33
.github/nginx-reverse-proxy-test.conf
vendored
Normal file
@@ -0,0 +1,33 @@
|
||||
server {
|
||||
listen 80;
|
||||
server_name localhost;
|
||||
|
||||
# Test basic reverse proxy to changedetection.io
|
||||
location / {
|
||||
proxy_pass http://changedet-app:5000;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
|
||||
# WebSocket support
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Upgrade $http_upgrade;
|
||||
proxy_set_header Connection "upgrade";
|
||||
}
|
||||
|
||||
# Test subpath deployment with X-Forwarded-Prefix
|
||||
location /changedet-sub/ {
|
||||
proxy_pass http://changedet-app:5000/;
|
||||
proxy_set_header X-Forwarded-Prefix /changedet-sub;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
|
||||
# WebSocket support
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Upgrade $http_upgrade;
|
||||
proxy_set_header Connection "upgrade";
|
||||
}
|
||||
}
|
||||
169
.github/workflows/test-stack-reusable-workflow.yml
vendored
169
.github/workflows/test-stack-reusable-workflow.yml
vendored
@@ -324,6 +324,175 @@ jobs:
|
||||
run: |
|
||||
docker run --rm --network changedet-network test-changedetectionio bash -c 'cd changedetectionio;pytest tests/smtp/test_notification_smtp.py'
|
||||
|
||||
nginx-reverse-proxy:
|
||||
runs-on: ubuntu-latest
|
||||
needs: build
|
||||
timeout-minutes: 10
|
||||
env:
|
||||
PYTHON_VERSION: ${{ inputs.python-version }}
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
|
||||
- name: Download Docker image artifact
|
||||
uses: actions/download-artifact@v7
|
||||
with:
|
||||
name: test-changedetectionio-${{ env.PYTHON_VERSION }}
|
||||
path: /tmp
|
||||
|
||||
- name: Load Docker image
|
||||
run: |
|
||||
docker load -i /tmp/test-changedetectionio.tar
|
||||
|
||||
- name: Spin up services
|
||||
run: |
|
||||
docker network create changedet-network
|
||||
|
||||
# Start changedetection.io container with X-Forwarded headers support
|
||||
docker run --name changedet-app --hostname changedet-app --network changedet-network \
|
||||
-e USE_X_SETTINGS=true \
|
||||
-d test-changedetectionio
|
||||
sleep 3
|
||||
|
||||
- name: Start nginx reverse proxy
|
||||
run: |
|
||||
# Start nginx with our test configuration
|
||||
docker run --name nginx-proxy --network changedet-network -d -p 8080:80 --rm \
|
||||
-v ${{ github.workspace }}/.github/nginx-reverse-proxy-test.conf:/etc/nginx/conf.d/default.conf:ro \
|
||||
nginx:alpine
|
||||
sleep 2
|
||||
|
||||
- name: Test reverse proxy - root path
|
||||
run: |
|
||||
echo "=== Testing nginx reverse proxy at root path ==="
|
||||
curl --retry-connrefused --retry 6 -s http://localhost:8080/ > /tmp/nginx-test-root.html
|
||||
|
||||
# Check for changedetection.io UI elements
|
||||
if grep -q "checkbox-uuid" /tmp/nginx-test-root.html; then
|
||||
echo "✓ Found checkbox-uuid in response"
|
||||
else
|
||||
echo "ERROR: checkbox-uuid not found in response"
|
||||
cat /tmp/nginx-test-root.html
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check for watchlist content
|
||||
if grep -q -i "watch" /tmp/nginx-test-root.html; then
|
||||
echo "✓ Found watch/watchlist content in response"
|
||||
else
|
||||
echo "ERROR: watchlist content not found"
|
||||
cat /tmp/nginx-test-root.html
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "✓ Root path reverse proxy working correctly"
|
||||
|
||||
- name: Test reverse proxy - subpath with X-Forwarded-Prefix
|
||||
run: |
|
||||
echo "=== Testing nginx reverse proxy at subpath /changedet-sub/ ==="
|
||||
curl --retry-connrefused --retry 6 -s http://localhost:8080/changedet-sub/ > /tmp/nginx-test-subpath.html
|
||||
|
||||
# Check for changedetection.io UI elements
|
||||
if grep -q "checkbox-uuid" /tmp/nginx-test-subpath.html; then
|
||||
echo "✓ Found checkbox-uuid in subpath response"
|
||||
else
|
||||
echo "ERROR: checkbox-uuid not found in subpath response"
|
||||
cat /tmp/nginx-test-subpath.html
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "✓ Subpath reverse proxy working correctly"
|
||||
|
||||
- name: Test API through reverse proxy subpath
|
||||
run: |
|
||||
echo "=== Testing API endpoints through nginx subpath /changedet-sub/ ==="
|
||||
|
||||
# Extract API key from the changedetection.io datastore
|
||||
API_KEY=$(docker exec changedet-app cat /datastore/changedetection.json | grep -o '"api_access_token": *"[^"]*"' | cut -d'"' -f4)
|
||||
|
||||
if [ -z "$API_KEY" ]; then
|
||||
echo "ERROR: Could not extract API key from datastore"
|
||||
docker exec changedet-app cat /datastore/changedetection.json
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "✓ Extracted API key: ${API_KEY:0:8}..."
|
||||
|
||||
# Create a watch via API through nginx proxy subpath
|
||||
echo "Creating watch via POST to /changedet-sub/api/v1/watch"
|
||||
RESPONSE=$(curl -s -w "\n%{http_code}" -X POST "http://localhost:8080/changedet-sub/api/v1/watch" \
|
||||
-H "x-api-key: ${API_KEY}" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"url": "https://example.com/test-nginx-proxy",
|
||||
"tag": "nginx-test"
|
||||
}')
|
||||
|
||||
HTTP_CODE=$(echo "$RESPONSE" | tail -n1)
|
||||
BODY=$(echo "$RESPONSE" | head -n-1)
|
||||
|
||||
if [ "$HTTP_CODE" != "201" ]; then
|
||||
echo "ERROR: Expected HTTP 201, got $HTTP_CODE"
|
||||
echo "Response: $BODY"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "✓ Watch created successfully (HTTP 201)"
|
||||
|
||||
# Extract the watch UUID from response
|
||||
WATCH_UUID=$(echo "$BODY" | grep -o '"uuid": *"[^"]*"' | cut -d'"' -f4)
|
||||
echo "✓ Watch UUID: $WATCH_UUID"
|
||||
|
||||
# Update the watch via PUT through nginx proxy subpath
|
||||
echo "Updating watch via PUT to /changedet-sub/api/v1/watch/${WATCH_UUID}"
|
||||
RESPONSE=$(curl -s -w "\n%{http_code}" -X PUT "http://localhost:8080/changedet-sub/api/v1/watch/${WATCH_UUID}" \
|
||||
-H "x-api-key: ${API_KEY}" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"paused": true
|
||||
}')
|
||||
|
||||
HTTP_CODE=$(echo "$RESPONSE" | tail -n1)
|
||||
BODY=$(echo "$RESPONSE" | head -n-1)
|
||||
|
||||
if [ "$HTTP_CODE" != "200" ]; then
|
||||
echo "ERROR: Expected HTTP 200, got $HTTP_CODE"
|
||||
echo "Response: $BODY"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if echo "$BODY" | grep -q 'OK'; then
|
||||
echo "✓ Watch updated successfully (HTTP 200, response: OK)"
|
||||
else
|
||||
echo "ERROR: Expected response 'OK', got: $BODY"
|
||||
echo "Response: $BODY"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Verify the watch is paused via GET
|
||||
echo "Verifying watch is paused via GET"
|
||||
RESPONSE=$(curl -s "http://localhost:8080/changedet-sub/api/v1/watch/${WATCH_UUID}" \
|
||||
-H "x-api-key: ${API_KEY}")
|
||||
|
||||
if echo "$RESPONSE" | grep -q '"paused": *true'; then
|
||||
echo "✓ Watch is paused as expected"
|
||||
else
|
||||
echo "ERROR: Watch paused state not confirmed"
|
||||
echo "Response: $RESPONSE"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "✓ API tests through nginx subpath completed successfully"
|
||||
|
||||
- name: Cleanup nginx test
|
||||
if: always()
|
||||
run: |
|
||||
docker logs nginx-proxy || true
|
||||
docker logs changedet-app || true
|
||||
docker stop nginx-proxy changedet-app || true
|
||||
docker rm nginx-proxy changedet-app || true
|
||||
|
||||
|
||||
|
||||
# Proxy tests
|
||||
proxy-tests:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
# Read more https://github.com/dgtlmoon/changedetection.io/wiki
|
||||
# Semver means never use .01, or 00. Should be .1.
|
||||
__version__ = '0.53.1'
|
||||
__version__ = '0.53.3'
|
||||
|
||||
from changedetectionio.strtobool import strtobool
|
||||
from json.decoder import JSONDecodeError
|
||||
@@ -610,7 +610,7 @@ def main():
|
||||
|
||||
@app.context_processor
|
||||
def inject_template_globals():
|
||||
return dict(right_sticky="v{}".format(datastore.data['version_tag']),
|
||||
return dict(right_sticky="v"+__version__,
|
||||
new_version_available=app.config['NEW_VERSION_AVAILABLE'],
|
||||
has_password=datastore.data['settings']['application']['password'] != False,
|
||||
socket_io_enabled=datastore.data['settings']['application'].get('ui', {}).get('socket_io_enabled', True),
|
||||
|
||||
@@ -103,6 +103,7 @@ def validate_openapi_request(operation_id):
|
||||
if request.method.upper() != 'GET':
|
||||
# Lazy import - only loaded when actually validating a request
|
||||
from openapi_core.contrib.flask import FlaskOpenAPIRequest
|
||||
from openapi_core.templating.paths.exceptions import ServerNotFound, PathNotFound, PathError
|
||||
|
||||
spec = get_openapi_spec()
|
||||
openapi_request = FlaskOpenAPIRequest(request)
|
||||
@@ -110,6 +111,16 @@ def validate_openapi_request(operation_id):
|
||||
if result.errors:
|
||||
error_details = []
|
||||
for error in result.errors:
|
||||
# Skip path/server validation errors for reverse proxy compatibility
|
||||
# Flask routing already validates that endpoints exist (returns 404 if not).
|
||||
# OpenAPI validation here is primarily for request body schema validation.
|
||||
# When behind nginx/reverse proxy, URLs may have path prefixes that don't
|
||||
# match the OpenAPI server definitions, causing false positives.
|
||||
if isinstance(error, PathError):
|
||||
logger.debug(f"API Call - Skipping path/server validation (delegated to Flask): {error}")
|
||||
continue
|
||||
|
||||
error_str = str(error)
|
||||
# Extract detailed schema errors from __cause__
|
||||
if hasattr(error, '__cause__') and hasattr(error.__cause__, 'schema_errors'):
|
||||
for schema_error in error.__cause__.schema_errors:
|
||||
@@ -117,9 +128,12 @@ def validate_openapi_request(operation_id):
|
||||
msg = schema_error.message if hasattr(schema_error, 'message') else str(schema_error)
|
||||
error_details.append(f"{field}: {msg}")
|
||||
else:
|
||||
error_details.append(str(error))
|
||||
error_details.append(error_str)
|
||||
|
||||
# Only raise if we have actual validation errors (not path/server issues)
|
||||
if error_details:
|
||||
logger.error(f"API Call - Validation failed: {'; '.join(error_details)}")
|
||||
raise BadRequest(f"Validation failed: {'; '.join(error_details)}")
|
||||
raise BadRequest(f"Validation failed: {'; '.join(error_details)}")
|
||||
except BadRequest:
|
||||
# Re-raise BadRequest exceptions (validation failures)
|
||||
raise
|
||||
|
||||
@@ -379,6 +379,4 @@ def construct_blueprint(datastore: ChangeDetectionStore):
|
||||
|
||||
return browser_steps_blueprint
|
||||
|
||||
return browser_steps_blueprint
|
||||
|
||||
|
||||
|
||||
@@ -304,12 +304,13 @@ html[data-darkmode="true"] .watch-tag-list.tag-{{ class_name }} {
|
||||
</span>
|
||||
{%- endif -%}
|
||||
|
||||
{%- if watch.get('restock') and watch['restock']['price'] != None -%}
|
||||
{%- if watch['restock']['price'] != None -%}
|
||||
{%- if watch.get('restock') and watch['restock'].get('price') -%}
|
||||
{%- if watch['restock']['price'] is number -%}
|
||||
<span class="restock-label price" title="{{ _('Price') }}">
|
||||
{{ watch['restock']['price']|format_number_locale if watch['restock'].get('price') else '' }} {{ watch['restock'].get('currency','') }}
|
||||
</span>
|
||||
{%- endif -%}
|
||||
{%- else -%} <!-- watch['restock']['price']' is not a number, cant output it -->
|
||||
{%- endif -%}
|
||||
{%- elif not watch.has_restock_info -%}
|
||||
<span class="restock-label error">{{ _('No information') }}</span>
|
||||
{%- endif -%}
|
||||
|
||||
@@ -712,8 +712,14 @@ def changedetection_app(config=None, datastore_o=None):
|
||||
def static_content(group, filename):
|
||||
from flask import make_response
|
||||
import re
|
||||
group = re.sub(r'[^\w.-]+', '', group.lower())
|
||||
filename = re.sub(r'[^\w.-]+', '', filename.lower())
|
||||
|
||||
# Strict sanitization: only allow a-z, 0-9, and underscore (blocks .. and other traversal)
|
||||
group = re.sub(r'[^a-z0-9_-]+', '', group.lower())
|
||||
filename = filename
|
||||
|
||||
# Additional safety: reject if sanitization resulted in empty strings
|
||||
if not group or not filename:
|
||||
abort(404)
|
||||
|
||||
if group == 'screenshot':
|
||||
# Could be sensitive, follow password requirements
|
||||
|
||||
@@ -360,16 +360,13 @@ $(document).ready(function () {
|
||||
// Add the extra buttons to the steps
|
||||
$('ul#browser_steps li').each(function (i) {
|
||||
var s = '<div class="control">' + '<a data-step-index=' + i + ' class="pure-button button-secondary button-green button-xsmall apply" >Apply</a> ';
|
||||
if (i > 0) {
|
||||
// The first step never gets these (Goto-site)
|
||||
s += `<a data-step-index="${i}" class="pure-button button-secondary button-xsmall clear" >Clear</a> ` +
|
||||
`<a data-step-index="${i}" class="pure-button button-secondary button-red button-xsmall remove" >Remove</a>`;
|
||||
s += `<a data-step-index="${i}" class="pure-button button-secondary button-xsmall clear" >Clear</a> ` +
|
||||
`<a data-step-index="${i}" class="pure-button button-secondary button-red button-xsmall remove" >Remove</a>`;
|
||||
|
||||
// if a screenshot is available
|
||||
if (browser_steps_available_screenshots.includes(i.toString())) {
|
||||
var d = (browser_steps_last_error_step === i+1) ? 'before' : 'after';
|
||||
s += ` <a data-step-index="${i}" class="pure-button button-secondary button-xsmall show-screenshot" title="Show screenshot from last run" data-type="${d}">Pic</a> `;
|
||||
}
|
||||
// if a screenshot is available
|
||||
if (browser_steps_available_screenshots.includes(i.toString())) {
|
||||
var d = (browser_steps_last_error_step === i+1) ? 'before' : 'after';
|
||||
s += ` <a data-step-index="${i}" class="pure-button button-secondary button-xsmall show-screenshot" title="Show screenshot from last run" data-type="${d}">Pic</a> `;
|
||||
}
|
||||
s += '</div>';
|
||||
$(this).append(s)
|
||||
|
||||
@@ -235,6 +235,8 @@ class ChangeDetectionStore(DatastoreUpdatesMixin, FileSavingDataStore):
|
||||
# No datastore yet - check if this is a fresh install or legacy migration
|
||||
self.init_fresh_install(include_default_watches=include_default_watches,
|
||||
version_tag=version_tag)
|
||||
# Maybe they copied a bunch of watch subdirs across too
|
||||
self._load_state()
|
||||
|
||||
def init_fresh_install(self, include_default_watches, version_tag):
|
||||
# Generate app_guid FIRST (required for all operations)
|
||||
|
||||
@@ -331,6 +331,7 @@ def prepare_test_function(live_server, datastore_path):
|
||||
# Cleanup: Clear watches and queue after test
|
||||
try:
|
||||
from changedetectionio.flask_app import update_q
|
||||
from pathlib import Path
|
||||
|
||||
# Clear the queue to prevent leakage to next test
|
||||
while not update_q.empty():
|
||||
@@ -340,6 +341,18 @@ def prepare_test_function(live_server, datastore_path):
|
||||
break
|
||||
|
||||
datastore.data['watching'] = {}
|
||||
|
||||
# Delete any old watch metadata JSON files
|
||||
base_path = Path(datastore.datastore_path).resolve()
|
||||
max_depth = 2
|
||||
|
||||
for file in base_path.rglob("*.json"):
|
||||
# Calculate depth relative to base path
|
||||
depth = len(file.relative_to(base_path).parts) - 1
|
||||
|
||||
if depth <= max_depth and file.is_file():
|
||||
file.unlink()
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"Error during datastore cleanup: {e}")
|
||||
|
||||
|
||||
@@ -9,7 +9,7 @@ by testing various scenarios that should trigger validation errors.
|
||||
import time
|
||||
import json
|
||||
from flask import url_for
|
||||
from .util import live_server_setup, wait_for_all_checks
|
||||
from .util import live_server_setup, wait_for_all_checks, delete_all_watches
|
||||
|
||||
|
||||
def test_openapi_validation_invalid_content_type_on_create_watch(client, live_server, measure_memory_usage, datastore_path):
|
||||
@@ -27,6 +27,7 @@ def test_openapi_validation_invalid_content_type_on_create_watch(client, live_se
|
||||
# Should get 400 error due to OpenAPI validation failure
|
||||
assert res.status_code == 400, f"Expected 400 but got {res.status_code}"
|
||||
assert b"Validation failed" in res.data, "Should contain validation error message"
|
||||
delete_all_watches(client)
|
||||
|
||||
|
||||
def test_openapi_validation_missing_required_field_create_watch(client, live_server, measure_memory_usage, datastore_path):
|
||||
@@ -44,6 +45,7 @@ def test_openapi_validation_missing_required_field_create_watch(client, live_ser
|
||||
# Should get 400 error due to missing required field
|
||||
assert res.status_code == 400, f"Expected 400 but got {res.status_code}"
|
||||
assert b"Validation failed" in res.data, "Should contain validation error message"
|
||||
delete_all_watches(client)
|
||||
|
||||
|
||||
def test_openapi_validation_invalid_field_in_request_body(client, live_server, measure_memory_usage, datastore_path):
|
||||
@@ -83,6 +85,7 @@ def test_openapi_validation_invalid_field_in_request_body(client, live_server, m
|
||||
# Backend validation now returns "Unknown field(s):" message
|
||||
assert b"Unknown field" in res.data, \
|
||||
"Should contain validation error about unknown fields"
|
||||
delete_all_watches(client)
|
||||
|
||||
|
||||
def test_openapi_validation_import_wrong_content_type(client, live_server, measure_memory_usage, datastore_path):
|
||||
@@ -100,6 +103,7 @@ def test_openapi_validation_import_wrong_content_type(client, live_server, measu
|
||||
# Should get 400 error due to content-type mismatch
|
||||
assert res.status_code == 400, f"Expected 400 but got {res.status_code}"
|
||||
assert b"Validation failed" in res.data, "Should contain validation error message"
|
||||
delete_all_watches(client)
|
||||
|
||||
|
||||
def test_openapi_validation_import_correct_content_type_succeeds(client, live_server, measure_memory_usage, datastore_path):
|
||||
@@ -117,6 +121,7 @@ def test_openapi_validation_import_correct_content_type_succeeds(client, live_se
|
||||
# Should succeed
|
||||
assert res.status_code == 200, f"Expected 200 but got {res.status_code}"
|
||||
assert len(res.json) == 2, "Should import 2 URLs"
|
||||
delete_all_watches(client)
|
||||
|
||||
|
||||
def test_openapi_validation_get_requests_bypass_validation(client, live_server, measure_memory_usage, datastore_path):
|
||||
@@ -141,6 +146,7 @@ def test_openapi_validation_get_requests_bypass_validation(client, live_server,
|
||||
|
||||
# Should return JSON with watch list (empty in this case)
|
||||
assert isinstance(res.json, dict), "Should return JSON dictionary for watch list"
|
||||
delete_all_watches(client)
|
||||
|
||||
|
||||
def test_openapi_validation_create_tag_missing_required_title(client, live_server, measure_memory_usage, datastore_path):
|
||||
@@ -158,10 +164,13 @@ def test_openapi_validation_create_tag_missing_required_title(client, live_serve
|
||||
# Should get 400 error due to missing required field
|
||||
assert res.status_code == 400, f"Expected 400 but got {res.status_code}"
|
||||
assert b"Validation failed" in res.data, "Should contain validation error message"
|
||||
delete_all_watches(client)
|
||||
|
||||
|
||||
def test_openapi_validation_watch_update_allows_partial_updates(client, live_server, measure_memory_usage, datastore_path):
|
||||
|
||||
"""Test that watch updates allow partial updates without requiring all fields (positive test)."""
|
||||
#xxx
|
||||
api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')
|
||||
|
||||
# First create a valid watch
|
||||
@@ -198,4 +207,5 @@ def test_openapi_validation_watch_update_allows_partial_updates(client, live_ser
|
||||
)
|
||||
assert res.status_code == 200
|
||||
assert res.json.get('title') == 'Updated Title Only', "Title should be updated"
|
||||
assert res.json.get('url') == 'https://example.com', "URL should remain unchanged"
|
||||
assert res.json.get('url') == 'https://example.com', "URL should remain unchanged"
|
||||
delete_all_watches(client)
|
||||
|
||||
@@ -6,8 +6,6 @@ from flask import url_for
|
||||
from .util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks, extract_rss_token_from_UI, \
|
||||
extract_UUID_from_client, delete_all_watches
|
||||
|
||||
sleep_time_for_fetch_thread = 3
|
||||
|
||||
|
||||
# Basic test to check inscriptus is not adding return line chars, basically works etc
|
||||
def test_inscriptus():
|
||||
|
||||
@@ -6,10 +6,6 @@ from urllib.request import urlopen
|
||||
from .util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks
|
||||
import os
|
||||
|
||||
sleep_time_for_fetch_thread = 3
|
||||
|
||||
|
||||
|
||||
def test_check_extract_text_from_diff(client, live_server, measure_memory_usage, datastore_path):
|
||||
import time
|
||||
with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
|
||||
|
||||
@@ -41,7 +41,6 @@ def set_modified_ignore_response(datastore_path):
|
||||
def test_render_anchor_tag_content_true(client, live_server, measure_memory_usage, datastore_path):
|
||||
"""Testing that the link changes are detected when
|
||||
render_anchor_tag_content setting is set to true"""
|
||||
sleep_time_for_fetch_thread = 3
|
||||
|
||||
# Give the endpoint time to spin up
|
||||
time.sleep(1)
|
||||
|
||||
@@ -100,7 +100,6 @@ def test_normal_page_check_works_with_ignore_status_code(client, live_server, me
|
||||
|
||||
# Tests the whole stack works with staus codes ignored
|
||||
def test_403_page_check_works_with_ignore_status_code(client, live_server, measure_memory_usage, datastore_path):
|
||||
sleep_time_for_fetch_thread = 3
|
||||
|
||||
set_original_response(datastore_path=datastore_path)
|
||||
|
||||
@@ -112,8 +111,7 @@ def test_403_page_check_works_with_ignore_status_code(client, live_server, measu
|
||||
uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)
|
||||
client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
|
||||
|
||||
# Give the thread time to pick it up
|
||||
time.sleep(sleep_time_for_fetch_thread)
|
||||
wait_for_all_checks(client)
|
||||
|
||||
# Goto the edit page, check our ignore option
|
||||
# Add our URL to the import page
|
||||
|
||||
@@ -2,10 +2,9 @@
|
||||
|
||||
import time
|
||||
from flask import url_for
|
||||
from . util import live_server_setup
|
||||
import os
|
||||
|
||||
|
||||
from .util import live_server_setup, delete_all_watches, wait_for_all_checks
|
||||
|
||||
|
||||
# Should be the same as set_original_ignore_response(datastore_path=datastore_path) but with a little more whitespacing
|
||||
@@ -50,10 +49,7 @@ def set_original_ignore_response(datastore_path):
|
||||
|
||||
# If there was only a change in the whitespacing, then we shouldnt have a change detected
|
||||
def test_check_ignore_whitespace(client, live_server, measure_memory_usage, datastore_path):
|
||||
sleep_time_for_fetch_thread = 3
|
||||
|
||||
# Give the endpoint time to spin up
|
||||
time.sleep(1)
|
||||
|
||||
set_original_ignore_response(datastore_path=datastore_path)
|
||||
|
||||
@@ -74,17 +70,17 @@ def test_check_ignore_whitespace(client, live_server, measure_memory_usage, data
|
||||
uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)
|
||||
client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
|
||||
|
||||
time.sleep(sleep_time_for_fetch_thread)
|
||||
wait_for_all_checks(client)
|
||||
# Trigger a check
|
||||
client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
|
||||
|
||||
set_original_ignore_response_but_with_whitespace(datastore_path)
|
||||
time.sleep(sleep_time_for_fetch_thread)
|
||||
wait_for_all_checks(client)
|
||||
# Trigger a check
|
||||
client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
|
||||
|
||||
# Give the thread time to pick it up
|
||||
time.sleep(sleep_time_for_fetch_thread)
|
||||
wait_for_all_checks(client)
|
||||
|
||||
# It should report nothing found (no new 'has-unread-changes' class)
|
||||
res = client.get(url_for("watchlist.index"))
|
||||
|
||||
@@ -24,6 +24,30 @@ def set_original_response(datastore_path):
|
||||
f.write(test_return_data)
|
||||
return None
|
||||
|
||||
|
||||
def test_favicon(client, live_server, measure_memory_usage, datastore_path):
|
||||
# Attempt to fetch it, make sure that works
|
||||
SVG_BASE64 = 'PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgMCAxIDEiLz4='
|
||||
uuid = client.application.config.get('DATASTORE').add_watch(url='https://localhost')
|
||||
live_server.app.config['DATASTORE'].data['watching'][uuid].bump_favicon(url="favicon-set-type.svg",
|
||||
favicon_base_64=SVG_BASE64
|
||||
)
|
||||
|
||||
res = client.get(url_for('static_content', group='favicon', filename=uuid))
|
||||
assert res.status_code == 200
|
||||
assert len(res.data) > 10
|
||||
|
||||
res = client.get(url_for('static_content', group='..', filename='__init__.py'))
|
||||
assert res.status_code != 200
|
||||
|
||||
|
||||
res = client.get(url_for('static_content', group='.', filename='../__init__.py'))
|
||||
assert res.status_code != 200
|
||||
|
||||
# Traverse by filename protection
|
||||
res = client.get(url_for('static_content', group='js', filename='../styles/styles.css'))
|
||||
assert res.status_code != 200
|
||||
|
||||
def test_bad_access(client, live_server, measure_memory_usage, datastore_path):
|
||||
|
||||
res = client.post(
|
||||
@@ -478,3 +502,80 @@ def test_logout_with_redirect(client, live_server, measure_memory_usage, datasto
|
||||
# Cleanup
|
||||
del client.application.config['DATASTORE'].data['settings']['application']['password']
|
||||
|
||||
|
||||
def test_static_directory_traversal(client, live_server, measure_memory_usage, datastore_path):
|
||||
"""
|
||||
Test that the static file serving route properly blocks directory traversal attempts.
|
||||
This tests the fix for GHSA-9jj8-v89v-xjvw (CVE pending).
|
||||
|
||||
The vulnerability was in /static/<group>/<filename> where the sanitization regex
|
||||
allowed dots, enabling "../" traversal to read application source files.
|
||||
|
||||
The fix changed the regex from r'[^\w.-]+' to r'[^a-z0-9_]+' which blocks dots.
|
||||
"""
|
||||
|
||||
# Test 1: Direct .. traversal attempt (URL-encoded)
|
||||
res = client.get(
|
||||
"/static/%2e%2e/flask_app.py",
|
||||
follow_redirects=False
|
||||
)
|
||||
# Should be blocked (404 or 403)
|
||||
assert res.status_code in [404, 403], f"Expected 404/403, got {res.status_code}"
|
||||
# Should NOT contain application source code
|
||||
assert b"def static_content" not in res.data
|
||||
assert b"changedetection_app" not in res.data
|
||||
|
||||
# Test 2: Direct .. traversal attempt (unencoded)
|
||||
res = client.get(
|
||||
"/static/../flask_app.py",
|
||||
follow_redirects=False
|
||||
)
|
||||
assert res.status_code in [404, 403], f"Expected 404/403, got {res.status_code}"
|
||||
assert b"def static_content" not in res.data
|
||||
|
||||
# Test 3: Multiple dots traversal
|
||||
res = client.get(
|
||||
"/static/..../flask_app.py",
|
||||
follow_redirects=False
|
||||
)
|
||||
assert res.status_code in [404, 403], f"Expected 404/403, got {res.status_code}"
|
||||
assert b"def static_content" not in res.data
|
||||
|
||||
# Test 4: Try to access other application files
|
||||
for filename in ["__init__.py", "datastore.py", "store.py"]:
|
||||
res = client.get(
|
||||
f"/static/%2e%2e/{filename}",
|
||||
follow_redirects=False
|
||||
)
|
||||
assert res.status_code in [404, 403], f"File {filename} should be blocked"
|
||||
# Should not contain Python code indicators
|
||||
assert b"import" not in res.data or b"# Test" in res.data # Allow "1 Imported" etc
|
||||
|
||||
# Test 5: Verify legitimate static files still work
|
||||
# Note: We can't test actual files without knowing what exists,
|
||||
# but we can verify the sanitization doesn't break valid groups
|
||||
res = client.get(
|
||||
"/static/images/test.png", # Will 404 if file doesn't exist, but won't traverse
|
||||
follow_redirects=False
|
||||
)
|
||||
# Should get 404 (file not found) not 403 (blocked)
|
||||
# This confirms the group name "images" is valid
|
||||
assert res.status_code == 404
|
||||
|
||||
# Test 6: Ensure hyphens and dots are blocked in group names
|
||||
res = client.get(
|
||||
"/static/../../../etc/passwd",
|
||||
follow_redirects=False
|
||||
)
|
||||
assert res.status_code in [404, 403]
|
||||
assert b"root:" not in res.data
|
||||
|
||||
# Test 7: Test that underscores still work (they're allowed)
|
||||
res = client.get(
|
||||
"/static/visual_selector_data/test.json",
|
||||
follow_redirects=False
|
||||
)
|
||||
# visual_selector_data is a real group, but requires auth
|
||||
# Should get 403 (not authenticated) or 404 (file not found), not a path traversal
|
||||
assert res.status_code in [403, 404]
|
||||
|
||||
|
||||
@@ -6,9 +6,6 @@ from urllib.request import urlopen
|
||||
from .util import set_original_response, set_modified_response, live_server_setup, delete_all_watches
|
||||
import re
|
||||
|
||||
sleep_time_for_fetch_thread = 3
|
||||
|
||||
|
||||
def test_share_watch(client, live_server, measure_memory_usage, datastore_path):
|
||||
set_original_response(datastore_path=datastore_path)
|
||||
|
||||
|
||||
@@ -6,7 +6,6 @@ from urllib.request import urlopen
|
||||
from .util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks
|
||||
from ..diff import ADDED_STYLE
|
||||
|
||||
sleep_time_for_fetch_thread = 3
|
||||
|
||||
def test_check_basic_change_detection_functionality_source(client, live_server, measure_memory_usage, datastore_path):
|
||||
set_original_response(datastore_path=datastore_path)
|
||||
@@ -72,7 +71,10 @@ def test_check_ignore_elements(client, live_server, measure_memory_usage, datast
|
||||
follow_redirects=True
|
||||
)
|
||||
|
||||
time.sleep(sleep_time_for_fetch_thread)
|
||||
client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
|
||||
|
||||
wait_for_all_checks(client)
|
||||
|
||||
|
||||
res = client.get(
|
||||
url_for("ui.ui_preview.preview_page", uuid="first"),
|
||||
|
||||
@@ -2,7 +2,8 @@
|
||||
|
||||
import time
|
||||
from flask import url_for
|
||||
from . util import live_server_setup, delete_all_watches
|
||||
|
||||
from .util import live_server_setup, delete_all_watches, wait_for_all_checks
|
||||
import os
|
||||
|
||||
|
||||
@@ -25,9 +26,6 @@ def set_original_ignore_response(datastore_path):
|
||||
|
||||
def test_trigger_regex_functionality_with_filter(client, live_server, measure_memory_usage, datastore_path):
|
||||
|
||||
# live_server_setup(live_server) # Setup on conftest per function
|
||||
sleep_time_for_fetch_thread = 3
|
||||
|
||||
set_original_ignore_response(datastore_path=datastore_path)
|
||||
|
||||
# Give the endpoint time to spin up
|
||||
@@ -38,8 +36,7 @@ def test_trigger_regex_functionality_with_filter(client, live_server, measure_me
|
||||
uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)
|
||||
client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
|
||||
|
||||
# it needs time to save the original version
|
||||
time.sleep(sleep_time_for_fetch_thread)
|
||||
wait_for_all_checks(client)
|
||||
|
||||
### test regex with filter
|
||||
res = client.post(
|
||||
@@ -52,8 +49,9 @@ def test_trigger_regex_functionality_with_filter(client, live_server, measure_me
|
||||
follow_redirects=True
|
||||
)
|
||||
|
||||
# Give the thread time to pick it up
|
||||
time.sleep(sleep_time_for_fetch_thread)
|
||||
client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
|
||||
|
||||
wait_for_all_checks(client)
|
||||
|
||||
client.get(url_for("ui.ui_diff.diff_history_page", uuid="first"))
|
||||
|
||||
@@ -62,7 +60,8 @@ def test_trigger_regex_functionality_with_filter(client, live_server, measure_me
|
||||
f.write("<html>some new noise with cool stuff2 ok</html>")
|
||||
|
||||
client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
|
||||
time.sleep(sleep_time_for_fetch_thread)
|
||||
|
||||
wait_for_all_checks(client)
|
||||
|
||||
# It should report nothing found (nothing should match the regex and filter)
|
||||
res = client.get(url_for("watchlist.index"))
|
||||
@@ -73,7 +72,8 @@ def test_trigger_regex_functionality_with_filter(client, live_server, measure_me
|
||||
f.write("<html>some new noise with <span id=in-here>cool stuff6</span> ok</html>")
|
||||
|
||||
client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
|
||||
time.sleep(sleep_time_for_fetch_thread)
|
||||
|
||||
wait_for_all_checks(client)
|
||||
res = client.get(url_for("watchlist.index"))
|
||||
assert b'has-unread-changes' in res.data
|
||||
|
||||
|
||||
@@ -160,6 +160,7 @@ def extract_UUID_from_client(client):
|
||||
return uuid.strip()
|
||||
|
||||
def delete_all_watches(client=None):
|
||||
wait_for_all_checks(client)
|
||||
|
||||
uuids = list(client.application.config.get('DATASTORE').data['watching'])
|
||||
for uuid in uuids:
|
||||
@@ -180,6 +181,23 @@ def delete_all_watches(client=None):
|
||||
|
||||
time.sleep(0.2)
|
||||
|
||||
# Delete any old watch metadata
|
||||
from pathlib import Path
|
||||
|
||||
base_path = Path(
|
||||
client.application.config.get('DATASTORE').datastore_path
|
||||
).resolve()
|
||||
|
||||
max_depth = 2
|
||||
|
||||
for file in base_path.rglob("*.json"):
|
||||
# Calculate depth relative to base path
|
||||
depth = len(file.relative_to(base_path).parts) - 1
|
||||
|
||||
if depth <= max_depth and file.is_file():
|
||||
file.unlink()
|
||||
|
||||
|
||||
def wait_for_all_checks(client=None):
|
||||
"""
|
||||
Waits until the queue is empty and workers are idle.
|
||||
|
||||
Reference in New Issue
Block a user