From 5b1f492cc2f2314c2710ffbcbbaa6a7b2fefca98 Mon Sep 17 00:00:00 2001 From: Alex Osborne Date: Thu, 5 Nov 2026 09:53:25 +0900 Subject: [PATCH] CrawlURI: add hasContentType() for media type comparison Compares the content type's type/subtype with the given media type, ignoring case and any parameters, so "Text/HTML; charset=utf-8" matches "text/html" but "text/htmlfoo" doesn't. This replaces ad hoc checks like toLowerCase().startsWith(...), which also match longer subtypes. --- .../crawler/datamodel/CrawlURITest.java | 23 ++++++++++++ .../java/org/archive/modules/CrawlURI.java | 35 ++++++++++++++++++- 2 files changed, 57 insertions(+), 1 deletion(-) diff --git a/engine/src/test/java/org/archive/crawler/datamodel/CrawlURITest.java b/engine/src/test/java/org/archive/crawler/datamodel/CrawlURITest.java index 5f2fba68..829addb8 100644 --- a/engine/src/test/java/org/archive/crawler/datamodel/CrawlURITest.java +++ b/engine/src/test/java/org/archive/crawler/datamodel/CrawlURITest.java @@ -38,6 +38,7 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; /** @@ -232,4 +233,26 @@ public class CrawlURITest { assertEquals(0, a.compareTo(b)); assertEquals(0, b.compareTo(a)); } + + @Test + public void testHasContentType() throws Exception { + assertTrue(withContentType("image/svg+xml").hasContentType("image/svg+xml")); + assertTrue(withContentType("Image/SVG+XML").hasContentType("image/svg+xml")); + assertTrue(withContentType("image/svg+xml;charset=utf-8").hasContentType("image/svg+xml")); + assertTrue(withContentType(" image/svg+xml ; charset=utf-8").hasContentType("image/svg+xml")); + + assertFalse(withContentType("image/svg+xmlfoo").hasContentType("image/svg+xml")); + assertFalse(withContentType("image/svg").hasContentType("image/svg+xml")); + assertFalse(withContentType("text/html").hasContentType("text/htm")); + assertFalse(withContentType("application/xml").hasContentType("text/xml")); + assertFalse(withContentType(null).hasContentType("text/html")); + assertFalse(new CrawlURI(UURIFactory.getInstance("https://example.org/")) + .hasContentType("text/html")); + } + + private static CrawlURI withContentType(String contentType) throws URIException { + CrawlURI curi = new CrawlURI(UURIFactory.getInstance("https://example.org/")); + curi.setContentType(contentType); + return curi; + } } diff --git a/modules/src/main/java/org/archive/modules/CrawlURI.java b/modules/src/main/java/org/archive/modules/CrawlURI.java index a43589d2..a0bf3e04 100644 --- a/modules/src/main/java/org/archive/modules/CrawlURI.java +++ b/modules/src/main/java/org/archive/modules/CrawlURI.java @@ -1741,7 +1741,40 @@ implements Reporter, Serializable, OverlayContext, Comparable { return caUri; } - + + /** + * Tests whether this URI's content type is the given media type, ignoring + * case and any parameters. For example, a content type of + * {@code "Text/HTML; charset=utf-8"} has the media type + * {@code "text/html"} but not {@code "text/htm"}. + * + * @param mediaType a type/subtype pair, e.g. "image/svg+xml" + * @return true if the content type is that media type, false if it's + * another media type or there's no content type + */ + public boolean hasContentType(String mediaType) { + String contentType = getContentType(); + if (contentType == null) { + return false; + } + int start = 0; + while (start < contentType.length() && isHttpWhitespace(contentType.charAt(start))) { + start++; + } + if (!contentType.regionMatches(true, start, mediaType, 0, mediaType.length())) { + return false; + } + int end = start + mediaType.length(); + while (end < contentType.length() && isHttpWhitespace(contentType.charAt(end))) { + end++; + } + return end == contentType.length() || contentType.charAt(end) == ';'; + } + + private static boolean isHttpWhitespace(char c) { + return c == ' ' || c == '\t'; + } + public boolean containsContentTypeCharsetDeclaration() { // TODO can this regex be improved? should the test consider if its legal? return getContentType().matches("(?i).*charset=.*");