From 5b1f492cc2f2314c2710ffbcbbaa6a7b2fefca98 Mon Sep 17 00:00:00 2001 From: Alex Osborne Date: Thu, 5 Nov 2026 09:53:25 +0900 Subject: [PATCH 1/2] CrawlURI: add hasContentType() for media type comparison Compares the content type's type/subtype with the given media type, ignoring case and any parameters, so "Text/HTML; charset=utf-8" matches "text/html" but "text/htmlfoo" doesn't. This replaces ad hoc checks like toLowerCase().startsWith(...), which also match longer subtypes. --- .../crawler/datamodel/CrawlURITest.java | 23 ++++++++++++ .../java/org/archive/modules/CrawlURI.java | 35 ++++++++++++++++++- 2 files changed, 57 insertions(+), 1 deletion(-) diff --git a/engine/src/test/java/org/archive/crawler/datamodel/CrawlURITest.java b/engine/src/test/java/org/archive/crawler/datamodel/CrawlURITest.java index 5f2fba68..829addb8 100644 --- a/engine/src/test/java/org/archive/crawler/datamodel/CrawlURITest.java +++ b/engine/src/test/java/org/archive/crawler/datamodel/CrawlURITest.java @@ -38,6 +38,7 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; /** @@ -232,4 +233,26 @@ public class CrawlURITest { assertEquals(0, a.compareTo(b)); assertEquals(0, b.compareTo(a)); } + + @Test + public void testHasContentType() throws Exception { + assertTrue(withContentType("image/svg+xml").hasContentType("image/svg+xml")); + assertTrue(withContentType("Image/SVG+XML").hasContentType("image/svg+xml")); + assertTrue(withContentType("image/svg+xml;charset=utf-8").hasContentType("image/svg+xml")); + assertTrue(withContentType(" image/svg+xml ; charset=utf-8").hasContentType("image/svg+xml")); + + assertFalse(withContentType("image/svg+xmlfoo").hasContentType("image/svg+xml")); + assertFalse(withContentType("image/svg").hasContentType("image/svg+xml")); + assertFalse(withContentType("text/html").hasContentType("text/htm")); + assertFalse(withContentType("application/xml").hasContentType("text/xml")); + assertFalse(withContentType(null).hasContentType("text/html")); + assertFalse(new CrawlURI(UURIFactory.getInstance("https://example.org/")) + .hasContentType("text/html")); + } + + private static CrawlURI withContentType(String contentType) throws URIException { + CrawlURI curi = new CrawlURI(UURIFactory.getInstance("https://example.org/")); + curi.setContentType(contentType); + return curi; + } } diff --git a/modules/src/main/java/org/archive/modules/CrawlURI.java b/modules/src/main/java/org/archive/modules/CrawlURI.java index a43589d2..a0bf3e04 100644 --- a/modules/src/main/java/org/archive/modules/CrawlURI.java +++ b/modules/src/main/java/org/archive/modules/CrawlURI.java @@ -1741,7 +1741,40 @@ implements Reporter, Serializable, OverlayContext, Comparable { return caUri; } - + + /** + * Tests whether this URI's content type is the given media type, ignoring + * case and any parameters. For example, a content type of + * {@code "Text/HTML; charset=utf-8"} has the media type + * {@code "text/html"} but not {@code "text/htm"}. + * + * @param mediaType a type/subtype pair, e.g. "image/svg+xml" + * @return true if the content type is that media type, false if it's + * another media type or there's no content type + */ + public boolean hasContentType(String mediaType) { + String contentType = getContentType(); + if (contentType == null) { + return false; + } + int start = 0; + while (start < contentType.length() && isHttpWhitespace(contentType.charAt(start))) { + start++; + } + if (!contentType.regionMatches(true, start, mediaType, 0, mediaType.length())) { + return false; + } + int end = start + mediaType.length(); + while (end < contentType.length() && isHttpWhitespace(contentType.charAt(end))) { + end++; + } + return end == contentType.length() || contentType.charAt(end) == ';'; + } + + private static boolean isHttpWhitespace(char c) { + return c == ' ' || c == '\t'; + } + public boolean containsContentTypeCharsetDeclaration() { // TODO can this regex be improved? should the test consider if its legal? return getContentType().matches("(?i).*charset=.*"); From 7a9e2689b40f5603a96704d7649f3946effb5de3 Mon Sep 17 00:00:00 2001 From: Alex Osborne Date: Thu, 5 Nov 2026 09:58:58 +0900 Subject: [PATCH 2/2] Use CrawlURI.hasContentType() for exact media type checks Replaces startsWith() prefix checks, which also matched longer subtypes, e.g. "application/vnd.wap.wml" matched the binary "application/vnd.wap.wmlc". The truncated "application/xhtml" and "application/vnd.wap.xhtml" checks now name the full "+xml" types. Also fixes case-sensitive comparisons in ExtractorPDF and ExtractorPDFContent, and drops BotBlockDetector's private equivalent. Substring heuristics like ExtractorJS's indexOf("javascript") and ExtractorXML's indexOf("xml") are intentionally left as they are. --- .../modules/extractor/ExtractorPDFContent.java | 3 +-- .../modules/extractor/ExtractorYoutubeDL.java | 11 +++++------ .../crawler/processor/BrowserProcessor.java | 3 +-- .../modules/extractor/ExtractorDOC.java | 6 +----- .../modules/extractor/ExtractorHTML.java | 11 +++++------ .../archive/modules/extractor/ExtractorJS.java | 2 +- .../modules/extractor/ExtractorPDF.java | 3 +-- .../modules/extractor/ExtractorSitemap.java | 4 ++-- .../modules/processor/BotBlockDetector.java | 18 +++--------------- .../modules/extractor/ExtractorHTMLTest.java | 2 +- 10 files changed, 21 insertions(+), 42 deletions(-) diff --git a/contrib/src/main/java/org/archive/modules/extractor/ExtractorPDFContent.java b/contrib/src/main/java/org/archive/modules/extractor/ExtractorPDFContent.java index 7972211c..335af41d 100644 --- a/contrib/src/main/java/org/archive/modules/extractor/ExtractorPDFContent.java +++ b/contrib/src/main/java/org/archive/modules/extractor/ExtractorPDFContent.java @@ -162,7 +162,6 @@ public class ExtractorPDFContent extends ContentExtractor { return false; } - String ct = uri.getContentType(); - return (ct != null) && (ct.startsWith("application/pdf")); + return uri.hasContentType("application/pdf"); } } diff --git a/contrib/src/main/java/org/archive/modules/extractor/ExtractorYoutubeDL.java b/contrib/src/main/java/org/archive/modules/extractor/ExtractorYoutubeDL.java index 5950fc7b..d3b001f5 100644 --- a/contrib/src/main/java/org/archive/modules/extractor/ExtractorYoutubeDL.java +++ b/contrib/src/main/java/org/archive/modules/extractor/ExtractorYoutubeDL.java @@ -608,12 +608,11 @@ public class ExtractorYoutubeDL extends Extractor return false; } - String mime = uri.getContentType().toLowerCase(); - if (mime.startsWith("text/html") - || mime.startsWith("application/xhtml") - || mime.startsWith("text/vnd.wap.wml") - || mime.startsWith("application/vnd.wap.wml") - || mime.startsWith("application/vnd.wap.xhtml")) { + if (uri.hasContentType("text/html") + || uri.hasContentType("application/xhtml+xml") + || uri.hasContentType("text/vnd.wap.wml") + || uri.hasContentType("application/vnd.wap.wml") + || uri.hasContentType("application/vnd.wap.xhtml+xml")) { return true; } diff --git a/engine/src/main/java/org/archive/crawler/processor/BrowserProcessor.java b/engine/src/main/java/org/archive/crawler/processor/BrowserProcessor.java index 534056fa..ff911499 100644 --- a/engine/src/main/java/org/archive/crawler/processor/BrowserProcessor.java +++ b/engine/src/main/java/org/archive/crawler/processor/BrowserProcessor.java @@ -131,8 +131,7 @@ public class BrowserProcessor extends Processor { if (curi.getFetchStatus() != 200) return false; String scheme = curi.getUURI().getScheme(); if (!scheme.equals("https") && !scheme.equals("http")) return false; - String mime = curi.getContentType().toLowerCase(); - if (!mime.startsWith("text/html")) return false; + if (!curi.hasContentType("text/html")) return false; if (StringUtils.startsWithIgnoreCase(curi.getHttpResponseHeader("Content-Disposition"), "attachment")) { return false; } diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorDOC.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorDOC.java index 8be14a77..3f0d5957 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorDOC.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorDOC.java @@ -56,11 +56,7 @@ public class ExtractorDOC extends ContentExtractor { @Override protected boolean shouldExtract(CrawlURI uri) { - String mimeType = uri.getContentType(); - if (mimeType == null) { - return false; - } - return mimeType.toLowerCase().startsWith("application/msword"); + return uri.hasContentType("application/msword"); } diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java index 5c47af1d..67e3e7c4 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java @@ -944,12 +944,11 @@ public class ExtractorHTML extends ContentExtractor implements InitializingBean } } - String mime = uri.getContentType().toLowerCase(); - if (mime.startsWith("text/html") - || mime.startsWith("application/xhtml") - || mime.startsWith("text/vnd.wap.wml") - || mime.startsWith("application/vnd.wap.wml") - || mime.startsWith("application/vnd.wap.xhtml")) { + if (uri.hasContentType("text/html") + || uri.hasContentType("application/xhtml+xml") + || uri.hasContentType("text/vnd.wap.wml") + || uri.hasContentType("application/vnd.wap.wml") + || uri.hasContentType("application/vnd.wap.xhtml+xml")) { return true; } diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorJS.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorJS.java index a4d9d5c2..8de567c7 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorJS.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorJS.java @@ -160,7 +160,7 @@ public class ExtractorJS extends ContentExtractor { return true; } - if (contentType.startsWith("application/json")) { + if (uri.hasContentType("application/json")) { return true; } diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorPDF.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorPDF.java index ae7476db..7e437384 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorPDF.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorPDF.java @@ -68,8 +68,7 @@ public class ExtractorPDF extends ContentExtractor { return false; } - String ct = uri.getContentType(); - return (ct != null) && (ct.startsWith("application/pdf")); + return uri.hasContentType("application/pdf"); } diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorSitemap.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorSitemap.java index 9ad6e6be..ff72ea6e 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorSitemap.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorSitemap.java @@ -70,8 +70,8 @@ public class ExtractorSitemap extends ContentExtractor { String mimeType = uri.getContentType(); if (mimeType != null ) { // Looks like XML: - if (mimeType.toLowerCase().startsWith("text/xml") - || mimeType.toLowerCase().startsWith("application/xml")) { + if (uri.hasContentType("text/xml") + || uri.hasContentType("application/xml")) { // check if content starts with xml preamble "