diff --git a/contrib/src/main/java/org/archive/modules/extractor/ExtractorPDFContent.java b/contrib/src/main/java/org/archive/modules/extractor/ExtractorPDFContent.java index 7972211c..335af41d 100644 --- a/contrib/src/main/java/org/archive/modules/extractor/ExtractorPDFContent.java +++ b/contrib/src/main/java/org/archive/modules/extractor/ExtractorPDFContent.java @@ -162,7 +162,6 @@ public class ExtractorPDFContent extends ContentExtractor { return false; } - String ct = uri.getContentType(); - return (ct != null) && (ct.startsWith("application/pdf")); + return uri.hasContentType("application/pdf"); } } diff --git a/contrib/src/main/java/org/archive/modules/extractor/ExtractorYoutubeDL.java b/contrib/src/main/java/org/archive/modules/extractor/ExtractorYoutubeDL.java index 5950fc7b..d3b001f5 100644 --- a/contrib/src/main/java/org/archive/modules/extractor/ExtractorYoutubeDL.java +++ b/contrib/src/main/java/org/archive/modules/extractor/ExtractorYoutubeDL.java @@ -608,12 +608,11 @@ public class ExtractorYoutubeDL extends Extractor return false; } - String mime = uri.getContentType().toLowerCase(); - if (mime.startsWith("text/html") - || mime.startsWith("application/xhtml") - || mime.startsWith("text/vnd.wap.wml") - || mime.startsWith("application/vnd.wap.wml") - || mime.startsWith("application/vnd.wap.xhtml")) { + if (uri.hasContentType("text/html") + || uri.hasContentType("application/xhtml+xml") + || uri.hasContentType("text/vnd.wap.wml") + || uri.hasContentType("application/vnd.wap.wml") + || uri.hasContentType("application/vnd.wap.xhtml+xml")) { return true; } diff --git a/engine/src/main/java/org/archive/crawler/processor/BrowserProcessor.java b/engine/src/main/java/org/archive/crawler/processor/BrowserProcessor.java index 534056fa..ff911499 100644 --- a/engine/src/main/java/org/archive/crawler/processor/BrowserProcessor.java +++ b/engine/src/main/java/org/archive/crawler/processor/BrowserProcessor.java @@ -131,8 +131,7 @@ public class BrowserProcessor extends Processor { if (curi.getFetchStatus() != 200) return false; String scheme = curi.getUURI().getScheme(); if (!scheme.equals("https") && !scheme.equals("http")) return false; - String mime = curi.getContentType().toLowerCase(); - if (!mime.startsWith("text/html")) return false; + if (!curi.hasContentType("text/html")) return false; if (StringUtils.startsWithIgnoreCase(curi.getHttpResponseHeader("Content-Disposition"), "attachment")) { return false; } diff --git a/engine/src/test/java/org/archive/crawler/datamodel/CrawlURITest.java b/engine/src/test/java/org/archive/crawler/datamodel/CrawlURITest.java index 5f2fba68..829addb8 100644 --- a/engine/src/test/java/org/archive/crawler/datamodel/CrawlURITest.java +++ b/engine/src/test/java/org/archive/crawler/datamodel/CrawlURITest.java @@ -38,6 +38,7 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; /** @@ -232,4 +233,26 @@ public class CrawlURITest { assertEquals(0, a.compareTo(b)); assertEquals(0, b.compareTo(a)); } + + @Test + public void testHasContentType() throws Exception { + assertTrue(withContentType("image/svg+xml").hasContentType("image/svg+xml")); + assertTrue(withContentType("Image/SVG+XML").hasContentType("image/svg+xml")); + assertTrue(withContentType("image/svg+xml;charset=utf-8").hasContentType("image/svg+xml")); + assertTrue(withContentType(" image/svg+xml ; charset=utf-8").hasContentType("image/svg+xml")); + + assertFalse(withContentType("image/svg+xmlfoo").hasContentType("image/svg+xml")); + assertFalse(withContentType("image/svg").hasContentType("image/svg+xml")); + assertFalse(withContentType("text/html").hasContentType("text/htm")); + assertFalse(withContentType("application/xml").hasContentType("text/xml")); + assertFalse(withContentType(null).hasContentType("text/html")); + assertFalse(new CrawlURI(UURIFactory.getInstance("https://example.org/")) + .hasContentType("text/html")); + } + + private static CrawlURI withContentType(String contentType) throws URIException { + CrawlURI curi = new CrawlURI(UURIFactory.getInstance("https://example.org/")); + curi.setContentType(contentType); + return curi; + } } diff --git a/modules/src/main/java/org/archive/modules/CrawlURI.java b/modules/src/main/java/org/archive/modules/CrawlURI.java index a43589d2..a0bf3e04 100644 --- a/modules/src/main/java/org/archive/modules/CrawlURI.java +++ b/modules/src/main/java/org/archive/modules/CrawlURI.java @@ -1741,7 +1741,40 @@ implements Reporter, Serializable, OverlayContext, Comparable { return caUri; } - + + /** + * Tests whether this URI's content type is the given media type, ignoring + * case and any parameters. For example, a content type of + * {@code "Text/HTML; charset=utf-8"} has the media type + * {@code "text/html"} but not {@code "text/htm"}. + * + * @param mediaType a type/subtype pair, e.g. "image/svg+xml" + * @return true if the content type is that media type, false if it's + * another media type or there's no content type + */ + public boolean hasContentType(String mediaType) { + String contentType = getContentType(); + if (contentType == null) { + return false; + } + int start = 0; + while (start < contentType.length() && isHttpWhitespace(contentType.charAt(start))) { + start++; + } + if (!contentType.regionMatches(true, start, mediaType, 0, mediaType.length())) { + return false; + } + int end = start + mediaType.length(); + while (end < contentType.length() && isHttpWhitespace(contentType.charAt(end))) { + end++; + } + return end == contentType.length() || contentType.charAt(end) == ';'; + } + + private static boolean isHttpWhitespace(char c) { + return c == ' ' || c == '\t'; + } + public boolean containsContentTypeCharsetDeclaration() { // TODO can this regex be improved? should the test consider if its legal? return getContentType().matches("(?i).*charset=.*"); diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorDOC.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorDOC.java index 8be14a77..3f0d5957 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorDOC.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorDOC.java @@ -56,11 +56,7 @@ public class ExtractorDOC extends ContentExtractor { @Override protected boolean shouldExtract(CrawlURI uri) { - String mimeType = uri.getContentType(); - if (mimeType == null) { - return false; - } - return mimeType.toLowerCase().startsWith("application/msword"); + return uri.hasContentType("application/msword"); } diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java index 5c47af1d..67e3e7c4 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java @@ -944,12 +944,11 @@ public class ExtractorHTML extends ContentExtractor implements InitializingBean } } - String mime = uri.getContentType().toLowerCase(); - if (mime.startsWith("text/html") - || mime.startsWith("application/xhtml") - || mime.startsWith("text/vnd.wap.wml") - || mime.startsWith("application/vnd.wap.wml") - || mime.startsWith("application/vnd.wap.xhtml")) { + if (uri.hasContentType("text/html") + || uri.hasContentType("application/xhtml+xml") + || uri.hasContentType("text/vnd.wap.wml") + || uri.hasContentType("application/vnd.wap.wml") + || uri.hasContentType("application/vnd.wap.xhtml+xml")) { return true; } diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorJS.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorJS.java index a4d9d5c2..8de567c7 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorJS.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorJS.java @@ -160,7 +160,7 @@ public class ExtractorJS extends ContentExtractor { return true; } - if (contentType.startsWith("application/json")) { + if (uri.hasContentType("application/json")) { return true; } diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorPDF.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorPDF.java index ae7476db..7e437384 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorPDF.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorPDF.java @@ -68,8 +68,7 @@ public class ExtractorPDF extends ContentExtractor { return false; } - String ct = uri.getContentType(); - return (ct != null) && (ct.startsWith("application/pdf")); + return uri.hasContentType("application/pdf"); } diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorSitemap.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorSitemap.java index 9ad6e6be..ff72ea6e 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorSitemap.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorSitemap.java @@ -70,8 +70,8 @@ public class ExtractorSitemap extends ContentExtractor { String mimeType = uri.getContentType(); if (mimeType != null ) { // Looks like XML: - if (mimeType.toLowerCase().startsWith("text/xml") - || mimeType.toLowerCase().startsWith("application/xml")) { + if (uri.hasContentType("text/xml") + || uri.hasContentType("application/xml")) { // check if content starts with xml preamble "