Merge pull request #811 from internetarchive/crawluri-has-content-type

Add CrawlURI.hasContentType() for media type checks
This commit is contained in:
Alex Osborne authored and GitHub committed 2026-11-05 16:17:25 +09:00
commit a35dcb86a6
12 files changed
+78 -43

No files matched your search

@@ -162,7 +162,6 @@ public class ExtractorPDFContent extends ContentExtractor {
return false;
}
String ct = uri.getContentType();
return (ct != null) && (ct.startsWith("application/pdf"));
return uri.hasContentType("application/pdf");
}
}
@@ -608,12 +608,11 @@ public class ExtractorYoutubeDL extends Extractor
return false;
}
String mime = uri.getContentType().toLowerCase();
if (mime.startsWith("text/html")
|| mime.startsWith("application/xhtml")
|| mime.startsWith("text/vnd.wap.wml")
|| mime.startsWith("application/vnd.wap.wml")
|| mime.startsWith("application/vnd.wap.xhtml")) {
if (uri.hasContentType("text/html")
|| uri.hasContentType("application/xhtml+xml")
|| uri.hasContentType("text/vnd.wap.wml")
|| uri.hasContentType("application/vnd.wap.wml")
|| uri.hasContentType("application/vnd.wap.xhtml+xml")) {
return true;
}
@@ -131,8 +131,7 @@ public class BrowserProcessor extends Processor {
if (curi.getFetchStatus() != 200) return false;
String scheme = curi.getUURI().getScheme();
if (!scheme.equals("https") && !scheme.equals("http")) return false;
String mime = curi.getContentType().toLowerCase();
if (!mime.startsWith("text/html")) return false;
if (!curi.hasContentType("text/html")) return false;
if (StringUtils.startsWithIgnoreCase(curi.getHttpResponseHeader("Content-Disposition"), "attachment")) {
return false;
}
@@ -38,6 +38,7 @@ import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
/**
@@ -232,4 +233,26 @@ public class CrawlURITest {
assertEquals(0, a.compareTo(b));
assertEquals(0, b.compareTo(a));
}
@Test
public void testHasContentType() throws Exception {
assertTrue(withContentType("image/svg+xml").hasContentType("image/svg+xml"));
assertTrue(withContentType("Image/SVG+XML").hasContentType("image/svg+xml"));
assertTrue(withContentType("image/svg+xml;charset=utf-8").hasContentType("image/svg+xml"));
assertTrue(withContentType(" image/svg+xml ; charset=utf-8").hasContentType("image/svg+xml"));
assertFalse(withContentType("image/svg+xmlfoo").hasContentType("image/svg+xml"));
assertFalse(withContentType("image/svg").hasContentType("image/svg+xml"));
assertFalse(withContentType("text/html").hasContentType("text/htm"));
assertFalse(withContentType("application/xml").hasContentType("text/xml"));
assertFalse(withContentType(null).hasContentType("text/html"));
assertFalse(new CrawlURI(UURIFactory.getInstance("https://example.org/"))
.hasContentType("text/html"));
}
private static CrawlURI withContentType(String contentType) throws URIException {
CrawlURI curi = new CrawlURI(UURIFactory.getInstance("https://example.org/"));
curi.setContentType(contentType);
return curi;
}
}
@@ -1741,7 +1741,40 @@ implements Reporter, Serializable, OverlayContext, Comparable<CrawlURI> {
return caUri;
}
/**
* Tests whether this URI's content type is the given media type, ignoring
* case and any parameters. For example, a content type of
* {@code "Text/HTML; charset=utf-8"} has the media type
* {@code "text/html"} but not {@code "text/htm"}.
*
* @param mediaType a type/subtype pair, e.g. "image/svg+xml"
* @return true if the content type is that media type, false if it's
* another media type or there's no content type
*/
public boolean hasContentType(String mediaType) {
String contentType = getContentType();
if (contentType == null) {
return false;
}
int start = 0;
while (start < contentType.length() && isHttpWhitespace(contentType.charAt(start))) {
start++;
}
if (!contentType.regionMatches(true, start, mediaType, 0, mediaType.length())) {
return false;
}
int end = start + mediaType.length();
while (end < contentType.length() && isHttpWhitespace(contentType.charAt(end))) {
end++;
}
return end == contentType.length() || contentType.charAt(end) == ';';
}
private static boolean isHttpWhitespace(char c) {
return c == ' ' || c == '\t';
}
public boolean containsContentTypeCharsetDeclaration() {
// TODO can this regex be improved? should the test consider if its legal?
return getContentType().matches("(?i).*charset=.*");
@@ -56,11 +56,7 @@ public class ExtractorDOC extends ContentExtractor {
@Override
protected boolean shouldExtract(CrawlURI uri) {
String mimeType = uri.getContentType();
if (mimeType == null) {
return false;
}
return mimeType.toLowerCase().startsWith("application/msword");
return uri.hasContentType("application/msword");
}
@@ -944,12 +944,11 @@ public class ExtractorHTML extends ContentExtractor implements InitializingBean
}
}
String mime = uri.getContentType().toLowerCase();
if (mime.startsWith("text/html")
|| mime.startsWith("application/xhtml")
|| mime.startsWith("text/vnd.wap.wml")
|| mime.startsWith("application/vnd.wap.wml")
|| mime.startsWith("application/vnd.wap.xhtml")) {
if (uri.hasContentType("text/html")
|| uri.hasContentType("application/xhtml+xml")
|| uri.hasContentType("text/vnd.wap.wml")
|| uri.hasContentType("application/vnd.wap.wml")
|| uri.hasContentType("application/vnd.wap.xhtml+xml")) {
return true;
}
@@ -160,7 +160,7 @@ public class ExtractorJS extends ContentExtractor {
return true;
}
if (contentType.startsWith("application/json")) {
if (uri.hasContentType("application/json")) {
return true;
}
@@ -68,8 +68,7 @@ public class ExtractorPDF extends ContentExtractor {
return false;
}
String ct = uri.getContentType();
return (ct != null) && (ct.startsWith("application/pdf"));
return uri.hasContentType("application/pdf");
}
@@ -70,8 +70,8 @@ public class ExtractorSitemap extends ContentExtractor {
String mimeType = uri.getContentType();
if (mimeType != null ) {
// Looks like XML:
if (mimeType.toLowerCase().startsWith("text/xml")
|| mimeType.toLowerCase().startsWith("application/xml")) {
if (uri.hasContentType("text/xml")
|| uri.hasContentType("application/xml")) {
// check if content starts with xml preamble "<?xml" and does
// contain "<urlset " or "<sitemapindex" early in the content
@@ -184,20 +184,8 @@ public class BotBlockDetector extends Processor {
}
private static boolean hasHtmlContentType(CrawlURI curi) {
String type = curi.getContentType();
return type == null ||
matchesMediaType(type, "text/html") ||
matchesMediaType(type, "application/xhtml+xml");
}
private static boolean matchesMediaType(String value, String expected) {
if (!value.regionMatches(true, 0, expected, 0, expected.length())) {
return false;
}
int i = expected.length();
while (i < value.length() && Character.isWhitespace(value.charAt(i))) {
i++;
}
return i == value.length() || value.charAt(i) == ';';
return curi.getContentType() == null ||
curi.hasContentType("text/html") ||
curi.hasContentType("application/xhtml+xml");
}
}
@@ -113,7 +113,7 @@ public class ExtractorHTMLTest extends StringExtractorTestBase {
euri = new CrawlURI(src, null, null, LinkContext.NAVLINK_MISC);
recorder = createRecorder(content, "UTF-8");
euri.setContentType("application/xhtml");
euri.setContentType("application/xhtml+xml");
euri.setRecorder(recorder);
euri.setContentSize(content.length());
result.add(new TestData(euri, link));