mirror of
https://github.com/internetarchive/heritrix3.git
synced 2026-11-22 13:47:02 +00:00
Merge pull request #811 from internetarchive/crawluri-has-content-type
Add CrawlURI.hasContentType() for media type checks
This commit is contained in:
12 files changed
+78
-43
No files matched your search
@@ -162,7 +162,6 @@ public class ExtractorPDFContent extends ContentExtractor {
|
||||
return false;
|
||||
}
|
||||
|
||||
String ct = uri.getContentType();
|
||||
return (ct != null) && (ct.startsWith("application/pdf"));
|
||||
return uri.hasContentType("application/pdf");
|
||||
}
|
||||
}
|
||||
@@ -608,12 +608,11 @@ public class ExtractorYoutubeDL extends Extractor
|
||||
return false;
|
||||
}
|
||||
|
||||
String mime = uri.getContentType().toLowerCase();
|
||||
if (mime.startsWith("text/html")
|
||||
|| mime.startsWith("application/xhtml")
|
||||
|| mime.startsWith("text/vnd.wap.wml")
|
||||
|| mime.startsWith("application/vnd.wap.wml")
|
||||
|| mime.startsWith("application/vnd.wap.xhtml")) {
|
||||
if (uri.hasContentType("text/html")
|
||||
|| uri.hasContentType("application/xhtml+xml")
|
||||
|| uri.hasContentType("text/vnd.wap.wml")
|
||||
|| uri.hasContentType("application/vnd.wap.wml")
|
||||
|| uri.hasContentType("application/vnd.wap.xhtml+xml")) {
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
@@ -131,8 +131,7 @@ public class BrowserProcessor extends Processor {
|
||||
if (curi.getFetchStatus() != 200) return false;
|
||||
String scheme = curi.getUURI().getScheme();
|
||||
if (!scheme.equals("https") && !scheme.equals("http")) return false;
|
||||
String mime = curi.getContentType().toLowerCase();
|
||||
if (!mime.startsWith("text/html")) return false;
|
||||
if (!curi.hasContentType("text/html")) return false;
|
||||
if (StringUtils.startsWithIgnoreCase(curi.getHttpResponseHeader("Content-Disposition"), "attachment")) {
|
||||
return false;
|
||||
}
|
||||
|
||||
@@ -38,6 +38,7 @@ import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.io.TempDir;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
/**
|
||||
@@ -232,4 +233,26 @@ public class CrawlURITest {
|
||||
assertEquals(0, a.compareTo(b));
|
||||
assertEquals(0, b.compareTo(a));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testHasContentType() throws Exception {
|
||||
assertTrue(withContentType("image/svg+xml").hasContentType("image/svg+xml"));
|
||||
assertTrue(withContentType("Image/SVG+XML").hasContentType("image/svg+xml"));
|
||||
assertTrue(withContentType("image/svg+xml;charset=utf-8").hasContentType("image/svg+xml"));
|
||||
assertTrue(withContentType(" image/svg+xml ; charset=utf-8").hasContentType("image/svg+xml"));
|
||||
|
||||
assertFalse(withContentType("image/svg+xmlfoo").hasContentType("image/svg+xml"));
|
||||
assertFalse(withContentType("image/svg").hasContentType("image/svg+xml"));
|
||||
assertFalse(withContentType("text/html").hasContentType("text/htm"));
|
||||
assertFalse(withContentType("application/xml").hasContentType("text/xml"));
|
||||
assertFalse(withContentType(null).hasContentType("text/html"));
|
||||
assertFalse(new CrawlURI(UURIFactory.getInstance("https://example.org/"))
|
||||
.hasContentType("text/html"));
|
||||
}
|
||||
|
||||
private static CrawlURI withContentType(String contentType) throws URIException {
|
||||
CrawlURI curi = new CrawlURI(UURIFactory.getInstance("https://example.org/"));
|
||||
curi.setContentType(contentType);
|
||||
return curi;
|
||||
}
|
||||
}
|
||||
@@ -1741,7 +1741,40 @@ implements Reporter, Serializable, OverlayContext, Comparable<CrawlURI> {
|
||||
|
||||
return caUri;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Tests whether this URI's content type is the given media type, ignoring
|
||||
* case and any parameters. For example, a content type of
|
||||
* {@code "Text/HTML; charset=utf-8"} has the media type
|
||||
* {@code "text/html"} but not {@code "text/htm"}.
|
||||
*
|
||||
* @param mediaType a type/subtype pair, e.g. "image/svg+xml"
|
||||
* @return true if the content type is that media type, false if it's
|
||||
* another media type or there's no content type
|
||||
*/
|
||||
public boolean hasContentType(String mediaType) {
|
||||
String contentType = getContentType();
|
||||
if (contentType == null) {
|
||||
return false;
|
||||
}
|
||||
int start = 0;
|
||||
while (start < contentType.length() && isHttpWhitespace(contentType.charAt(start))) {
|
||||
start++;
|
||||
}
|
||||
if (!contentType.regionMatches(true, start, mediaType, 0, mediaType.length())) {
|
||||
return false;
|
||||
}
|
||||
int end = start + mediaType.length();
|
||||
while (end < contentType.length() && isHttpWhitespace(contentType.charAt(end))) {
|
||||
end++;
|
||||
}
|
||||
return end == contentType.length() || contentType.charAt(end) == ';';
|
||||
}
|
||||
|
||||
private static boolean isHttpWhitespace(char c) {
|
||||
return c == ' ' || c == '\t';
|
||||
}
|
||||
|
||||
public boolean containsContentTypeCharsetDeclaration() {
|
||||
// TODO can this regex be improved? should the test consider if its legal?
|
||||
return getContentType().matches("(?i).*charset=.*");
|
||||
|
||||
@@ -56,11 +56,7 @@ public class ExtractorDOC extends ContentExtractor {
|
||||
|
||||
@Override
|
||||
protected boolean shouldExtract(CrawlURI uri) {
|
||||
String mimeType = uri.getContentType();
|
||||
if (mimeType == null) {
|
||||
return false;
|
||||
}
|
||||
return mimeType.toLowerCase().startsWith("application/msword");
|
||||
return uri.hasContentType("application/msword");
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -944,12 +944,11 @@ public class ExtractorHTML extends ContentExtractor implements InitializingBean
|
||||
}
|
||||
}
|
||||
|
||||
String mime = uri.getContentType().toLowerCase();
|
||||
if (mime.startsWith("text/html")
|
||||
|| mime.startsWith("application/xhtml")
|
||||
|| mime.startsWith("text/vnd.wap.wml")
|
||||
|| mime.startsWith("application/vnd.wap.wml")
|
||||
|| mime.startsWith("application/vnd.wap.xhtml")) {
|
||||
if (uri.hasContentType("text/html")
|
||||
|| uri.hasContentType("application/xhtml+xml")
|
||||
|| uri.hasContentType("text/vnd.wap.wml")
|
||||
|| uri.hasContentType("application/vnd.wap.wml")
|
||||
|| uri.hasContentType("application/vnd.wap.xhtml+xml")) {
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
@@ -160,7 +160,7 @@ public class ExtractorJS extends ContentExtractor {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (contentType.startsWith("application/json")) {
|
||||
if (uri.hasContentType("application/json")) {
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
@@ -68,8 +68,7 @@ public class ExtractorPDF extends ContentExtractor {
|
||||
return false;
|
||||
}
|
||||
|
||||
String ct = uri.getContentType();
|
||||
return (ct != null) && (ct.startsWith("application/pdf"));
|
||||
return uri.hasContentType("application/pdf");
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -70,8 +70,8 @@ public class ExtractorSitemap extends ContentExtractor {
|
||||
String mimeType = uri.getContentType();
|
||||
if (mimeType != null ) {
|
||||
// Looks like XML:
|
||||
if (mimeType.toLowerCase().startsWith("text/xml")
|
||||
|| mimeType.toLowerCase().startsWith("application/xml")) {
|
||||
if (uri.hasContentType("text/xml")
|
||||
|| uri.hasContentType("application/xml")) {
|
||||
|
||||
// check if content starts with xml preamble "<?xml" and does
|
||||
// contain "<urlset " or "<sitemapindex" early in the content
|
||||
|
||||
@@ -184,20 +184,8 @@ public class BotBlockDetector extends Processor {
|
||||
}
|
||||
|
||||
private static boolean hasHtmlContentType(CrawlURI curi) {
|
||||
String type = curi.getContentType();
|
||||
return type == null ||
|
||||
matchesMediaType(type, "text/html") ||
|
||||
matchesMediaType(type, "application/xhtml+xml");
|
||||
}
|
||||
|
||||
private static boolean matchesMediaType(String value, String expected) {
|
||||
if (!value.regionMatches(true, 0, expected, 0, expected.length())) {
|
||||
return false;
|
||||
}
|
||||
int i = expected.length();
|
||||
while (i < value.length() && Character.isWhitespace(value.charAt(i))) {
|
||||
i++;
|
||||
}
|
||||
return i == value.length() || value.charAt(i) == ';';
|
||||
return curi.getContentType() == null ||
|
||||
curi.hasContentType("text/html") ||
|
||||
curi.hasContentType("application/xhtml+xml");
|
||||
}
|
||||
}
|
||||
@@ -113,7 +113,7 @@ public class ExtractorHTMLTest extends StringExtractorTestBase {
|
||||
|
||||
euri = new CrawlURI(src, null, null, LinkContext.NAVLINK_MISC);
|
||||
recorder = createRecorder(content, "UTF-8");
|
||||
euri.setContentType("application/xhtml");
|
||||
euri.setContentType("application/xhtml+xml");
|
||||
euri.setRecorder(recorder);
|
||||
euri.setContentSize(content.length());
|
||||
result.add(new TestData(euri, link));
|
||||
|
||||
Reference in new issue
Block a user