mirror of
https://github.com/internetarchive/heritrix3.git
synced 2026-09-23 14:15:47 +00:00
ExtractorChrome: Don't capture data: URIs
They are already captured as part of their containing document. There's no need to record them separately. #430
This commit is contained in:
@@ -29,6 +29,7 @@ import org.archive.modules.ProcessorChain;
|
||||
import org.archive.net.chrome.*;
|
||||
import org.archive.spring.KeyedProperties;
|
||||
import org.archive.util.Recorder;
|
||||
import org.archive.util.UriUtils;
|
||||
import org.json.JSONArray;
|
||||
import org.springframework.context.ApplicationEventPublisher;
|
||||
|
||||
@@ -220,7 +221,7 @@ public class ExtractorChrome extends ContentExtractor {
|
||||
}
|
||||
|
||||
private void handleCapturedRequest(CrawlURI via, ChromeRequest request) {
|
||||
if (request.isResponseFulfilledByInterception()) {
|
||||
if (request.isResponseFulfilledByInterception() || UriUtils.isDataUri(request.getUrl())) {
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
@@ -67,6 +67,7 @@ public class ExtractorChromeTest {
|
||||
response.setContentType("text/html");
|
||||
response.getWriter().write("<a href=http://example.org/page2.html>link</a>" +
|
||||
"<img src=/blue.png>" +
|
||||
"<img src='data:image/gif;base64,R0lGODlhAQABAAAAACH5BAEKAAEALAAAAAABAAEAAAICTAEAOw=='>" +
|
||||
"<script>fetch('/post', {method: 'POST', body: 'hello'});</script>" +
|
||||
"<link rel=stylesheet href=style.css>");
|
||||
baseRequest.setHandled(true);
|
||||
|
||||
Reference in New Issue
Block a user