lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-09-24 22:55:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

implement shouldFetchBodyRule

Browse Source
This commit is contained in:
Noah Levitt
2012-09-05 14:19:17 -07:00
parent d703fce31a
commit fbd25b789a
2 changed files with 93 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/src/main/java/org/archive/modules/fetcher/FetchHTTP2.java
+79 -21
View File
@@ -87,6 +87,9 @@ import org.archive.modules.credential.Credential;
import org.archive.modules.credential.CredentialStore;
import org.archive.modules.credential.HtmlFormCredential;
import org.archive.modules.credential.HttpAuthenticationCredential;
import org.archive.modules.deciderules.AcceptDecideRule;
import org.archive.modules.deciderules.DecideResult;
import org.archive.modules.deciderules.DecideRule;
import org.archive.modules.extractor.LinkContext;
import org.archive.modules.net.CrawlHost;
import org.archive.modules.net.CrawlServer;
@@ -153,6 +156,9 @@ public class FetchHTTP2 extends AbstractFetchHTTP implements Lifecycle {
kp.put("userAgentProvider",provider);
}
protected static final Header HEADER_SEND_CONNECTION_CLOSE = new BasicHeader(
HTTP.CONN_DIRECTIVE, HTTP.CONN_CLOSE);
{
setSendConnectionClose(true);
}
@@ -225,8 +231,8 @@ public class FetchHTTP2 extends AbstractFetchHTTP implements Lifecycle {
* remote server and can be of assistance to webmasters trying to figure how
* a crawler got to a particular area on a site.
*/
public void setSendReferer(boolean sendClose) {
kp.put("sendReferer",sendClose);
public void setSendReferer(boolean sendReferer) {
kp.put("sendReferer",sendReferer);
}
{
@@ -447,8 +453,21 @@ public class FetchHTTP2 extends AbstractFetchHTTP implements Lifecycle {
kp.put("sendIfNoneMatch",sendIfNoneMatch);
}
protected static final Header HEADER_SEND_CONNECTION_CLOSE = new BasicHeader(
HTTP.CONN_DIRECTIVE, HTTP.CONN_CLOSE);
{
setShouldFetchBodyRule(new AcceptDecideRule());
}
public DecideRule getShouldFetchBodyRule() {
return (DecideRule) kp.get("shouldFetchBodyRule");
}
/**
* DecideRules applied after receipt of HTTP response headers but before we
* start to download the body. If any filter returns FALSE, the fetch is
* aborted. Prerequisites such as robots.txt by-pass filtering (i.e. they
* cannot be midfetch aborted.
*/
public void setShouldFetchBodyRule(DecideRule rule) {
kp.put("shouldFetchBodyRule", rule);
}
/**
* Can this processor fetch the given CrawlURI. May set a fetch status
@@ -485,14 +504,16 @@ public class FetchHTTP2 extends AbstractFetchHTTP implements Lifecycle {
*/
protected void setOtherCodings(CrawlURI uri, final Recorder rec,
final HttpResponse response) {
rec.setInputIsChunked(response.getEntity().isChunked());
Header contentEncodingHeader = response.getEntity().getContentEncoding();
if (contentEncodingHeader != null) {
String ce = contentEncodingHeader.getValue().trim();
try {
rec.setContentEncoding(ce);
} catch (IllegalArgumentException e) {
uri.getAnnotations().add("unsatisfiableContentEncoding:" + StringUtils.stripToEmpty(ce));
if (response.getEntity() != null) {
rec.setInputIsChunked(response.getEntity().isChunked());
Header contentEncodingHeader = response.getEntity().getContentEncoding();
if (contentEncodingHeader != null) {
String ce = contentEncodingHeader.getValue().trim();
try {
rec.setContentEncoding(ce);
} catch (IllegalArgumentException e) {
uri.getAnnotations().add("unsatisfiableContentEncoding:" + StringUtils.stripToEmpty(ce));
}
}
}
}
@@ -527,8 +548,44 @@ public class FetchHTTP2 extends AbstractFetchHTTP implements Lifecycle {
}
}
protected boolean checkMidfetchAbort(CrawlURI curi) {
if (curi.isPrerequisite()) {
return false;
}
DecideResult r = getShouldFetchBodyRule().decisionFor(curi);
if (r != DecideResult.REJECT) {
return false;
}
// method.markContentBegin(conn);
return true;
}
protected void doAbort(CrawlURI curi, HttpRequestBase request,
String annotation) {
curi.getAnnotations().add(annotation);
curi.getRecorder().close();
request.abort();
}
// XXX Unfortunately the place where midfetch abort happens is deep in the
// bowels of the http library code and it would be tricky and ugly to get
// these necessary variables in there. We keep these threadlocals instead.
private ThreadLocal<CrawlURI> threadActiveCrawlURI = new ThreadLocal<CrawlURI>();
private ThreadLocal<HttpRequestBase> threadActiveRequest = new ThreadLocal<HttpRequestBase>();
protected boolean maybeMidfetchAbort() {
if (checkMidfetchAbort(threadActiveCrawlURI.get())) {
doAbort(threadActiveCrawlURI.get(), threadActiveRequest.get(), "midFetchAbort");
return true;
} else {
return false;
}
}
@Override
protected void innerProcess(final CrawlURI curi) throws InterruptedException {
threadActiveCrawlURI.set(curi);
// could call after fetch in finally block, but right here feels even more sure
resetHttpClient();
@@ -559,6 +616,7 @@ public class FetchHTTP2 extends AbstractFetchHTTP implements Lifecycle {
curi.setFetchType(FetchType.HTTP_GET);
}
threadActiveRequest.set(request);
configureRequest(curi, request);
HttpHost targetHost = URIUtils.extractHost(request.getURI());
@@ -624,6 +682,11 @@ public class FetchHTTP2 extends AbstractFetchHTTP implements Lifecycle {
}
// Note completion time
curi.setFetchCompletedTime(System.currentTimeMillis());
// finished with these (midfetch abort would have happened already)
threadActiveCrawlURI.set(null);
threadActiveRequest.set(null);
// Set the response charset into the HttpRecord if available.
setCharacterEncoding(curi, rec, response);
setSizes(curi, rec);
@@ -1148,7 +1211,7 @@ public class FetchHTTP2 extends AbstractFetchHTTP implements Lifecycle {
protected RecordingHttpClient httpClient() {
if (httpClient == null) {
httpClient = new RecordingHttpClient(getServerCache());
httpClient = new RecordingHttpClient(this, getServerCache());
// some http client config
HttpClientParams.setRedirecting(httpClient.getParams(), false);
@@ -1161,13 +1224,6 @@ public class FetchHTTP2 extends AbstractFetchHTTP implements Lifecycle {
return httpClient;
}
protected void doAbort(CrawlURI curi, HttpRequestBase request,
String annotation) {
curi.getAnnotations().add(annotation);
curi.getRecorder().close();
request.abort();
}
/**
* Update CrawlURI internal sizes based on current transaction (and
* in the case of 304s, history)
@@ -1248,6 +1304,9 @@ public class FetchHTTP2 extends AbstractFetchHTTP implements Lifecycle {
curi.getNonFatalFailures().add(exception);
curi.setFetchStatus(status);
curi.getRecorder().close();
threadActiveCrawlURI.set(null);
threadActiveRequest.set(null);
}
public void start() {
@@ -1296,5 +1355,4 @@ public class FetchHTTP2 extends AbstractFetchHTTP implements Lifecycle {
return null;
}
}
}
modules/src/main/java/org/archive/modules/fetcher/RecordingHttpClient.java
+14 -5
View File
@@ -55,14 +55,17 @@ import org.archive.util.Recorder;
*/
public class RecordingHttpClient extends DefaultHttpClient {
private ServerCache serverCache;
private FetchHTTP2 fetcher;
/**
*
* @param fetchHTTP2
* @param serverCache
*/
public RecordingHttpClient(ServerCache serverCache) {
public RecordingHttpClient(FetchHTTP2 fetchHTTP2, ServerCache serverCache) {
super();
this.fetcher = fetchHTTP2;
this.setServerCache(serverCache);
// XXX uhh? see HeritrixHttpMethodRetryHandler ??
@@ -97,7 +100,8 @@ public class RecordingHttpClient extends DefaultHttpClient {
@Override
protected ClientConnectionOperator createConnectionOperator(SchemeRegistry schreg) {
return new RecordingClientConnectionOperator(schreg,
new ServerCacheResolver(RecordingHttpClient.this.getServerCache()));
new ServerCacheResolver(RecordingHttpClient.this.getServerCache()),
RecordingHttpClient.this.fetcher);
}
};
}
@@ -168,9 +172,12 @@ public class RecordingHttpClient extends DefaultHttpClient {
* @contributor nlevitt
*/
protected static class RecordingClientConnectionOperator extends DefaultClientConnectionOperator {
private FetchHTTP2 fetcher;
public RecordingClientConnectionOperator(SchemeRegistry schemes,
DnsResolver dnsResolver) {
DnsResolver dnsResolver, FetchHTTP2 fetcher) {
super(schemes, dnsResolver);
this.fetcher = fetcher;
}
@Override
@@ -189,13 +196,15 @@ public class RecordingHttpClient extends DefaultHttpClient {
@Override
public void receiveResponseEntity(HttpResponse response)
throws HttpException, IOException {
// XXX is this null check necessary? what happens if proxied, etc?
Recorder recorder = Recorder.getHttpRecorder();
if (recorder != null) {
recorder.markContentBegin();
}
super.receiveResponseEntity(response);
if (!fetcher.maybeMidfetchAbort()) {
super.receiveResponseEntity(response);
}
}
};
}