lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-09-25 07:05:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

Altered FetchStats to also keep track of the number of URIs that have

Browse Source 
an HTTP status between 200 and 299.

Altered the QuotaEnforcer to enforce quotas on the number of URIs that
have an HTTP status between 200 and 299.

Removed confusingly named variables (SUCCESS, SUCCESS_KB, RESPONSES,
RESPONSES_KB) and replaced their references.
This commit is contained in:
Christopher Miles
2011-11-17 12:34:05 -05:00
parent 86cf3e1fcd
commit 4c1740ce1e
2 changed files with 239 additions and 87 deletions
Download Patch File Download Diff File Expand all files Collapse all files
engine/src/main/java/org/archive/crawler/prefetch/QuotaEnforcer.java
+121 -38
View File
@@ -1,8 +1,8 @@
/*
* This file is part of the Heritrix web crawler (crawler.archive.org).
*
* Licensed to the Internet Archive (IA) by one or more individual
* contributors.
* Licensed to the Internet Archive (IA) by one or more individual
* contributors.
*
* The IA licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
@@ -35,9 +35,9 @@ import org.springframework.beans.factory.annotation.Autowired;
/**
* A simple quota enforcer. If the host, server, or frontier group
* associated with the current CrawlURI is already over its quotas,
* associated with the current CrawlURI is already over its quotas,
* blocks the current URI's processing with S_BLOCKED_BY_QUOTA.
*
*
* @author gojomo
* @version $Date$, $Revision$
*/
@@ -46,54 +46,54 @@ public class QuotaEnforcer extends Processor {
private static final Logger LOGGER =
Logger.getLogger(QuotaEnforcer.class.getName());
// indexed table of reused string categorical names/keys
protected static final int SERVER = 0;
protected static final int HOST = 1;
protected static final int GROUP = 2;
protected static final int SUCCESSES = 0;
protected static final int SUCCESS_KB = 1;
protected static final int RESPONSES = 2;
protected static final int RESPONSE_KB = 3;
private static final String SERVER_MAX_FETCH_SUCCESSES = "serverMaxFetchSuccesses";
private static final String SERVER_MAX_SUCCESS_KB = "serverMaxSuccessKb";
private static final String SERVER_MAX_FETCH_RESPONSES = "serverMaxFetchResponses";
private static final String SERVER_MAX_ALL_KB = "serverMaxAllKb";
private static final String SERVER_MAX_FETCH_HTTP_SUCCESSES = "serverMaxFetchHttpSuccesses";
private static final String HOST_MAX_FETCH_SUCCESSES = "hostMaxFetchSuccesses";
private static final String HOST_MAX_SUCCESS_KB = "hostMaxSuccessKb";
private static final String HOST_MAX_FETCH_RESPONSES = "hostMaxFetchResponses";
private static final String HOST_MAX_ALL_KB = "hostMaxAllKb";
private static final String HOST_MAX_FETCH_HTTP_SUCCESSES = "hostMaxFetchHttpSuccesses";
private static final String GROUP_MAX_FETCH_SUCCESSES = "groupMaxFetchSuccesses";
private static final String GROUP_MAX_SUCCESS_KB = "groupMaxSuccessKb";
private static final String GROUP_MAX_FETCH_RESPONSES = "groupMaxFetchResponses";
private static final String GROUP_MAX_ALL_KB = "groupMaxAllKb";
private static final String GROUP_MAX_FETCH_HTTP_SUCCESSES = "groupMaxFetchHttpSuccesses";
protected static final String[][] keys = new String[][] {
{
//"server",
SERVER_MAX_FETCH_SUCCESSES,
SERVER_MAX_SUCCESS_KB,
SERVER_MAX_FETCH_RESPONSES,
SERVER_MAX_ALL_KB
SERVER_MAX_ALL_KB,
SERVER_MAX_FETCH_HTTP_SUCCESSES
},
{
//"host"
HOST_MAX_FETCH_SUCCESSES,
HOST_MAX_SUCCESS_KB,
HOST_MAX_FETCH_RESPONSES,
HOST_MAX_ALL_KB
,
HOST_MAX_ALL_KB,
HOST_MAX_FETCH_HTTP_SUCCESSES
},
{
//"group"
GROUP_MAX_FETCH_SUCCESSES,
GROUP_MAX_SUCCESS_KB,
GROUP_MAX_FETCH_RESPONSES,
GROUP_MAX_ALL_KB
GROUP_MAX_ALL_KB,
GROUP_MAX_FETCH_HTTP_SUCCESSES
}
};
@@ -114,6 +114,31 @@ public class QuotaEnforcer extends Processor {
kp.put(SERVER_MAX_FETCH_SUCCESSES,max);
}
{
setServerMaxFetchHttpSuccesses(-1L); // no limit
}
/**
* Maximum number of HTTP fetch successes (URI with an HTTP status
* code between 200 and 299) to collect from one server. Default
* is -1, meaning no limit.
*
* @param long Maximum number of fetch successes
*/
public void setServerMaxFetchHttpSuccesses(long max) {
kp.put(SERVER_MAX_FETCH_HTTP_SUCCESSES, max);
}
/**
* Returns the maximum number of fetch successes (URI with an HTTP
* status code between 200 and 299) to collect from one
* server. The default value is -1, indicating no limit.
*/
public long getServerMaxFetchHttpSuccesses() {
return((Long) kp.get(SERVER_MAX_FETCH_HTTP_SUCCESSES));
}
/**
* Maximum amount of fetch success content (e.g. 200 responses) in KB to
@@ -171,6 +196,32 @@ public class QuotaEnforcer extends Processor {
kp.put(HOST_MAX_FETCH_SUCCESSES,max);
}
{
setHostMaxFetchHttpSuccesses(-1L); // no limit
}
/**
* Maximum number of HTTP fetch successes (URI with an HTTP status
* code between 200 and 299) to collect from one server. Default
* is -1, meaning no limit.
*
* @param long Maximum number of fetch successes
*/
public void setHostMaxFetchHttpSuccesses(long max) {
kp.put(HOST_MAX_FETCH_HTTP_SUCCESSES, max);
}
/**
* Returns the maximum number of fetch successes (URI with an HTTP
* status code between 200 and 299) to collect from one
* server. The default value is -1, indicating no limit.
*/
public long getHostMaxFetchHttpSuccesses() {
return((Long) kp.get(HOST_MAX_FETCH_HTTP_SUCCESSES));
}
/**
* Maximum amount of fetch success content (e.g. 200 responses) in KB to
* collect from one host. Default is -1, meaning no limit.
@@ -227,6 +278,32 @@ public class QuotaEnforcer extends Processor {
kp.put(GROUP_MAX_FETCH_SUCCESSES,max);
}
{
setGroupMaxFetchHttpSuccesses(-1L); // no limit
}
/**
* Maximum number of HTTP fetch successes (URI with an HTTP status
* code between 200 and 299) to collect from one server. Default
* is -1, meaning no limit.
*
* @param long Maximum number of fetch successes
*/
public void setGroupMaxFetchHttpSuccesses(long max) {
kp.put(GROUP_MAX_FETCH_HTTP_SUCCESSES, max);
}
/**
* Returns the maximum number of fetch successes (URI with an HTTP
* status code between 200 and 299) to collect from one
* server. The default value is -1, indicating no limit.
*/
public long getGroupMaxFetchHttpSuccesses() {
return((Long) kp.get(GROUP_MAX_FETCH_HTTP_SUCCESSES));
}
/**
* Maximum amount of fetch success content (e.g. 200 responses) in KB to
* collect from one group. Default is -1, meaning no limit.
@@ -285,7 +362,7 @@ public class QuotaEnforcer extends Processor {
public void setForceRetire(boolean force) {
kp.put("forceRetire",force);
}
protected ServerCache serverCache;
public ServerCache getServerCache() {
return this.serverCache;
@@ -294,7 +371,7 @@ public class QuotaEnforcer extends Processor {
public void setServerCache(ServerCache serverCache) {
this.serverCache = serverCache;
}
protected Frontier frontier;
public Frontier getFrontier() {
return this.frontier;
@@ -303,7 +380,7 @@ public class QuotaEnforcer extends Processor {
public void setFrontier(Frontier frontier) {
this.frontier = frontier;
}
protected boolean shouldProcess(CrawlURI puri) {
return puri instanceof CrawlURI;
}
@@ -311,31 +388,31 @@ public class QuotaEnforcer extends Processor {
protected void innerProcess(CrawlURI puri) {
throw new AssertionError();
}
protected ProcessResult innerProcessResult(CrawlURI puri) {
CrawlURI curi = (CrawlURI)puri;
protected ProcessResult innerProcessResult(CrawlURI curi) {
final CrawlServer server = serverCache.getServerFor(curi.getUURI());
final CrawlHost host = serverCache.getHostFor(curi.getUURI());
FetchStats.HasFetchStats[] haveStats =
FetchStats.HasFetchStats[] haveStats =
new FetchStats.HasFetchStats[] {
server,
host,
server,
host,
frontier.getGroup(curi)
};
for(int cat=SERVER;cat<=GROUP;cat++) {
if (checkQuotas(curi,haveStats[cat],cat)) {
return ProcessResult.FINISH;
}
}
return ProcessResult.PROCEED;
}
/**
* Check all quotas for the given substats and category (server, host, or
* group).
*
* group).
*
* @param curi CrawlURI to mark up with results
* @param hasStats holds CrawlSubstats with actual values to test
* @param CAT category index (SERVER, HOST, GROUP) to quota settings keys
@@ -344,41 +421,47 @@ public class QuotaEnforcer extends Processor {
protected boolean checkQuotas(final CrawlURI curi,
final FetchStats.HasFetchStats hasStats,
final int CAT) {
if (hasStats == null) {
if (LOGGER.isLoggable(Level.FINE)) {
LOGGER.fine(curi.toString() + " null stats category: " + CAT);
}
return false;
}
FetchStats substats = hasStats.getSubstats();
long[] actuals = new long[] {
substats.getFetchSuccesses(),
substats.getSuccessBytes()/1024,
substats.getFetchResponses(),
substats.getTotalBytes()/1024,
substats.getFetchHttpSuccesses()
};
for(int q=SUCCESSES; q<=RESPONSE_KB; q++) {
for(int q = 0; q < keys[CAT].length; q++) {
String key = keys[CAT][q];
if (applyQuota(curi, key, actuals[q])) {
return true;
return true;
}
}
return false;
return false;
}
/**
* Apply the quota specified by the given key against the actual
* value provided. If the quota and actual values rule out processing the
* given CrawlURI, mark up the CrawlURI appropriately.
*
* Apply the quota specified by the given key against the actual
* value provided. If the quota and actual values rule out processing the
* given CrawlURI, mark up the CrawlURI appropriately.
*
* @param curi CrawlURI whose processing is subject to a potential quota
* limitation
* @param quotaKey settings key to get applicable quota
* @param actual current value to compare to quota
* @param actual current value to compare to quota
* @return true is CrawlURI is blocked by a quota, false otherwise
*/
protected boolean applyQuota(CrawlURI curi, String key, long actual) {
long quota = (Long)kp.get(key);
if (quota >= 0 && actual >= quota) {
curi.getAnnotations().add("Q:"+key);
if (getForceRetire()) {
@@ -390,6 +473,6 @@ public class QuotaEnforcer extends Processor {
}
return true;
}
return false;
return false;
}
}
modules/src/main/java/org/archive/modules/fetcher/FetchStats.java
+118 -49
View File
@@ -1,8 +1,8 @@
/*
* This file is part of the Heritrix web crawler (crawler.archive.org).
*
* Licensed to the Internet Archive (IA) by one or more individual
* contributors.
* Licensed to the Internet Archive (IA) by one or more individual
* contributors.
*
* The IA licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
@@ -26,51 +26,99 @@ import java.util.Map;
import org.archive.modules.CrawlURI;
import org.archive.util.ArchiveUtils;
import org.archive.util.MultiReporter;
import org.apache.commons.httpclient.HttpStatus;
import org.apache.commons.httpclient.HttpStatus;
import org.archive.modules.deciderules.recrawl.IdenticalDigestDecideRule;
import org.apache.commons.httpclient.HttpMethod;
/**
* Collector of statistics for a 'subset' of a crawl,
* such as a server (host:port), host, or frontier group
* (eg queue).
*
* such as a server (host:port), host, or frontier group
* (eg queue).
*
* @author gojomo
*/
public class FetchStats implements Serializable, FetchStatusCodes, MultiReporter {
/** lower bound on HTTP status codes that we'll interpret as successes */
private static final int HTTP_STATUS_CODE_SUCCESS_LOWER = 200;
/** upper bound on HTTP status codes that we'll interpret as successes */
private static final int HTTP_STATUS_CODE_SUCCESS_UPPER = 299;
private static final long serialVersionUID = 8624425657056569036L;
public enum Stage {SCHEDULED, RELOCATED, RETRIED,
public enum Stage {SCHEDULED, RELOCATED, RETRIED,
SUCCEEDED, DISREGARDED, FAILED};
public interface HasFetchStats {
public FetchStats getSubstats();
}
public interface CollectsFetchStats {
public void tally(CrawlURI curi, Stage stage);
public void tally(CrawlURI curi, Stage stage);
}
long totalScheduled; // anything initially scheduled
// (totalScheduled - (fetchSuccesses + fetchFailures)
long fetchSuccesses; // anything disposed-success
// (HTTP 2XX response codes, other non-errors)
long fetchFailures; // anything disposed-failure
long fetchDisregards; // anything disposed-disregard
long fetchResponses; // all positive responses (incl. 3XX, 4XX, 5XX)
long robotsDenials; // all robots-precluded failures
long successBytes; // total size of all success responses
long totalBytes; // total size of all responses
long fetchNonResponses; // processing attempts resulting in no response
// (both failures and temp deferrals)
long novelBytes;
/**
* anyting initially scheduled, (totalScheduled - (fetchSuccesses
* + fetchFailures)
*/
long totalScheduled;
/**
* anything disposed-success, any non-error HTTP status coed
*/
long fetchSuccesses;
/**
* anything with an HTTP status code between 200 and 299
*/
long fetchHttpSuccesses;
/**
* anything disposed-failure
*/
long fetchFailures;
/**
* anything disposed-disregard
*/
long fetchDisregards;
/**
* all positive responses (incl. 3XX, 4XX, 5XX)
*/
long fetchResponses;
/**
* all robots-precluded failures
*/
long robotsDenials;
/**
* total size of all success responses
*/
long successBytes;
/**
* total size of all responses
*/
long totalBytes;
/**
* processing attempts resulting in no response (both failures and
* temp deferrals)
*/
long fetchNonResponses;
long novelBytes;
long novelUrls;
long notModifiedBytes;
long notModifiedUrls;
long dupByHashBytes;
long dupByHashUrls;
long lastSuccessTime;
long dupByHashUrls;
long lastSuccessTime;
public synchronized void tally(CrawlURI curi, Stage stage) {
switch(stage) {
case SCHEDULED:
@@ -86,7 +134,7 @@ public class FetchStats implements Serializable, FetchStatusCodes, MultiReporter
fetchResponses++;
totalBytes += curi.getContentSize();
successBytes += curi.getContentSize();
if (curi.getFetchStatus() == HttpStatus.SC_NOT_MODIFIED) {
notModifiedBytes += curi.getContentSize();
notModifiedUrls++;
@@ -96,9 +144,23 @@ public class FetchStats implements Serializable, FetchStatusCodes, MultiReporter
} else {
novelBytes += curi.getContentSize();
novelUrls++;
}
}
lastSuccessTime = curi.getFetchCompletedTime();
HttpMethod httpMethod = curi.getHttpMethod();
if(httpMethod != null) {
int statusCode = httpMethod.getStatusCode();
if(statusCode >= HTTP_STATUS_CODE_SUCCESS_LOWER &&
statusCode <= HTTP_STATUS_CODE_SUCCESS_UPPER) {
fetchHttpSuccesses++;
}
}
break;
case DISREGARDED:
fetchDisregards++;
@@ -112,8 +174,8 @@ public class FetchStats implements Serializable, FetchStatusCodes, MultiReporter
} else {
fetchResponses++;
totalBytes += curi.getContentSize();
if (curi.getFetchStatus() == HttpStatus.SC_NOT_MODIFIED) {
if (curi.getFetchStatus() == HttpStatus.SC_NOT_MODIFIED) {
notModifiedBytes += curi.getContentSize();
notModifiedUrls++;
} else if (IdenticalDigestDecideRule.
@@ -123,17 +185,20 @@ public class FetchStats implements Serializable, FetchStatusCodes, MultiReporter
} else {
novelBytes += curi.getContentSize();
novelUrls++;
}
}
}
fetchFailures++;
break;
}
}
public long getFetchSuccesses() {
return fetchSuccesses;
}
public long getFetchHttpSuccesses() {
return fetchHttpSuccesses;
}
public long getFetchResponses() {
return fetchResponses;
}
@@ -155,7 +220,7 @@ public class FetchStats implements Serializable, FetchStatusCodes, MultiReporter
public long getRobotsDenials() {
return robotsDenials;
}
public long getRemaining() {
return totalScheduled - (fetchSuccesses + fetchFailures + fetchDisregards);
}
@@ -163,7 +228,7 @@ public class FetchStats implements Serializable, FetchStatusCodes, MultiReporter
return fetchSuccesses + fetchFailures;
}
public long getNovelBytes() {
public long getNovelBytes() {
return novelBytes;
}
@@ -185,8 +250,8 @@ public class FetchStats implements Serializable, FetchStatusCodes, MultiReporter
public long getDupByHashUrls() {
return dupByHashUrls;
}
}
public String[] getReports() {
// TODO Auto-generated method stub
return null;
@@ -209,9 +274,10 @@ public class FetchStats implements Serializable, FetchStatusCodes, MultiReporter
}
public String shortReportLegend() {
return "totalScheduled fetchSuccesses fetchFailures fetchDisregards " +
"fetchResponses robotsDenials successBytes totalBytes " +
"fetchNonResponses lastSuccessTime";
return "totalScheduled fetchSuccesses fetchHttpSuccesses " +
"fetchFailures fetchDisregards fetchResponses " +
"robotsDenials successBytes totalBytes fetchNonResponses " +
"lastSuccessTime";
}
public String shortReportLine() {
@@ -223,20 +289,22 @@ public class FetchStats implements Serializable, FetchStatusCodes, MultiReporter
writer.print(" ");
writer.print(fetchSuccesses);
writer.print(" ");
writer.print(fetchHttpSuccesses);
writer.print(" ");
writer.print(fetchFailures);
writer.print(" ");
writer.print(" ");
writer.print(fetchDisregards);
writer.print(" ");
writer.print(" ");
writer.print(fetchResponses);
writer.print(" ");
writer.print(" ");
writer.print(robotsDenials);
writer.print(" ");
writer.print(" ");
writer.print(successBytes);
writer.print(" ");
writer.print(" ");
writer.print(totalBytes);
writer.print(" ");
writer.print(" ");
writer.print(fetchNonResponses);
writer.print(" ");
writer.print(" ");
writer.print(ArchiveUtils.getLog17Date(lastSuccessTime));
}
@@ -244,6 +312,7 @@ public class FetchStats implements Serializable, FetchStatusCodes, MultiReporter
Map<String,Object> map = new LinkedHashMap<String, Object>();
map.put("totalScheduled", totalScheduled);
map.put("fetchSuccesses", fetchSuccesses);
map.put("fetchHttpSuccesses", fetchHttpSuccesses);
map.put("fetchFailures", fetchFailures);
map.put("fetchDisregards", fetchDisregards);
map.put("fetchResponses", fetchResponses);