diff --git a/engine/src/main/java/org/archive/crawler/prefetch/QuotaEnforcer.java b/engine/src/main/java/org/archive/crawler/prefetch/QuotaEnforcer.java index e403c5d1..4cd8418b 100644 --- a/engine/src/main/java/org/archive/crawler/prefetch/QuotaEnforcer.java +++ b/engine/src/main/java/org/archive/crawler/prefetch/QuotaEnforcer.java @@ -1,8 +1,8 @@ /* * This file is part of the Heritrix web crawler (crawler.archive.org). * - * Licensed to the Internet Archive (IA) by one or more individual - * contributors. + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. * * The IA licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with @@ -35,9 +35,9 @@ import org.springframework.beans.factory.annotation.Autowired; /** * A simple quota enforcer. If the host, server, or frontier group - * associated with the current CrawlURI is already over its quotas, + * associated with the current CrawlURI is already over its quotas, * blocks the current URI's processing with S_BLOCKED_BY_QUOTA. - * + * * @author gojomo * @version $Date$, $Revision$ */ @@ -46,54 +46,54 @@ public class QuotaEnforcer extends Processor { private static final Logger LOGGER = Logger.getLogger(QuotaEnforcer.class.getName()); - + // indexed table of reused string categorical names/keys protected static final int SERVER = 0; protected static final int HOST = 1; protected static final int GROUP = 2; - - protected static final int SUCCESSES = 0; - protected static final int SUCCESS_KB = 1; - protected static final int RESPONSES = 2; - protected static final int RESPONSE_KB = 3; - + private static final String SERVER_MAX_FETCH_SUCCESSES = "serverMaxFetchSuccesses"; private static final String SERVER_MAX_SUCCESS_KB = "serverMaxSuccessKb"; private static final String SERVER_MAX_FETCH_RESPONSES = "serverMaxFetchResponses"; private static final String SERVER_MAX_ALL_KB = "serverMaxAllKb"; + private static final String SERVER_MAX_FETCH_HTTP_SUCCESSES = "serverMaxFetchHttpSuccesses"; private static final String HOST_MAX_FETCH_SUCCESSES = "hostMaxFetchSuccesses"; private static final String HOST_MAX_SUCCESS_KB = "hostMaxSuccessKb"; private static final String HOST_MAX_FETCH_RESPONSES = "hostMaxFetchResponses"; private static final String HOST_MAX_ALL_KB = "hostMaxAllKb"; + private static final String HOST_MAX_FETCH_HTTP_SUCCESSES = "hostMaxFetchHttpSuccesses"; private static final String GROUP_MAX_FETCH_SUCCESSES = "groupMaxFetchSuccesses"; private static final String GROUP_MAX_SUCCESS_KB = "groupMaxSuccessKb"; private static final String GROUP_MAX_FETCH_RESPONSES = "groupMaxFetchResponses"; private static final String GROUP_MAX_ALL_KB = "groupMaxAllKb"; - + private static final String GROUP_MAX_FETCH_HTTP_SUCCESSES = "groupMaxFetchHttpSuccesses"; + protected static final String[][] keys = new String[][] { { //"server", SERVER_MAX_FETCH_SUCCESSES, SERVER_MAX_SUCCESS_KB, SERVER_MAX_FETCH_RESPONSES, - SERVER_MAX_ALL_KB + SERVER_MAX_ALL_KB, + SERVER_MAX_FETCH_HTTP_SUCCESSES }, { //"host" HOST_MAX_FETCH_SUCCESSES, HOST_MAX_SUCCESS_KB, HOST_MAX_FETCH_RESPONSES, - HOST_MAX_ALL_KB - , + HOST_MAX_ALL_KB, + HOST_MAX_FETCH_HTTP_SUCCESSES }, { //"group" GROUP_MAX_FETCH_SUCCESSES, GROUP_MAX_SUCCESS_KB, GROUP_MAX_FETCH_RESPONSES, - GROUP_MAX_ALL_KB + GROUP_MAX_ALL_KB, + GROUP_MAX_FETCH_HTTP_SUCCESSES } }; @@ -114,6 +114,31 @@ public class QuotaEnforcer extends Processor { kp.put(SERVER_MAX_FETCH_SUCCESSES,max); } + { + setServerMaxFetchHttpSuccesses(-1L); // no limit + } + + /** + * Maximum number of HTTP fetch successes (URI with an HTTP status + * code between 200 and 299) to collect from one server. Default + * is -1, meaning no limit. + * + * @param long Maximum number of fetch successes + */ + public void setServerMaxFetchHttpSuccesses(long max) { + + kp.put(SERVER_MAX_FETCH_HTTP_SUCCESSES, max); + } + + /** + * Returns the maximum number of fetch successes (URI with an HTTP + * status code between 200 and 299) to collect from one + * server. The default value is -1, indicating no limit. + */ + public long getServerMaxFetchHttpSuccesses() { + + return((Long) kp.get(SERVER_MAX_FETCH_HTTP_SUCCESSES)); + } /** * Maximum amount of fetch success content (e.g. 200 responses) in KB to @@ -171,6 +196,32 @@ public class QuotaEnforcer extends Processor { kp.put(HOST_MAX_FETCH_SUCCESSES,max); } + { + setHostMaxFetchHttpSuccesses(-1L); // no limit + } + + /** + * Maximum number of HTTP fetch successes (URI with an HTTP status + * code between 200 and 299) to collect from one server. Default + * is -1, meaning no limit. + * + * @param long Maximum number of fetch successes + */ + public void setHostMaxFetchHttpSuccesses(long max) { + + kp.put(HOST_MAX_FETCH_HTTP_SUCCESSES, max); + } + + /** + * Returns the maximum number of fetch successes (URI with an HTTP + * status code between 200 and 299) to collect from one + * server. The default value is -1, indicating no limit. + */ + public long getHostMaxFetchHttpSuccesses() { + + return((Long) kp.get(HOST_MAX_FETCH_HTTP_SUCCESSES)); + } + /** * Maximum amount of fetch success content (e.g. 200 responses) in KB to * collect from one host. Default is -1, meaning no limit. @@ -227,6 +278,32 @@ public class QuotaEnforcer extends Processor { kp.put(GROUP_MAX_FETCH_SUCCESSES,max); } + { + setGroupMaxFetchHttpSuccesses(-1L); // no limit + } + + /** + * Maximum number of HTTP fetch successes (URI with an HTTP status + * code between 200 and 299) to collect from one server. Default + * is -1, meaning no limit. + * + * @param long Maximum number of fetch successes + */ + public void setGroupMaxFetchHttpSuccesses(long max) { + + kp.put(GROUP_MAX_FETCH_HTTP_SUCCESSES, max); + } + + /** + * Returns the maximum number of fetch successes (URI with an HTTP + * status code between 200 and 299) to collect from one + * server. The default value is -1, indicating no limit. + */ + public long getGroupMaxFetchHttpSuccesses() { + + return((Long) kp.get(GROUP_MAX_FETCH_HTTP_SUCCESSES)); + } + /** * Maximum amount of fetch success content (e.g. 200 responses) in KB to * collect from one group. Default is -1, meaning no limit. @@ -285,7 +362,7 @@ public class QuotaEnforcer extends Processor { public void setForceRetire(boolean force) { kp.put("forceRetire",force); } - + protected ServerCache serverCache; public ServerCache getServerCache() { return this.serverCache; @@ -294,7 +371,7 @@ public class QuotaEnforcer extends Processor { public void setServerCache(ServerCache serverCache) { this.serverCache = serverCache; } - + protected Frontier frontier; public Frontier getFrontier() { return this.frontier; @@ -303,7 +380,7 @@ public class QuotaEnforcer extends Processor { public void setFrontier(Frontier frontier) { this.frontier = frontier; } - + protected boolean shouldProcess(CrawlURI puri) { return puri instanceof CrawlURI; } @@ -311,31 +388,31 @@ public class QuotaEnforcer extends Processor { protected void innerProcess(CrawlURI puri) { throw new AssertionError(); } - - protected ProcessResult innerProcessResult(CrawlURI puri) { - CrawlURI curi = (CrawlURI)puri; + + protected ProcessResult innerProcessResult(CrawlURI curi) { + final CrawlServer server = serverCache.getServerFor(curi.getUURI()); final CrawlHost host = serverCache.getHostFor(curi.getUURI()); - FetchStats.HasFetchStats[] haveStats = + FetchStats.HasFetchStats[] haveStats = new FetchStats.HasFetchStats[] { - server, - host, + server, + host, frontier.getGroup(curi) }; - + for(int cat=SERVER;cat<=GROUP;cat++) { if (checkQuotas(curi,haveStats[cat],cat)) { return ProcessResult.FINISH; } } - + return ProcessResult.PROCEED; } /** * Check all quotas for the given substats and category (server, host, or - * group). - * + * group). + * * @param curi CrawlURI to mark up with results * @param hasStats holds CrawlSubstats with actual values to test * @param CAT category index (SERVER, HOST, GROUP) to quota settings keys @@ -344,41 +421,47 @@ public class QuotaEnforcer extends Processor { protected boolean checkQuotas(final CrawlURI curi, final FetchStats.HasFetchStats hasStats, final int CAT) { + if (hasStats == null) { if (LOGGER.isLoggable(Level.FINE)) { LOGGER.fine(curi.toString() + " null stats category: " + CAT); } return false; } + FetchStats substats = hasStats.getSubstats(); + long[] actuals = new long[] { substats.getFetchSuccesses(), substats.getSuccessBytes()/1024, substats.getFetchResponses(), substats.getTotalBytes()/1024, + substats.getFetchHttpSuccesses() }; - for(int q=SUCCESSES; q<=RESPONSE_KB; q++) { + + for(int q = 0; q < keys[CAT].length; q++) { String key = keys[CAT][q]; if (applyQuota(curi, key, actuals[q])) { - return true; + return true; } } - return false; + return false; } /** - * Apply the quota specified by the given key against the actual - * value provided. If the quota and actual values rule out processing the - * given CrawlURI, mark up the CrawlURI appropriately. - * + * Apply the quota specified by the given key against the actual + * value provided. If the quota and actual values rule out processing the + * given CrawlURI, mark up the CrawlURI appropriately. + * * @param curi CrawlURI whose processing is subject to a potential quota * limitation * @param quotaKey settings key to get applicable quota - * @param actual current value to compare to quota + * @param actual current value to compare to quota * @return true is CrawlURI is blocked by a quota, false otherwise */ protected boolean applyQuota(CrawlURI curi, String key, long actual) { long quota = (Long)kp.get(key); + if (quota >= 0 && actual >= quota) { curi.getAnnotations().add("Q:"+key); if (getForceRetire()) { @@ -390,6 +473,6 @@ public class QuotaEnforcer extends Processor { } return true; } - return false; + return false; } } diff --git a/modules/src/main/java/org/archive/modules/fetcher/FetchStats.java b/modules/src/main/java/org/archive/modules/fetcher/FetchStats.java index 861caf25..aa89aca6 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/FetchStats.java +++ b/modules/src/main/java/org/archive/modules/fetcher/FetchStats.java @@ -1,8 +1,8 @@ /* * This file is part of the Heritrix web crawler (crawler.archive.org). * - * Licensed to the Internet Archive (IA) by one or more individual - * contributors. + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. * * The IA licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with @@ -26,51 +26,99 @@ import java.util.Map; import org.archive.modules.CrawlURI; import org.archive.util.ArchiveUtils; import org.archive.util.MultiReporter; -import org.apache.commons.httpclient.HttpStatus; +import org.apache.commons.httpclient.HttpStatus; import org.archive.modules.deciderules.recrawl.IdenticalDigestDecideRule; +import org.apache.commons.httpclient.HttpMethod; /** * Collector of statistics for a 'subset' of a crawl, - * such as a server (host:port), host, or frontier group - * (eg queue). - * + * such as a server (host:port), host, or frontier group + * (eg queue). + * * @author gojomo */ public class FetchStats implements Serializable, FetchStatusCodes, MultiReporter { + + /** lower bound on HTTP status codes that we'll interpret as successes */ + private static final int HTTP_STATUS_CODE_SUCCESS_LOWER = 200; + + /** upper bound on HTTP status codes that we'll interpret as successes */ + private static final int HTTP_STATUS_CODE_SUCCESS_UPPER = 299; + private static final long serialVersionUID = 8624425657056569036L; - public enum Stage {SCHEDULED, RELOCATED, RETRIED, + public enum Stage {SCHEDULED, RELOCATED, RETRIED, SUCCEEDED, DISREGARDED, FAILED}; - + public interface HasFetchStats { public FetchStats getSubstats(); } + public interface CollectsFetchStats { - public void tally(CrawlURI curi, Stage stage); + public void tally(CrawlURI curi, Stage stage); } - - long totalScheduled; // anything initially scheduled - // (totalScheduled - (fetchSuccesses + fetchFailures) - long fetchSuccesses; // anything disposed-success - // (HTTP 2XX response codes, other non-errors) - long fetchFailures; // anything disposed-failure - long fetchDisregards; // anything disposed-disregard - long fetchResponses; // all positive responses (incl. 3XX, 4XX, 5XX) - long robotsDenials; // all robots-precluded failures - long successBytes; // total size of all success responses - long totalBytes; // total size of all responses - long fetchNonResponses; // processing attempts resulting in no response - // (both failures and temp deferrals) - - long novelBytes; + + /** + * anyting initially scheduled, (totalScheduled - (fetchSuccesses + * + fetchFailures) + */ + long totalScheduled; + + /** + * anything disposed-success, any non-error HTTP status coed + */ + long fetchSuccesses; + + /** + * anything with an HTTP status code between 200 and 299 + */ + long fetchHttpSuccesses; + + /** + * anything disposed-failure + */ + long fetchFailures; + + /** + * anything disposed-disregard + */ + long fetchDisregards; + + /** + * all positive responses (incl. 3XX, 4XX, 5XX) + */ + long fetchResponses; + + /** + * all robots-precluded failures + */ + long robotsDenials; + + /** + * total size of all success responses + */ + long successBytes; + + /** + * total size of all responses + */ + long totalBytes; + + /** + * processing attempts resulting in no response (both failures and + * temp deferrals) + */ + long fetchNonResponses; + + long novelBytes; long novelUrls; long notModifiedBytes; long notModifiedUrls; long dupByHashBytes; - long dupByHashUrls; - - long lastSuccessTime; - + long dupByHashUrls; + + long lastSuccessTime; + public synchronized void tally(CrawlURI curi, Stage stage) { switch(stage) { case SCHEDULED: @@ -86,7 +134,7 @@ public class FetchStats implements Serializable, FetchStatusCodes, MultiReporter fetchResponses++; totalBytes += curi.getContentSize(); successBytes += curi.getContentSize(); - + if (curi.getFetchStatus() == HttpStatus.SC_NOT_MODIFIED) { notModifiedBytes += curi.getContentSize(); notModifiedUrls++; @@ -96,9 +144,23 @@ public class FetchStats implements Serializable, FetchStatusCodes, MultiReporter } else { novelBytes += curi.getContentSize(); novelUrls++; - } - + } + lastSuccessTime = curi.getFetchCompletedTime(); + + HttpMethod httpMethod = curi.getHttpMethod(); + + if(httpMethod != null) { + + int statusCode = httpMethod.getStatusCode(); + + if(statusCode >= HTTP_STATUS_CODE_SUCCESS_LOWER && + statusCode <= HTTP_STATUS_CODE_SUCCESS_UPPER) { + + fetchHttpSuccesses++; + } + } + break; case DISREGARDED: fetchDisregards++; @@ -112,8 +174,8 @@ public class FetchStats implements Serializable, FetchStatusCodes, MultiReporter } else { fetchResponses++; totalBytes += curi.getContentSize(); - - if (curi.getFetchStatus() == HttpStatus.SC_NOT_MODIFIED) { + + if (curi.getFetchStatus() == HttpStatus.SC_NOT_MODIFIED) { notModifiedBytes += curi.getContentSize(); notModifiedUrls++; } else if (IdenticalDigestDecideRule. @@ -123,17 +185,20 @@ public class FetchStats implements Serializable, FetchStatusCodes, MultiReporter } else { novelBytes += curi.getContentSize(); novelUrls++; - } + } } fetchFailures++; break; } } - + public long getFetchSuccesses() { return fetchSuccesses; } + public long getFetchHttpSuccesses() { + return fetchHttpSuccesses; + } public long getFetchResponses() { return fetchResponses; } @@ -155,7 +220,7 @@ public class FetchStats implements Serializable, FetchStatusCodes, MultiReporter public long getRobotsDenials() { return robotsDenials; } - + public long getRemaining() { return totalScheduled - (fetchSuccesses + fetchFailures + fetchDisregards); } @@ -163,7 +228,7 @@ public class FetchStats implements Serializable, FetchStatusCodes, MultiReporter return fetchSuccesses + fetchFailures; } - public long getNovelBytes() { + public long getNovelBytes() { return novelBytes; } @@ -185,8 +250,8 @@ public class FetchStats implements Serializable, FetchStatusCodes, MultiReporter public long getDupByHashUrls() { return dupByHashUrls; - } - + } + public String[] getReports() { // TODO Auto-generated method stub return null; @@ -209,9 +274,10 @@ public class FetchStats implements Serializable, FetchStatusCodes, MultiReporter } public String shortReportLegend() { - return "totalScheduled fetchSuccesses fetchFailures fetchDisregards " + - "fetchResponses robotsDenials successBytes totalBytes " + - "fetchNonResponses lastSuccessTime"; + return "totalScheduled fetchSuccesses fetchHttpSuccesses " + + "fetchFailures fetchDisregards fetchResponses " + + "robotsDenials successBytes totalBytes fetchNonResponses " + + "lastSuccessTime"; } public String shortReportLine() { @@ -223,20 +289,22 @@ public class FetchStats implements Serializable, FetchStatusCodes, MultiReporter writer.print(" "); writer.print(fetchSuccesses); writer.print(" "); + writer.print(fetchHttpSuccesses); + writer.print(" "); writer.print(fetchFailures); - writer.print(" "); + writer.print(" "); writer.print(fetchDisregards); - writer.print(" "); + writer.print(" "); writer.print(fetchResponses); - writer.print(" "); + writer.print(" "); writer.print(robotsDenials); - writer.print(" "); + writer.print(" "); writer.print(successBytes); - writer.print(" "); + writer.print(" "); writer.print(totalBytes); - writer.print(" "); + writer.print(" "); writer.print(fetchNonResponses); - writer.print(" "); + writer.print(" "); writer.print(ArchiveUtils.getLog17Date(lastSuccessTime)); } @@ -244,6 +312,7 @@ public class FetchStats implements Serializable, FetchStatusCodes, MultiReporter Map map = new LinkedHashMap(); map.put("totalScheduled", totalScheduled); map.put("fetchSuccesses", fetchSuccesses); + map.put("fetchHttpSuccesses", fetchHttpSuccesses); map.put("fetchFailures", fetchFailures); map.put("fetchDisregards", fetchDisregards); map.put("fetchResponses", fetchResponses);