diff --git a/contrib/src/main/java/org/archive/crawler/prefetch/HostQuotaEnforcer.java b/contrib/src/main/java/org/archive/crawler/prefetch/HostQuotaEnforcer.java index 355e2707..09c2f960 100644 --- a/contrib/src/main/java/org/archive/crawler/prefetch/HostQuotaEnforcer.java +++ b/contrib/src/main/java/org/archive/crawler/prefetch/HostQuotaEnforcer.java @@ -31,6 +31,8 @@ import org.archive.modules.net.CrawlHost; import org.archive.modules.net.ServerCache; import org.springframework.beans.factory.annotation.Autowired; +import com.google.common.net.InternetDomainName; + /** * Enforces quotas on a host. Should be configured early in {@link FetchChain} in * crawler-beans.cxml. Supports quotas on any of the fields tracked in @@ -59,6 +61,18 @@ public class HostQuotaEnforcer extends Processor { this.host = host; } + protected boolean applyToSubdomains = false; + public boolean getApplyToSubdomains() { + return applyToSubdomains; + } + /** + * Whether to apply the quotas to each subdomain of {@link #host} + * (separately, not cumulatively). + */ + public void setApplyToSubdomains(boolean applyToSubdomains) { + this.applyToSubdomains = applyToSubdomains; + } + protected Map quotas = new HashMap(); public Map getQuotas() { return quotas; @@ -72,7 +86,25 @@ public class HostQuotaEnforcer extends Processor { @Override protected boolean shouldProcess(CrawlURI curi) { - return serverCache.getHostFor(curi.getUURI()) == serverCache.getHostFor(host); + if (getApplyToSubdomains()) { + InternetDomainName h = InternetDomainName.from(host); + InternetDomainName uriHostOrAncestor = InternetDomainName.from(serverCache.getHostFor(curi.getUURI()).getHostName()); + while (true) { + if (uriHostOrAncestor.equals(h)) { + return true; + } + if (uriHostOrAncestor.hasParent()) { + uriHostOrAncestor = uriHostOrAncestor.parent(); + } else { + break; + } + } + + return false; + } else { + return serverCache.getHostFor(curi.getUURI()) == serverCache.getHostFor(host); + } + } @Override