lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-07-21 06:51:38 +00:00
Code Issues Packages Projects Releases Wiki Activity

change class originally known as SeedLimitsEnforcer to

Browse Source 
SourceQuotaEnforcer; make it a Processor instead of a DecideRule
(because checking quota at link scoping time doesn't work, since many
urls which would go over quota can be added to the frontier); support
quotas on any of the fields tracked by CrawledBytesHistotable
This commit is contained in:
Noah Levitt
2015-12-08 13:45:25 -08:00
parent 7929e16d6a
commit ce31cada5f
2 changed files with 120 additions and 101 deletions
Download Patch File Download Diff File Expand all files Collapse all files
contrib/src/main/java/org/archive/crawler/prefetch/SourceQuotaEnforcer.java
+120
View File
@@ -0,0 +1,120 @@
/*
* This file is part of the Heritrix web crawler (crawler.archive.org).
*
* Licensed to the Internet Archive (IA) by one or more individual
* contributors.
*
* The IA licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.archive.crawler.prefetch;
import java.util.HashMap;
import java.util.Map;
import java.util.Map.Entry;
import org.archive.crawler.prefetch.QuotaEnforcer;
import org.archive.crawler.prefetch.RuntimeLimitEnforcer;
import org.archive.crawler.reporting.StatisticsTracker;
import org.archive.crawler.util.CrawledBytesHistotable;
import org.archive.modules.CoreAttributeConstants;
import org.archive.modules.CrawlURI;
import org.archive.modules.FetchChain;
import org.archive.modules.ProcessResult;
import org.archive.modules.Processor;
import org.archive.modules.fetcher.FetchStatusCodes;
import org.archive.modules.seeds.SeedModule;
import org.springframework.beans.factory.annotation.Autowired;
/**
* Processor for enforcing quotas by source tag (normally the seed url if
* enabled). Should be configured early in {@link FetchChain} in
* crawler-beans.cxml. Supports quotas on any of the fields tracked in
* {@link CrawledBytesHistotable}. Only takes effect if
* {@link SeedModule#getSourceTagSeeds()} (disabled by default) and
* {@link StatisticsTracker#getTrackSources()} (enabled by default) are both
* enabled.
*
* @contributor nlevitt
*
* @see {@link QuotaEnforcer}
* @see {@link RuntimeLimitEnforcer}
*/
public class SourceQuotaEnforcer extends Processor {
protected String sourceTag;
public void setSourceTag(String sourceTag) {
this.sourceTag = sourceTag;
}
public String getSourceTag() {
return sourceTag;
}
protected Map<String, Long> quotas = new HashMap<String, Long>();
public Map<String, Long> getQuotas() {
return quotas;
}
/**
* Keys can be any of the {@link CrawledBytesHistotable} keys:
* <ul>
* <li>{@value CrawledBytesHistotable#NOVEL}
* <li>{@value CrawledBytesHistotable#NOVELCOUNT}
* <li>{@value CrawledBytesHistotable#DUPLICATE}
* <li>{@value CrawledBytesHistotable#DUPLICATECOUNT}
* <li>{@value CrawledBytesHistotable#NOTMODIFIED}
* <li>{@value CrawledBytesHistotable#NOTMODIFIEDCOUNT}
* <li>{@value CrawledBytesHistotable#OTHERDUPLICATE}
* <li>{@value CrawledBytesHistotable#OTHERDUPLICATECOUNT}
* </ul>
*/
public void setQuotas(Map<String, Long> quotas) {
this.quotas = quotas;
}
protected StatisticsTracker statisticsTracker;
public StatisticsTracker getStatisticsTracker() {
return this.statisticsTracker;
}
@Autowired
public void setStatisticsTracker(StatisticsTracker statisticsTracker) {
this.statisticsTracker = statisticsTracker;
}
@Override
protected boolean shouldProcess(CrawlURI curi) {
return curi.containsDataKey(CoreAttributeConstants.A_SOURCE_TAG)
&& statisticsTracker.getSourceStats(curi.getSourceTag()) != null;
}
protected void innerProcess(CrawlURI curi) {
throw new AssertionError();
}
protected ProcessResult innerProcessResult(CrawlURI curi) {
if (!shouldProcess(curi)) {
return ProcessResult.PROCEED;
}
CrawledBytesHistotable stats = statisticsTracker.getSourceStats(curi.getSourceTag());
for (Entry<String, Long> quota: quotas.entrySet()) {
if (stats.get(quota.getKey()) >= quota.getValue()) {
curi.getAnnotations().add("sourceQuota:" + quota.getKey());
curi.setFetchStatus(FetchStatusCodes.S_BLOCKED_BY_QUOTA);
return ProcessResult.FINISH;
}
}
return ProcessResult.PROCEED;
}
}
contrib/src/main/java/org/archive/modules/deciderules/SeedLimitsEnforcer.java
-101
View File
@@ -1,101 +0,0 @@
/*
* This file is part of the Heritrix web crawler (crawler.archive.org).
*
* Licensed to the Internet Archive (IA) by one or more individual
* contributors.
*
* The IA licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.archive.modules.deciderules;
import org.archive.crawler.prefetch.QuotaEnforcer;
import org.archive.crawler.prefetch.RuntimeLimitEnforcer;
import org.archive.crawler.reporting.StatisticsTracker;
import org.archive.crawler.util.CrawledBytesHistotable;
import org.archive.modules.CoreAttributeConstants;
import org.archive.modules.CrawlURI;
import org.springframework.beans.factory.annotation.Autowired;
/**
* DecideRule that rejects any CrawlURI with {@code sourceTag} matching the
* configured {@code seed}, if more than {@code maxBytesDownload} bytes or
* {@code maxDocumentsDownload} urls have been downloaded from the same source
* seed.
*
* @contributor nlevitt
*
* @see {@link QuotaEnforcer}
* @see {@link RuntimeLimitEnforcer}
*/
public class SeedLimitsEnforcer extends DecideRule {
private static final long serialVersionUID = 1l;
private String seed;
public void setSeed(String seed) {
this.seed = seed;
}
public String getSeed() {
return seed;
}
protected Long maxBytesDownload;
public Long getMaxBytesDownload() {
return maxBytesDownload;
}
public void setMaxBytesDownload(Long maxBytesDownload) {
this.maxBytesDownload = maxBytesDownload;
}
protected Long maxDocumentsDownload;
public Long getMaxDocumentsDownload() {
return maxDocumentsDownload;
}
public void setMaxDocumentsDownload(Long maxDocumentsDownload) {
this.maxDocumentsDownload = maxDocumentsDownload;
}
protected StatisticsTracker statisticsTracker;
public StatisticsTracker getStatisticsTracker() {
return this.statisticsTracker;
}
@Autowired
public void setStatisticsTracker(StatisticsTracker statisticsTracker) {
this.statisticsTracker = statisticsTracker;
}
@Override
protected DecideResult innerDecide(CrawlURI curi) {
if (!curi.getData().containsKey(CoreAttributeConstants.A_SOURCE_TAG)) {
return DecideResult.NONE;
}
CrawledBytesHistotable stats = statisticsTracker.getSourceStats(curi.getSourceTag());
if (stats == null) {
return DecideResult.NONE;
}
if (maxBytesDownload != null
&& stats.get(CrawledBytesHistotable.NOVEL) >= maxBytesDownload) {
return DecideResult.REJECT;
}
if (maxDocumentsDownload != null
&& stats.get(CrawledBytesHistotable.NOVELCOUNT) >= maxDocumentsDownload) {
return DecideResult.REJECT;
}
return DecideResult.NONE;
}
}