lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-09-26 07:35:39 +00:00
Code Issues Packages Projects Releases Wiki Activity

* StatisticsTracker.java

Browse Source 
make expensive seeds-result tracking optional
This commit is contained in:
gojomo
2009-10-26 12:08:49 +00:00
parent 648ef7deef
commit 7cf25a844e
1 changed files with 24 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
engine/src/main/java/org/archive/crawler/reporting/StatisticsTracker.java
+24 -10
View File
@@ -237,6 +237,18 @@ public class StatisticsTracker
return new AtomicLong(0);
}};
/**
* Whether to maintain seed disposition records (expensive in
* crawls with millions of seeds)
*/
boolean trackSeeds = true;
public boolean getTrackSeeds() {
return this.trackSeeds;
}
public void setTrackSeeds(boolean trackSeeds) {
this.trackSeeds = trackSeeds;
}
/**
* The interval between writing progress information to log.
*/
@@ -733,16 +745,18 @@ public class StatisticsTracker
* @param curi The CrawlURI that may be a seed.
* @param disposition The disposition of the CrawlURI.
*/
private void handleSeed(final CrawlURI curi, final String disposition) {
if(curi.isSeed()){
SeedRecord sr = processedSeedsRecords.getOrUse(
curi.getURI(),
new Supplier<SeedRecord>() {
public SeedRecord get() {
return new SeedRecord(curi, disposition);
}});
sr.updateWith(curi,disposition);
}
protected void handleSeed(final CrawlURI curi, final String disposition) {
if(getTrackSeeds()) {
if(curi.isSeed()){
SeedRecord sr = processedSeedsRecords.getOrUse(
curi.getURI(),
new Supplier<SeedRecord>() {
public SeedRecord get() {
return new SeedRecord(curi, disposition);
}});
sr.updateWith(curi,disposition);
}
} // else ignore
}
public void crawledURISuccessful(CrawlURI curi) {