From 3cfc315de9f9f34015560f1c2d5b0ed655ebe87b Mon Sep 17 00:00:00 2001 From: nlevitt Date: Tue, 2 Aug 2011 01:07:11 +0000 Subject: [PATCH] Fix bug reported by Travis where each url in action directory ".include" file is reported as "not canonicalized" in heritrix_out, and some end up crawled in spite of being considered included * WorkQueueFrontier.java considerIncluded(CrawlURI) - call FrontierPreparer.prepare(CrawlURI) so that CrawlURI is canonicalized etc before being considered included (similar to what schedule() does) * CrawlURI.java getCanonicalString(), getPolitenessDelay() - use logger instead of System.err --- .../org/archive/crawler/frontier/WorkQueueFrontier.java | 6 +++++- modules/src/main/java/org/archive/modules/CrawlURI.java | 4 ++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/engine/src/main/java/org/archive/crawler/frontier/WorkQueueFrontier.java b/engine/src/main/java/org/archive/crawler/frontier/WorkQueueFrontier.java index b2a6c8cd..56aa8067 100644 --- a/engine/src/main/java/org/archive/crawler/frontier/WorkQueueFrontier.java +++ b/engine/src/main/java/org/archive/crawler/frontier/WorkQueueFrontier.java @@ -1504,8 +1504,12 @@ implements Closeable, } public void considerIncluded(CrawlURI curi) { - this.uriUniqFilter.note(curi.getCanonicalString()); sheetOverlaysManager.applyOverlaysTo(curi); + if(curi.getClassKey()==null) { + // remedial processing + preparer.prepare(curi); + } + this.uriUniqFilter.note(curi.getCanonicalString()); try { KeyedProperties.loadOverridesFrom(curi); curi.setClassKey(getClassKey(curi)); diff --git a/modules/src/main/java/org/archive/modules/CrawlURI.java b/modules/src/main/java/org/archive/modules/CrawlURI.java index 41a1d878..98d9711e 100644 --- a/modules/src/main/java/org/archive/modules/CrawlURI.java +++ b/modules/src/main/java/org/archive/modules/CrawlURI.java @@ -1772,7 +1772,7 @@ implements MultiReporter, Serializable, OverlayContext { } public String getCanonicalString() { if(StringUtils.isEmpty(canonicalString)){ - System.err.println("not canonicalized "+this); + logger.warning("canonicalString unset, returning uncanonicalized " + getURI()); return getURI(); } return canonicalString; @@ -1784,7 +1784,7 @@ implements MultiReporter, Serializable, OverlayContext { } public long getPolitenessDelay() { if(politenessDelay<0) { - System.err.println("not politeness-calc "+this); + logger.warning("politessDelay unset, returning default 5000 for " + this); return 5000; } return this.politenessDelay;