diff --git a/engine/src/main/java/org/archive/crawler/frontier/WorkQueueFrontier.java b/engine/src/main/java/org/archive/crawler/frontier/WorkQueueFrontier.java index b2a6c8cd..56aa8067 100644 --- a/engine/src/main/java/org/archive/crawler/frontier/WorkQueueFrontier.java +++ b/engine/src/main/java/org/archive/crawler/frontier/WorkQueueFrontier.java @@ -1504,8 +1504,12 @@ implements Closeable, } public void considerIncluded(CrawlURI curi) { - this.uriUniqFilter.note(curi.getCanonicalString()); sheetOverlaysManager.applyOverlaysTo(curi); + if(curi.getClassKey()==null) { + // remedial processing + preparer.prepare(curi); + } + this.uriUniqFilter.note(curi.getCanonicalString()); try { KeyedProperties.loadOverridesFrom(curi); curi.setClassKey(getClassKey(curi)); diff --git a/modules/src/main/java/org/archive/modules/CrawlURI.java b/modules/src/main/java/org/archive/modules/CrawlURI.java index 41a1d878..98d9711e 100644 --- a/modules/src/main/java/org/archive/modules/CrawlURI.java +++ b/modules/src/main/java/org/archive/modules/CrawlURI.java @@ -1772,7 +1772,7 @@ implements MultiReporter, Serializable, OverlayContext { } public String getCanonicalString() { if(StringUtils.isEmpty(canonicalString)){ - System.err.println("not canonicalized "+this); + logger.warning("canonicalString unset, returning uncanonicalized " + getURI()); return getURI(); } return canonicalString; @@ -1784,7 +1784,7 @@ implements MultiReporter, Serializable, OverlayContext { } public long getPolitenessDelay() { if(politenessDelay<0) { - System.err.println("not politeness-calc "+this); + logger.warning("politessDelay unset, returning default 5000 for " + this); return 5000; } return this.politenessDelay;