mirror of
https://github.com/internetarchive/heritrix3.git
synced 2026-11-09 07:24:41 +00:00
Fix bug reported by Travis where each url in action directory ".include" file is reported as "not canonicalized" in heritrix_out, and some end up crawled in spite of being considered included
* WorkQueueFrontier.java
considerIncluded(CrawlURI) - call FrontierPreparer.prepare(CrawlURI) so that CrawlURI is canonicalized etc before being considered included (similar to what schedule() does)
* CrawlURI.java
getCanonicalString(), getPolitenessDelay() - use logger instead of System.err
This commit is contained in:
1 parent
dcbc259776
commit
3cfc315de9
2 files changed
+7
-3
No files matched your search
@@ -1504,8 +1504,12 @@ implements Closeable,
|
||||
}
|
||||
|
||||
public void considerIncluded(CrawlURI curi) {
|
||||
this.uriUniqFilter.note(curi.getCanonicalString());
|
||||
sheetOverlaysManager.applyOverlaysTo(curi);
|
||||
if(curi.getClassKey()==null) {
|
||||
// remedial processing
|
||||
preparer.prepare(curi);
|
||||
}
|
||||
this.uriUniqFilter.note(curi.getCanonicalString());
|
||||
try {
|
||||
KeyedProperties.loadOverridesFrom(curi);
|
||||
curi.setClassKey(getClassKey(curi));
|
||||
|
||||
Reference in new issue
Block a user