mirror of
https://github.com/internetarchive/heritrix3.git
synced 2026-09-26 23:55:50 +00:00
[HER-1714] fixed-interval rescheduling
* AbstractFrontier.java, WorkQueueFrontier.java, BdbFrontier.java
add 'futureUris' map of CrawlURIs to be reenqueued at specific future dates
the usual findEligibleUri also checks for newly-eligible rescheduled URIs
the frontier is no longer empty unless the future count is also 0
* CrawlURI.java
add rescheduleTime property
* ReschedulingProcessor.java
optional post-processor to set a CrawlURI's reschedulingTime
* CrawlStatSnapshot.java, CrawlJob.java
add futureUriCount to stats/uriTotalsReport
This commit is contained in:
@@ -1826,4 +1826,15 @@ implements MultiReporter, Serializable, OverlayContext {
|
||||
public CrawlURI getFullVia() {
|
||||
return fullVia;
|
||||
}
|
||||
|
||||
/**
|
||||
* A future time at which this CrawlURI should be reenqueued.
|
||||
*/
|
||||
protected long rescheduleTime = -1;
|
||||
public void setRescheduleTime(long time) {
|
||||
this.rescheduleTime = time;
|
||||
}
|
||||
public long getRescheduleTime() {
|
||||
return this.rescheduleTime;
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user