mirror of
https://github.com/internetarchive/heritrix3.git
synced 2026-11-09 07:24:41 +00:00
[HER-1796] H3: interval rescheduling becomes erratic after URI rescheduled more than maxRetries times
* CrawlURI.java
improve comments for fetchAttempts methods
(resetForRescheduling) added to clear per-scheduling state
* WorkQueueFrontier.java
call resetForRescheduling just before scheduling-at-specific-time
* FrontierJournal.java, AbstractFrontier.java
renamings: prefer 'reenqueue' instead of 'reschedule' for describing simple (non-timed) retries
This commit is contained in:
1 parent
274e522f85
commit
cfc4b347a6
4 files changed
+35
-18
No files matched your search
@@ -695,10 +695,10 @@ public abstract class AbstractFrontier
|
||||
}
|
||||
}
|
||||
|
||||
protected void doJournalRescheduled(CrawlURI c) {
|
||||
protected void doJournalReenqueued(CrawlURI c) {
|
||||
tally(c,Stage.RETRIED);
|
||||
if (this.recover != null) {
|
||||
this.recover.rescheduled(c);
|
||||
this.recover.reenqueued(c);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -55,7 +55,7 @@ public class FrontierJournal extends CrawlerJournal {
|
||||
public final static String F_EMIT = "Fe ";
|
||||
public final static String F_INCLUDE = "Fi ";
|
||||
public final static String F_DISREGARD = "Fd ";
|
||||
public final static String F_RESCHEDULE = "Fr ";
|
||||
public final static String F_REENQUEUED = "Fr ";
|
||||
public final static String F_SUCCESS = "Fs ";
|
||||
public final static String F_FAILURE = "Ff ";
|
||||
|
||||
@@ -117,8 +117,8 @@ public class FrontierJournal extends CrawlerJournal {
|
||||
writeLine(F_DISREGARD, curi.toString());
|
||||
}
|
||||
|
||||
public void rescheduled(CrawlURI curi) {
|
||||
writeLine(F_RESCHEDULE, curi.toString());
|
||||
public void reenqueued(CrawlURI curi) {
|
||||
writeLine(F_REENQUEUED, curi.toString());
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -1019,7 +1019,7 @@ implements Closeable,
|
||||
// Let everyone interested know that it will be retried.
|
||||
appCtx.publishEvent(
|
||||
new CrawlURIDispositionEvent(this,curi,DEFERRED_FOR_RETRY));
|
||||
doJournalRescheduled(curi);
|
||||
doJournalReenqueued(curi);
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -1082,6 +1082,7 @@ implements Closeable,
|
||||
if(curi.getRescheduleTime()>0) {
|
||||
// marked up for forced-revisit at a set time
|
||||
curi.processingCleanup();
|
||||
curi.resetForRescheduling();
|
||||
futureUris.put(curi.getRescheduleTime(),curi);
|
||||
futureUriCount.incrementAndGet();
|
||||
} else {
|
||||
|
||||
Reference in new issue
Block a user