[HER-1796] H3: interval rescheduling becomes erratic after URI rescheduled more than maxRetries times

* CrawlURI.java
    improve comments for fetchAttempts methods
    (resetForRescheduling) added to clear per-scheduling state
* WorkQueueFrontier.java
    call resetForRescheduling just before scheduling-at-specific-time
* FrontierJournal.java, AbstractFrontier.java
    renamings: prefer 'reenqueue' instead of 'reschedule' for describing simple (non-timed) retries
This commit is contained in:
gojomo committed 2010-07-09 01:45:26 +00:00
1 parent 274e522f85
commit cfc4b347a6
4 files changed
+35 -18

No files matched your search

@@ -695,10 +695,10 @@ public abstract class AbstractFrontier
}
}
protected void doJournalRescheduled(CrawlURI c) {
protected void doJournalReenqueued(CrawlURI c) {
tally(c,Stage.RETRIED);
if (this.recover != null) {
this.recover.rescheduled(c);
this.recover.reenqueued(c);
}
}
@@ -55,7 +55,7 @@ public class FrontierJournal extends CrawlerJournal {
public final static String F_EMIT = "Fe ";
public final static String F_INCLUDE = "Fi ";
public final static String F_DISREGARD = "Fd ";
public final static String F_RESCHEDULE = "Fr ";
public final static String F_REENQUEUED = "Fr ";
public final static String F_SUCCESS = "Fs ";
public final static String F_FAILURE = "Ff ";
@@ -117,8 +117,8 @@ public class FrontierJournal extends CrawlerJournal {
writeLine(F_DISREGARD, curi.toString());
}
public void rescheduled(CrawlURI curi) {
writeLine(F_RESCHEDULE, curi.toString());
public void reenqueued(CrawlURI curi) {
writeLine(F_REENQUEUED, curi.toString());
}
@@ -1019,7 +1019,7 @@ implements Closeable,
// Let everyone interested know that it will be retried.
appCtx.publishEvent(
new CrawlURIDispositionEvent(this,curi,DEFERRED_FOR_RETRY));
doJournalRescheduled(curi);
doJournalReenqueued(curi);
return;
}
@@ -1082,6 +1082,7 @@ implements Closeable,
if(curi.getRescheduleTime()>0) {
// marked up for forced-revisit at a set time
curi.processingCleanup();
curi.resetForRescheduling();
futureUris.put(curi.getRescheduleTime(),curi);
futureUriCount.incrementAndGet();
} else {