diff --git a/engine/src/main/java/org/archive/crawler/frontier/AbstractFrontier.java b/engine/src/main/java/org/archive/crawler/frontier/AbstractFrontier.java index e54ad283..b5f8a427 100644 --- a/engine/src/main/java/org/archive/crawler/frontier/AbstractFrontier.java +++ b/engine/src/main/java/org/archive/crawler/frontier/AbstractFrontier.java @@ -695,10 +695,10 @@ public abstract class AbstractFrontier } } - protected void doJournalRescheduled(CrawlURI c) { + protected void doJournalReenqueued(CrawlURI c) { tally(c,Stage.RETRIED); if (this.recover != null) { - this.recover.rescheduled(c); + this.recover.reenqueued(c); } } diff --git a/engine/src/main/java/org/archive/crawler/frontier/FrontierJournal.java b/engine/src/main/java/org/archive/crawler/frontier/FrontierJournal.java index 9e0e6432..54dc6ca6 100644 --- a/engine/src/main/java/org/archive/crawler/frontier/FrontierJournal.java +++ b/engine/src/main/java/org/archive/crawler/frontier/FrontierJournal.java @@ -55,7 +55,7 @@ public class FrontierJournal extends CrawlerJournal { public final static String F_EMIT = "Fe "; public final static String F_INCLUDE = "Fi "; public final static String F_DISREGARD = "Fd "; - public final static String F_RESCHEDULE = "Fr "; + public final static String F_REENQUEUED = "Fr "; public final static String F_SUCCESS = "Fs "; public final static String F_FAILURE = "Ff "; @@ -117,8 +117,8 @@ public class FrontierJournal extends CrawlerJournal { writeLine(F_DISREGARD, curi.toString()); } - public void rescheduled(CrawlURI curi) { - writeLine(F_RESCHEDULE, curi.toString()); + public void reenqueued(CrawlURI curi) { + writeLine(F_REENQUEUED, curi.toString()); } diff --git a/engine/src/main/java/org/archive/crawler/frontier/WorkQueueFrontier.java b/engine/src/main/java/org/archive/crawler/frontier/WorkQueueFrontier.java index 172215fd..971462a7 100644 --- a/engine/src/main/java/org/archive/crawler/frontier/WorkQueueFrontier.java +++ b/engine/src/main/java/org/archive/crawler/frontier/WorkQueueFrontier.java @@ -1019,7 +1019,7 @@ implements Closeable, // Let everyone interested know that it will be retried. appCtx.publishEvent( new CrawlURIDispositionEvent(this,curi,DEFERRED_FOR_RETRY)); - doJournalRescheduled(curi); + doJournalReenqueued(curi); return; } @@ -1082,6 +1082,7 @@ implements Closeable, if(curi.getRescheduleTime()>0) { // marked up for forced-revisit at a set time curi.processingCleanup(); + curi.resetForRescheduling(); futureUris.put(curi.getRescheduleTime(),curi); futureUriCount.incrementAndGet(); } else { diff --git a/modules/src/main/java/org/archive/modules/CrawlURI.java b/modules/src/main/java/org/archive/modules/CrawlURI.java index 14fd8e19..2dd153d9 100644 --- a/modules/src/main/java/org/archive/modules/CrawlURI.java +++ b/modules/src/main/java/org/archive/modules/CrawlURI.java @@ -493,26 +493,31 @@ implements MultiReporter, Serializable, OverlayContext { } /** - * Get the number of attempts at getting the document referenced by this - * URI. + * Get the count of attempts (trips through the processing + * loop) at getting the document referenced by this URI. + * Compared against a configured maximum to determine when + * to stop retrying. + * + * TODO: Consider renaming as something more generic, as all + * processing-loops do not necessarily include an attempted + * network-fetch (for example, when processing is aborted + * early to enqueue a prerequisite), and this counter may be + * reset if a URI is starting a fresh series of tries (as when + * rescheduled at a future time). Perhaps simply 'tryCount' + * or 'attempts'? * - * @return the number of attempts at getting the document referenced by this - * URI. + * @return attempts count */ public int getFetchAttempts() { return fetchAttempts; } /** - * Increment the number of attempts at getting the document referenced by - * this URI. - * - * @return the number of attempts at getting the document referenced by this - * URI. + * Increment the count of attempts (trips through the processing + * loop) at getting the document referenced by this URI. */ - public int incrementFetchAttempts() { - // TODO: rename, this is actually processing-loop-attempts - return fetchAttempts++; + public void incrementFetchAttempts() { + fetchAttempts++; } /** @@ -1847,4 +1852,15 @@ implements MultiReporter, Serializable, OverlayContext { public long getRescheduleTime() { return this.rescheduleTime; } + + /** + * Reset state that that should not persist when a URI is + * rescheduled for a specific future time. + */ + public void resetForRescheduling() { + // retries from now don't count against future try + resetFetchAttempts(); + // deferrals from now don't count against future try + resetDeferrals(); + } }