diff --git a/engine/src/main/java/org/archive/crawler/frontier/AbstractFrontier.java b/engine/src/main/java/org/archive/crawler/frontier/AbstractFrontier.java index 2fcca4a1..851ccd7b 100644 --- a/engine/src/main/java/org/archive/crawler/frontier/AbstractFrontier.java +++ b/engine/src/main/java/org/archive/crawler/frontier/AbstractFrontier.java @@ -568,6 +568,12 @@ public abstract class AbstractFrontier /** * Arrange for the given CrawlURI to be visited, if it is not * already scheduled/completed. + * + * This implementation defers uniqueness-testing into the frontier + * managerThread with a ScheduleIfUnique InEvent; this may cause + * unnecessary contention/single-threading. WorkQueueFrontier currently + * overrides as an experiment in decreasing contention. TODO: settle on + * one approach. * * @see org.archive.crawler.framework.Frontier#schedule(org.archive.modules.CrawlURI) */ diff --git a/engine/src/main/java/org/archive/crawler/frontier/WorkQueueFrontier.java b/engine/src/main/java/org/archive/crawler/frontier/WorkQueueFrontier.java index d2081698..2bd18f22 100644 --- a/engine/src/main/java/org/archive/crawler/frontier/WorkQueueFrontier.java +++ b/engine/src/main/java/org/archive/crawler/frontier/WorkQueueFrontier.java @@ -333,6 +333,32 @@ implements Closeable, sendToQueue(curi); } + + /** + * Arrange for the given CrawlURI to be visited, if it is not + * already enqueued/completed. + * + * Differs from superclass in that it operates in calling thread, rather + * than deferring operations via in-queue to managerThread. TODO: settle + * on either defer or in-thread approach after testing. + * + * @see org.archive.crawler.framework.Frontier#schedule(org.archive.modules.CrawlURI) + */ + @Override + public void schedule(CrawlURI curi) { + sheetOverlaysManager.applyOverridesTo(curi); + try { + KeyedProperties.loadOverridesFrom(curi); + if(curi.getClassKey()==null) { + // remedial processing + preparer.prepare(curi); + } + processScheduleIfUnique(curi); + } finally { + KeyedProperties.clearOverridesFrom(curi); + } + } + /** * Arrange for the given CrawlURI to be visited, if it is not * already scheduled/completed. @@ -340,7 +366,7 @@ implements Closeable, * @see org.archive.crawler.framework.Frontier#schedule(org.archive.modules.CrawlURI) */ protected void processScheduleIfUnique(CrawlURI curi) { - assert Thread.currentThread() == managerThread; +// assert Thread.currentThread() == managerThread; assert KeyedProperties.overridesActiveFrom(curi); // Canonicalization may set forceFetch flag. See