From fcd0ec66991956314d9556afdb7e27111372f08a Mon Sep 17 00:00:00 2001
From: gojomo <gojomo@daa5b2f2-a927-0410-8b2d-f5f262fa301a>
Date: Tue, 27 Apr 2010 21:37:24 +0000
Subject: [PATCH] HER-1762 H3 performance tuning * WorkQueueFrontier.java    
 override schedule() to keep unique-testing in calling thread; may remove
 bottleneck shown by time-profiling and thread-observation (most ToeThreads
 stuck in candidatesProcessor, schedule()ing to frontier) *
 AbstractFrontier.java     add comment about implementation vs. subclass
 override

---
 .../crawler/frontier/AbstractFrontier.java    |  6 ++++
 .../crawler/frontier/WorkQueueFrontier.java   | 28 ++++++++++++++++++-
 2 files changed, 33 insertions(+), 1 deletion(-)

diff --git a/engine/src/main/java/org/archive/crawler/frontier/AbstractFrontier.java b/engine/src/main/java/org/archive/crawler/frontier/AbstractFrontier.java
index 2fcca4a1..851ccd7b 100644
--- a/engine/src/main/java/org/archive/crawler/frontier/AbstractFrontier.java
+++ b/engine/src/main/java/org/archive/crawler/frontier/AbstractFrontier.java
@@ -568,6 +568,12 @@ public abstract class AbstractFrontier
     /**
      * Arrange for the given CrawlURI to be visited, if it is not
      * already scheduled/completed.
+     * 
+     * This implementation defers uniqueness-testing into the frontier 
+     * managerThread with a ScheduleIfUnique InEvent; this may cause 
+     * unnecessary contention/single-threading. WorkQueueFrontier currently
+     * overrides as an experiment in decreasing contention. TODO: settle on
+     * one approach. 
      *
      * @see org.archive.crawler.framework.Frontier#schedule(org.archive.modules.CrawlURI)
      */
diff --git a/engine/src/main/java/org/archive/crawler/frontier/WorkQueueFrontier.java b/engine/src/main/java/org/archive/crawler/frontier/WorkQueueFrontier.java
index d2081698..2bd18f22 100644
--- a/engine/src/main/java/org/archive/crawler/frontier/WorkQueueFrontier.java
+++ b/engine/src/main/java/org/archive/crawler/frontier/WorkQueueFrontier.java
@@ -333,6 +333,32 @@ implements Closeable,
         sendToQueue(curi);
     }
     
+    
+    /**
+     * Arrange for the given CrawlURI to be visited, if it is not
+     * already enqueued/completed. 
+     * 
+     * Differs from superclass in that it operates in calling thread, rather 
+     * than deferring operations via in-queue to managerThread. TODO: settle
+     * on either defer or in-thread approach after testing. 
+     *
+     * @see org.archive.crawler.framework.Frontier#schedule(org.archive.modules.CrawlURI)
+     */
+    @Override
+    public void schedule(CrawlURI curi) {
+        sheetOverlaysManager.applyOverridesTo(curi);
+        try {
+            KeyedProperties.loadOverridesFrom(curi);
+            if(curi.getClassKey()==null) {
+                // remedial processing
+                preparer.prepare(curi);
+            }
+            processScheduleIfUnique(curi);
+        } finally {
+            KeyedProperties.clearOverridesFrom(curi); 
+        }
+    }
+
     /**
      * Arrange for the given CrawlURI to be visited, if it is not
      * already scheduled/completed.
@@ -340,7 +366,7 @@ implements Closeable,
      * @see org.archive.crawler.framework.Frontier#schedule(org.archive.modules.CrawlURI)
      */
     protected void processScheduleIfUnique(CrawlURI curi) {
-        assert Thread.currentThread() == managerThread;
+//        assert Thread.currentThread() == managerThread;
         assert KeyedProperties.overridesActiveFrom(curi); 
         
         // Canonicalization may set forceFetch flag.  See