From ea49e55c892a888ef9a601ec0ead1dabdb154e40 Mon Sep 17 00:00:00 2001
From: gojomo <gojomo@daa5b2f2-a927-0410-8b2d-f5f262fa301a>
Date: Thu, 9 Dec 2010 22:28:20 +0000
Subject: [PATCH] [HER-1850] IllegalStateException from
 WorkQueueFrontier.wakeQueues in BlockIterator * WorkQueueFrontier.java    
 double in-memory snoozedClassQueues size     protect all adds/removes to
 snoozedOverflow in synchronized blocks

---
 .../crawler/frontier/WorkQueueFrontier.java   | 31 ++++++++++++-------
 1 file changed, 19 insertions(+), 12 deletions(-)

diff --git a/engine/src/main/java/org/archive/crawler/frontier/WorkQueueFrontier.java b/engine/src/main/java/org/archive/crawler/frontier/WorkQueueFrontier.java
index e714c629..7abf7ac7 100644
--- a/engine/src/main/java/org/archive/crawler/frontier/WorkQueueFrontier.java
+++ b/engine/src/main/java/org/archive/crawler/frontier/WorkQueueFrontier.java
@@ -201,7 +201,7 @@ implements Closeable,
     transient protected DelayQueue<DelayedWorkQueue> snoozedClassQueues;
     protected StoredSortedMap<Long,DelayedWorkQueue> snoozedOverflow; 
     protected AtomicInteger snoozedOverflowCount = new AtomicInteger(0); 
-    protected static int MAX_SNOOZED_IN_MEMORY = 5000; 
+    protected static int MAX_SNOOZED_IN_MEMORY = 10000; 
     
     /** URIs scheduled to be re-enqueued at future date */
     protected StoredSortedMap<Long, CrawlURI> futureUris; 
@@ -844,6 +844,7 @@ implements Closeable,
             queue.setWakeTime(0);
             reenqueueQueue(queue);
             iterOverflow.remove(); 
+            snoozedOverflowCount.decrementAndGet();
         }
     }
     
@@ -858,15 +859,19 @@ implements Closeable,
             reenqueueQueue(queue);
         }
         // also consider overflow (usually empty)
-        Iterator<DelayedWorkQueue> iter = 
-            snoozedOverflow.headMap(System.currentTimeMillis()).values().iterator();
-        while(iter.hasNext()) {
-            DelayedWorkQueue dq = iter.next();
-            iter.remove();
-            snoozedOverflowCount.decrementAndGet();
-            WorkQueue queue = dq.getWorkQueue(this);
-            queue.setWakeTime(0);
-            reenqueueQueue(queue);
+        if(!snoozedOverflow.isEmpty()) {
+            synchronized(snoozedOverflow) {
+                Iterator<DelayedWorkQueue> iter = 
+                    snoozedOverflow.headMap(System.currentTimeMillis()).values().iterator();
+                while(iter.hasNext()) {
+                    DelayedWorkQueue dq = iter.next();
+                    iter.remove();
+                    snoozedOverflowCount.decrementAndGet();
+                    WorkQueue queue = dq.getWorkQueue(this);
+                    queue.setWakeTime(0);
+                    reenqueueQueue(queue);
+                }
+            }
         }
     }
     
@@ -1012,8 +1017,10 @@ implements Closeable,
         if(snoozedClassQueues.size()<MAX_SNOOZED_IN_MEMORY) {
             snoozedClassQueues.add(dq);
         } else {
-            snoozedOverflow.put(nextTime, dq);
-            snoozedOverflowCount.incrementAndGet();
+            synchronized(snoozedOverflow) {
+                snoozedOverflow.put(nextTime, dq);
+                snoozedOverflowCount.incrementAndGet();
+            }
         }
     }