From a82838cd27d36a4c978c364cc8ed184435cacfc4 Mon Sep 17 00:00:00 2001
From: gojomo <gojomo@daa5b2f2-a927-0410-8b2d-f5f262fa301a>
Date: Tue, 13 Apr 2010 02:10:56 +0000
Subject: [PATCH] HER-1756 improved crawl status reporting: more definitive
 start-after-setup, conclusion-after-logging of crawling *
 AbstractFrontier.java     call new finalTasks just before announcing FINISH
 state * WorkQueueFrontier.java     eliminate closeQueue * BdbFrontier.java   
  move dumpAllPendingToLog from closeQueue (triggered by CrawlController
 FINISHED) to finalTasks (before Frontier FINISH)     replace closeQueue with
 close() override

---
 .../crawler/frontier/AbstractFrontier.java       |  9 +++++++++
 .../archive/crawler/frontier/BdbFrontier.java    | 16 +++++++++++++++-
 .../crawler/frontier/WorkQueueFrontier.java      |  5 +----
 3 files changed, 25 insertions(+), 5 deletions(-)

diff --git a/engine/src/main/java/org/archive/crawler/frontier/AbstractFrontier.java b/engine/src/main/java/org/archive/crawler/frontier/AbstractFrontier.java
index 3aeab3c8..2fcca4a1 100644
--- a/engine/src/main/java/org/archive/crawler/frontier/AbstractFrontier.java
+++ b/engine/src/main/java/org/archive/crawler/frontier/AbstractFrontier.java
@@ -408,6 +408,7 @@ public abstract class AbstractFrontier
                             // continue to process discovered and finished URIs
                             inbound.take().process();
                         }
+                        finalTasks(); 
                         // TODO: more cleanup?
                         reachedState(State.FINISH);
                         break loop;
@@ -435,6 +436,14 @@ public abstract class AbstractFrontier
     }
 
 
+    /**
+     * Perform any tasks necessary before entering 
+     * FINISH frontier state/FINISHED crawl state
+     */
+    protected void finalTasks() {
+        // by default; nothing
+    }
+    
     /**
      * Fill the outbound queue with eligible CrawlURIs, to capacity
      * or as much as possible. 
diff --git a/engine/src/main/java/org/archive/crawler/frontier/BdbFrontier.java b/engine/src/main/java/org/archive/crawler/frontier/BdbFrontier.java
index 45cb2283..815aea6c 100644
--- a/engine/src/main/java/org/archive/crawler/frontier/BdbFrontier.java
+++ b/engine/src/main/java/org/archive/crawler/frontier/BdbFrontier.java
@@ -181,7 +181,13 @@ implements Checkpointable, BeanNameAware {
         }
     }
     
-    protected void closeQueue() {
+    
+    /* (non-Javadoc)
+     * @see org.archive.crawler.frontier.AbstractFrontier#finalTasks()
+     */
+    @Override
+    protected void finalTasks() {
+        super.finalTasks();
         // before closing/releasing, dump if requested
         if (getDumpPendingAtClose()) {
             try {
@@ -190,7 +196,15 @@ implements Checkpointable, BeanNameAware {
                 logger.log(Level.WARNING, "dump pending problem", e);
             }
         }
+    }
+    
+    /* (non-Javadoc)
+     * @see org.archive.crawler.frontier.WorkQueueFrontier#close()
+     */
+    @Override 
+    public void close() {
         ArchiveUtils.closeQuietly(pendingUris);
+        super.close(); 
     }
         
     protected BdbMultipleWorkQueues getWorkQueues() {
diff --git a/engine/src/main/java/org/archive/crawler/frontier/WorkQueueFrontier.java b/engine/src/main/java/org/archive/crawler/frontier/WorkQueueFrontier.java
index fcd66680..d2081698 100644
--- a/engine/src/main/java/org/archive/crawler/frontier/WorkQueueFrontier.java
+++ b/engine/src/main/java/org/archive/crawler/frontier/WorkQueueFrontier.java
@@ -312,8 +312,7 @@ implements Closeable,
      * Release resources only needed when running
      */
     public void close() {
-        ArchiveUtils.closeQuietly(uriUniqFilter);
-        closeQueue();        
+        ArchiveUtils.closeQuietly(uriUniqFilter);     
         ArchiveUtils.closeQuietly(allQueues);
     }
     
@@ -1551,8 +1550,6 @@ implements Closeable,
         }
     }
     
-    protected abstract void closeQueue();
-    
     /**
      * Returns <code>true</code> if the WorkQueue implementation of this
      * Frontier stores its workload on disk instead of relying