diff --git a/engine/src/main/java/org/archive/crawler/frontier/AbstractFrontier.java b/engine/src/main/java/org/archive/crawler/frontier/AbstractFrontier.java index 3aeab3c8..2fcca4a1 100644 --- a/engine/src/main/java/org/archive/crawler/frontier/AbstractFrontier.java +++ b/engine/src/main/java/org/archive/crawler/frontier/AbstractFrontier.java @@ -408,6 +408,7 @@ public abstract class AbstractFrontier // continue to process discovered and finished URIs inbound.take().process(); } + finalTasks(); // TODO: more cleanup? reachedState(State.FINISH); break loop; @@ -435,6 +436,14 @@ public abstract class AbstractFrontier } + /** + * Perform any tasks necessary before entering + * FINISH frontier state/FINISHED crawl state + */ + protected void finalTasks() { + // by default; nothing + } + /** * Fill the outbound queue with eligible CrawlURIs, to capacity * or as much as possible. diff --git a/engine/src/main/java/org/archive/crawler/frontier/BdbFrontier.java b/engine/src/main/java/org/archive/crawler/frontier/BdbFrontier.java index 45cb2283..815aea6c 100644 --- a/engine/src/main/java/org/archive/crawler/frontier/BdbFrontier.java +++ b/engine/src/main/java/org/archive/crawler/frontier/BdbFrontier.java @@ -181,7 +181,13 @@ implements Checkpointable, BeanNameAware { } } - protected void closeQueue() { + + /* (non-Javadoc) + * @see org.archive.crawler.frontier.AbstractFrontier#finalTasks() + */ + @Override + protected void finalTasks() { + super.finalTasks(); // before closing/releasing, dump if requested if (getDumpPendingAtClose()) { try { @@ -190,7 +196,15 @@ implements Checkpointable, BeanNameAware { logger.log(Level.WARNING, "dump pending problem", e); } } + } + + /* (non-Javadoc) + * @see org.archive.crawler.frontier.WorkQueueFrontier#close() + */ + @Override + public void close() { ArchiveUtils.closeQuietly(pendingUris); + super.close(); } protected BdbMultipleWorkQueues getWorkQueues() { diff --git a/engine/src/main/java/org/archive/crawler/frontier/WorkQueueFrontier.java b/engine/src/main/java/org/archive/crawler/frontier/WorkQueueFrontier.java index fcd66680..d2081698 100644 --- a/engine/src/main/java/org/archive/crawler/frontier/WorkQueueFrontier.java +++ b/engine/src/main/java/org/archive/crawler/frontier/WorkQueueFrontier.java @@ -312,8 +312,7 @@ implements Closeable, * Release resources only needed when running */ public void close() { - ArchiveUtils.closeQuietly(uriUniqFilter); - closeQueue(); + ArchiveUtils.closeQuietly(uriUniqFilter); ArchiveUtils.closeQuietly(allQueues); } @@ -1551,8 +1550,6 @@ implements Closeable, } } - protected abstract void closeQueue(); - /** * Returns true if the WorkQueue implementation of this * Frontier stores its workload on disk instead of relying