From 721e89ca1058939092be7f32ff8719097ebdca94 Mon Sep 17 00:00:00 2001
From: Alex Osborne <aosborne@nla.gov.au>
Date: Mon, 26 Jul 2021 15:26:09 +0900
Subject: [PATCH] ToeThread: ensure currentCuri is finished before exiting

Thread interruption and certain other exceptions can cause a toe thread
to exit without informing the frontier that the current CrawlURI is
finished. This causes the job to get permanently stuck in the STOPPING
state.

This change adds a section to the finally block that will finish any
unfinished CrawlURI.

We also move the continueCheck() call after setCurrentCuri() to ensure
there's no window where InterruptedException can be thrown after the
frontier returns the next CrawlURI but before it gets assigned to
currentCuri.

Fixes #420
---
 .../archive/crawler/framework/ToeThread.java   | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/engine/src/main/java/org/archive/crawler/framework/ToeThread.java b/engine/src/main/java/org/archive/crawler/framework/ToeThread.java
index 4e9c4791..cc26ce79 100644
--- a/engine/src/main/java/org/archive/crawler/framework/ToeThread.java
+++ b/engine/src/main/java/org/archive/crawler/framework/ToeThread.java
@@ -132,11 +132,10 @@ implements Reporter, ProgressStatisticsReporter,
                 setStep(Step.ABOUT_TO_GET_URI, null);
 
                 CrawlURI curi = controller.getFrontier().next();
-                
-                
+
                 synchronized(this) {
-                    ArchiveUtils.continueCheck();
                     setCurrentCuri(curi);
+                    ArchiveUtils.continueCheck();
                     currentCuri.setThreadNumber(this.serialNumber);
                     lastStartTime = System.currentTimeMillis();
                     currentCuri.setRecorder(httpRecorder);
@@ -197,9 +196,6 @@ implements Reporter, ProgressStatisticsReporter,
                 }
             }
         } catch (InterruptedException e) {
-            if(currentCuri!=null){
-                logger.log(Level.SEVERE,"Interrupt leaving unfinished CrawlURI "+getName()+" - job may hang",e);
-            }
             // thread interrupted, ok to end
             logger.log(Level.FINE,this.getName()+ " ended with Interruption");
         } catch (Exception e) {
@@ -208,8 +204,16 @@ implements Reporter, ProgressStatisticsReporter,
         } catch (OutOfMemoryError err) {
             seriousError(err);
         } finally {
+            synchronized (this) {
+                if (currentCuri != null) {
+                    logger.log(Level.WARNING,"Leaving with unfinished CrawlURI " + getName() +
+                            " - attempting to finish");
+                    currentCuri.setFetchStatus(S_PROCESSING_THREAD_KILLED);
+                    controller.getFrontier().finished(currentCuri);
+                    setCurrentCuri(null);
+                }
+            }
             controller.getFrontier().endDisposition();
-
         }
 
         setCurrentCuri(null);