Improve ability to force-terminate crawls

* RecordingInputStream.java
    check for interrupt on each socket-timeout
* CrawlController.java
    on second requestCrawlStop, interrupt threads via ToePool.cleanup
* ToePool.java
   adjust for new earlier/repeated cleanup
* ToeThread.java
   better warnings/recovery on forced-interrupts

commons/src/main/java/org/archive/io/RecordingInputStream.java

@@ -219,6 +219,10 @@ public class RecordingInputStream
                 if (logger.isLoggable(Level.FINE)) {
                     logger.log(Level.FINE, "socket timeout", e); 
                 }
+                // check for interrupt
+                if (Thread.interrupted()) {
+                    throw new InterruptedException("Interrupted during IO");
+                }
                 // check for overall timeout
                 recordingOutputStream.checkLimits();
             } catch (SocketException se) {

engine/src/main/java/org/archive/crawler/framework/CrawlController.java

@@ -398,6 +398,10 @@ implements Serializable,
      * Operator requested for crawl to stop.
      */
     public synchronized void requestCrawlStop() {
+        if(state == State.STOPPING) {
+            // second stop request; nudge the threads with interrupts
+            getToePool().cleanup();
+        }
         requestCrawlStop(CrawlStatus.ABORTED);
     }
     

engine/src/main/java/org/archive/crawler/framework/ToePool.java

@@ -70,7 +70,7 @@ public class ToePool extends ThreadGroup implements MultiReporter {
                 toe.interrupt();
             }
         }
-        this.controller = null;
+//        this.controller = null;
     }
 
     /**
@@ -297,6 +297,9 @@ public class ToePool extends ThreadGroup implements MultiReporter {
         w.print(" threads: ");
         
         TreeSet<Map.Entry<Object,Long>> sortedSteps = (TreeSet<Entry<Object, Long>>) map.get("steps");
+        if(sortedSteps.size()==0) {
+            return;
+        }
         w.print(Histotable.entryString(sortedSteps.first()));
         if(sortedSteps.size()>1) {
             Iterator<Map.Entry<Object,Long>> iter = sortedSteps.iterator();

engine/src/main/java/org/archive/crawler/framework/ToeThread.java

@@ -162,6 +162,13 @@ implements MultiReporter, ProgressStatisticsReporter,
                     recoverableProblem(ae);
                 } catch (RuntimeException e) {
                     recoverableProblem(e);
+                } catch (InterruptedException e) {
+                    if(currentCuri!=null) {
+                        recoverableProblem(e);
+                        Thread.interrupted(); // clear interrupt status
+                    } else {
+                        throw e;
+                    }
                 } catch (StackOverflowError err) {
                     recoverableProblem(err);
                 } catch (Error err) {
@@ -187,6 +194,9 @@ implements MultiReporter, ProgressStatisticsReporter,
                 }
             }
         } catch (InterruptedException e) {
+            if(currentCuri!=null){
+                logger.log(Level.SEVERE,"Interrupt leaving unfinished CrawlURI "+getName()+" - job may hang",e);
+            }
             // thread interrupted, ok to end
             logger.log(Level.FINE,this.getName()+ " ended with Interruption");
         } catch (Exception e) {
@@ -288,7 +298,7 @@ implements MultiReporter, ProgressStatisticsReporter,
     private void recoverableProblem(Throwable e) {
         Object previousStep = step;
         setStep(Step.HANDLING_RUNTIME_EXCEPTION, null);
-        e.printStackTrace(System.err);
+        //e.printStackTrace(System.err);
         currentCuri.setFetchStatus(S_RUNTIME_EXCEPTION);
         // store exception temporarily for logging
         currentCuri.getAnnotations().add("err="+e.getClass().getName());