From 513c5fc4871cec1cba84d66ef8d4a0edce59fe90 Mon Sep 17 00:00:00 2001
From: Alex Osborne <aosborne@nla.gov.au>
Date: Mon, 1 Nov 2021 15:40:29 +0900
Subject: [PATCH 1/2] FetchDNS: Move DNS cache init from CrawlController

Since the cache is global we only need to configure it once, not every
time we start a job. By moving it we keep the code that interfaces with
dnsjava centralized in FetchDNS.

Looks like dnsjava no longer uses a cleaner thread so remove the comment
referring to it.
---
 .../java/org/archive/crawler/framework/CrawlController.java | 6 ------
 .../src/main/java/org/archive/modules/fetcher/FetchDNS.java | 5 +++++
 2 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/engine/src/main/java/org/archive/crawler/framework/CrawlController.java b/engine/src/main/java/org/archive/crawler/framework/CrawlController.java
index 0c32cc3a..a54ec16d 100644
--- a/engine/src/main/java/org/archive/crawler/framework/CrawlController.java
+++ b/engine/src/main/java/org/archive/crawler/framework/CrawlController.java
@@ -51,8 +51,6 @@ import org.springframework.context.ApplicationContextAware;
 import org.springframework.context.ApplicationEvent;
 import org.springframework.context.Lifecycle;
 import org.springframework.context.support.AbstractApplicationContext;
-import org.xbill.DNS.DClass;
-import org.xbill.DNS.Lookup;
 
 /**
  * CrawlController collects all the classes which cooperate to
@@ -289,10 +287,6 @@ implements Serializable,
        
         sExit = CrawlStatus.FINISHED_ABNORMAL;
 
-        // force creation of DNS Cache now -- avoids CacheCleaner in toe-threads group
-        // also cap size at 1 (we never wanta cached value; 0 is non-operative)
-        Lookup.getDefaultCache(DClass.IN).setMaxEntries(1);
-        
         reserveMemory = new LinkedList<byte[]>();
         for(int i = 0; i < RESERVE_BLOCKS; i++) {
             reserveMemory.add(new byte[RESERVE_BLOCK_SIZE]);
diff --git a/modules/src/main/java/org/archive/modules/fetcher/FetchDNS.java b/modules/src/main/java/org/archive/modules/fetcher/FetchDNS.java
index 8d3c97b1..9a38142b 100644
--- a/modules/src/main/java/org/archive/modules/fetcher/FetchDNS.java
+++ b/modules/src/main/java/org/archive/modules/fetcher/FetchDNS.java
@@ -66,6 +66,11 @@ public class FetchDNS extends Processor {
 
     private static Logger logger = Logger.getLogger(FetchDNS.class.getName());
 
+    static {
+        // cap size at 1 (we never want a cached value; 0 is non-operative)
+        Lookup.getDefaultCache(DClass.IN).setMaxEntries(1);
+    }
+
     // Defaults.
     private short ClassType = DClass.IN;
     private short TypeType = Type.A;

From 1832c5ef89534a930c86e9aa5af5b4cbd55921c8 Mon Sep 17 00:00:00 2001
From: Alex Osborne <aosborne@nla.gov.au>
Date: Mon, 1 Nov 2021 15:56:46 +0900
Subject: [PATCH 2/2] FetchDNS: Keep dnsjava selector thread out of ToePool

dnsjava will create its global selector thread on first DNS lookup.
Since the first lookup occurs on a toe thread then this will cause the
selector thead to join the job's toe thread group. This is a problem
because when the job is terminated ToePool sends an interrupt to all
threads in the thread group which the dnsjava selector thread is
not expected and never clears. The interrupted state then causes
select() to return immediately instead of sleeping so it consumes
100% CPU.

So workaround this by doing a dummy lookup when FetchDNS is loaded,
this ensures the first lookup happens on the web server thread that
handles the job build action instead of a toe thread.

Fixes: 045b2516da91 ("Update to latest version of dnsjava")
---
 .../main/java/org/archive/modules/fetcher/FetchDNS.java   | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/modules/src/main/java/org/archive/modules/fetcher/FetchDNS.java b/modules/src/main/java/org/archive/modules/fetcher/FetchDNS.java
index 9a38142b..e1a1c641 100644
--- a/modules/src/main/java/org/archive/modules/fetcher/FetchDNS.java
+++ b/modules/src/main/java/org/archive/modules/fetcher/FetchDNS.java
@@ -69,6 +69,14 @@ public class FetchDNS extends Processor {
     static {
         // cap size at 1 (we never want a cached value; 0 is non-operative)
         Lookup.getDefaultCache(DClass.IN).setMaxEntries(1);
+
+        // do a dummy lookup to force the creation of dnsjava NIO selector thread
+        // ensures it doesn't end up in the toe-threads group
+        try {
+            new Lookup("localhost").run();
+        } catch (TextParseException e) {
+            throw new RuntimeException(e);
+        }
     }
 
     // Defaults.