Merge pull request #444 from internetarchive/fix-dnsjava-selector-thread

FetchDNS: Keep dnsjava selector thread out of ToePool

engine/src/main/java/org/archive/crawler/framework/CrawlController.java

@@ -51,8 +51,6 @@ import org.springframework.context.ApplicationContextAware;
 import org.springframework.context.ApplicationEvent;
 import org.springframework.context.Lifecycle;
 import org.springframework.context.support.AbstractApplicationContext;
-import org.xbill.DNS.DClass;
-import org.xbill.DNS.Lookup;
 
 /**
  * CrawlController collects all the classes which cooperate to
@@ -289,10 +287,6 @@ implements Serializable,
        
         sExit = CrawlStatus.FINISHED_ABNORMAL;
 
-        // force creation of DNS Cache now -- avoids CacheCleaner in toe-threads group
-        // also cap size at 1 (we never wanta cached value; 0 is non-operative)
-        Lookup.getDefaultCache(DClass.IN).setMaxEntries(1);
-        
         reserveMemory = new LinkedList<byte[]>();
         for(int i = 0; i < RESERVE_BLOCKS; i++) {
             reserveMemory.add(new byte[RESERVE_BLOCK_SIZE]);

modules/src/main/java/org/archive/modules/fetcher/FetchDNS.java

@@ -66,6 +66,19 @@ public class FetchDNS extends Processor {
 
     private static Logger logger = Logger.getLogger(FetchDNS.class.getName());
 
+    static {
+        // cap size at 1 (we never want a cached value; 0 is non-operative)
+        Lookup.getDefaultCache(DClass.IN).setMaxEntries(1);
+
+        // do a dummy lookup to force the creation of dnsjava NIO selector thread
+        // ensures it doesn't end up in the toe-threads group
+        try {
+            new Lookup("localhost").run();
+        } catch (TextParseException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
     // Defaults.
     private short ClassType = DClass.IN;
     private short TypeType = Type.A;