Fix for [HER-1683] h3: inconsistent host classification for dns: urls

* DefaultServerCache.java
    getHostFor(UURI): special case dns uris
* ServerCache.java
    add getHostFor(UURI) to public interface
* ServerCacheUtil.java
    remove now redundant wrapper method
* DispositionProcessor.java AbstractFrontier.java IPQueueAssignmentPolicy.java
  BucketQueueAssignmentPolicy.java PreconditionEnforcer.java QuotaEnforcer.java
  StatisticsTracker.java ServerCacheTest.java FetchHTTP.java
  Kw3WriterProcessor.java WriterPoolProcessor.java 
    changed to use serverCache.getHostFor(UURI) (a few cases remain that use
    getHostFor(String))

engine/src/main/java/org/archive/crawler/frontier/AbstractFrontier.java

@@ -679,8 +679,7 @@ public abstract class AbstractFrontier
         if (server != null) {
             server.getSubstats().tally(curi, stage);
         }
-        CrawlHost host = ServerCacheUtil.getHostFor(
-                getServerCache(),curi.getUURI());
+        CrawlHost host = getServerCache().getHostFor(curi.getUURI());
         if (host != null) {
             host.getSubstats().tally(curi, stage);
         }

engine/src/main/java/org/archive/crawler/frontier/BucketQueueAssignmentPolicy.java

@@ -25,7 +25,6 @@
  */
 package org.archive.crawler.frontier;
 
-import org.apache.commons.httpclient.URIException;
 import org.archive.modules.CrawlURI;
 import org.archive.modules.net.CrawlHost;
 import org.archive.modules.net.ServerCache;
@@ -56,13 +55,7 @@ public class BucketQueueAssignmentPolicy extends QueueAssignmentPolicy {
     public String getClassKey(final CrawlURI curi) {
         
         CrawlHost host;
-        try {
-            host = serverCache.getHostFor(curi.getUURI().getReferencedHost());
-        } catch (URIException e) {
-            // FIXME error handling
-            e.printStackTrace();
-            host = null;
-        }
+        host = serverCache.getHostFor(curi.getUURI());
         if(host == null) {
             return "NO-HOST";
         } else if(host.getIP() == null) {

engine/src/main/java/org/archive/crawler/frontier/IPQueueAssignmentPolicy.java

@@ -21,7 +21,6 @@ package org.archive.crawler.frontier;
 import org.archive.modules.CrawlURI;
 import org.archive.modules.net.CrawlHost;
 import org.archive.modules.net.ServerCache;
-import org.archive.modules.net.ServerCacheUtil;
 import org.springframework.beans.factory.annotation.Autowired;
 
 
@@ -45,8 +44,7 @@ public class IPQueueAssignmentPolicy
     }
     
     public String getClassKey(CrawlURI cauri) {
-        CrawlHost host = ServerCacheUtil.getHostFor(serverCache, 
-                cauri.getUURI());
+        CrawlHost host = serverCache.getHostFor(cauri.getUURI());
         if (host == null || host.getIP() == null) {
             // if no server or no IP, use superclass implementation
             return super.getClassKey(cauri);

engine/src/main/java/org/archive/crawler/postprocessor/DispositionProcessor.java

@@ -247,7 +247,7 @@ public class DispositionProcessor extends Processor {
             if (maxBandwidthKB > 0) {
                 // Enforce bandwidth limit
                 ServerCache cache = this.getServerCache();
-                CrawlHost host = ServerCacheUtil.getHostFor(cache, curi.getUURI());
+                CrawlHost host = cache.getHostFor(curi.getUURI());
                 long minDurationToWait = host.getEarliestNextURIEmitTime()
                         - now;
                 float maxBandwidth = maxBandwidthKB * 1.024F; // kilo factor

engine/src/main/java/org/archive/crawler/prefetch/PreconditionEnforcer.java

@@ -287,7 +287,7 @@ public class PreconditionEnforcer extends Processor  {
         // If we've done a dns lookup and it didn't resolve a host
         // cancel further fetch-processing of this URI, because
         // the domain is unresolvable
-        CrawlHost ch = getHostFor(curi);
+        CrawlHost ch = serverCache.getHostFor(curi.getUURI());
         if (ch == null || ch.hasBeenLookedUp() && ch.getIP() == null) {
             if (logger.isLoggable(Level.FINE)) {
                 logger.fine( "no dns for " + ch +
@@ -322,7 +322,7 @@ public class PreconditionEnforcer extends Processor  {
      * @return true if ip should be looked up.
      */
     public boolean isIpExpired(CrawlURI curi) {
-        CrawlHost host = getHostFor(curi);
+        CrawlHost host = serverCache.getHostFor(curi.getUURI());
         if (!host.hasBeenLookedUp()) {
             // IP has not been looked up yet.
             return true;
@@ -492,13 +492,7 @@ public class PreconditionEnforcer extends Processor  {
         //skipToPostProcessing();
     }
     
-    
     private CrawlServer getServerFor(CrawlURI curi) {
         return ServerCacheUtil.getServerFor(serverCache, curi.getUURI());
     }
-    
-    
-    private CrawlHost getHostFor(CrawlURI curi) {
-        return ServerCacheUtil.getHostFor(serverCache, curi.getUURI());
-    }
 }

engine/src/main/java/org/archive/crawler/prefetch/QuotaEnforcer.java

@@ -317,8 +317,7 @@ public class QuotaEnforcer extends Processor {
         CrawlURI curi = (CrawlURI)puri;
         final CrawlServer server = ServerCacheUtil.getServerFor(serverCache, 
                 curi.getUURI());
-        final CrawlHost host = ServerCacheUtil.getHostFor(serverCache, 
-                curi.getUURI());
+        final CrawlHost host = serverCache.getHostFor(curi.getUURI());
         FetchStats.HasFetchStats[] haveStats = 
             new FetchStats.HasFetchStats[] {
                 server, 

engine/src/main/java/org/archive/crawler/reporting/StatisticsTracker.java

@@ -50,7 +50,6 @@ import org.archive.crawler.util.CrawledBytesHistotable;
 import org.archive.crawler.util.TopNSet;
 import org.archive.modules.CrawlURI;
 import org.archive.modules.net.ServerCache;
-import org.archive.modules.net.ServerCacheUtil;
 import org.archive.modules.seeds.SeedListener;
 import org.archive.modules.seeds.SeedModule;
 import org.archive.spring.ConfigPath;
@@ -775,13 +774,12 @@ public class StatisticsTracker
 
         // Save hosts stats.
         ServerCache sc = serverCache;
-        saveHostStats((curi.getFetchStatus() == 1)? "dns:":
-                ServerCacheUtil.getHostFor(sc, curi.getUURI()).getHostName(),
+        saveHostStats(sc.getHostFor(curi.getUURI()).getHostName(),
                 curi.getContentSize());
         
         if (curi.getData().containsKey(A_SOURCE_TAG)) {
         	saveSourceStats((String)curi.getData().get(A_SOURCE_TAG),
-                        ServerCacheUtil.getHostFor(sc, curi.getUURI()).
+                        sc.getHostFor(curi.getUURI()).
                     getHostName()); 
         }
     }