diff --git a/engine/src/main/java/org/archive/crawler/frontier/AbstractFrontier.java b/engine/src/main/java/org/archive/crawler/frontier/AbstractFrontier.java index fb8b1f09..bb74973f 100644 --- a/engine/src/main/java/org/archive/crawler/frontier/AbstractFrontier.java +++ b/engine/src/main/java/org/archive/crawler/frontier/AbstractFrontier.java @@ -679,8 +679,7 @@ public abstract class AbstractFrontier if (server != null) { server.getSubstats().tally(curi, stage); } - CrawlHost host = ServerCacheUtil.getHostFor( - getServerCache(),curi.getUURI()); + CrawlHost host = getServerCache().getHostFor(curi.getUURI()); if (host != null) { host.getSubstats().tally(curi, stage); } diff --git a/engine/src/main/java/org/archive/crawler/frontier/BucketQueueAssignmentPolicy.java b/engine/src/main/java/org/archive/crawler/frontier/BucketQueueAssignmentPolicy.java index d0b9561d..b89b4fe2 100644 --- a/engine/src/main/java/org/archive/crawler/frontier/BucketQueueAssignmentPolicy.java +++ b/engine/src/main/java/org/archive/crawler/frontier/BucketQueueAssignmentPolicy.java @@ -25,7 +25,6 @@ */ package org.archive.crawler.frontier; -import org.apache.commons.httpclient.URIException; import org.archive.modules.CrawlURI; import org.archive.modules.net.CrawlHost; import org.archive.modules.net.ServerCache; @@ -56,13 +55,7 @@ public class BucketQueueAssignmentPolicy extends QueueAssignmentPolicy { public String getClassKey(final CrawlURI curi) { CrawlHost host; - try { - host = serverCache.getHostFor(curi.getUURI().getReferencedHost()); - } catch (URIException e) { - // FIXME error handling - e.printStackTrace(); - host = null; - } + host = serverCache.getHostFor(curi.getUURI()); if(host == null) { return "NO-HOST"; } else if(host.getIP() == null) { diff --git a/engine/src/main/java/org/archive/crawler/frontier/IPQueueAssignmentPolicy.java b/engine/src/main/java/org/archive/crawler/frontier/IPQueueAssignmentPolicy.java index 878afef5..d16418b8 100644 --- a/engine/src/main/java/org/archive/crawler/frontier/IPQueueAssignmentPolicy.java +++ b/engine/src/main/java/org/archive/crawler/frontier/IPQueueAssignmentPolicy.java @@ -21,7 +21,6 @@ package org.archive.crawler.frontier; import org.archive.modules.CrawlURI; import org.archive.modules.net.CrawlHost; import org.archive.modules.net.ServerCache; -import org.archive.modules.net.ServerCacheUtil; import org.springframework.beans.factory.annotation.Autowired; @@ -45,8 +44,7 @@ public class IPQueueAssignmentPolicy } public String getClassKey(CrawlURI cauri) { - CrawlHost host = ServerCacheUtil.getHostFor(serverCache, - cauri.getUURI()); + CrawlHost host = serverCache.getHostFor(cauri.getUURI()); if (host == null || host.getIP() == null) { // if no server or no IP, use superclass implementation return super.getClassKey(cauri); diff --git a/engine/src/main/java/org/archive/crawler/postprocessor/DispositionProcessor.java b/engine/src/main/java/org/archive/crawler/postprocessor/DispositionProcessor.java index b2d27bab..f6a487ef 100644 --- a/engine/src/main/java/org/archive/crawler/postprocessor/DispositionProcessor.java +++ b/engine/src/main/java/org/archive/crawler/postprocessor/DispositionProcessor.java @@ -247,7 +247,7 @@ public class DispositionProcessor extends Processor { if (maxBandwidthKB > 0) { // Enforce bandwidth limit ServerCache cache = this.getServerCache(); - CrawlHost host = ServerCacheUtil.getHostFor(cache, curi.getUURI()); + CrawlHost host = cache.getHostFor(curi.getUURI()); long minDurationToWait = host.getEarliestNextURIEmitTime() - now; float maxBandwidth = maxBandwidthKB * 1.024F; // kilo factor diff --git a/engine/src/main/java/org/archive/crawler/prefetch/PreconditionEnforcer.java b/engine/src/main/java/org/archive/crawler/prefetch/PreconditionEnforcer.java index faba4218..e078d3af 100644 --- a/engine/src/main/java/org/archive/crawler/prefetch/PreconditionEnforcer.java +++ b/engine/src/main/java/org/archive/crawler/prefetch/PreconditionEnforcer.java @@ -287,7 +287,7 @@ public class PreconditionEnforcer extends Processor { // If we've done a dns lookup and it didn't resolve a host // cancel further fetch-processing of this URI, because // the domain is unresolvable - CrawlHost ch = getHostFor(curi); + CrawlHost ch = serverCache.getHostFor(curi.getUURI()); if (ch == null || ch.hasBeenLookedUp() && ch.getIP() == null) { if (logger.isLoggable(Level.FINE)) { logger.fine( "no dns for " + ch + @@ -322,7 +322,7 @@ public class PreconditionEnforcer extends Processor { * @return true if ip should be looked up. */ public boolean isIpExpired(CrawlURI curi) { - CrawlHost host = getHostFor(curi); + CrawlHost host = serverCache.getHostFor(curi.getUURI()); if (!host.hasBeenLookedUp()) { // IP has not been looked up yet. return true; @@ -492,13 +492,7 @@ public class PreconditionEnforcer extends Processor { //skipToPostProcessing(); } - private CrawlServer getServerFor(CrawlURI curi) { return ServerCacheUtil.getServerFor(serverCache, curi.getUURI()); } - - - private CrawlHost getHostFor(CrawlURI curi) { - return ServerCacheUtil.getHostFor(serverCache, curi.getUURI()); - } } diff --git a/engine/src/main/java/org/archive/crawler/prefetch/QuotaEnforcer.java b/engine/src/main/java/org/archive/crawler/prefetch/QuotaEnforcer.java index 9546a82d..9758fb01 100644 --- a/engine/src/main/java/org/archive/crawler/prefetch/QuotaEnforcer.java +++ b/engine/src/main/java/org/archive/crawler/prefetch/QuotaEnforcer.java @@ -317,8 +317,7 @@ public class QuotaEnforcer extends Processor { CrawlURI curi = (CrawlURI)puri; final CrawlServer server = ServerCacheUtil.getServerFor(serverCache, curi.getUURI()); - final CrawlHost host = ServerCacheUtil.getHostFor(serverCache, - curi.getUURI()); + final CrawlHost host = serverCache.getHostFor(curi.getUURI()); FetchStats.HasFetchStats[] haveStats = new FetchStats.HasFetchStats[] { server, diff --git a/engine/src/main/java/org/archive/crawler/reporting/StatisticsTracker.java b/engine/src/main/java/org/archive/crawler/reporting/StatisticsTracker.java index 36add394..bec46bb9 100644 --- a/engine/src/main/java/org/archive/crawler/reporting/StatisticsTracker.java +++ b/engine/src/main/java/org/archive/crawler/reporting/StatisticsTracker.java @@ -50,7 +50,6 @@ import org.archive.crawler.util.CrawledBytesHistotable; import org.archive.crawler.util.TopNSet; import org.archive.modules.CrawlURI; import org.archive.modules.net.ServerCache; -import org.archive.modules.net.ServerCacheUtil; import org.archive.modules.seeds.SeedListener; import org.archive.modules.seeds.SeedModule; import org.archive.spring.ConfigPath; @@ -775,13 +774,12 @@ public class StatisticsTracker // Save hosts stats. ServerCache sc = serverCache; - saveHostStats((curi.getFetchStatus() == 1)? "dns:": - ServerCacheUtil.getHostFor(sc, curi.getUURI()).getHostName(), + saveHostStats(sc.getHostFor(curi.getUURI()).getHostName(), curi.getContentSize()); if (curi.getData().containsKey(A_SOURCE_TAG)) { saveSourceStats((String)curi.getData().get(A_SOURCE_TAG), - ServerCacheUtil.getHostFor(sc, curi.getUURI()). + sc.getHostFor(curi.getUURI()). getHostName()); } } diff --git a/modules/src/main/java/org/archive/modules/fetcher/DefaultServerCache.java b/modules/src/main/java/org/archive/modules/fetcher/DefaultServerCache.java index 302b3ed5..178315a4 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/DefaultServerCache.java +++ b/modules/src/main/java/org/archive/modules/fetcher/DefaultServerCache.java @@ -142,7 +142,11 @@ public class DefaultServerCache implements ServerCache, Closeable, Serializable public CrawlHost getHostFor(UURI uuri) { CrawlHost h = null; try { - h = getHostFor(uuri.getReferencedHost()); + if (uuri.getScheme().equals("dns")) { + h = getHostFor("dns:"); + } else { + h = getHostFor(uuri.getReferencedHost()); + } } catch (URIException e) { e.printStackTrace(); } diff --git a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java index 30f7b451..eec8ee14 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java +++ b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java @@ -103,7 +103,6 @@ import org.archive.modules.extractor.LinkContext; import org.archive.modules.net.CrawlHost; import org.archive.modules.net.CrawlServer; import org.archive.modules.net.ServerCache; -import org.archive.net.UURI; import org.archive.util.Recorder; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.Lifecycle; @@ -831,7 +830,7 @@ public class FetchHTTP extends Processor implements Lifecycle { return false; } - CrawlHost host = getHostFor(curi.getUURI()); + CrawlHost host = serverCache.getHostFor(curi.getUURI()); if (host.getIP() == null && host.hasBeenLookedUp()) { curi.setFetchStatus(S_DOMAIN_PREREQUISITE_FAILURE); return false; @@ -1507,22 +1506,4 @@ public class FetchHTTP extends Processor implements Lifecycle { return null; } } - - /** - * Get the {@link CrawlHost} associated with curi. - * - * @param uuri - * CandidateURI we're to return Host for. - * @return CandidateURI instance that matches the passed Host name. - */ - private CrawlHost getHostFor(UURI uuri) { - CrawlHost h = null; - try { - h = serverCache.getHostFor(uuri.getReferencedHost()); - } catch (URIException e) { - e.printStackTrace(); - } - return h; - } - } diff --git a/modules/src/main/java/org/archive/modules/net/ServerCache.java b/modules/src/main/java/org/archive/modules/net/ServerCache.java index 4efb50d5..0a60c87e 100644 --- a/modules/src/main/java/org/archive/modules/net/ServerCache.java +++ b/modules/src/main/java/org/archive/modules/net/ServerCache.java @@ -20,6 +20,7 @@ package org.archive.modules.net; import org.apache.commons.collections.Closure; +import org.archive.net.UURI; /** * Interface for crawl-global registry of CrawlServer (host:port) @@ -32,6 +33,8 @@ public interface ServerCache { CrawlHost getHostFor(String host); + CrawlHost getHostFor(UURI uuri); + CrawlServer getServerFor(String serverKey); diff --git a/modules/src/main/java/org/archive/modules/net/ServerCacheUtil.java b/modules/src/main/java/org/archive/modules/net/ServerCacheUtil.java index b68399d3..b591d87e 100644 --- a/modules/src/main/java/org/archive/modules/net/ServerCacheUtil.java +++ b/modules/src/main/java/org/archive/modules/net/ServerCacheUtil.java @@ -33,17 +33,6 @@ public class ServerCacheUtil { private ServerCacheUtil() { } - - public static CrawlHost getHostFor(ServerCache cache, UURI uuri) { - try { - return cache.getHostFor(uuri.getReferencedHost()); - } catch (URIException e) { - logger.log(Level.SEVERE, uuri.toString(), e); - return null; - } - } - - public static CrawlServer getServerFor(ServerCache cache, UURI uuri) { CrawlServer cs = null; try { diff --git a/modules/src/main/java/org/archive/modules/writer/Kw3WriterProcessor.java b/modules/src/main/java/org/archive/modules/writer/Kw3WriterProcessor.java index c1991085..ce97f6bd 100644 --- a/modules/src/main/java/org/archive/modules/writer/Kw3WriterProcessor.java +++ b/modules/src/main/java/org/archive/modules/writer/Kw3WriterProcessor.java @@ -43,11 +43,10 @@ import java.util.logging.Level; import java.util.logging.Logger; import org.archive.io.ReplayInputStream; -import org.archive.modules.Processor; import org.archive.modules.CrawlURI; +import org.archive.modules.Processor; import org.archive.modules.net.CrawlHost; import org.archive.modules.net.ServerCache; -import org.archive.modules.net.ServerCacheUtil; import org.archive.spring.ConfigPath; import org.springframework.beans.factory.annotation.Autowired; @@ -425,7 +424,7 @@ public class Kw3WriterProcessor extends Processor { } private String getHostAddress(CrawlURI curi) { - CrawlHost h = ServerCacheUtil.getHostFor(serverCache, curi.getUURI()); + CrawlHost h = serverCache.getHostFor(curi.getUURI()); if (h == null) { throw new NullPointerException("Crawlhost is null for " + curi + " " + curi.getVia()); diff --git a/modules/src/main/java/org/archive/modules/writer/WriterPoolProcessor.java b/modules/src/main/java/org/archive/modules/writer/WriterPoolProcessor.java index 9f827d38..bbde604e 100644 --- a/modules/src/main/java/org/archive/modules/writer/WriterPoolProcessor.java +++ b/modules/src/main/java/org/archive/modules/writer/WriterPoolProcessor.java @@ -45,7 +45,6 @@ import org.archive.modules.Processor; import org.archive.modules.deciderules.recrawl.IdenticalDigestDecideRule; import org.archive.modules.net.CrawlHost; import org.archive.modules.net.ServerCache; -import org.archive.modules.net.ServerCacheUtil; import org.archive.spring.ConfigPath; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.Lifecycle; @@ -332,7 +331,7 @@ implements Lifecycle { // otherwise, host referenced in URI // TODO:FIXME: have fetcher insert exact IP contacted into curi, // use that rather than inferred by CrawlHost lookup - CrawlHost h = ServerCacheUtil.getHostFor(getServerCache(), curi.getUURI()); + CrawlHost h = getServerCache().getHostFor(curi.getUURI()); if (h == null) { throw new NullPointerException("Crawlhost is null for " + curi + " " + curi.getVia()); diff --git a/modules/src/test/java/org/archive/modules/net/ServerCacheTest.java b/modules/src/test/java/org/archive/modules/net/ServerCacheTest.java index 2320cd1b..2ebf1031 100644 --- a/modules/src/test/java/org/archive/modules/net/ServerCacheTest.java +++ b/modules/src/test/java/org/archive/modules/net/ServerCacheTest.java @@ -55,7 +55,7 @@ public class ServerCacheTest extends TestCase { throws URIException { UURI uuri = UURIFactory.getInstance(uri); ServerCacheUtil.getServerFor(servers, uuri); - ServerCacheUtil.getHostFor(servers, uuri);; + servers.getHostFor(uuri); assertTrue("cache lost server", servers.containsServer(CrawlServer.getServerKey(uuri))); assertTrue("cache lost host",