diff --git a/contrib/src/main/java/org/archive/util/OneLineSimpleLayout.java b/commons/src/main/java/org/archive/util/OneLineSimpleLayout.java similarity index 100% rename from contrib/src/main/java/org/archive/util/OneLineSimpleLayout.java rename to commons/src/main/java/org/archive/util/OneLineSimpleLayout.java diff --git a/engine/src/test/java/org/archive/crawler/selftest/StatisticsSelfTest.java b/engine/src/test/java/org/archive/crawler/selftest/StatisticsSelfTest.java index d7d5d10e..8d880bf3 100644 --- a/engine/src/test/java/org/archive/crawler/selftest/StatisticsSelfTest.java +++ b/engine/src/test/java/org/archive/crawler/selftest/StatisticsSelfTest.java @@ -47,14 +47,14 @@ public class StatisticsSelfTest extends SelfTestBase { protected void verifyWarcStats() { StatisticsTracker stats = heritrix.getEngine().getJob("selftest-job").getCrawlController().getStatisticsTracker(); assertNotNull(stats); - assertEquals(14, (long) stats.getCrawledBytes().get(CrawledBytesHistotable.WARC_NOVEL_URLS)); + assertEquals(13, (long) stats.getCrawledBytes().get(CrawledBytesHistotable.WARC_NOVEL_URLS)); assertEquals(12669, (long) stats.getCrawledBytes().get(CrawledBytesHistotable.WARC_NOVEL_CONTENT_BYTES) - stats.getBytesPerHost("dns:")); assertEquals(3, (long) stats.getServerCache().getHostFor("127.0.0.1").getSubstats().get(CrawledBytesHistotable.WARC_NOVEL_URLS)); assertEquals(2942, (long) stats.getServerCache().getHostFor("127.0.0.1").getSubstats().get(CrawledBytesHistotable.WARC_NOVEL_CONTENT_BYTES)); assertEquals(10, (long) stats.getServerCache().getHostFor("localhost").getSubstats().get(CrawledBytesHistotable.WARC_NOVEL_URLS)); assertEquals(9727, (long) stats.getServerCache().getHostFor("localhost").getSubstats().get(CrawledBytesHistotable.WARC_NOVEL_CONTENT_BYTES)); - assertEquals(1, (long) stats.getServerCache().getHostFor("dns:").getSubstats().get(CrawledBytesHistotable.WARC_NOVEL_URLS)); + assertEquals(0, (long) stats.getServerCache().getHostFor("dns:").getSubstats().get(CrawledBytesHistotable.WARC_NOVEL_URLS)); } protected void verifySourceStats() throws Exception { @@ -77,7 +77,7 @@ public class StatisticsSelfTest extends SelfTestBase { assertEquals(9727l, (long) sourceStats.get("novel") - stats.getBytesPerHost("dns:")); assertEquals(11l, (long) sourceStats.get("novelCount")); assertEquals(9727l, (long) sourceStats.get("warcNovelContentBytes") - stats.getBytesPerHost("dns:")); - assertEquals(11l, (long) sourceStats.get("warcNovelUrls")); + assertEquals(10l, (long) sourceStats.get("warcNovelUrls")); } } diff --git a/engine/testdata/selftest/conf/selftest-crawler-beans.cxml b/engine/testdata/selftest/conf/selftest-crawler-beans.cxml index 2332bfb2..f73d0863 100644 --- a/engine/testdata/selftest/conf/selftest-crawler-beans.cxml +++ b/engine/testdata/selftest/conf/selftest-crawler-beans.cxml @@ -73,7 +73,9 @@ crawlController.pauseAtStart=false - + + + diff --git a/modules/src/main/java/org/archive/modules/fetcher/FetchDNS.java b/modules/src/main/java/org/archive/modules/fetcher/FetchDNS.java index 1c6dcac2..ea34cbde 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/FetchDNS.java +++ b/modules/src/main/java/org/archive/modules/fetcher/FetchDNS.java @@ -72,15 +72,35 @@ public class FetchDNS extends Processor { protected InetAddress serverInetAddr = null; /** - * If a DNS lookup fails, whether or not to fallback to InetAddress + * If a DNS lookup fails, whether or not to fall back to InetAddress * resolution, which may use local 'hosts' files or other mechanisms. */ - protected boolean acceptNonDnsResolves = false; + { + setAcceptNonDnsResolves(false); + } public boolean getAcceptNonDnsResolves() { - return acceptNonDnsResolves; + return (Boolean) kp.get("acceptNonDnsResolves"); } public void setAcceptNonDnsResolves(boolean acceptNonDnsResolves) { - this.acceptNonDnsResolves = acceptNonDnsResolves; + kp.put("acceptNonDnsResolves",acceptNonDnsResolves); + } + + /** + * Optionally, only allow InetAddress resolution, precisely because it + * may use local 'hosts' files or other mechanisms. + * + * This should not generally be used in production as it will prevent + * DNS lookups from being recorded properly. + * + */ + { + setDisableJavaDnsResolves(false); + } + public boolean getDisableJavaDnsResolves() { + return (Boolean) kp.get("disableJavaDnsResolves"); + } + public void setDisableJavaDnsResolves(boolean disableJavaDnsResolves) { + kp.put("disableJavaDnsResolves",disableJavaDnsResolves); } /** @@ -158,10 +178,13 @@ public class FetchDNS extends Processor { // Try to get the records for this host (assume domain name) // TODO: Bug #935119 concerns potential hang here String lookupName = dnsName.endsWith(".") ? dnsName : dnsName + "."; - try { - rrecordSet = (new Lookup(lookupName, TypeType, ClassType)).run(); - } catch (TextParseException e) { - rrecordSet = null; + // If we have not disabled JavaDNS, use that: + if (!getDisableJavaDnsResolves()) { + try { + rrecordSet = (new Lookup(lookupName, TypeType, ClassType)).run(); + } catch (TextParseException e) { + rrecordSet = null; + } } curi.setContentType("text/dns"); if (rrecordSet != null) { @@ -173,7 +196,7 @@ public class FetchDNS extends Processor { if (logger.isLoggable(Level.FINE)) { logger.fine("Failed find of recordset for " + lookupName); } - if (getAcceptNonDnsResolves()||"localhost".equals(dnsName)) { + if (getAcceptNonDnsResolves()||getDisableJavaDnsResolves()||"localhost".equals(dnsName)) { // Do lookup that bypasses javadns. InetAddress address = null; try { @@ -184,6 +207,7 @@ public class FetchDNS extends Processor { if (address != null) { targetHost.setIP(address, DEFAULT_TTL_FOR_NON_DNS_RESOLVES); curi.setFetchStatus(S_GETBYNAME_SUCCESS); + curi.setContentSize(0); if (logger.isLoggable(Level.FINE)) { logger.fine("Found address for " + dnsName + " using native dns."); diff --git a/pom.xml b/pom.xml index c3d5c4ee..a86861c7 100644 --- a/pom.xml +++ b/pom.xml @@ -372,8 +372,19 @@ http://maven.apache.org/guides/mini/guide-m1-m2.html UTF-8 - - + + org.apache.maven.plugins + maven-surefire-plugin + 2.9 + + -Xmx1g + + ${project.build.directory}/heritrix-junit-tests + localhost + + + + UTF-8