diff --git a/contrib/src/main/java/org/archive/util/OneLineSimpleLayout.java b/commons/src/main/java/org/archive/util/OneLineSimpleLayout.java
similarity index 100%
rename from contrib/src/main/java/org/archive/util/OneLineSimpleLayout.java
rename to commons/src/main/java/org/archive/util/OneLineSimpleLayout.java
diff --git a/engine/src/test/java/org/archive/crawler/selftest/StatisticsSelfTest.java b/engine/src/test/java/org/archive/crawler/selftest/StatisticsSelfTest.java
index d7d5d10e..8d880bf3 100644
--- a/engine/src/test/java/org/archive/crawler/selftest/StatisticsSelfTest.java
+++ b/engine/src/test/java/org/archive/crawler/selftest/StatisticsSelfTest.java
@@ -47,14 +47,14 @@ public class StatisticsSelfTest extends SelfTestBase {
protected void verifyWarcStats() {
StatisticsTracker stats = heritrix.getEngine().getJob("selftest-job").getCrawlController().getStatisticsTracker();
assertNotNull(stats);
- assertEquals(14, (long) stats.getCrawledBytes().get(CrawledBytesHistotable.WARC_NOVEL_URLS));
+ assertEquals(13, (long) stats.getCrawledBytes().get(CrawledBytesHistotable.WARC_NOVEL_URLS));
assertEquals(12669, (long) stats.getCrawledBytes().get(CrawledBytesHistotable.WARC_NOVEL_CONTENT_BYTES) - stats.getBytesPerHost("dns:"));
assertEquals(3, (long) stats.getServerCache().getHostFor("127.0.0.1").getSubstats().get(CrawledBytesHistotable.WARC_NOVEL_URLS));
assertEquals(2942, (long) stats.getServerCache().getHostFor("127.0.0.1").getSubstats().get(CrawledBytesHistotable.WARC_NOVEL_CONTENT_BYTES));
assertEquals(10, (long) stats.getServerCache().getHostFor("localhost").getSubstats().get(CrawledBytesHistotable.WARC_NOVEL_URLS));
assertEquals(9727, (long) stats.getServerCache().getHostFor("localhost").getSubstats().get(CrawledBytesHistotable.WARC_NOVEL_CONTENT_BYTES));
- assertEquals(1, (long) stats.getServerCache().getHostFor("dns:").getSubstats().get(CrawledBytesHistotable.WARC_NOVEL_URLS));
+ assertEquals(0, (long) stats.getServerCache().getHostFor("dns:").getSubstats().get(CrawledBytesHistotable.WARC_NOVEL_URLS));
}
protected void verifySourceStats() throws Exception {
@@ -77,7 +77,7 @@ public class StatisticsSelfTest extends SelfTestBase {
assertEquals(9727l, (long) sourceStats.get("novel") - stats.getBytesPerHost("dns:"));
assertEquals(11l, (long) sourceStats.get("novelCount"));
assertEquals(9727l, (long) sourceStats.get("warcNovelContentBytes") - stats.getBytesPerHost("dns:"));
- assertEquals(11l, (long) sourceStats.get("warcNovelUrls"));
+ assertEquals(10l, (long) sourceStats.get("warcNovelUrls"));
}
}
diff --git a/engine/testdata/selftest/conf/selftest-crawler-beans.cxml b/engine/testdata/selftest/conf/selftest-crawler-beans.cxml
index 2332bfb2..f73d0863 100644
--- a/engine/testdata/selftest/conf/selftest-crawler-beans.cxml
+++ b/engine/testdata/selftest/conf/selftest-crawler-beans.cxml
@@ -73,7 +73,9 @@ crawlController.pauseAtStart=false
-
+
+
+
diff --git a/modules/src/main/java/org/archive/modules/fetcher/FetchDNS.java b/modules/src/main/java/org/archive/modules/fetcher/FetchDNS.java
index 1c6dcac2..ea34cbde 100644
--- a/modules/src/main/java/org/archive/modules/fetcher/FetchDNS.java
+++ b/modules/src/main/java/org/archive/modules/fetcher/FetchDNS.java
@@ -72,15 +72,35 @@ public class FetchDNS extends Processor {
protected InetAddress serverInetAddr = null;
/**
- * If a DNS lookup fails, whether or not to fallback to InetAddress
+ * If a DNS lookup fails, whether or not to fall back to InetAddress
* resolution, which may use local 'hosts' files or other mechanisms.
*/
- protected boolean acceptNonDnsResolves = false;
+ {
+ setAcceptNonDnsResolves(false);
+ }
public boolean getAcceptNonDnsResolves() {
- return acceptNonDnsResolves;
+ return (Boolean) kp.get("acceptNonDnsResolves");
}
public void setAcceptNonDnsResolves(boolean acceptNonDnsResolves) {
- this.acceptNonDnsResolves = acceptNonDnsResolves;
+ kp.put("acceptNonDnsResolves",acceptNonDnsResolves);
+ }
+
+ /**
+ * Optionally, only allow InetAddress resolution, precisely because it
+ * may use local 'hosts' files or other mechanisms.
+ *
+ * This should not generally be used in production as it will prevent
+ * DNS lookups from being recorded properly.
+ *
+ */
+ {
+ setDisableJavaDnsResolves(false);
+ }
+ public boolean getDisableJavaDnsResolves() {
+ return (Boolean) kp.get("disableJavaDnsResolves");
+ }
+ public void setDisableJavaDnsResolves(boolean disableJavaDnsResolves) {
+ kp.put("disableJavaDnsResolves",disableJavaDnsResolves);
}
/**
@@ -158,10 +178,13 @@ public class FetchDNS extends Processor {
// Try to get the records for this host (assume domain name)
// TODO: Bug #935119 concerns potential hang here
String lookupName = dnsName.endsWith(".") ? dnsName : dnsName + ".";
- try {
- rrecordSet = (new Lookup(lookupName, TypeType, ClassType)).run();
- } catch (TextParseException e) {
- rrecordSet = null;
+ // If we have not disabled JavaDNS, use that:
+ if (!getDisableJavaDnsResolves()) {
+ try {
+ rrecordSet = (new Lookup(lookupName, TypeType, ClassType)).run();
+ } catch (TextParseException e) {
+ rrecordSet = null;
+ }
}
curi.setContentType("text/dns");
if (rrecordSet != null) {
@@ -173,7 +196,7 @@ public class FetchDNS extends Processor {
if (logger.isLoggable(Level.FINE)) {
logger.fine("Failed find of recordset for " + lookupName);
}
- if (getAcceptNonDnsResolves()||"localhost".equals(dnsName)) {
+ if (getAcceptNonDnsResolves()||getDisableJavaDnsResolves()||"localhost".equals(dnsName)) {
// Do lookup that bypasses javadns.
InetAddress address = null;
try {
@@ -184,6 +207,7 @@ public class FetchDNS extends Processor {
if (address != null) {
targetHost.setIP(address, DEFAULT_TTL_FOR_NON_DNS_RESOLVES);
curi.setFetchStatus(S_GETBYNAME_SUCCESS);
+ curi.setContentSize(0);
if (logger.isLoggable(Level.FINE)) {
logger.fine("Found address for " + dnsName +
" using native dns.");
diff --git a/pom.xml b/pom.xml
index c3d5c4ee..a86861c7 100644
--- a/pom.xml
+++ b/pom.xml
@@ -372,8 +372,19 @@ http://maven.apache.org/guides/mini/guide-m1-m2.html
UTF-8
-
-
+
+ org.apache.maven.plugins
+ maven-surefire-plugin
+ 2.9
+
+ -Xmx1g
+
+ ${project.build.directory}/heritrix-junit-tests
+ localhost
+
+
+
+
UTF-8