From 1de6f7d2a08ce5ad42122408773c1f6b3d2ba5d5 Mon Sep 17 00:00:00 2001 From: Andrew Jackson Date: Mon, 9 Oct 2017 09:54:16 +0100 Subject: [PATCH 01/10] Update Travis CI file to suitable Java 7 + 8 variants. --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 16244cd6..08e91b4b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,9 +3,9 @@ sudo: false language: java jdk: - - oraclejdk7 - - oraclejdk8 - openjdk7 + - openjdk8 + - oraclejdk8 before_install: - "export JAVA_OPTS=-Xmx1500m" From 401e27fc642ec2c2b86ba28273b7faff6656ce0e Mon Sep 17 00:00:00 2001 From: Andrew Jackson Date: Mon, 9 Oct 2017 21:50:05 +0100 Subject: [PATCH 02/10] Adapt test to cope with variation in how localhost gets resolved. --- .../crawler/selftest/StatisticsSelfTest.java | 71 ++++++++++++++++--- 1 file changed, 62 insertions(+), 9 deletions(-) diff --git a/engine/src/test/java/org/archive/crawler/selftest/StatisticsSelfTest.java b/engine/src/test/java/org/archive/crawler/selftest/StatisticsSelfTest.java index d7d5d10e..4adc8586 100644 --- a/engine/src/test/java/org/archive/crawler/selftest/StatisticsSelfTest.java +++ b/engine/src/test/java/org/archive/crawler/selftest/StatisticsSelfTest.java @@ -47,14 +47,56 @@ public class StatisticsSelfTest extends SelfTestBase { protected void verifyWarcStats() { StatisticsTracker stats = heritrix.getEngine().getJob("selftest-job").getCrawlController().getStatisticsTracker(); assertNotNull(stats); - assertEquals(14, (long) stats.getCrawledBytes().get(CrawledBytesHistotable.WARC_NOVEL_URLS)); - assertEquals(12669, (long) stats.getCrawledBytes().get(CrawledBytesHistotable.WARC_NOVEL_CONTENT_BYTES) - stats.getBytesPerHost("dns:")); - assertEquals(3, (long) stats.getServerCache().getHostFor("127.0.0.1").getSubstats().get(CrawledBytesHistotable.WARC_NOVEL_URLS)); - assertEquals(2942, (long) stats.getServerCache().getHostFor("127.0.0.1").getSubstats().get(CrawledBytesHistotable.WARC_NOVEL_CONTENT_BYTES)); - assertEquals(10, (long) stats.getServerCache().getHostFor("localhost").getSubstats().get(CrawledBytesHistotable.WARC_NOVEL_URLS)); - assertEquals(9727, (long) stats.getServerCache().getHostFor("localhost").getSubstats().get(CrawledBytesHistotable.WARC_NOVEL_CONTENT_BYTES)); - assertEquals(1, (long) stats.getServerCache().getHostFor("dns:").getSubstats().get(CrawledBytesHistotable.WARC_NOVEL_URLS)); + long dnsCount = stats.getBytesPerHost("dns:"); + // Cope when DNS lookup may or may not be used for localhost: + if (dnsCount == -1) { + // No DNS record: + assertEquals(13, (long) stats.getCrawledBytes() + .get(CrawledBytesHistotable.WARC_NOVEL_URLS)); + assertEquals(12669, (long) stats.getCrawledBytes() + .get(CrawledBytesHistotable.WARC_NOVEL_CONTENT_BYTES)); + + assertEquals(3, + (long) stats.getServerCache().getHostFor("127.0.0.1") + .getSubstats() + .get(CrawledBytesHistotable.WARC_NOVEL_URLS)); + assertEquals(2942, (long) stats.getServerCache() + .getHostFor("127.0.0.1").getSubstats() + .get(CrawledBytesHistotable.WARC_NOVEL_CONTENT_BYTES)); + assertEquals(10, + (long) stats.getServerCache().getHostFor("localhost") + .getSubstats() + .get(CrawledBytesHistotable.WARC_NOVEL_URLS)); + assertEquals(9727, (long) stats.getServerCache() + .getHostFor("localhost").getSubstats() + .get(CrawledBytesHistotable.WARC_NOVEL_CONTENT_BYTES)); + } else { + // DNS record exists: + assertEquals(14, (long) stats.getCrawledBytes() + .get(CrawledBytesHistotable.WARC_NOVEL_URLS)); + assertEquals(12669, + (long) stats.getCrawledBytes() + .get(CrawledBytesHistotable.WARC_NOVEL_CONTENT_BYTES) + - stats.getBytesPerHost("dns:")); + + assertEquals(3, + (long) stats.getServerCache().getHostFor("127.0.0.1") + .getSubstats() + .get(CrawledBytesHistotable.WARC_NOVEL_URLS)); + assertEquals(2942, (long) stats.getServerCache() + .getHostFor("127.0.0.1").getSubstats() + .get(CrawledBytesHistotable.WARC_NOVEL_CONTENT_BYTES)); + assertEquals(10, + (long) stats.getServerCache().getHostFor("localhost") + .getSubstats() + .get(CrawledBytesHistotable.WARC_NOVEL_URLS)); + assertEquals(9727, (long) stats.getServerCache() + .getHostFor("localhost").getSubstats() + .get(CrawledBytesHistotable.WARC_NOVEL_CONTENT_BYTES)); + assertEquals(1, (long) stats.getServerCache().getHostFor("dns:") + .getSubstats().get(CrawledBytesHistotable.WARC_NOVEL_URLS)); + } } protected void verifySourceStats() throws Exception { @@ -76,8 +118,19 @@ public class StatisticsSelfTest extends SelfTestBase { assertEquals(4, sourceStats.keySet().size()); assertEquals(9727l, (long) sourceStats.get("novel") - stats.getBytesPerHost("dns:")); assertEquals(11l, (long) sourceStats.get("novelCount")); - assertEquals(9727l, (long) sourceStats.get("warcNovelContentBytes") - stats.getBytesPerHost("dns:")); - assertEquals(11l, (long) sourceStats.get("warcNovelUrls")); + long dnsCount = stats.getBytesPerHost("dns:"); + // Cope when DNS lookup may or may not be used for localhost: + if (dnsCount == -1) { + // No DNS record: + assertEquals(9727l, + (long) sourceStats.get("warcNovelContentBytes")); + assertEquals(10l, (long) sourceStats.get("warcNovelUrls")); + } else { + // DNS record exists: + assertEquals(9727l, + (long) sourceStats.get("warcNovelContentBytes") - dnsCount); + assertEquals(11l, (long) sourceStats.get("warcNovelUrls")); + } } } From a5381440fd9ebc466fc92ed8fcc7eae82c83a299 Mon Sep 17 00:00:00 2001 From: Andrew Jackson Date: Mon, 9 Oct 2017 22:13:53 +0100 Subject: [PATCH 03/10] Use build folder for temp files as tmp gets wiped on Travis CI sometimes. --- pom.xml | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index c3d5c4ee..a86861c7 100644 --- a/pom.xml +++ b/pom.xml @@ -372,8 +372,19 @@ http://maven.apache.org/guides/mini/guide-m1-m2.html UTF-8 - - + + org.apache.maven.plugins + maven-surefire-plugin + 2.9 + + -Xmx1g + + ${project.build.directory}/heritrix-junit-tests + localhost + + + + UTF-8 From 46fa54da0c081c5e3e4bcb21281ea39ca3f8d263 Mon Sep 17 00:00:00 2001 From: Andrew Jackson Date: Wed, 11 Oct 2017 17:08:15 +0100 Subject: [PATCH 04/10] Set an appropriate Content Size when using non-DNS resolves. --- modules/src/main/java/org/archive/modules/fetcher/FetchDNS.java | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/src/main/java/org/archive/modules/fetcher/FetchDNS.java b/modules/src/main/java/org/archive/modules/fetcher/FetchDNS.java index 1c6dcac2..26b562c1 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/FetchDNS.java +++ b/modules/src/main/java/org/archive/modules/fetcher/FetchDNS.java @@ -184,6 +184,7 @@ public class FetchDNS extends Processor { if (address != null) { targetHost.setIP(address, DEFAULT_TTL_FOR_NON_DNS_RESOLVES); curi.setFetchStatus(S_GETBYNAME_SUCCESS); + curi.setContentSize(0); if (logger.isLoggable(Level.FINE)) { logger.fine("Found address for " + dnsName + " using native dns."); From 09ee95f37bcd48bde6d39cf81c3ab9e58fa8eeef Mon Sep 17 00:00:00 2001 From: Andrew Jackson Date: Wed, 11 Oct 2017 23:30:17 +0100 Subject: [PATCH 05/10] Allow results with or without DNS lookup of localhost. --- .../crawler/selftest/StatisticsSelfTest.java | 85 ++++++------------- 1 file changed, 27 insertions(+), 58 deletions(-) diff --git a/engine/src/test/java/org/archive/crawler/selftest/StatisticsSelfTest.java b/engine/src/test/java/org/archive/crawler/selftest/StatisticsSelfTest.java index 4adc8586..f8612870 100644 --- a/engine/src/test/java/org/archive/crawler/selftest/StatisticsSelfTest.java +++ b/engine/src/test/java/org/archive/crawler/selftest/StatisticsSelfTest.java @@ -48,55 +48,31 @@ public class StatisticsSelfTest extends SelfTestBase { StatisticsTracker stats = heritrix.getEngine().getJob("selftest-job").getCrawlController().getStatisticsTracker(); assertNotNull(stats); - long dnsCount = stats.getBytesPerHost("dns:"); // Cope when DNS lookup may or may not be used for localhost: - if (dnsCount == -1) { - // No DNS record: - assertEquals(13, (long) stats.getCrawledBytes() - .get(CrawledBytesHistotable.WARC_NOVEL_URLS)); - assertEquals(12669, (long) stats.getCrawledBytes() - .get(CrawledBytesHistotable.WARC_NOVEL_CONTENT_BYTES)); + long novelUrls = (long) stats.getCrawledBytes() + .get(CrawledBytesHistotable.WARC_NOVEL_URLS); + assertTrue(novelUrls == 14 || novelUrls == 13); + assertEquals(12669, + (long) stats.getCrawledBytes() + .get(CrawledBytesHistotable.WARC_NOVEL_CONTENT_BYTES) + - stats.getBytesPerHost("dns:")); - assertEquals(3, - (long) stats.getServerCache().getHostFor("127.0.0.1") - .getSubstats() - .get(CrawledBytesHistotable.WARC_NOVEL_URLS)); - assertEquals(2942, (long) stats.getServerCache() - .getHostFor("127.0.0.1").getSubstats() - .get(CrawledBytesHistotable.WARC_NOVEL_CONTENT_BYTES)); - assertEquals(10, - (long) stats.getServerCache().getHostFor("localhost") - .getSubstats() - .get(CrawledBytesHistotable.WARC_NOVEL_URLS)); - assertEquals(9727, (long) stats.getServerCache() - .getHostFor("localhost").getSubstats() - .get(CrawledBytesHistotable.WARC_NOVEL_CONTENT_BYTES)); - } else { - // DNS record exists: - assertEquals(14, (long) stats.getCrawledBytes() - .get(CrawledBytesHistotable.WARC_NOVEL_URLS)); - assertEquals(12669, - (long) stats.getCrawledBytes() - .get(CrawledBytesHistotable.WARC_NOVEL_CONTENT_BYTES) - - stats.getBytesPerHost("dns:")); - - assertEquals(3, - (long) stats.getServerCache().getHostFor("127.0.0.1") - .getSubstats() - .get(CrawledBytesHistotable.WARC_NOVEL_URLS)); - assertEquals(2942, (long) stats.getServerCache() - .getHostFor("127.0.0.1").getSubstats() - .get(CrawledBytesHistotable.WARC_NOVEL_CONTENT_BYTES)); - assertEquals(10, - (long) stats.getServerCache().getHostFor("localhost") - .getSubstats() - .get(CrawledBytesHistotable.WARC_NOVEL_URLS)); - assertEquals(9727, (long) stats.getServerCache() - .getHostFor("localhost").getSubstats() - .get(CrawledBytesHistotable.WARC_NOVEL_CONTENT_BYTES)); - assertEquals(1, (long) stats.getServerCache().getHostFor("dns:") - .getSubstats().get(CrawledBytesHistotable.WARC_NOVEL_URLS)); - } + assertEquals(3, (long) stats.getServerCache().getHostFor("127.0.0.1") + .getSubstats().get(CrawledBytesHistotable.WARC_NOVEL_URLS)); + assertEquals(2942, + (long) stats.getServerCache().getHostFor("127.0.0.1") + .getSubstats() + .get(CrawledBytesHistotable.WARC_NOVEL_CONTENT_BYTES)); + assertEquals(10, (long) stats.getServerCache().getHostFor("localhost") + .getSubstats().get(CrawledBytesHistotable.WARC_NOVEL_URLS)); + assertEquals(9727, + (long) stats.getServerCache().getHostFor("localhost") + .getSubstats() + .get(CrawledBytesHistotable.WARC_NOVEL_CONTENT_BYTES)); + // Cope when DNS lookup may or may not be used for localhost: + long novelDnsUrls = (long) stats.getServerCache().getHostFor("dns:") + .getSubstats().get(CrawledBytesHistotable.WARC_NOVEL_URLS); + assertTrue(novelDnsUrls == 1 | novelDnsUrls == 0); } protected void verifySourceStats() throws Exception { @@ -119,18 +95,11 @@ public class StatisticsSelfTest extends SelfTestBase { assertEquals(9727l, (long) sourceStats.get("novel") - stats.getBytesPerHost("dns:")); assertEquals(11l, (long) sourceStats.get("novelCount")); long dnsCount = stats.getBytesPerHost("dns:"); - // Cope when DNS lookup may or may not be used for localhost: - if (dnsCount == -1) { - // No DNS record: - assertEquals(9727l, - (long) sourceStats.get("warcNovelContentBytes")); - assertEquals(10l, (long) sourceStats.get("warcNovelUrls")); - } else { - // DNS record exists: - assertEquals(9727l, + assertEquals(9727l, (long) sourceStats.get("warcNovelContentBytes") - dnsCount); - assertEquals(11l, (long) sourceStats.get("warcNovelUrls")); - } + long urls = (long) sourceStats.get("warcNovelUrls"); + // Cope when DNS lookup may or may not be used for localhost: + assertTrue(urls == 10 || urls == 11); } } From b0594ac0f48725e2090f391b8365ac4650995d20 Mon Sep 17 00:00:00 2001 From: Andrew Jackson Date: Thu, 12 Oct 2017 21:29:34 +0100 Subject: [PATCH 06/10] Allow JavaDNS to be disabled and use that to avoid issues in StatisticsSelfTest. --- .../crawler/selftest/StatisticsSelfTest.java | 40 +++++-------------- .../selftest/conf/selftest-crawler-beans.cxml | 3 +- .../org/archive/modules/fetcher/FetchDNS.java | 23 +++++++++-- 3 files changed, 30 insertions(+), 36 deletions(-) diff --git a/engine/src/test/java/org/archive/crawler/selftest/StatisticsSelfTest.java b/engine/src/test/java/org/archive/crawler/selftest/StatisticsSelfTest.java index f8612870..8d880bf3 100644 --- a/engine/src/test/java/org/archive/crawler/selftest/StatisticsSelfTest.java +++ b/engine/src/test/java/org/archive/crawler/selftest/StatisticsSelfTest.java @@ -47,32 +47,14 @@ public class StatisticsSelfTest extends SelfTestBase { protected void verifyWarcStats() { StatisticsTracker stats = heritrix.getEngine().getJob("selftest-job").getCrawlController().getStatisticsTracker(); assertNotNull(stats); + assertEquals(13, (long) stats.getCrawledBytes().get(CrawledBytesHistotable.WARC_NOVEL_URLS)); + assertEquals(12669, (long) stats.getCrawledBytes().get(CrawledBytesHistotable.WARC_NOVEL_CONTENT_BYTES) - stats.getBytesPerHost("dns:")); - // Cope when DNS lookup may or may not be used for localhost: - long novelUrls = (long) stats.getCrawledBytes() - .get(CrawledBytesHistotable.WARC_NOVEL_URLS); - assertTrue(novelUrls == 14 || novelUrls == 13); - assertEquals(12669, - (long) stats.getCrawledBytes() - .get(CrawledBytesHistotable.WARC_NOVEL_CONTENT_BYTES) - - stats.getBytesPerHost("dns:")); - - assertEquals(3, (long) stats.getServerCache().getHostFor("127.0.0.1") - .getSubstats().get(CrawledBytesHistotable.WARC_NOVEL_URLS)); - assertEquals(2942, - (long) stats.getServerCache().getHostFor("127.0.0.1") - .getSubstats() - .get(CrawledBytesHistotable.WARC_NOVEL_CONTENT_BYTES)); - assertEquals(10, (long) stats.getServerCache().getHostFor("localhost") - .getSubstats().get(CrawledBytesHistotable.WARC_NOVEL_URLS)); - assertEquals(9727, - (long) stats.getServerCache().getHostFor("localhost") - .getSubstats() - .get(CrawledBytesHistotable.WARC_NOVEL_CONTENT_BYTES)); - // Cope when DNS lookup may or may not be used for localhost: - long novelDnsUrls = (long) stats.getServerCache().getHostFor("dns:") - .getSubstats().get(CrawledBytesHistotable.WARC_NOVEL_URLS); - assertTrue(novelDnsUrls == 1 | novelDnsUrls == 0); + assertEquals(3, (long) stats.getServerCache().getHostFor("127.0.0.1").getSubstats().get(CrawledBytesHistotable.WARC_NOVEL_URLS)); + assertEquals(2942, (long) stats.getServerCache().getHostFor("127.0.0.1").getSubstats().get(CrawledBytesHistotable.WARC_NOVEL_CONTENT_BYTES)); + assertEquals(10, (long) stats.getServerCache().getHostFor("localhost").getSubstats().get(CrawledBytesHistotable.WARC_NOVEL_URLS)); + assertEquals(9727, (long) stats.getServerCache().getHostFor("localhost").getSubstats().get(CrawledBytesHistotable.WARC_NOVEL_CONTENT_BYTES)); + assertEquals(0, (long) stats.getServerCache().getHostFor("dns:").getSubstats().get(CrawledBytesHistotable.WARC_NOVEL_URLS)); } protected void verifySourceStats() throws Exception { @@ -94,12 +76,8 @@ public class StatisticsSelfTest extends SelfTestBase { assertEquals(4, sourceStats.keySet().size()); assertEquals(9727l, (long) sourceStats.get("novel") - stats.getBytesPerHost("dns:")); assertEquals(11l, (long) sourceStats.get("novelCount")); - long dnsCount = stats.getBytesPerHost("dns:"); - assertEquals(9727l, - (long) sourceStats.get("warcNovelContentBytes") - dnsCount); - long urls = (long) sourceStats.get("warcNovelUrls"); - // Cope when DNS lookup may or may not be used for localhost: - assertTrue(urls == 10 || urls == 11); + assertEquals(9727l, (long) sourceStats.get("warcNovelContentBytes") - stats.getBytesPerHost("dns:")); + assertEquals(10l, (long) sourceStats.get("warcNovelUrls")); } } diff --git a/engine/testdata/selftest/conf/selftest-crawler-beans.cxml b/engine/testdata/selftest/conf/selftest-crawler-beans.cxml index 2332bfb2..4aa0ec67 100644 --- a/engine/testdata/selftest/conf/selftest-crawler-beans.cxml +++ b/engine/testdata/selftest/conf/selftest-crawler-beans.cxml @@ -73,7 +73,8 @@ crawlController.pauseAtStart=false - + + diff --git a/modules/src/main/java/org/archive/modules/fetcher/FetchDNS.java b/modules/src/main/java/org/archive/modules/fetcher/FetchDNS.java index 26b562c1..2cdd3cb9 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/FetchDNS.java +++ b/modules/src/main/java/org/archive/modules/fetcher/FetchDNS.java @@ -83,6 +83,18 @@ public class FetchDNS extends Processor { this.acceptNonDnsResolves = acceptNonDnsResolves; } + /** + * Optionally, only allow InetAddress resolution, precisely because it + * may use local 'hosts' files or other mechanisms. + */ + protected boolean disableJavaDnsResolves = false; + public boolean getDisableJavaDnsResolves() { + return disableJavaDnsResolves; + } + public void setDisableJavaDnsResolves(boolean disableJavaDnsResolves) { + this.disableJavaDnsResolves = disableJavaDnsResolves; + } + /** * Used to do DNS lookups. */ @@ -158,10 +170,13 @@ public class FetchDNS extends Processor { // Try to get the records for this host (assume domain name) // TODO: Bug #935119 concerns potential hang here String lookupName = dnsName.endsWith(".") ? dnsName : dnsName + "."; - try { - rrecordSet = (new Lookup(lookupName, TypeType, ClassType)).run(); - } catch (TextParseException e) { - rrecordSet = null; + // If we have not disabled JavaDNS, use that: + if( ! this.getDisableJavaDnsResolves()) { + try { + rrecordSet = (new Lookup(lookupName, TypeType, ClassType)).run(); + } catch (TextParseException e) { + rrecordSet = null; + } } curi.setContentType("text/dns"); if (rrecordSet != null) { From d85d2a3fe81ed381b9250b8f33e74674d117b0e3 Mon Sep 17 00:00:00 2001 From: Andrew Jackson Date: Thu, 12 Oct 2017 22:10:58 +0100 Subject: [PATCH 07/10] Move OneLineSimpleLayout into commons so all can use it. --- .../src/main/java/org/archive/util/OneLineSimpleLayout.java | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename {contrib => commons}/src/main/java/org/archive/util/OneLineSimpleLayout.java (100%) diff --git a/contrib/src/main/java/org/archive/util/OneLineSimpleLayout.java b/commons/src/main/java/org/archive/util/OneLineSimpleLayout.java similarity index 100% rename from contrib/src/main/java/org/archive/util/OneLineSimpleLayout.java rename to commons/src/main/java/org/archive/util/OneLineSimpleLayout.java From d178c689de968e1513136f23f2625df8b302a428 Mon Sep 17 00:00:00 2001 From: Andrew Jackson Date: Thu, 12 Oct 2017 22:13:00 +0100 Subject: [PATCH 08/10] Add some comments to try to avoid misuse. --- engine/testdata/selftest/conf/selftest-crawler-beans.cxml | 1 + .../src/main/java/org/archive/modules/fetcher/FetchDNS.java | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/engine/testdata/selftest/conf/selftest-crawler-beans.cxml b/engine/testdata/selftest/conf/selftest-crawler-beans.cxml index 4aa0ec67..f73d0863 100644 --- a/engine/testdata/selftest/conf/selftest-crawler-beans.cxml +++ b/engine/testdata/selftest/conf/selftest-crawler-beans.cxml @@ -74,6 +74,7 @@ crawlController.pauseAtStart=false + diff --git a/modules/src/main/java/org/archive/modules/fetcher/FetchDNS.java b/modules/src/main/java/org/archive/modules/fetcher/FetchDNS.java index 2cdd3cb9..47f293c1 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/FetchDNS.java +++ b/modules/src/main/java/org/archive/modules/fetcher/FetchDNS.java @@ -86,6 +86,12 @@ public class FetchDNS extends Processor { /** * Optionally, only allow InetAddress resolution, precisely because it * may use local 'hosts' files or other mechanisms. + * + * This will not work unless you also set allowNonDnsResolves. + * + * This should not generally be used in production as it will prevent + * DNS lookups from being recorded properly. + * */ protected boolean disableJavaDnsResolves = false; public boolean getDisableJavaDnsResolves() { From e7ce1f56b422d119b24e40664f43152eb93730ee Mon Sep 17 00:00:00 2001 From: Andrew Jackson Date: Thu, 12 Oct 2017 23:33:54 +0100 Subject: [PATCH 09/10] Ensure disabling JavaDNS always fails over to the InetAddress approach. --- .../src/main/java/org/archive/modules/fetcher/FetchDNS.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/modules/src/main/java/org/archive/modules/fetcher/FetchDNS.java b/modules/src/main/java/org/archive/modules/fetcher/FetchDNS.java index 47f293c1..9755ab73 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/FetchDNS.java +++ b/modules/src/main/java/org/archive/modules/fetcher/FetchDNS.java @@ -87,8 +87,6 @@ public class FetchDNS extends Processor { * Optionally, only allow InetAddress resolution, precisely because it * may use local 'hosts' files or other mechanisms. * - * This will not work unless you also set allowNonDnsResolves. - * * This should not generally be used in production as it will prevent * DNS lookups from being recorded properly. * @@ -177,7 +175,7 @@ public class FetchDNS extends Processor { // TODO: Bug #935119 concerns potential hang here String lookupName = dnsName.endsWith(".") ? dnsName : dnsName + "."; // If we have not disabled JavaDNS, use that: - if( ! this.getDisableJavaDnsResolves()) { + if (!getDisableJavaDnsResolves()) { try { rrecordSet = (new Lookup(lookupName, TypeType, ClassType)).run(); } catch (TextParseException e) { @@ -194,7 +192,7 @@ public class FetchDNS extends Processor { if (logger.isLoggable(Level.FINE)) { logger.fine("Failed find of recordset for " + lookupName); } - if (getAcceptNonDnsResolves()||"localhost".equals(dnsName)) { + if (getAcceptNonDnsResolves()||getDisableJavaDnsResolves()||"localhost".equals(dnsName)) { // Do lookup that bypasses javadns. InetAddress address = null; try { From aee83dfe26ea5a36a4eb3092380e1b0d7b242aab Mon Sep 17 00:00:00 2001 From: Andrew Jackson Date: Fri, 13 Oct 2017 10:29:15 +0100 Subject: [PATCH 10/10] Allow DNS settings to be overridden via sheets. --- .../org/archive/modules/fetcher/FetchDNS.java | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/modules/src/main/java/org/archive/modules/fetcher/FetchDNS.java b/modules/src/main/java/org/archive/modules/fetcher/FetchDNS.java index 9755ab73..ea34cbde 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/FetchDNS.java +++ b/modules/src/main/java/org/archive/modules/fetcher/FetchDNS.java @@ -72,15 +72,17 @@ public class FetchDNS extends Processor { protected InetAddress serverInetAddr = null; /** - * If a DNS lookup fails, whether or not to fallback to InetAddress + * If a DNS lookup fails, whether or not to fall back to InetAddress * resolution, which may use local 'hosts' files or other mechanisms. */ - protected boolean acceptNonDnsResolves = false; + { + setAcceptNonDnsResolves(false); + } public boolean getAcceptNonDnsResolves() { - return acceptNonDnsResolves; + return (Boolean) kp.get("acceptNonDnsResolves"); } public void setAcceptNonDnsResolves(boolean acceptNonDnsResolves) { - this.acceptNonDnsResolves = acceptNonDnsResolves; + kp.put("acceptNonDnsResolves",acceptNonDnsResolves); } /** @@ -91,12 +93,14 @@ public class FetchDNS extends Processor { * DNS lookups from being recorded properly. * */ - protected boolean disableJavaDnsResolves = false; + { + setDisableJavaDnsResolves(false); + } public boolean getDisableJavaDnsResolves() { - return disableJavaDnsResolves; + return (Boolean) kp.get("disableJavaDnsResolves"); } public void setDisableJavaDnsResolves(boolean disableJavaDnsResolves) { - this.disableJavaDnsResolves = disableJavaDnsResolves; + kp.put("disableJavaDnsResolves",disableJavaDnsResolves); } /**