mirror of
https://github.com/internetarchive/heritrix3.git
synced 2026-09-28 08:35:52 +00:00
Merge pull request #190 from ukwa/fix-ci-and-add-java-8
Allow JavaDNS to be disabled as part of resolving outstanding build and test issues
This commit is contained in:
Noah Levitt
@@ -47,14 +47,14 @@ public class StatisticsSelfTest extends SelfTestBase {
|
||||
protected void verifyWarcStats() {
|
||||
StatisticsTracker stats = heritrix.getEngine().getJob("selftest-job").getCrawlController().getStatisticsTracker();
|
||||
assertNotNull(stats);
|
||||
assertEquals(14, (long) stats.getCrawledBytes().get(CrawledBytesHistotable.WARC_NOVEL_URLS));
|
||||
assertEquals(13, (long) stats.getCrawledBytes().get(CrawledBytesHistotable.WARC_NOVEL_URLS));
|
||||
assertEquals(12669, (long) stats.getCrawledBytes().get(CrawledBytesHistotable.WARC_NOVEL_CONTENT_BYTES) - stats.getBytesPerHost("dns:"));
|
||||
|
||||
assertEquals(3, (long) stats.getServerCache().getHostFor("127.0.0.1").getSubstats().get(CrawledBytesHistotable.WARC_NOVEL_URLS));
|
||||
assertEquals(2942, (long) stats.getServerCache().getHostFor("127.0.0.1").getSubstats().get(CrawledBytesHistotable.WARC_NOVEL_CONTENT_BYTES));
|
||||
assertEquals(10, (long) stats.getServerCache().getHostFor("localhost").getSubstats().get(CrawledBytesHistotable.WARC_NOVEL_URLS));
|
||||
assertEquals(9727, (long) stats.getServerCache().getHostFor("localhost").getSubstats().get(CrawledBytesHistotable.WARC_NOVEL_CONTENT_BYTES));
|
||||
assertEquals(1, (long) stats.getServerCache().getHostFor("dns:").getSubstats().get(CrawledBytesHistotable.WARC_NOVEL_URLS));
|
||||
assertEquals(0, (long) stats.getServerCache().getHostFor("dns:").getSubstats().get(CrawledBytesHistotable.WARC_NOVEL_URLS));
|
||||
}
|
||||
|
||||
protected void verifySourceStats() throws Exception {
|
||||
@@ -77,7 +77,7 @@ public class StatisticsSelfTest extends SelfTestBase {
|
||||
assertEquals(9727l, (long) sourceStats.get("novel") - stats.getBytesPerHost("dns:"));
|
||||
assertEquals(11l, (long) sourceStats.get("novelCount"));
|
||||
assertEquals(9727l, (long) sourceStats.get("warcNovelContentBytes") - stats.getBytesPerHost("dns:"));
|
||||
assertEquals(11l, (long) sourceStats.get("warcNovelUrls"));
|
||||
assertEquals(10l, (long) sourceStats.get("warcNovelUrls"));
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -73,7 +73,9 @@ crawlController.pauseAtStart=false
|
||||
</bean>
|
||||
<bean id="preconditionEnforcer" class="org.archive.crawler.prefetch.PreconditionEnforcer">
|
||||
</bean>
|
||||
<bean id="fetchDns" class="org.archive.modules.fetcher.FetchDNS">
|
||||
<bean id="fetchDns" class="org.archive.modules.fetcher.FetchDNS">
|
||||
<!-- This setting is not recommended for production crawling -->
|
||||
<property name="disableJavaDnsResolves" value="true"/>
|
||||
</bean>
|
||||
<bean id="fetchHttp" class="org.archive.modules.fetcher.FetchHTTP">
|
||||
</bean>
|
||||
|
||||
@@ -72,15 +72,35 @@ public class FetchDNS extends Processor {
|
||||
protected InetAddress serverInetAddr = null;
|
||||
|
||||
/**
|
||||
* If a DNS lookup fails, whether or not to fallback to InetAddress
|
||||
* If a DNS lookup fails, whether or not to fall back to InetAddress
|
||||
* resolution, which may use local 'hosts' files or other mechanisms.
|
||||
*/
|
||||
protected boolean acceptNonDnsResolves = false;
|
||||
{
|
||||
setAcceptNonDnsResolves(false);
|
||||
}
|
||||
public boolean getAcceptNonDnsResolves() {
|
||||
return acceptNonDnsResolves;
|
||||
return (Boolean) kp.get("acceptNonDnsResolves");
|
||||
}
|
||||
public void setAcceptNonDnsResolves(boolean acceptNonDnsResolves) {
|
||||
this.acceptNonDnsResolves = acceptNonDnsResolves;
|
||||
kp.put("acceptNonDnsResolves",acceptNonDnsResolves);
|
||||
}
|
||||
|
||||
/**
|
||||
* Optionally, only allow InetAddress resolution, precisely because it
|
||||
* may use local 'hosts' files or other mechanisms.
|
||||
*
|
||||
* This should not generally be used in production as it will prevent
|
||||
* DNS lookups from being recorded properly.
|
||||
*
|
||||
*/
|
||||
{
|
||||
setDisableJavaDnsResolves(false);
|
||||
}
|
||||
public boolean getDisableJavaDnsResolves() {
|
||||
return (Boolean) kp.get("disableJavaDnsResolves");
|
||||
}
|
||||
public void setDisableJavaDnsResolves(boolean disableJavaDnsResolves) {
|
||||
kp.put("disableJavaDnsResolves",disableJavaDnsResolves);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -158,10 +178,13 @@ public class FetchDNS extends Processor {
|
||||
// Try to get the records for this host (assume domain name)
|
||||
// TODO: Bug #935119 concerns potential hang here
|
||||
String lookupName = dnsName.endsWith(".") ? dnsName : dnsName + ".";
|
||||
try {
|
||||
rrecordSet = (new Lookup(lookupName, TypeType, ClassType)).run();
|
||||
} catch (TextParseException e) {
|
||||
rrecordSet = null;
|
||||
// If we have not disabled JavaDNS, use that:
|
||||
if (!getDisableJavaDnsResolves()) {
|
||||
try {
|
||||
rrecordSet = (new Lookup(lookupName, TypeType, ClassType)).run();
|
||||
} catch (TextParseException e) {
|
||||
rrecordSet = null;
|
||||
}
|
||||
}
|
||||
curi.setContentType("text/dns");
|
||||
if (rrecordSet != null) {
|
||||
@@ -173,7 +196,7 @@ public class FetchDNS extends Processor {
|
||||
if (logger.isLoggable(Level.FINE)) {
|
||||
logger.fine("Failed find of recordset for " + lookupName);
|
||||
}
|
||||
if (getAcceptNonDnsResolves()||"localhost".equals(dnsName)) {
|
||||
if (getAcceptNonDnsResolves()||getDisableJavaDnsResolves()||"localhost".equals(dnsName)) {
|
||||
// Do lookup that bypasses javadns.
|
||||
InetAddress address = null;
|
||||
try {
|
||||
@@ -184,6 +207,7 @@ public class FetchDNS extends Processor {
|
||||
if (address != null) {
|
||||
targetHost.setIP(address, DEFAULT_TTL_FOR_NON_DNS_RESOLVES);
|
||||
curi.setFetchStatus(S_GETBYNAME_SUCCESS);
|
||||
curi.setContentSize(0);
|
||||
if (logger.isLoggable(Level.FINE)) {
|
||||
logger.fine("Found address for " + dnsName +
|
||||
" using native dns.");
|
||||
|
||||
@@ -372,8 +372,19 @@ http://maven.apache.org/guides/mini/guide-m1-m2.html
|
||||
<encoding>UTF-8</encoding>
|
||||
</configuration>
|
||||
</plugin>
|
||||
|
||||
</plugins>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-surefire-plugin</artifactId>
|
||||
<version>2.9</version>
|
||||
<configuration>
|
||||
<argLine>-Xmx1g</argLine>
|
||||
<systemPropertyVariables>
|
||||
<testtmpdir>${project.build.directory}/heritrix-junit-tests</testtmpdir>
|
||||
<heritrix.hostname>localhost</heritrix.hostname>
|
||||
</systemPropertyVariables>
|
||||
</configuration>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
<properties>
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
|
||||
Reference in New Issue
Block a user