diff --git a/commons/src/main/java/org/archive/net/UURIFactory.java b/commons/src/main/java/org/archive/net/UURIFactory.java index 62cb79e3..57b3da00 100644 --- a/commons/src/main/java/org/archive/net/UURIFactory.java +++ b/commons/src/main/java/org/archive/net/UURIFactory.java @@ -484,6 +484,9 @@ public class UURIFactory extends URI { + ((uriQuery != null)? uriQuery.length(): 0)); appendNonNull(s, uriScheme, ":", true); appendNonNull(s, uriAuthority, "//", false); + if ("whois".equals(uriScheme) && uriAuthority == null) { + s.append("//"); // XXX :( is this generalizable somehow? + } appendNonNull(s, uriPath, "", false); appendNonNull(s, uriQuery, "?", false); return s.toString(); diff --git a/engine/src/main/java/org/archive/crawler/frontier/URIAuthorityBasedQueueAssignmentPolicy.java b/engine/src/main/java/org/archive/crawler/frontier/URIAuthorityBasedQueueAssignmentPolicy.java index cbc49aeb..ea90cc75 100644 --- a/engine/src/main/java/org/archive/crawler/frontier/URIAuthorityBasedQueueAssignmentPolicy.java +++ b/engine/src/main/java/org/archive/crawler/frontier/URIAuthorityBasedQueueAssignmentPolicy.java @@ -86,6 +86,11 @@ implements return getForceQueueAssignment(); } + // all whois urls in the same queue + if (curi.getUURI().getScheme().equals("whois")) { + return "whois"; + } + UURI basis = curi.getPolicyBasisUURI(); String candidate = getCoreKey(basis); diff --git a/engine/src/main/java/org/archive/crawler/prefetch/PreconditionEnforcer.java b/engine/src/main/java/org/archive/crawler/prefetch/PreconditionEnforcer.java index 58a98e61..d1de5341 100644 --- a/engine/src/main/java/org/archive/crawler/prefetch/PreconditionEnforcer.java +++ b/engine/src/main/java/org/archive/crawler/prefetch/PreconditionEnforcer.java @@ -18,7 +18,6 @@ */ package org.archive.crawler.prefetch; -import static org.archive.modules.fetcher.FetchStatusCodes.S_DEFERRED; import static org.archive.modules.fetcher.FetchStatusCodes.S_DOMAIN_PREREQUISITE_FAILURE; import static org.archive.modules.fetcher.FetchStatusCodes.S_ROBOTS_PRECLUDED; import static org.archive.modules.fetcher.FetchStatusCodes.S_ROBOTS_PREREQUISITE_FAILURE; @@ -36,15 +35,11 @@ import org.archive.modules.ProcessResult; import org.archive.modules.Processor; import org.archive.modules.credential.Credential; import org.archive.modules.credential.CredentialStore; -import org.archive.modules.extractor.Hop; -import org.archive.modules.extractor.Link; -import org.archive.modules.extractor.LinkContext; import org.archive.modules.net.CrawlHost; import org.archive.modules.net.CrawlServer; import org.archive.modules.net.RobotsPolicy; import org.archive.modules.net.ServerCache; import org.archive.net.UURI; -import org.archive.net.UURIFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -232,7 +227,7 @@ public class PreconditionEnforcer extends Processor { // crawled. try { String prereq = curi.getUURI().resolve("/robots.txt").toString(); - markPrerequisite(curi, prereq); + curi.markPrerequisite(prereq); } catch (URIException e1) { logger.severe("Failed resolve using " + curi); @@ -280,6 +275,8 @@ public class PreconditionEnforcer extends Processor { // DNS URIs never have a DNS precondition curi.setPrerequisite(true); return false; + } else if (curi.getUURI().getScheme().equals("whois")) { + return false; } CrawlServer cs = serverCache.getServerFor(curi.getUURI()); @@ -310,7 +307,7 @@ public class PreconditionEnforcer extends Processor { + " for dns lookup."); String preq = "dns:" + ch.getHostName(); try { - markPrerequisite(curi, preq); + curi.markPrerequisite(preq); } catch (URIException e) { throw new RuntimeException(e); // shouldn't ever happen } @@ -421,7 +418,7 @@ public class PreconditionEnforcer extends Processor { + " is null."); } else { try { - markPrerequisite(curi, prereq); + curi.markPrerequisite(prereq); } catch (URIException e) { logger.severe("unable to set credentials prerequisite "+prereq); loggerModule.logUriError(e,curi.getUURI(),prereq); @@ -461,36 +458,4 @@ public class PreconditionEnforcer extends Processor { return false; } - - /** - * Do all actions associated with setting a CrawlURI as - * requiring a prerequisite. - * - * @param lastProcessorChain Last processor chain reference. This chain is - * where this CrawlURI goes next. - * @param preq Object to set a prerequisite. - * @throws URIException - */ - protected void markPrerequisite(CrawlURI curi, String preq) - throws URIException { - UURI src = curi.getUURI(); - UURI dest = UURIFactory.getInstance(preq); - LinkContext lc = LinkContext.PREREQ_MISC; - Hop hop = Hop.PREREQ; - Link link = new Link(src, dest, lc, hop); - CrawlURI caUri = curi.createCrawlURI(curi.getBaseURI(), link); - // TODO: consider moving some of this to candidate-handling - int prereqPriority = curi.getSchedulingDirective() - 1; - if (prereqPriority < 0) { - prereqPriority = 0; - logger.severe("Unable to promote prerequisite " + caUri + - " above " + this); - } - caUri.setSchedulingDirective(prereqPriority); - caUri.setForceFetch(true); - curi.setPrerequisiteUri(caUri); - curi.incrementDeferrals(); - curi.setFetchStatus(S_DEFERRED); - //skipToPostProcessing(); - } } diff --git a/engine/src/main/resources/org/archive/crawler/restlet/profile-crawler-beans.cxml b/engine/src/main/resources/org/archive/crawler/restlet/profile-crawler-beans.cxml index d7534c92..64a8eec4 100644 --- a/engine/src/main/resources/org/archive/crawler/restlet/profile-crawler-beans.cxml +++ b/engine/src/main/resources/org/archive/crawler/restlet/profile-crawler-beans.cxml @@ -242,6 +242,15 @@ http://example.example/example + @@ -298,6 +307,7 @@ http://example.example/example + diff --git a/modules/src/main/java/org/archive/modules/CrawlURI.java b/modules/src/main/java/org/archive/modules/CrawlURI.java index 82b8c113..681b1e6e 100644 --- a/modules/src/main/java/org/archive/modules/CrawlURI.java +++ b/modules/src/main/java/org/archive/modules/CrawlURI.java @@ -69,6 +69,7 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.CopyOnWriteArrayList; +import java.util.logging.Logger; import org.apache.commons.httpclient.HttpMethod; import org.apache.commons.httpclient.HttpStatus; @@ -109,6 +110,9 @@ public class CrawlURI implements MultiReporter, Serializable, OverlayContext { private static final long serialVersionUID = 3L; + private static final Logger logger = + Logger.getLogger(CrawlURI.class.getName()); + public static final int UNCALCULATED = -1; public static enum FetchType { HTTP_GET, HTTP_POST, UNKNOWN }; @@ -1829,4 +1833,38 @@ implements MultiReporter, Serializable, OverlayContext { kryo.autoregister(org.archive.modules.credential.HtmlFormCredential.class); kryo.setRegistrationOptional(true); } + + /** + * Do all actions associated with setting a CrawlURI as + * requiring a prerequisite. + * + * @param lastProcessorChain Last processor chain reference. This chain is + * where this CrawlURI goes next. + * @param preq Object to set a prerequisite. + * @return the newly created prerequisite CrawlURI + * @throws URIException + */ + public CrawlURI markPrerequisite(String preq) + throws URIException { + UURI src = getUURI(); + UURI dest = UURIFactory.getInstance(preq); + LinkContext lc = LinkContext.PREREQ_MISC; + Hop hop = Hop.PREREQ; + Link link = new Link(src, dest, lc, hop); + CrawlURI caUri = createCrawlURI(getBaseURI(), link); + // TODO: consider moving some of this to candidate-handling + int prereqPriority = getSchedulingDirective() - 1; + if (prereqPriority < 0) { + prereqPriority = 0; + logger.severe("Unable to promote prerequisite " + caUri + " above " + this); + } + caUri.setSchedulingDirective(prereqPriority); + caUri.setForceFetch(true); + setPrerequisiteUri(caUri); + incrementDeferrals(); + setFetchStatus(S_DEFERRED); + + return caUri; + } + } diff --git a/modules/src/main/java/org/archive/modules/deciderules/SchemeNotInSetDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/SchemeNotInSetDecideRule.java index 9c076625..e13b91c2 100644 --- a/modules/src/main/java/org/archive/modules/deciderules/SchemeNotInSetDecideRule.java +++ b/modules/src/main/java/org/archive/modules/deciderules/SchemeNotInSetDecideRule.java @@ -48,7 +48,7 @@ public class SchemeNotInSetDecideRule extends PredicatedDecideRule { protected Set schemes = new HashSet(); { // default set are those schemes Heritrix supports in usual configuration - schemes.addAll(Arrays.asList(new String[] {"http","https","ftp","dns"})); + schemes.addAll(Arrays.asList(new String[] {"http","https","ftp","dns","whois"})); } public Set getSchemes() { return schemes; diff --git a/modules/src/main/java/org/archive/modules/fetcher/FetchStatusCodes.java b/modules/src/main/java/org/archive/modules/fetcher/FetchStatusCodes.java index 255f2694..ec319768 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/FetchStatusCodes.java +++ b/modules/src/main/java/org/archive/modules/fetcher/FetchStatusCodes.java @@ -113,7 +113,11 @@ public interface FetchStatusCodes { public static final int S_DNS_SUCCESS = 1; /** InetAddress.getByName success */ public static final int S_GETBYNAME_SUCCESS = 1001; - + /** WHOIS success */ + public static final int S_WHOIS_SUCCESS = 2001; + /** Finished all fetches for serverless WHOIS url (whois:///foo.org) */ + public static final int S_WHOIS_GENERIC_FINISHED = 2002; + /** HTTP 404 NOT FOUND */ public static final int S_NOT_FOUND = 404; // } diff --git a/modules/src/main/java/org/archive/modules/fetcher/FetchWhois.java b/modules/src/main/java/org/archive/modules/fetcher/FetchWhois.java new file mode 100644 index 00000000..0ddb8a62 --- /dev/null +++ b/modules/src/main/java/org/archive/modules/fetcher/FetchWhois.java @@ -0,0 +1,347 @@ +package org.archive.modules.fetcher; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.UnsupportedEncodingException; +import java.net.URLEncoder; +import java.util.HashMap; +import java.util.Map; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.regex.Matcher; + +import org.apache.commons.httpclient.URIException; +import org.apache.commons.net.whois.WhoisClient; +import org.archive.bdb.BdbModule; +import org.archive.modules.CoreAttributeConstants; +import org.archive.modules.CrawlURI; +import org.archive.modules.ProcessResult; +import org.archive.modules.Processor; +import org.archive.modules.extractor.Hop; +import org.archive.modules.extractor.Link; +import org.archive.modules.extractor.LinkContext; +import org.archive.modules.net.CrawlHost; +import org.archive.modules.net.ServerCache; +import org.archive.util.Recorder; +import org.archive.util.TextUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.Lifecycle; + +import com.google.common.net.InternetDomainName; +import com.sleepycat.bind.tuple.IntegerBinding; +import com.sleepycat.bind.tuple.StringBinding; +import com.sleepycat.collections.StoredSortedMap; +import com.sleepycat.je.Database; +import com.sleepycat.je.DatabaseException; + +/** + * WHOIS Fetcher (RFC 3912). If this fetcher is enabled, heritrix will attempt + * whois lookups on the topmost assigned domain and the IP address of each URL. + * + * @contributor nlevitt + */ +public class FetchWhois extends Processor implements CoreAttributeConstants, + FetchStatusCodes, Lifecycle { + private static final long serialVersionUID = 1L; + + private static Logger logger = Logger.getLogger(FetchWhois.class.getName()); + + public static final String IP_ADDRESS_REGEX = "\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}"; + + // North America-centric, but it should refer us to the right server + // e.g. "ReferralServer: whois://whois.apnic.net" + protected static final String DEFAULT_IP_WHOIS_SERVER = "whois.arin.net"; + + // look up "com" "net" "fr" "info" etc + protected static final String ULTRA_SUFFIX_WHOIS_SERVER = "whois.iana.org"; + + // [whois://whois.arin.net/192.102.239.53] ReferralServer: whois://whois.apnic.net + // [whois://whois.arin.net/208.49.199.10] ReferralServer: rwhois://rwhois.gblx.net:4321 + // [whois://whois.arin.net/195.154.120.129] ReferralServer: whois://whois.ripe.net:43 + // (obsolete) [whois://whois.iana.org/fr] Whois Server (port 43): whois.nic.fr + // [whois://whois.iana.org/fr] whois: whois.nic.fr + // [whois://whois.verisign-grs.com/domain%201stbattalion9thmarinesfirebase.net] Whois Server: whois.fastdomain.com + // (false positive fixed) WHOIS lookup made at 23:48:04 13-Jan-2011 + protected static String WHOIS_SERVER_REGEX = "(?i)^\\s*(?:whois server|ReferralServer|whois)[^:]*:.*?([a-zA-Z0-9-]+\\.[a-zA-Z0-9.:-]+)/*$"; + + protected enum UrlStatus {IN_PROGRESS, DONE}; + private transient Database whoisDb; + private transient StoredSortedMap referralServers; + private transient StoredSortedMap urlProgress; + + protected BdbModule bdb; + + @Autowired + public void setBdbModule(BdbModule bdb) { + this.bdb = bdb; + } + + protected Map specialQueryTemplates; + { + // Default special templates. Keep commented out section of + // profile-crawler-beans.xml in synch with this. + specialQueryTemplates = new HashMap(); + specialQueryTemplates.put("whois.verisign-grs.com", "domain %s"); + specialQueryTemplates.put("whois.arin.net", "z + %s"); + specialQueryTemplates.put("whois.denic.de", "-T dn %s"); + } + public void setSpecialQueryTemplates(Map m) { + this.specialQueryTemplates.clear(); + this.specialQueryTemplates.putAll(m); + } + + private boolean isRunning = false; + public void start() { + if(isRunning()) { + return; + } + + try { + BdbModule.BdbConfig dbConfig = new BdbModule.BdbConfig(); + dbConfig.setTransactional(false); + + boolean isRecovery = recoveryCheckpoint != null; + dbConfig.setAllowCreate(!isRecovery); + whoisDb = bdb.openDatabase("whoisKnowledge", dbConfig, isRecovery); + + referralServers = new StoredSortedMap(whoisDb, + new StringBinding(), new StringBinding(), true); + urlProgress = new StoredSortedMap(whoisDb, + new StringBinding(), new IntegerBinding(), true); + } catch (DatabaseException e) { + throw new RuntimeException(e); + } + + isRunning = true; + } + + public boolean isRunning() { + return isRunning; + } + + public void stop() { + isRunning = false; + + // BdbModule will handle closing of DB + bdb = null; + } + + @Override + protected ProcessResult innerProcessResult(CrawlURI curi) throws InterruptedException { + if (curi.getUURI().getScheme().equals("whois")) { + curi.setFetchBeginTime(System.currentTimeMillis()); + + String whoisServer = getWhoisServer(curi); + String whoisQuery = getWhoisQuery(curi); + + if (whoisServer == null) { + // e.g. whois:///foo.org + ProcessResult ret = deferOrFinishGeneric(curi, whoisQuery); + return ret; + } else { + // e.g. whois://whois.pir.org/foo.org + fetch(curi, whoisServer, whoisQuery); + return ProcessResult.PROCEED; + } + + } else { + addWhoisLinks(curi); + return ProcessResult.PROCEED; + + } + } + + // handle server-less whois url + protected ProcessResult deferOrFinishGeneric(CrawlURI curi, String domainOrIp) { + String tryThis = null; + String ultraSuffix = domainOrIp.substring(domainOrIp.lastIndexOf('.') + 1).toLowerCase(); + + if (referralServers.containsKey(domainOrIp)) { + tryThis = "whois://" + referralServers.get(domainOrIp) + '/' + domainOrIp; + } else if (TextUtils.getMatcher(IP_ADDRESS_REGEX, domainOrIp).matches()) { + tryThis = makeWhoisUrl(DEFAULT_IP_WHOIS_SERVER, domainOrIp); + } else if (referralServers.containsKey(ultraSuffix)) { + tryThis = makeWhoisUrl(referralServers.get(ultraSuffix), domainOrIp); + } else if (urlProgress.get(makeWhoisUrl(ULTRA_SUFFIX_WHOIS_SERVER, ultraSuffix)) == null) { + tryThis = makeWhoisUrl(ULTRA_SUFFIX_WHOIS_SERVER, ultraSuffix); + } + + assert(tryThis != null); + Integer progress = urlProgress.get(tryThis); + if (tryThis == null || (progress != null && progress == UrlStatus.DONE.ordinal())) { + if (logger.isLoggable(Level.FINE)) { + logger.fine("finished with generic serverless whois uri " + curi); + } + curi.setFetchStatus(S_WHOIS_GENERIC_FINISHED); + return ProcessResult.PROCEED; + } else { + if (progress == null) { + try { + if (logger.isLoggable(Level.FINE)) { + logger.fine(curi + " marking prerequisite " + tryThis + " and deferring"); + } + CrawlURI caUri = curi.markPrerequisite(tryThis); + caUri.setForceFetch(false); + urlProgress.put(tryThis, UrlStatus.IN_PROGRESS.ordinal()); + } catch (URIException e) { + throw new RuntimeException(e); + } + + } else { + // this shouldn't happen because prereqs should be crawled before we're attempted again + curi.incrementDeferrals(); + curi.setFetchStatus(S_DEFERRED); + if (logger.isLoggable(Level.FINE)) { + logger.fine(curi + ": prerequisite " + tryThis + " is in progress, deferring"); + } + } + + return ProcessResult.FINISH; + } + } + + protected String makeWhoisUrl(String server, String principle) { + try { + String query; + String template = specialQueryTemplates.get(server.toLowerCase()); + if (template != null) { + query = template.replaceAll("%s", principle); + } else { + query = principle; + } + + return "whois://" + server + "/" + URLEncoder.encode(query, "UTF-8"); + } catch (UnsupportedEncodingException e) { + throw new RuntimeException(e); + } + } + + protected void fetch(CrawlURI curi, String whoisServer, String whoisQuery) { + WhoisClient client = new WhoisClient(); + Recorder recorder = curi.getRecorder(); + + try { + if (curi.getUURI().getPort() > 0) { + client.connect(whoisServer, curi.getUURI().getPort()); + } else { + client.connect(whoisServer); + } + + recorder.inputWrap(client.getInputStream(whoisQuery)); + + // look for info about whois server in the response + // XXX run regex on the whole thing, rather than line by line? + BufferedReader reader = new BufferedReader(new InputStreamReader(recorder.getRecordedInput(), "ASCII")); + for (String line = reader.readLine(); line != null; line = reader.readLine()) { + Matcher matcher = TextUtils.getMatcher(WHOIS_SERVER_REGEX, line); + if (matcher.find()) { + // gets rid of "domain " for whois.verisign-grs.com queries + String key = whoisQuery.replaceFirst("(\\S+\\s+)+", "").toLowerCase(); + referralServers.put(key, matcher.group(1).toLowerCase()); + if (logger.isLoggable(Level.INFO)) { + logger.fine("added referral server " + matcher.group(1) + " to server list for " + key); + } + } + } + + curi.setContentType("text/plain"); + curi.setFetchStatus(S_WHOIS_SUCCESS); + } catch (IOException e) { + if (logger.isLoggable(Level.FINE)) { + logger.fine("failed to connect to whois server for uri " + curi + ": " + e); + } + curi.getNonFatalFailures().add(e); + curi.setFetchStatus(S_CONNECT_FAILED); + } finally { + recorder.close(); + curi.setContentSize(recorder.getRecordedInput().getSize()); + logger.fine(curi + ": " + recorder.getRecordedInput().getSize() + " bytes read"); + + if (client != null && client.isConnected()) try { + client.disconnect(); + } catch (IOException e) { + logger.fine("problem closing connection to whois server for uri " + curi + ": " + e); + } + + urlProgress.put(curi.toString(), UrlStatus.DONE.ordinal()); + } + } + + protected String getWhoisQuery(CrawlURI curi) { + try { + return curi.getUURI().getPathQuery().toString().substring(1); + } catch (URIException e) { + logger.log(Level.SEVERE, "Failed to get path/query from uri " + curi, e); + return null; + } + } + + protected String getWhoisServer(CrawlURI curi) { + String whoisServer = null; + + try { + whoisServer = curi.getUURI().getHost(); + if (whoisServer.length() == 0) { + whoisServer = null; + } + } catch (URIException e) { + logger.warning("Failed to get host from uri " + curi + ": " + e); + whoisServer = null; + } + + return whoisServer; + } + + @Override + protected boolean shouldProcess(CrawlURI uri) { + // process all uris - non-whois uris get whois outlinks added + return true; + } + + protected ServerCache serverCache; + public ServerCache getServerCache() { + return this.serverCache; + } + @Autowired + public void setServerCache(ServerCache serverCache) { + this.serverCache = serverCache; + } + + protected void addWhoisLink(CrawlURI curi, String query) { + String whoisUrl = "whois:///" + query; + try { + Link.add(curi, Integer.MAX_VALUE, whoisUrl, LinkContext.INFERRED_MISC, Hop.INFERRED); + } catch (URIException e) { + logger.log(Level.WARNING, "problem with url " + whoisUrl, e); + } + } + + /** + * Adds outlinks to whois:///{domain} and whois:///{ipAddress} + */ + protected void addWhoisLinks(CrawlURI curi) throws InterruptedException { + CrawlHost ch = serverCache.getHostFor(curi.getUURI()); + + if (ch == null) { + return; + } + + if (ch.getIP() != null) { + // do a whois lookup on the ip address + addWhoisLink(curi, ch.getIP().getHostAddress()); + } + + if (InternetDomainName.isValidLenient(ch.getHostName())) { + // do a whois lookup on the domain + String topmostAssigned = InternetDomainName.fromLenient(ch.getHostName()).topPrivateDomain().name(); + addWhoisLink(curi, topmostAssigned); + } + } + + @Override + protected void innerProcess(CrawlURI uri) throws InterruptedException { + throw new RuntimeException("this method shouldn't be called - should use innerProcessResult()"); + } + +} diff --git a/modules/src/main/java/org/archive/modules/net/ServerCache.java b/modules/src/main/java/org/archive/modules/net/ServerCache.java index e8d389f3..c8b89a59 100644 --- a/modules/src/main/java/org/archive/modules/net/ServerCache.java +++ b/modules/src/main/java/org/archive/modules/net/ServerCache.java @@ -58,6 +58,8 @@ public abstract class ServerCache { try { if (uuri.getScheme().equals("dns")) { h = getHostFor("dns:"); + } else if (uuri.getScheme().equals("whois")) { + h = getHostFor("whois:"); } else { h = getHostFor(uuri.getReferencedHost()); } diff --git a/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java b/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java index f91f43dc..19f683c7 100644 --- a/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java +++ b/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java @@ -249,6 +249,8 @@ public class WARCWriterProcessor extends WriterPoolProcessor implements WARCWrit writeDnsRecords(curi, writer, baseid, timestamp); } else if (lowerCaseScheme.equals("ftp")) { writeFtpRecords(writer, curi, baseid, timestamp); + } else if (lowerCaseScheme.equals("whois")) { + writeWhoisRecords(writer, curi, baseid, timestamp); } else { logger.warning("No handler for scheme " + lowerCaseScheme); } @@ -310,7 +312,12 @@ public class WARCWriterProcessor extends WriterPoolProcessor implements WARCWrit writeResponse(w, timestamp, curi.getContentType(), baseid, curi, headers); } - + + private void writeWhoisRecords(WARCWriter w, CrawlURI curi, URI baseid, + String timestamp) throws IOException { + writeResponse(w, timestamp, curi.getContentType(), baseid, curi, null); + } + private void writeHttpRecords(final CrawlURI curi, WARCWriter w, final URI baseid, final String timestamp) throws IOException { // Add named fields for ip, checksum, and relate the metadata diff --git a/modules/src/main/java/org/archive/modules/writer/WriterPoolProcessor.java b/modules/src/main/java/org/archive/modules/writer/WriterPoolProcessor.java index 8fac715e..d81e9d66 100644 --- a/modules/src/main/java/org/archive/modules/writer/WriterPoolProcessor.java +++ b/modules/src/main/java/org/archive/modules/writer/WriterPoolProcessor.java @@ -21,6 +21,7 @@ package org.archive.modules.writer; import static org.archive.modules.CoreAttributeConstants.A_DNS_SERVER_IP_LABEL; import static org.archive.modules.fetcher.FetchStatusCodes.S_DNS_SUCCESS; +import static org.archive.modules.fetcher.FetchStatusCodes.S_WHOIS_SUCCESS; import java.io.File; import java.io.IOException; @@ -325,6 +326,8 @@ implements Lifecycle, Checkpointable, WriterPoolSettings { // TODO: possibly move this sort of isSuccess() test into CrawlURI if (scheme.equals("dns")) { retVal = curi.getFetchStatus() == S_DNS_SUCCESS; + } else if (scheme.equals("whois")) { + retVal = curi.getFetchStatus() == S_WHOIS_SUCCESS; } else if (scheme.equals("http") || scheme.equals("https")) { retVal = curi.getFetchStatus() > 0 && curi.getHttpMethod() != null; } else if (scheme.equals("ftp")) {