From dff143b301fbfd549bdaf9ea45efb3cd3d66b82b Mon Sep 17 00:00:00 2001 From: Noah Levitt Date: Tue, 1 Nov 2011 16:57:22 -0700 Subject: [PATCH] HER-1964 record whois server ip address in warc * FetchWhois.java fetch() - record server ip address in CrawlURI attributes list * CoreAttributeConstants.java new string constant A_WHOIS_SERVER_IP, key for whois server ip address CrawlURI attribute * WARCWriterProcessor.java writeWhoisRecords() - include warc header WARC-IP-Address --- .../java/org/archive/modules/CoreAttributeConstants.java | 2 ++ .../main/java/org/archive/modules/fetcher/FetchWhois.java | 4 ++++ .../org/archive/modules/writer/WARCWriterProcessor.java | 8 +++++++- 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/modules/src/main/java/org/archive/modules/CoreAttributeConstants.java b/modules/src/main/java/org/archive/modules/CoreAttributeConstants.java index 3398576c..8f6d389e 100644 --- a/modules/src/main/java/org/archive/modules/CoreAttributeConstants.java +++ b/modules/src/main/java/org/archive/modules/CoreAttributeConstants.java @@ -112,4 +112,6 @@ public interface CoreAttributeConstants { public static final String A_FTP_CONTROL_CONVERSATION = "ftp-control-conversation"; public static final String A_FTP_FETCH_STATUS = "ftp-fetch-status"; + + public static final String A_WHOIS_SERVER_IP = "whois-server-ip"; } diff --git a/modules/src/main/java/org/archive/modules/fetcher/FetchWhois.java b/modules/src/main/java/org/archive/modules/fetcher/FetchWhois.java index d90f622b..7fa2306b 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/FetchWhois.java +++ b/modules/src/main/java/org/archive/modules/fetcher/FetchWhois.java @@ -121,6 +121,7 @@ import com.sleepycat.je.DatabaseException; */ public class FetchWhois extends Processor implements CoreAttributeConstants, FetchStatusCodes, Lifecycle { + private static final long serialVersionUID = 1L; private static Logger logger = Logger.getLogger(FetchWhois.class.getName()); @@ -329,6 +330,9 @@ public class FetchWhois extends Processor implements CoreAttributeConstants, client.setSoTimeout(getSoTimeoutMs()); // must be after connect() + curi.getData().put(CoreAttributeConstants.A_WHOIS_SERVER_IP, + client.getRemoteAddress().getHostAddress()); + recorder.inputWrap(client.getInputStream(whoisQuery)); // look for info about whois server in the response diff --git a/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java b/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java index 525e4700..3b661d62 100644 --- a/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java +++ b/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java @@ -77,6 +77,7 @@ import org.archive.io.ReplayInputStream; import org.archive.io.warc.WARCWriter; import org.archive.io.warc.WARCWriterPool; import org.archive.io.warc.WARCWriterPoolSettings; +import org.archive.modules.CoreAttributeConstants; import org.archive.modules.CrawlMetadata; import org.archive.modules.CrawlURI; import org.archive.modules.ProcessResult; @@ -330,7 +331,12 @@ public class WARCWriterProcessor extends WriterPoolProcessor implements WARCWrit private void writeWhoisRecords(WARCWriter w, CrawlURI curi, URI baseid, String timestamp) throws IOException { - writeResponse(w, timestamp, curi.getContentType(), baseid, curi, null); + ANVLRecord headers = new ANVLRecord(1); + Object whoisServerIP = curi.getData().get(CoreAttributeConstants.A_WHOIS_SERVER_IP); + if (whoisServerIP != null) { + headers.addLabelValue(HEADER_KEY_IP, whoisServerIP.toString()); + } + writeResponse(w, timestamp, curi.getContentType(), baseid, curi, headers); } private void writeHttpRecords(final CrawlURI curi, WARCWriter w,