lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-07-18 05:27:11 +00:00
Code Issues Packages Projects Releases Wiki Activity

HER-1964 record whois server ip address in warc

Browse Source 
* FetchWhois.java
    fetch() - record server ip address in CrawlURI attributes list
* CoreAttributeConstants.java
    new string constant A_WHOIS_SERVER_IP, key for whois server ip address CrawlURI attribute
* WARCWriterProcessor.java
    writeWhoisRecords() - include warc header WARC-IP-Address
This commit is contained in:
Noah Levitt
2011-11-01 16:57:22 -07:00
parent 48434a6293
commit dff143b301
3 changed files with 13 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/src/main/java/org/archive/modules/CoreAttributeConstants.java
+2
View File
@@ -112,4 +112,6 @@ public interface CoreAttributeConstants {
public static final String A_FTP_CONTROL_CONVERSATION = "ftp-control-conversation";
public static final String A_FTP_FETCH_STATUS = "ftp-fetch-status";
public static final String A_WHOIS_SERVER_IP = "whois-server-ip";
}
modules/src/main/java/org/archive/modules/fetcher/FetchWhois.java
+4
View File
@@ -121,6 +121,7 @@ import com.sleepycat.je.DatabaseException;
*/
public class FetchWhois extends Processor implements CoreAttributeConstants,
FetchStatusCodes, Lifecycle {
private static final long serialVersionUID = 1L;
private static Logger logger = Logger.getLogger(FetchWhois.class.getName());
@@ -329,6 +330,9 @@ public class FetchWhois extends Processor implements CoreAttributeConstants,
client.setSoTimeout(getSoTimeoutMs()); // must be after connect()
curi.getData().put(CoreAttributeConstants.A_WHOIS_SERVER_IP,
client.getRemoteAddress().getHostAddress());
recorder.inputWrap(client.getInputStream(whoisQuery));
// look for info about whois server in the response
modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java
+7 -1
View File
@@ -77,6 +77,7 @@ import org.archive.io.ReplayInputStream;
import org.archive.io.warc.WARCWriter;
import org.archive.io.warc.WARCWriterPool;
import org.archive.io.warc.WARCWriterPoolSettings;
import org.archive.modules.CoreAttributeConstants;
import org.archive.modules.CrawlMetadata;
import org.archive.modules.CrawlURI;
import org.archive.modules.ProcessResult;
@@ -330,7 +331,12 @@ public class WARCWriterProcessor extends WriterPoolProcessor implements WARCWrit
private void writeWhoisRecords(WARCWriter w, CrawlURI curi, URI baseid,
String timestamp) throws IOException {
writeResponse(w, timestamp, curi.getContentType(), baseid, curi, null);
ANVLRecord headers = new ANVLRecord(1);
Object whoisServerIP = curi.getData().get(CoreAttributeConstants.A_WHOIS_SERVER_IP);
if (whoisServerIP != null) {
headers.addLabelValue(HEADER_KEY_IP, whoisServerIP.toString());
}
writeResponse(w, timestamp, curi.getContentType(), baseid, curi, headers);
}
private void writeHttpRecords(final CrawlURI curi, WARCWriter w,