lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-09-27 08:05:46 +00:00
Code Issues Packages Projects Releases Wiki Activity

make WARCRecordBuilder an interface

Browse Source 
this way other classes that extend other classes can also implement
WARCRecordBuilder
This commit is contained in:
Noah Levitt
2019-06-12 15:04:39 -07:00
parent 9e67a8dab4
commit 9ddd281ef2
12 changed files with 82 additions and 72 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/src/main/java/org/archive/modules/warc/BaseWARCRecordBuilder.java
+66
View File
@@ -0,0 +1,66 @@
package org.archive.modules.warc;
import static org.archive.modules.CoreAttributeConstants.A_DNS_SERVER_IP_LABEL;
import java.net.InetAddress;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.UUID;
import org.archive.modules.CrawlURI;
import org.archive.modules.net.CrawlHost;
import org.archive.modules.net.ServerCache;
import org.springframework.beans.factory.annotation.Autowired;
public abstract class BaseWARCRecordBuilder implements WARCRecordBuilder {
transient protected ServerCache serverCache;
public ServerCache getServerCache() {
return this.serverCache;
}
@Autowired
public void setServerCache(ServerCache serverCache) {
this.serverCache = serverCache;
}
public URI generateRecordID() {
try {
return new URI("urn:uuid:" + UUID.randomUUID());
} catch (URISyntaxException e) {
throw new RuntimeException(e); // impossible
}
}
/**
* Return IP address of given URI suitable for recording (as in a
* classic ARC 5-field header line).
*
* @param curi CrawlURI
* @return String of IP address
*/
protected String getHostAddress(CrawlURI curi) {
// special handling for DNS URIs: want address of DNS server
if (curi.getUURI().getScheme().toLowerCase().equals("dns")) {
return (String)curi.getData().get(A_DNS_SERVER_IP_LABEL);
}
// otherwise, host referenced in URI
// TODO:FIXME: have fetcher insert exact IP contacted into curi,
// use that rather than inferred by CrawlHost lookup
CrawlHost h = getServerCache().getHostFor(curi.getUURI());
if (h == null) {
throw new NullPointerException("Crawlhost is null for " +
curi + " " + curi.getVia());
}
InetAddress a = h.getIP();
if (a == null) {
throw new NullPointerException("Address is null for " +
curi + " " + curi.getVia() + ". Address " +
((h.getIpFetched() == CrawlHost.IP_NEVER_LOOKED_UP)?
"was never looked up.":
(System.currentTimeMillis() - h.getIpFetched()) +
" ms ago."));
}
return h.getIP().getHostAddress();
}
}
modules/src/main/java/org/archive/modules/warc/DnsResponseRecordBuilder.java
+1 -1
View File
@@ -12,7 +12,7 @@ import org.archive.io.warc.WARCRecordInfo;
import org.archive.modules.CrawlURI;
import org.archive.util.ArchiveUtils;
public class DnsResponseRecordBuilder extends WARCRecordBuilder {
public class DnsResponseRecordBuilder extends BaseWARCRecordBuilder {
@Override
public boolean shouldProcess(CrawlURI curi) {
modules/src/main/java/org/archive/modules/warc/FtpControlConversationRecordBuilder.java
+1 -1
View File
@@ -14,7 +14,7 @@ import org.archive.modules.CrawlURI;
import org.archive.util.ArchiveUtils;
import org.archive.util.anvl.ANVLRecord;
public class FtpControlConversationRecordBuilder extends WARCRecordBuilder {
public class FtpControlConversationRecordBuilder extends BaseWARCRecordBuilder {
@Override
public boolean shouldProcess(CrawlURI curi) {
modules/src/main/java/org/archive/modules/warc/FtpResponseRecordBuilder.java
+1 -1
View File
@@ -13,7 +13,7 @@ import org.archive.io.warc.WARCRecordInfo;
import org.archive.modules.CrawlURI;
import org.archive.util.ArchiveUtils;
public class FtpResponseRecordBuilder extends WARCRecordBuilder {
public class FtpResponseRecordBuilder extends BaseWARCRecordBuilder {
@Override
public boolean shouldProcess(CrawlURI curi) {
modules/src/main/java/org/archive/modules/warc/HttpRequestRecordBuilder.java
+1 -1
View File
@@ -12,7 +12,7 @@ import org.archive.io.warc.WARCRecordInfo;
import org.archive.modules.CrawlURI;
import org.archive.util.ArchiveUtils;
public class HttpRequestRecordBuilder extends WARCRecordBuilder {
public class HttpRequestRecordBuilder extends BaseWARCRecordBuilder {
@Override
public boolean shouldProcess(CrawlURI curi) {
modules/src/main/java/org/archive/modules/warc/HttpResponseRecordBuilder.java
+1 -1
View File
@@ -22,7 +22,7 @@ import org.archive.io.warc.WARCRecordInfo;
import org.archive.modules.CrawlURI;
import org.archive.util.ArchiveUtils;
public class HttpResponseRecordBuilder extends WARCRecordBuilder {
public class HttpResponseRecordBuilder extends BaseWARCRecordBuilder {
@Override
public boolean shouldProcess(CrawlURI curi) {
modules/src/main/java/org/archive/modules/warc/MetadataRecordBuilder.java
+1 -1
View File
@@ -16,7 +16,7 @@ import org.archive.modules.CrawlURI;
import org.archive.util.ArchiveUtils;
import org.archive.util.anvl.ANVLRecord;
public class MetadataRecordBuilder extends WARCRecordBuilder {
public class MetadataRecordBuilder extends BaseWARCRecordBuilder {
/**
* If you don't want metadata records, take this class out of the chain.
modules/src/main/java/org/archive/modules/warc/RevisitRecordBuilder.java
+1 -1
View File
@@ -18,7 +18,7 @@ import org.archive.modules.CrawlURI;
import org.archive.modules.revisit.RevisitProfile;
import org.archive.util.ArchiveUtils;
public class RevisitRecordBuilder extends WARCRecordBuilder {
public class RevisitRecordBuilder extends BaseWARCRecordBuilder {
@Override
public boolean shouldProcess(CrawlURI curi) {
modules/src/main/java/org/archive/modules/warc/WARCRecordBuilder.java
+4 -60
View File
@@ -1,72 +1,16 @@
package org.archive.modules.warc;
import static org.archive.modules.CoreAttributeConstants.A_DNS_SERVER_IP_LABEL;
import java.io.IOException;
import java.net.InetAddress;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.UUID;
import org.archive.io.warc.WARCRecordInfo;
import org.archive.modules.CrawlURI;
import org.archive.modules.net.CrawlHost;
import org.archive.modules.net.ServerCache;
import org.springframework.beans.factory.annotation.Autowired;
public abstract class WARCRecordBuilder {
public interface WARCRecordBuilder {
transient protected ServerCache serverCache;
public ServerCache getServerCache() {
return this.serverCache;
}
@Autowired
public void setServerCache(ServerCache serverCache) {
this.serverCache = serverCache;
}
boolean shouldProcess(CrawlURI curi);
public abstract boolean shouldProcess(CrawlURI curi);
public abstract WARCRecordInfo buildRecord(CrawlURI curi, URI concurrentTo)
WARCRecordInfo buildRecord(CrawlURI curi, URI concurrentTo)
throws IOException;
public URI generateRecordID() {
try {
return new URI("urn:uuid:" + UUID.randomUUID());
} catch (URISyntaxException e) {
throw new RuntimeException(e); // impossible
}
}
/**
* Return IP address of given URI suitable for recording (as in a
* classic ARC 5-field header line).
*
* @param curi CrawlURI
* @return String of IP address
*/
protected String getHostAddress(CrawlURI curi) {
// special handling for DNS URIs: want address of DNS server
if (curi.getUURI().getScheme().toLowerCase().equals("dns")) {
return (String)curi.getData().get(A_DNS_SERVER_IP_LABEL);
}
// otherwise, host referenced in URI
// TODO:FIXME: have fetcher insert exact IP contacted into curi,
// use that rather than inferred by CrawlHost lookup
CrawlHost h = getServerCache().getHostFor(curi.getUURI());
if (h == null) {
throw new NullPointerException("Crawlhost is null for " +
curi + " " + curi.getVia());
}
InetAddress a = h.getIP();
if (a == null) {
throw new NullPointerException("Address is null for " +
curi + " " + curi.getVia() + ". Address " +
((h.getIpFetched() == CrawlHost.IP_NEVER_LOOKED_UP)?
"was never looked up.":
(System.currentTimeMillis() - h.getIpFetched()) +
" ms ago."));
}
return h.getIP().getHostAddress();
}
}
}
modules/src/main/java/org/archive/modules/warc/WhoisResponseRecordBuilder.java
+1 -1
View File
@@ -12,7 +12,7 @@ import org.archive.modules.CoreAttributeConstants;
import org.archive.modules.CrawlURI;
import org.archive.util.ArchiveUtils;
public class WhoisResponseRecordBuilder extends WARCRecordBuilder {
public class WhoisResponseRecordBuilder extends BaseWARCRecordBuilder {
@Override
public boolean shouldProcess(CrawlURI curi) {
modules/src/main/java/org/archive/modules/writer/WARCWriterChainProcessor.java
+2 -2
View File
@@ -39,10 +39,10 @@ public class WARCWriterChainProcessor extends BaseWARCWriterProcessor implements
new MetadataRecordBuilder()));
}
@SuppressWarnings("unchecked")
public List<WARCRecordBuilder> getChain() {
public List<? extends WARCRecordBuilder> getChain() {
return (List<WARCRecordBuilder>) kp.get("chain");
}
public void setChain(List<WARCRecordBuilder> chain) {
public void setChain(List<? extends WARCRecordBuilder> chain) {
kp.put("chain", chain);
}
modules/src/main/java/org/archive/modules/writer/WriterPoolProcessor.java
+2 -2
View File
@@ -45,7 +45,7 @@ import org.archive.modules.Processor;
import org.archive.modules.deciderules.recrawl.IdenticalDigestDecideRule;
import org.archive.modules.net.CrawlHost;
import org.archive.modules.net.ServerCache;
import org.archive.modules.warc.WARCRecordBuilder;
import org.archive.modules.warc.BaseWARCRecordBuilder;
import org.archive.spring.ConfigPath;
import org.archive.util.FileUtils;
import org.json.JSONException;
@@ -372,7 +372,7 @@ implements Lifecycle, Checkpointable, WriterPoolSettings {
* @param curi CrawlURI
* @return String of IP address
*
* @deprecated WARCRecordBuilder instances use {@link WARCRecordBuilder#getHostAddress(CrawlURI)}
* @deprecated WARCRecordBuilder instances use {@link BaseWARCRecordBuilder#getHostAddress(CrawlURI)}
*/
@Deprecated
protected String getHostAddress(CrawlURI curi) {