From 9ddd281ef28301959573bf2df4e37537f5868998 Mon Sep 17 00:00:00 2001 From: Noah Levitt Date: Wed, 12 Jun 2019 15:04:39 -0700 Subject: [PATCH] make WARCRecordBuilder an interface this way other classes that extend other classes can also implement WARCRecordBuilder --- .../modules/warc/BaseWARCRecordBuilder.java | 66 +++++++++++++++++++ .../warc/DnsResponseRecordBuilder.java | 2 +- .../FtpControlConversationRecordBuilder.java | 2 +- .../warc/FtpResponseRecordBuilder.java | 2 +- .../warc/HttpRequestRecordBuilder.java | 2 +- .../warc/HttpResponseRecordBuilder.java | 2 +- .../modules/warc/MetadataRecordBuilder.java | 2 +- .../modules/warc/RevisitRecordBuilder.java | 2 +- .../modules/warc/WARCRecordBuilder.java | 64 ++---------------- .../warc/WhoisResponseRecordBuilder.java | 2 +- .../writer/WARCWriterChainProcessor.java | 4 +- .../modules/writer/WriterPoolProcessor.java | 4 +- 12 files changed, 82 insertions(+), 72 deletions(-) create mode 100644 modules/src/main/java/org/archive/modules/warc/BaseWARCRecordBuilder.java diff --git a/modules/src/main/java/org/archive/modules/warc/BaseWARCRecordBuilder.java b/modules/src/main/java/org/archive/modules/warc/BaseWARCRecordBuilder.java new file mode 100644 index 00000000..40eba843 --- /dev/null +++ b/modules/src/main/java/org/archive/modules/warc/BaseWARCRecordBuilder.java @@ -0,0 +1,66 @@ +package org.archive.modules.warc; + +import static org.archive.modules.CoreAttributeConstants.A_DNS_SERVER_IP_LABEL; + +import java.net.InetAddress; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.UUID; + +import org.archive.modules.CrawlURI; +import org.archive.modules.net.CrawlHost; +import org.archive.modules.net.ServerCache; +import org.springframework.beans.factory.annotation.Autowired; + +public abstract class BaseWARCRecordBuilder implements WARCRecordBuilder { + + transient protected ServerCache serverCache; + public ServerCache getServerCache() { + return this.serverCache; + } + @Autowired + public void setServerCache(ServerCache serverCache) { + this.serverCache = serverCache; + } + + public URI generateRecordID() { + try { + return new URI("urn:uuid:" + UUID.randomUUID()); + } catch (URISyntaxException e) { + throw new RuntimeException(e); // impossible + } + } + + /** + * Return IP address of given URI suitable for recording (as in a + * classic ARC 5-field header line). + * + * @param curi CrawlURI + * @return String of IP address + */ + protected String getHostAddress(CrawlURI curi) { + // special handling for DNS URIs: want address of DNS server + if (curi.getUURI().getScheme().toLowerCase().equals("dns")) { + return (String)curi.getData().get(A_DNS_SERVER_IP_LABEL); + } + // otherwise, host referenced in URI + // TODO:FIXME: have fetcher insert exact IP contacted into curi, + // use that rather than inferred by CrawlHost lookup + CrawlHost h = getServerCache().getHostFor(curi.getUURI()); + if (h == null) { + throw new NullPointerException("Crawlhost is null for " + + curi + " " + curi.getVia()); + } + InetAddress a = h.getIP(); + if (a == null) { + throw new NullPointerException("Address is null for " + + curi + " " + curi.getVia() + ". Address " + + ((h.getIpFetched() == CrawlHost.IP_NEVER_LOOKED_UP)? + "was never looked up.": + (System.currentTimeMillis() - h.getIpFetched()) + + " ms ago.")); + } + return h.getIP().getHostAddress(); + } + +} diff --git a/modules/src/main/java/org/archive/modules/warc/DnsResponseRecordBuilder.java b/modules/src/main/java/org/archive/modules/warc/DnsResponseRecordBuilder.java index 2751b3a0..c8394215 100644 --- a/modules/src/main/java/org/archive/modules/warc/DnsResponseRecordBuilder.java +++ b/modules/src/main/java/org/archive/modules/warc/DnsResponseRecordBuilder.java @@ -12,7 +12,7 @@ import org.archive.io.warc.WARCRecordInfo; import org.archive.modules.CrawlURI; import org.archive.util.ArchiveUtils; -public class DnsResponseRecordBuilder extends WARCRecordBuilder { +public class DnsResponseRecordBuilder extends BaseWARCRecordBuilder { @Override public boolean shouldProcess(CrawlURI curi) { diff --git a/modules/src/main/java/org/archive/modules/warc/FtpControlConversationRecordBuilder.java b/modules/src/main/java/org/archive/modules/warc/FtpControlConversationRecordBuilder.java index 3d85c2eb..e5481080 100644 --- a/modules/src/main/java/org/archive/modules/warc/FtpControlConversationRecordBuilder.java +++ b/modules/src/main/java/org/archive/modules/warc/FtpControlConversationRecordBuilder.java @@ -14,7 +14,7 @@ import org.archive.modules.CrawlURI; import org.archive.util.ArchiveUtils; import org.archive.util.anvl.ANVLRecord; -public class FtpControlConversationRecordBuilder extends WARCRecordBuilder { +public class FtpControlConversationRecordBuilder extends BaseWARCRecordBuilder { @Override public boolean shouldProcess(CrawlURI curi) { diff --git a/modules/src/main/java/org/archive/modules/warc/FtpResponseRecordBuilder.java b/modules/src/main/java/org/archive/modules/warc/FtpResponseRecordBuilder.java index 60366ec0..f3068569 100644 --- a/modules/src/main/java/org/archive/modules/warc/FtpResponseRecordBuilder.java +++ b/modules/src/main/java/org/archive/modules/warc/FtpResponseRecordBuilder.java @@ -13,7 +13,7 @@ import org.archive.io.warc.WARCRecordInfo; import org.archive.modules.CrawlURI; import org.archive.util.ArchiveUtils; -public class FtpResponseRecordBuilder extends WARCRecordBuilder { +public class FtpResponseRecordBuilder extends BaseWARCRecordBuilder { @Override public boolean shouldProcess(CrawlURI curi) { diff --git a/modules/src/main/java/org/archive/modules/warc/HttpRequestRecordBuilder.java b/modules/src/main/java/org/archive/modules/warc/HttpRequestRecordBuilder.java index b71567aa..f2f55e54 100644 --- a/modules/src/main/java/org/archive/modules/warc/HttpRequestRecordBuilder.java +++ b/modules/src/main/java/org/archive/modules/warc/HttpRequestRecordBuilder.java @@ -12,7 +12,7 @@ import org.archive.io.warc.WARCRecordInfo; import org.archive.modules.CrawlURI; import org.archive.util.ArchiveUtils; -public class HttpRequestRecordBuilder extends WARCRecordBuilder { +public class HttpRequestRecordBuilder extends BaseWARCRecordBuilder { @Override public boolean shouldProcess(CrawlURI curi) { diff --git a/modules/src/main/java/org/archive/modules/warc/HttpResponseRecordBuilder.java b/modules/src/main/java/org/archive/modules/warc/HttpResponseRecordBuilder.java index b6f69e88..06a1b732 100644 --- a/modules/src/main/java/org/archive/modules/warc/HttpResponseRecordBuilder.java +++ b/modules/src/main/java/org/archive/modules/warc/HttpResponseRecordBuilder.java @@ -22,7 +22,7 @@ import org.archive.io.warc.WARCRecordInfo; import org.archive.modules.CrawlURI; import org.archive.util.ArchiveUtils; -public class HttpResponseRecordBuilder extends WARCRecordBuilder { +public class HttpResponseRecordBuilder extends BaseWARCRecordBuilder { @Override public boolean shouldProcess(CrawlURI curi) { diff --git a/modules/src/main/java/org/archive/modules/warc/MetadataRecordBuilder.java b/modules/src/main/java/org/archive/modules/warc/MetadataRecordBuilder.java index 51a1547a..9158810f 100644 --- a/modules/src/main/java/org/archive/modules/warc/MetadataRecordBuilder.java +++ b/modules/src/main/java/org/archive/modules/warc/MetadataRecordBuilder.java @@ -16,7 +16,7 @@ import org.archive.modules.CrawlURI; import org.archive.util.ArchiveUtils; import org.archive.util.anvl.ANVLRecord; -public class MetadataRecordBuilder extends WARCRecordBuilder { +public class MetadataRecordBuilder extends BaseWARCRecordBuilder { /** * If you don't want metadata records, take this class out of the chain. diff --git a/modules/src/main/java/org/archive/modules/warc/RevisitRecordBuilder.java b/modules/src/main/java/org/archive/modules/warc/RevisitRecordBuilder.java index bbed451a..aaaaac55 100644 --- a/modules/src/main/java/org/archive/modules/warc/RevisitRecordBuilder.java +++ b/modules/src/main/java/org/archive/modules/warc/RevisitRecordBuilder.java @@ -18,7 +18,7 @@ import org.archive.modules.CrawlURI; import org.archive.modules.revisit.RevisitProfile; import org.archive.util.ArchiveUtils; -public class RevisitRecordBuilder extends WARCRecordBuilder { +public class RevisitRecordBuilder extends BaseWARCRecordBuilder { @Override public boolean shouldProcess(CrawlURI curi) { diff --git a/modules/src/main/java/org/archive/modules/warc/WARCRecordBuilder.java b/modules/src/main/java/org/archive/modules/warc/WARCRecordBuilder.java index a9ec884b..ecd02db4 100644 --- a/modules/src/main/java/org/archive/modules/warc/WARCRecordBuilder.java +++ b/modules/src/main/java/org/archive/modules/warc/WARCRecordBuilder.java @@ -1,72 +1,16 @@ package org.archive.modules.warc; -import static org.archive.modules.CoreAttributeConstants.A_DNS_SERVER_IP_LABEL; - import java.io.IOException; -import java.net.InetAddress; import java.net.URI; -import java.net.URISyntaxException; -import java.util.UUID; import org.archive.io.warc.WARCRecordInfo; import org.archive.modules.CrawlURI; -import org.archive.modules.net.CrawlHost; -import org.archive.modules.net.ServerCache; -import org.springframework.beans.factory.annotation.Autowired; -public abstract class WARCRecordBuilder { +public interface WARCRecordBuilder { - transient protected ServerCache serverCache; - public ServerCache getServerCache() { - return this.serverCache; - } - @Autowired - public void setServerCache(ServerCache serverCache) { - this.serverCache = serverCache; - } + boolean shouldProcess(CrawlURI curi); - public abstract boolean shouldProcess(CrawlURI curi); - public abstract WARCRecordInfo buildRecord(CrawlURI curi, URI concurrentTo) + WARCRecordInfo buildRecord(CrawlURI curi, URI concurrentTo) throws IOException; - - public URI generateRecordID() { - try { - return new URI("urn:uuid:" + UUID.randomUUID()); - } catch (URISyntaxException e) { - throw new RuntimeException(e); // impossible - } - } - - /** - * Return IP address of given URI suitable for recording (as in a - * classic ARC 5-field header line). - * - * @param curi CrawlURI - * @return String of IP address - */ - protected String getHostAddress(CrawlURI curi) { - // special handling for DNS URIs: want address of DNS server - if (curi.getUURI().getScheme().toLowerCase().equals("dns")) { - return (String)curi.getData().get(A_DNS_SERVER_IP_LABEL); - } - // otherwise, host referenced in URI - // TODO:FIXME: have fetcher insert exact IP contacted into curi, - // use that rather than inferred by CrawlHost lookup - CrawlHost h = getServerCache().getHostFor(curi.getUURI()); - if (h == null) { - throw new NullPointerException("Crawlhost is null for " + - curi + " " + curi.getVia()); - } - InetAddress a = h.getIP(); - if (a == null) { - throw new NullPointerException("Address is null for " + - curi + " " + curi.getVia() + ". Address " + - ((h.getIpFetched() == CrawlHost.IP_NEVER_LOOKED_UP)? - "was never looked up.": - (System.currentTimeMillis() - h.getIpFetched()) + - " ms ago.")); - } - return h.getIP().getHostAddress(); - } -} +} \ No newline at end of file diff --git a/modules/src/main/java/org/archive/modules/warc/WhoisResponseRecordBuilder.java b/modules/src/main/java/org/archive/modules/warc/WhoisResponseRecordBuilder.java index cefc1da8..8ddbb9d1 100644 --- a/modules/src/main/java/org/archive/modules/warc/WhoisResponseRecordBuilder.java +++ b/modules/src/main/java/org/archive/modules/warc/WhoisResponseRecordBuilder.java @@ -12,7 +12,7 @@ import org.archive.modules.CoreAttributeConstants; import org.archive.modules.CrawlURI; import org.archive.util.ArchiveUtils; -public class WhoisResponseRecordBuilder extends WARCRecordBuilder { +public class WhoisResponseRecordBuilder extends BaseWARCRecordBuilder { @Override public boolean shouldProcess(CrawlURI curi) { diff --git a/modules/src/main/java/org/archive/modules/writer/WARCWriterChainProcessor.java b/modules/src/main/java/org/archive/modules/writer/WARCWriterChainProcessor.java index 84e1b6f3..6c04c06b 100644 --- a/modules/src/main/java/org/archive/modules/writer/WARCWriterChainProcessor.java +++ b/modules/src/main/java/org/archive/modules/writer/WARCWriterChainProcessor.java @@ -39,10 +39,10 @@ public class WARCWriterChainProcessor extends BaseWARCWriterProcessor implements new MetadataRecordBuilder())); } @SuppressWarnings("unchecked") - public List getChain() { + public List getChain() { return (List) kp.get("chain"); } - public void setChain(List chain) { + public void setChain(List chain) { kp.put("chain", chain); } diff --git a/modules/src/main/java/org/archive/modules/writer/WriterPoolProcessor.java b/modules/src/main/java/org/archive/modules/writer/WriterPoolProcessor.java index 12a9cabd..a5a030e8 100644 --- a/modules/src/main/java/org/archive/modules/writer/WriterPoolProcessor.java +++ b/modules/src/main/java/org/archive/modules/writer/WriterPoolProcessor.java @@ -45,7 +45,7 @@ import org.archive.modules.Processor; import org.archive.modules.deciderules.recrawl.IdenticalDigestDecideRule; import org.archive.modules.net.CrawlHost; import org.archive.modules.net.ServerCache; -import org.archive.modules.warc.WARCRecordBuilder; +import org.archive.modules.warc.BaseWARCRecordBuilder; import org.archive.spring.ConfigPath; import org.archive.util.FileUtils; import org.json.JSONException; @@ -372,7 +372,7 @@ implements Lifecycle, Checkpointable, WriterPoolSettings { * @param curi CrawlURI * @return String of IP address * - * @deprecated WARCRecordBuilder instances use {@link WARCRecordBuilder#getHostAddress(CrawlURI)} + * @deprecated WARCRecordBuilder instances use {@link BaseWARCRecordBuilder#getHostAddress(CrawlURI)} */ @Deprecated protected String getHostAddress(CrawlURI curi) {