Make logging of metadata record optional, but default.

This commit is contained in:
Adam Miller
2024-06-13 23:28:05 +00:00
parent 2f3605540d
commit 5483407829
@@ -27,6 +27,7 @@ import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.RandomAccessFile;
import java.io.Reader;
import java.io.UnsupportedEncodingException;
@@ -187,6 +188,19 @@ public class ExtractorYoutubeDL extends Extractor
this.controller = controller;
}
{
setLogMetadataRecord(true);
}
public boolean getLogMetadataRecord() {
return (Boolean) kp.get("logMetadataRecord");
}
/**
* Whether or not to create a crawl.log entry for any WARC Metadata Records written.
*/
public void setLogMetadataRecord(boolean logMetadataRecord) {
kp.put("logMetadataRecord",logMetadataRecord);
}
@Override
public void start() {
if (!isRunning) {
@@ -649,6 +663,9 @@ public class ExtractorYoutubeDL extends Extractor
*/
@Override
public void postWrite(WARCRecordInfo recordInfo, CrawlURI curi) {
if(!this.getLogMetadataRecord())
return;
CrawlURI pseudoCuri = null;
try {
pseudoCuri = curi.createCrawlURI(recordInfo.getUrl(), LinkContext.EMBED_MISC, Hop.INFERRED);