mirror of
https://github.com/internetarchive/heritrix3.git
synced 2026-09-23 14:15:47 +00:00
Make logging of metadata record optional, but default.
This commit is contained in:
@@ -27,6 +27,7 @@ import java.io.FileInputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.InputStreamReader;
|
||||
import java.io.OutputStream;
|
||||
import java.io.RandomAccessFile;
|
||||
import java.io.Reader;
|
||||
import java.io.UnsupportedEncodingException;
|
||||
@@ -187,6 +188,19 @@ public class ExtractorYoutubeDL extends Extractor
|
||||
this.controller = controller;
|
||||
}
|
||||
|
||||
{
|
||||
setLogMetadataRecord(true);
|
||||
}
|
||||
public boolean getLogMetadataRecord() {
|
||||
return (Boolean) kp.get("logMetadataRecord");
|
||||
}
|
||||
/**
|
||||
* Whether or not to create a crawl.log entry for any WARC Metadata Records written.
|
||||
*/
|
||||
public void setLogMetadataRecord(boolean logMetadataRecord) {
|
||||
kp.put("logMetadataRecord",logMetadataRecord);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void start() {
|
||||
if (!isRunning) {
|
||||
@@ -649,6 +663,9 @@ public class ExtractorYoutubeDL extends Extractor
|
||||
*/
|
||||
@Override
|
||||
public void postWrite(WARCRecordInfo recordInfo, CrawlURI curi) {
|
||||
if(!this.getLogMetadataRecord())
|
||||
return;
|
||||
|
||||
CrawlURI pseudoCuri = null;
|
||||
try {
|
||||
pseudoCuri = curi.createCrawlURI(recordInfo.getUrl(), LinkContext.EMBED_MISC, Hop.INFERRED);
|
||||
|
||||
Reference in New Issue
Block a user