From 5e1571f8ddbcc45b1c0ef024211fd5fdcb932ab8 Mon Sep 17 00:00:00 2001 From: nlevitt Date: Wed, 25 May 2011 22:11:18 +0000 Subject: [PATCH] HER-1790 - record warc where url was saved in crawl.log * CrawlURI.java new field JSONObject extraInfo, methods getExtraInfo() and addExtraInfo() * WARCWriterProcessor.java write() - on success, add warcFilename to CrawlURI extraInfo * CrawlerLoggerModule.java new config option logExtraInfo, default false * UriProcessingFormatter.java new field logExtraInfo format() - include CrawlURI extraInfo if logExtraInfo is enabled also include "-" if CrawlURI has no annotations, since this is no longer the last field on the line * profile-crawler-beans.cxml * NonFatalErrorFormatter.java, RuntimeErrorFormatter.java new constructor to handle logExtraInfo since these subclass UriProcessingFormatter --- .../crawler/io/NonFatalErrorFormatter.java | 4 +++ .../crawler/io/RuntimeErrorFormatter.java | 5 ++++ .../crawler/io/UriProcessingFormatter.java | 17 +++++++++++-- .../reporting/CrawlerLoggerModule.java | 25 +++++++++++++------ .../restlet/profile-crawler-beans.cxml | 1 + .../java/org/archive/modules/CrawlURI.java | 21 ++++++++++++++++ .../modules/writer/WARCWriterProcessor.java | 9 ++++++- 7 files changed, 72 insertions(+), 10 deletions(-) diff --git a/engine/src/main/java/org/archive/crawler/io/NonFatalErrorFormatter.java b/engine/src/main/java/org/archive/crawler/io/NonFatalErrorFormatter.java index c64448e6..bc3b7759 100644 --- a/engine/src/main/java/org/archive/crawler/io/NonFatalErrorFormatter.java +++ b/engine/src/main/java/org/archive/crawler/io/NonFatalErrorFormatter.java @@ -30,6 +30,10 @@ import org.archive.modules.CoreAttributeConstants; */ public class NonFatalErrorFormatter extends UriProcessingFormatter implements CoreAttributeConstants { + public NonFatalErrorFormatter(boolean logExtraInfo) { + super(logExtraInfo); + } + /* (non-Javadoc) * @see java.util.logging.Formatter#format(java.util.logging.LogRecord) */ diff --git a/engine/src/main/java/org/archive/crawler/io/RuntimeErrorFormatter.java b/engine/src/main/java/org/archive/crawler/io/RuntimeErrorFormatter.java index 7b6477c1..537fbca2 100644 --- a/engine/src/main/java/org/archive/crawler/io/RuntimeErrorFormatter.java +++ b/engine/src/main/java/org/archive/crawler/io/RuntimeErrorFormatter.java @@ -35,6 +35,11 @@ import org.archive.modules.CrawlURI; */ public class RuntimeErrorFormatter extends UriProcessingFormatter implements CoreAttributeConstants { + + public RuntimeErrorFormatter(boolean logExtraInfo) { + super(logExtraInfo); + } + public String format(LogRecord lr) { Object [] parameters = lr.getParameters(); String stackTrace = "None retrieved"; diff --git a/engine/src/main/java/org/archive/crawler/io/UriProcessingFormatter.java b/engine/src/main/java/org/archive/crawler/io/UriProcessingFormatter.java index 933857a8..6be14cd4 100644 --- a/engine/src/main/java/org/archive/crawler/io/UriProcessingFormatter.java +++ b/engine/src/main/java/org/archive/crawler/io/UriProcessingFormatter.java @@ -58,8 +58,13 @@ extends Formatter implements CoreAttributeConstants { }; protected final ThreadLocal cachedRecord = new ThreadLocal(); - protected final ThreadLocal cachedFormat = new ThreadLocal(); + protected final ThreadLocal cachedFormat = new ThreadLocal(); + protected boolean logExtraInfo; + public UriProcessingFormatter(boolean logExtraInfo) { + this.logExtraInfo = logExtraInfo; + } + public String format(LogRecord lr) { if(lr==cachedRecord.get()) { return cachedFormat.get(); @@ -138,14 +143,22 @@ extends Formatter implements CoreAttributeConstants { buffer.append(','); buffer.append(iter.next()); } + } else { + buffer.append(NA); } + + if (logExtraInfo) { + // XXX would we rather have "-" if info's empty? + buffer.append(" ").append(curi.getExtraInfo()); + } + buffer.append("\n"); cachedRecord.set(lr); String formatted = buffer.toString(); cachedFormat.set(formatted); return formatted; } - + /** * @param str String to check. * @return Return passed string or NA if null. diff --git a/engine/src/main/java/org/archive/crawler/reporting/CrawlerLoggerModule.java b/engine/src/main/java/org/archive/crawler/reporting/CrawlerLoggerModule.java index 8c5114dd..3cf128ac 100644 --- a/engine/src/main/java/org/archive/crawler/reporting/CrawlerLoggerModule.java +++ b/engine/src/main/java/org/archive/crawler/reporting/CrawlerLoggerModule.java @@ -69,15 +69,26 @@ public class CrawlerLoggerModule public void setPath(ConfigPath cp) { this.path.merge(cp); } + + /** + * Whether to include the "extra info" field for each entry in crawl.log. + * "Extra info" is arbitrary JSON. It is the last field of the log line. + */ + boolean logExtraInfo = false; + public boolean getLogExtraInfo() { + return logExtraInfo; + } + public void setLogExtraInfo(boolean logExtraInfo) { + this.logExtraInfo = logExtraInfo; + } // manifest support - /** abbrieviation label for config files in manifest */ + /** abbreviation label for config files in manifest */ public static final char MANIFEST_CONFIG_FILE = 'C'; - /** abbrieviation label for report files in manifest */ + /** abbreviation label for report files in manifest */ public static final char MANIFEST_REPORT_FILE = 'R'; - /** abbrieviation label for log files in manifest */ + /** abbreviation label for log files in manifest */ public static final char MANIFEST_LOG_FILE = 'L'; - // key log names private static final String LOGNAME_CRAWL = "crawl"; @@ -239,15 +250,15 @@ public class CrawlerLoggerModule this.fileHandlers = new HashMap(); setupLogFile(uriProcessing, getCrawlLogPath().getFile().getAbsolutePath(), - new UriProcessingFormatter(), true); + new UriProcessingFormatter(getLogExtraInfo()), true); setupLogFile(runtimeErrors, getRuntimeErrorsLogPath().getFile().getAbsolutePath(), - new RuntimeErrorFormatter(), true); + new RuntimeErrorFormatter(getLogExtraInfo()), true); setupLogFile(nonfatalErrors, getNonfatalErrorsLogPath().getFile().getAbsolutePath(), - new NonFatalErrorFormatter(), true); + new NonFatalErrorFormatter(getLogExtraInfo()), true); setupLogFile(uriErrors, getUriErrorsLogPath().getFile().getAbsolutePath(), diff --git a/engine/src/main/resources/org/archive/crawler/restlet/profile-crawler-beans.cxml b/engine/src/main/resources/org/archive/crawler/restlet/profile-crawler-beans.cxml index 5fa2e30a..d3accb21 100644 --- a/engine/src/main/resources/org/archive/crawler/restlet/profile-crawler-beans.cxml +++ b/engine/src/main/resources/org/archive/crawler/restlet/profile-crawler-beans.cxml @@ -627,6 +627,7 @@ http://example.example/example +