HER-1790 - record warc where url was saved in crawl.log

* CrawlURI.java
    new field JSONObject extraInfo, methods getExtraInfo() and addExtraInfo()
* WARCWriterProcessor.java
    write() - on success, add warcFilename to CrawlURI extraInfo
* CrawlerLoggerModule.java
    new config option logExtraInfo, default false 
* UriProcessingFormatter.java
    new field logExtraInfo
    format() - include CrawlURI extraInfo if logExtraInfo is enabled
               also include "-" if CrawlURI has no annotations, since this is no longer the last field on the line
* profile-crawler-beans.cxml
    <!-- <property name="logExtraInfo" value="false" /> -->
* NonFatalErrorFormatter.java, RuntimeErrorFormatter.java
    new constructor to handle logExtraInfo since these subclass UriProcessingFormatter

engine/src/main/java/org/archive/crawler/io/NonFatalErrorFormatter.java

@@ -30,6 +30,10 @@ import org.archive.modules.CoreAttributeConstants;
  */
 public class NonFatalErrorFormatter extends UriProcessingFormatter implements CoreAttributeConstants {
 
+    public NonFatalErrorFormatter(boolean logExtraInfo) {
+        super(logExtraInfo);
+    }
+
     /* (non-Javadoc)
      * @see java.util.logging.Formatter#format(java.util.logging.LogRecord)
      */

engine/src/main/java/org/archive/crawler/io/RuntimeErrorFormatter.java

@@ -35,6 +35,11 @@ import org.archive.modules.CrawlURI;
  */
 public class RuntimeErrorFormatter extends UriProcessingFormatter
 implements CoreAttributeConstants {
+
+    public RuntimeErrorFormatter(boolean logExtraInfo) {
+        super(logExtraInfo);
+    }
+
     public String format(LogRecord lr) {
         Object [] parameters = lr.getParameters();
         String stackTrace = "None retrieved";

engine/src/main/java/org/archive/crawler/io/UriProcessingFormatter.java

@@ -58,8 +58,13 @@ extends Formatter implements CoreAttributeConstants {
     };
     
     protected final ThreadLocal<LogRecord> cachedRecord = new ThreadLocal<LogRecord>(); 
-    protected final ThreadLocal<String> cachedFormat = new ThreadLocal<String>(); 
+    protected final ThreadLocal<String> cachedFormat = new ThreadLocal<String>();
+    protected boolean logExtraInfo; 
     
+    public UriProcessingFormatter(boolean logExtraInfo) {
+        this.logExtraInfo = logExtraInfo;
+    }
+
     public String format(LogRecord lr) {
         if(lr==cachedRecord.get()) {
             return cachedFormat.get();
@@ -138,14 +143,22 @@ extends Formatter implements CoreAttributeConstants {
             	buffer.append(',');
             	buffer.append(iter.next());
             }
+        } else {
+            buffer.append(NA);
         }
+        
+        if (logExtraInfo) {
+            // XXX would we rather have "-" if info's empty?
+            buffer.append(" ").append(curi.getExtraInfo());
+        }
+        
         buffer.append("\n");
         cachedRecord.set(lr); 
         String formatted = buffer.toString(); 
         cachedFormat.set(formatted);
         return formatted;
     }
-    
+
     /**
      * @param str String to check.
      * @return Return passed string or <code>NA</code> if null.

engine/src/main/java/org/archive/crawler/reporting/CrawlerLoggerModule.java

@@ -69,15 +69,26 @@ public class CrawlerLoggerModule
     public void setPath(ConfigPath cp) {
         this.path.merge(cp);
     }
+
+    /**
+     * Whether to include the "extra info" field for each entry in crawl.log.
+     * "Extra info" is arbitrary JSON. It is the last field of the log line.
+     */
+    boolean logExtraInfo = false;
+    public boolean getLogExtraInfo() {
+        return logExtraInfo;
+    }
+    public void setLogExtraInfo(boolean logExtraInfo) {
+        this.logExtraInfo = logExtraInfo;
+    }
     
     // manifest support
-    /** abbrieviation label for config files in manifest */
+    /** abbreviation label for config files in manifest */
     public static final char MANIFEST_CONFIG_FILE = 'C';
-    /** abbrieviation label for report files in manifest */
+    /** abbreviation label for report files in manifest */
     public static final char MANIFEST_REPORT_FILE = 'R';
-    /** abbrieviation label for log files in manifest */
+    /** abbreviation label for log files in manifest */
     public static final char MANIFEST_LOG_FILE = 'L';
-
     
     // key log names
     private static final String LOGNAME_CRAWL = "crawl";
@@ -239,15 +250,15 @@ public class CrawlerLoggerModule
         this.fileHandlers = new HashMap<Logger,FileHandler>();
         setupLogFile(uriProcessing,
             getCrawlLogPath().getFile().getAbsolutePath(),
-            new UriProcessingFormatter(), true);
+            new UriProcessingFormatter(getLogExtraInfo()), true);
 
         setupLogFile(runtimeErrors,
             getRuntimeErrorsLogPath().getFile().getAbsolutePath(),
-            new RuntimeErrorFormatter(), true);
+            new RuntimeErrorFormatter(getLogExtraInfo()), true);
 
         setupLogFile(nonfatalErrors,
             getNonfatalErrorsLogPath().getFile().getAbsolutePath(),
-            new NonFatalErrorFormatter(), true);
+            new NonFatalErrorFormatter(getLogExtraInfo()), true);
 
         setupLogFile(uriErrors,
             getUriErrorsLogPath().getFile().getAbsolutePath(),