HER-1790 - record warc where url was saved in crawl.log

* CrawlURI.java
    new field JSONObject extraInfo, methods getExtraInfo() and addExtraInfo()
* WARCWriterProcessor.java
    write() - on success, add warcFilename to CrawlURI extraInfo
* CrawlerLoggerModule.java
    new config option logExtraInfo, default false 
* UriProcessingFormatter.java
    new field logExtraInfo
    format() - include CrawlURI extraInfo if logExtraInfo is enabled
               also include "-" if CrawlURI has no annotations, since this is no longer the last field on the line
* profile-crawler-beans.cxml
    <!-- <property name="logExtraInfo" value="false" /> -->
* NonFatalErrorFormatter.java, RuntimeErrorFormatter.java
    new constructor to handle logExtraInfo since these subclass UriProcessingFormatter

engine/src/main/java/org/archive/crawler/io/NonFatalErrorFormatter.java

@@ -30,6 +30,10 @@ import org.archive.modules.CoreAttributeConstants;
  */
 public class NonFatalErrorFormatter extends UriProcessingFormatter implements CoreAttributeConstants {
 
+    public NonFatalErrorFormatter(boolean logExtraInfo) {
+        super(logExtraInfo);
+    }
+
     /* (non-Javadoc)
      * @see java.util.logging.Formatter#format(java.util.logging.LogRecord)
      */

engine/src/main/java/org/archive/crawler/io/RuntimeErrorFormatter.java

@@ -35,6 +35,11 @@ import org.archive.modules.CrawlURI;
  */
 public class RuntimeErrorFormatter extends UriProcessingFormatter
 implements CoreAttributeConstants {
+
+    public RuntimeErrorFormatter(boolean logExtraInfo) {
+        super(logExtraInfo);
+    }
+
     public String format(LogRecord lr) {
         Object [] parameters = lr.getParameters();
         String stackTrace = "None retrieved";

engine/src/main/java/org/archive/crawler/io/UriProcessingFormatter.java

@@ -58,8 +58,13 @@ extends Formatter implements CoreAttributeConstants {
     };
     
     protected final ThreadLocal<LogRecord> cachedRecord = new ThreadLocal<LogRecord>(); 
-    protected final ThreadLocal<String> cachedFormat = new ThreadLocal<String>(); 
+    protected final ThreadLocal<String> cachedFormat = new ThreadLocal<String>();
+    protected boolean logExtraInfo; 
     
+    public UriProcessingFormatter(boolean logExtraInfo) {
+        this.logExtraInfo = logExtraInfo;
+    }
+
     public String format(LogRecord lr) {
         if(lr==cachedRecord.get()) {
             return cachedFormat.get();
@@ -138,14 +143,22 @@ extends Formatter implements CoreAttributeConstants {
             	buffer.append(',');
             	buffer.append(iter.next());
             }
+        } else {
+            buffer.append(NA);
         }
+        
+        if (logExtraInfo) {
+            // XXX would we rather have "-" if info's empty?
+            buffer.append(" ").append(curi.getExtraInfo());
+        }
+        
         buffer.append("\n");
         cachedRecord.set(lr); 
         String formatted = buffer.toString(); 
         cachedFormat.set(formatted);
         return formatted;
     }
-    
+
     /**
      * @param str String to check.
      * @return Return passed string or <code>NA</code> if null.

engine/src/main/java/org/archive/crawler/reporting/CrawlerLoggerModule.java

@@ -69,15 +69,26 @@ public class CrawlerLoggerModule
     public void setPath(ConfigPath cp) {
         this.path.merge(cp);
     }
+
+    /**
+     * Whether to include the "extra info" field for each entry in crawl.log.
+     * "Extra info" is arbitrary JSON. It is the last field of the log line.
+     */
+    boolean logExtraInfo = false;
+    public boolean getLogExtraInfo() {
+        return logExtraInfo;
+    }
+    public void setLogExtraInfo(boolean logExtraInfo) {
+        this.logExtraInfo = logExtraInfo;
+    }
     
     // manifest support
-    /** abbrieviation label for config files in manifest */
+    /** abbreviation label for config files in manifest */
     public static final char MANIFEST_CONFIG_FILE = 'C';
-    /** abbrieviation label for report files in manifest */
+    /** abbreviation label for report files in manifest */
     public static final char MANIFEST_REPORT_FILE = 'R';
-    /** abbrieviation label for log files in manifest */
+    /** abbreviation label for log files in manifest */
     public static final char MANIFEST_LOG_FILE = 'L';
-
     
     // key log names
     private static final String LOGNAME_CRAWL = "crawl";
@@ -239,15 +250,15 @@ public class CrawlerLoggerModule
         this.fileHandlers = new HashMap<Logger,FileHandler>();
         setupLogFile(uriProcessing,
             getCrawlLogPath().getFile().getAbsolutePath(),
-            new UriProcessingFormatter(), true);
+            new UriProcessingFormatter(getLogExtraInfo()), true);
 
         setupLogFile(runtimeErrors,
             getRuntimeErrorsLogPath().getFile().getAbsolutePath(),
-            new RuntimeErrorFormatter(), true);
+            new RuntimeErrorFormatter(getLogExtraInfo()), true);
 
         setupLogFile(nonfatalErrors,
             getNonfatalErrorsLogPath().getFile().getAbsolutePath(),
-            new NonFatalErrorFormatter(), true);
+            new NonFatalErrorFormatter(getLogExtraInfo()), true);
 
         setupLogFile(uriErrors,
             getUriErrorsLogPath().getFile().getAbsolutePath(),

engine/src/main/resources/org/archive/crawler/restlet/profile-crawler-beans.cxml

@@ -627,6 +627,7 @@ http://example.example/example
   <!-- <property name="uriErrorsLogPath" value="uri-errors.log" /> -->
   <!-- <property name="runtimeErrorsLogPath" value="runtime-errors.log" /> -->
   <!-- <property name="nonfatalErrorsLogPath" value="nonfatal-errors.log" /> -->
+  <!-- <property name="logExtraInfo" value="false" /> -->
  </bean>
  
  <!-- SHEETOVERLAYMANAGER: manager of sheets of contextual overlays

modules/src/main/java/org/archive/modules/CrawlURI.java

@@ -69,6 +69,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.logging.Level;
 import java.util.logging.Logger;
 
 import org.apache.commons.httpclient.HttpMethod;
@@ -92,6 +93,8 @@ import org.archive.util.ArchiveUtils;
 import org.archive.util.Base32;
 import org.archive.util.MultiReporter;
 import org.archive.util.Recorder;
+import org.json.JSONException;
+import org.json.JSONObject;
 
 
 /**
@@ -1793,6 +1796,9 @@ implements MultiReporter, Serializable, OverlayContext {
      * A future time at which this CrawlURI should be reenqueued.
      */
     protected long rescheduleTime = -1;
+
+    protected JSONObject extraInfo;
+    
     public void setRescheduleTime(long time) {
         this.rescheduleTime = time;
     }
@@ -1866,5 +1872,20 @@ implements MultiReporter, Serializable, OverlayContext {
         
         return caUri;
     }
+    
+    public JSONObject getExtraInfo() {
+        if (extraInfo == null) {
+            extraInfo = new JSONObject();
+        }
+        return extraInfo;
+    }
+
+    public void addExtraInfo(String key, Object value) {
+        try {
+            getExtraInfo().put(key, value);
+        } catch (JSONException e) {
+            logger.log(Level.WARNING, "failed to add extra info", e);
+        }
+    }
 
 }

modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java

@@ -70,6 +70,7 @@ import org.apache.commons.httpclient.HttpMethod;
 import org.apache.commons.httpclient.HttpStatus;
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang.StringUtils;
+import org.archive.io.ArchiveFileConstants;
 import org.archive.io.ReplayInputStream;
 import org.archive.io.warc.WARCWriter;
 import org.archive.io.warc.WARCWriterPool;
@@ -164,7 +165,7 @@ public class WARCWriterProcessor extends WriterPoolProcessor implements WARCWrit
     public void setWriteRevisitForNotModified(boolean writeRevisits) {
         kp.put("writeRevisitForNotModified",writeRevisits);
     }
-    
+
     /**
      * Generator for record IDs
      */
@@ -276,6 +277,12 @@ public class WARCWriterProcessor extends WriterPoolProcessor implements WARCWrit
             	setTotalBytesWritten(getTotalBytesWritten() +
             	     (writer.getPosition() - position));
                 getPool().returnFile(writer);
+
+                String filename = writer.getFile().getName();
+                if (filename.endsWith(ArchiveFileConstants.OCCUPIED_SUFFIX)) {
+                    filename = filename.substring(0, filename.length() - ArchiveFileConstants.OCCUPIED_SUFFIX.length());
+                }
+                curi.addExtraInfo("warcFilename", filename);
             }
         }
         return checkBytesWritten();