From 7312ad597ebb616b61e52f4d68e3252d2c40b3b6 Mon Sep 17 00:00:00 2001
From: Noah Levitt <nlevitt@archive.org>
Date: Fri, 7 Nov 2014 13:21:42 -0800
Subject: [PATCH] so that it can appear in the crawl log, add contentSize to
 CrawlURI extraInfo, for http transactions (the only case in heritrix
 currently where contentLength and contentSize differ)

---
 .../java/org/archive/crawler/io/UriProcessingFormatter.java   | 4 +---
 .../src/main/java/org/archive/modules/fetcher/FetchHTTP.java  | 4 ++++
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/engine/src/main/java/org/archive/crawler/io/UriProcessingFormatter.java b/engine/src/main/java/org/archive/crawler/io/UriProcessingFormatter.java
index 02cb9a67..5c12e4ae 100644
--- a/engine/src/main/java/org/archive/crawler/io/UriProcessingFormatter.java
+++ b/engine/src/main/java/org/archive/crawler/io/UriProcessingFormatter.java
@@ -83,14 +83,12 @@ extends Formatter implements Preformatter, CoreAttributeConstants {
             } else if (curi.getContentSize() > 0) {
                 length = Long.toString(curi.getContentSize());
             }
-            mime = curi.getContentType();
         } else {
             if (curi.getContentSize() > 0) {
                 length = Long.toString(curi.getContentSize());
             } 
-            mime = curi.getContentType();
         }
-        mime = MimetypeUtils.truncate(mime);
+        mime = MimetypeUtils.truncate(curi.getContentType());
 
         long time = System.currentTimeMillis();
 
diff --git a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java
index 249334df..fed4e501 100644
--- a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java
+++ b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java
@@ -962,6 +962,10 @@ public class FetchHTTP extends Processor implements Lifecycle {
     protected void setSizes(CrawlURI curi, Recorder rec) {
         // set reporting size
         curi.setContentSize(rec.getRecordedInput().getSize());
+
+        // add contentSize to extraInfo so it's available to log in the crawl log
+        curi.addExtraInfo("contentSize", rec.getRecordedInput().getSize());
+
         // special handling for 304-not modified
         if (curi.getFetchStatus() == HttpStatus.SC_NOT_MODIFIED
                 && curi.getFetchHistory() != null) {