diff --git a/engine/src/main/java/org/archive/crawler/io/UriProcessingFormatter.java b/engine/src/main/java/org/archive/crawler/io/UriProcessingFormatter.java index 02cb9a67..5c12e4ae 100644 --- a/engine/src/main/java/org/archive/crawler/io/UriProcessingFormatter.java +++ b/engine/src/main/java/org/archive/crawler/io/UriProcessingFormatter.java @@ -83,14 +83,12 @@ extends Formatter implements Preformatter, CoreAttributeConstants { } else if (curi.getContentSize() > 0) { length = Long.toString(curi.getContentSize()); } - mime = curi.getContentType(); } else { if (curi.getContentSize() > 0) { length = Long.toString(curi.getContentSize()); } - mime = curi.getContentType(); } - mime = MimetypeUtils.truncate(mime); + mime = MimetypeUtils.truncate(curi.getContentType()); long time = System.currentTimeMillis(); diff --git a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java index 249334df..fed4e501 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java +++ b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java @@ -962,6 +962,10 @@ public class FetchHTTP extends Processor implements Lifecycle { protected void setSizes(CrawlURI curi, Recorder rec) { // set reporting size curi.setContentSize(rec.getRecordedInput().getSize()); + + // add contentSize to extraInfo so it's available to log in the crawl log + curi.addExtraInfo("contentSize", rec.getRecordedInput().getSize()); + // special handling for 304-not modified if (curi.getFetchStatus() == HttpStatus.SC_NOT_MODIFIED && curi.getFetchHistory() != null) {