lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-09-26 15:46:31 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'extra-info-size' into aitfive-203

Browse Source
This commit is contained in:
Noah Levitt
2014-11-07 15:58:08 -08:00
parent 23fa29d673 7312ad597e
commit 4527d5af29
2 changed files with 5 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
engine/src/main/java/org/archive/crawler/io/UriProcessingFormatter.java
+1 -3
View File
@@ -83,14 +83,12 @@ extends Formatter implements Preformatter, CoreAttributeConstants {
} else if (curi.getContentSize() > 0) {
length = Long.toString(curi.getContentSize());
}
mime = curi.getContentType();
} else {
if (curi.getContentSize() > 0) {
length = Long.toString(curi.getContentSize());
}
mime = curi.getContentType();
}
mime = MimetypeUtils.truncate(mime);
mime = MimetypeUtils.truncate(curi.getContentType());
long time = System.currentTimeMillis();
modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java
+4
View File
@@ -962,6 +962,10 @@ public class FetchHTTP extends Processor implements Lifecycle {
protected void setSizes(CrawlURI curi, Recorder rec) {
// set reporting size
curi.setContentSize(rec.getRecordedInput().getSize());
// add contentSize to extraInfo so it's available to log in the crawl log
curi.addExtraInfo("contentSize", rec.getRecordedInput().getSize());
// special handling for 304-not modified
if (curi.getFetchStatus() == HttpStatus.SC_NOT_MODIFIED
&& curi.getFetchHistory() != null) {