lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-08-26 08:26:53 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add sizeOnDisk on job status (#699)

Browse Source
This commit is contained in:
Leslie Bellony
2025-12-09 15:59:03 +01:00
parent fae274a23b
commit 8a7db14a51
1 changed files with 35 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
engine/src/main/java/org/archive/crawler/restlet/models/CrawlJobModel.java
+35 -1
View File
@@ -11,6 +11,7 @@ import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.logging.Level;
import org.apache.commons.io.filefilter.IOFileFilter;
import org.archive.checkpointing.Checkpoint;
@@ -25,6 +26,8 @@ import org.archive.util.FileUtils;
@SuppressWarnings("serial")
public class CrawlJobModel extends LinkedHashMap<String, Object> implements Serializable{
private static final String CURRENT_WARC = ".open";
private static final String LATEST_WARCS = "/latest/warcs";
private CrawlJob crawlJob;
public static final IOFileFilter EDIT_FILTER = FileUtils
.getRegexFileFilter(".*\\.((c?xml)|(txt))$");
@@ -85,6 +88,13 @@ public class CrawlJobModel extends LinkedHashMap<String, Object> implements Seri
if (!sizeTotalsReportData.containsKey("totalCount")) {
sizeTotalsReportData.put("totalCount", 0L);
}
File warcsDir = new File(crawlJob.getJobDir(), LATEST_WARCS);
if (warcsDir.exists()) {
sizeTotalsReportData.put("sizeOnDisk", sizeOfDir(warcsDir));
} else {
sizeTotalsReportData.put("sizeOnDisk", 0L);
}
this.put("sizeTotalsReport", sizeTotalsReportData);
this.put("rateReport", crawlJob.rateReportData());
@@ -147,6 +157,30 @@ public class CrawlJobModel extends LinkedHashMap<String, Object> implements Seri
}
this.put("reports", generateReports());
}
private long sizeOfDir(File warcsDir) {
long size = 0;
File[] warcsFiles = warcsDir.listFiles();
if (warcsFiles != null) {
for (File file : warcsFiles) {
if (!file.exists()) {
File closedWarc = new File(warcsDir.getAbsolutePath(), file.getName().replace(CURRENT_WARC, ""));
if (closedWarc.exists()) {
crawlJob.getJobLogger().log(Level.INFO, "sizeTotalsReportData - filename is changed from " + file.getName() + " to "+ closedWarc.getName());
file = closedWarc;
} else {
crawlJob.getJobLogger().log(Level.WARNING, "sizeTotalsReportData - file " + file.getName() + " not exist" );
continue;
}
}
if (!file.isDirectory()) {
size += file.length();
}
}
}
return size;
}
public String formatBytes(Long bytes){
return ArchiveUtils.formatBytesForDisplay(bytes);
}
@@ -222,4 +256,4 @@ public class CrawlJobModel extends LinkedHashMap<String, Object> implements Seri
}
return referencedPaths;
}
}
}