From 8a7db14a517ecdce069daf1e82637e8bf68e6bbd Mon Sep 17 00:00:00 2001
From: Leslie Bellony <leslie.bellony-ext@bnf.fr>
Date: Tue, 9 Dec 2025 15:59:03 +0100
Subject: [PATCH] Add sizeOnDisk on job status (#699)

---
 .../crawler/restlet/models/CrawlJobModel.java | 36 ++++++++++++++++++-
 1 file changed, 35 insertions(+), 1 deletion(-)

diff --git a/engine/src/main/java/org/archive/crawler/restlet/models/CrawlJobModel.java b/engine/src/main/java/org/archive/crawler/restlet/models/CrawlJobModel.java
index fc3bc44f..fc8d4c41 100644
--- a/engine/src/main/java/org/archive/crawler/restlet/models/CrawlJobModel.java
+++ b/engine/src/main/java/org/archive/crawler/restlet/models/CrawlJobModel.java
@@ -11,6 +11,7 @@ import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.logging.Level;
 
 import org.apache.commons.io.filefilter.IOFileFilter;
 import org.archive.checkpointing.Checkpoint;
@@ -25,6 +26,8 @@ import org.archive.util.FileUtils;
 @SuppressWarnings("serial")
 public class CrawlJobModel extends LinkedHashMap<String, Object> implements Serializable{
 
+	private static final String CURRENT_WARC = ".open";
+	private static final String LATEST_WARCS = "/latest/warcs";
     private CrawlJob crawlJob;
     public static final IOFileFilter EDIT_FILTER = FileUtils
             .getRegexFileFilter(".*\\.((c?xml)|(txt))$");
@@ -85,6 +88,13 @@ public class CrawlJobModel extends LinkedHashMap<String, Object> implements Seri
         if (!sizeTotalsReportData.containsKey("totalCount")) {
             sizeTotalsReportData.put("totalCount", 0L);
         }
+        File warcsDir = new File(crawlJob.getJobDir(), LATEST_WARCS);
+        if (warcsDir.exists()) {
+            sizeTotalsReportData.put("sizeOnDisk", sizeOfDir(warcsDir));
+        } else {
+            sizeTotalsReportData.put("sizeOnDisk", 0L);
+        }
+        
         this.put("sizeTotalsReport", sizeTotalsReportData);
         
         this.put("rateReport", crawlJob.rateReportData());
@@ -147,6 +157,30 @@ public class CrawlJobModel extends LinkedHashMap<String, Object> implements Seri
         }
         this.put("reports", generateReports());
     }
+
+    private long sizeOfDir(File warcsDir) {
+        long size = 0;
+        File[] warcsFiles = warcsDir.listFiles();
+        if (warcsFiles != null) {
+            for (File file : warcsFiles) {
+                if (!file.exists()) {
+                    File closedWarc = new File(warcsDir.getAbsolutePath(), file.getName().replace(CURRENT_WARC, ""));
+                    if (closedWarc.exists()) {
+                        crawlJob.getJobLogger().log(Level.INFO, "sizeTotalsReportData - filename is changed from " + file.getName() + " to "+ closedWarc.getName());
+                        file = closedWarc;
+                    } else {
+                        crawlJob.getJobLogger().log(Level.WARNING, "sizeTotalsReportData - file " + file.getName() + " not exist" );
+                        continue;
+                    }
+                }
+                if (!file.isDirectory()) {
+                   size += file.length();
+                }
+            }
+        }
+        return size;
+    }
+
     public String formatBytes(Long bytes){
         return ArchiveUtils.formatBytesForDisplay(bytes);
     }
@@ -222,4 +256,4 @@ public class CrawlJobModel extends LinkedHashMap<String, Object> implements Seri
         }
         return referencedPaths;
     }
-}
+}
\ No newline at end of file