From 0ab8eee24e79c7119bbd438cb5b282072fe83fdc Mon Sep 17 00:00:00 2001
From: Noah Levitt <nlevitt@archive.org>
Date: Thu, 29 Aug 2013 10:56:12 -0700
Subject: [PATCH] forgetting old checkpoints - roll up previous checkpointed
 CrawlerJournals (e.g. frontier.recover.gz) into new checkpoint

---
 .../java/org/archive/io/CrawlerJournal.java   | 36 +++++++++++++++++--
 1 file changed, 34 insertions(+), 2 deletions(-)

diff --git a/commons/src/main/java/org/archive/io/CrawlerJournal.java b/commons/src/main/java/org/archive/io/CrawlerJournal.java
index aee9f0f3..2575e29c 100644
--- a/commons/src/main/java/org/archive/io/CrawlerJournal.java
+++ b/commons/src/main/java/org/archive/io/CrawlerJournal.java
@@ -25,17 +25,21 @@ import java.io.Closeable;
 import java.io.File;
 import java.io.FileNotFoundException;
 import java.io.FileOutputStream;
+import java.io.FilenameFilter;
 import java.io.IOException;
 import java.io.OutputStreamWriter;
 import java.io.Writer;
+import java.util.Arrays;
 import java.util.logging.Level;
 import java.util.logging.Logger;
+import java.util.regex.Pattern;
 import java.util.zip.GZIPOutputStream;
 
 import org.apache.commons.lang.StringUtils;
 import org.archive.checkpointing.Checkpoint;
 import org.archive.util.ArchiveUtils;
 import org.archive.util.FileUtils;
+import org.archive.util.TextUtils;
 
 /**
  * Utility class for a crawler journal/log that is compressed and 
@@ -196,12 +200,40 @@ public class CrawlerJournal implements Closeable {
             return;
         }
         close();
-        // Rename gzipFile with the checkpoint name as suffix.
+
         File newName = new File(this.gzipFile.getParentFile(),
                 this.gzipFile.getName() + "." + checkpointInProgress.getName());
         try {
             FileUtils.moveAsideIfExists(newName); 
-            this.gzipFile.renameTo(newName);
+            if (checkpointInProgress.getForgetAllButLatest()) {
+                // merge any earlier checkpointed files into new checkpoint
+                // file, taking advantage of the legality of concatenating gzips
+                
+                File[] oldCheckpointeds = this.gzipFile.getParentFile().listFiles(new FilenameFilter() {
+                    @Override
+                    public boolean accept(File dir, String name) {
+                        String regex = "^" + Pattern.quote(gzipFile.getName()) + "\\.cp\\d{5}-\\d{14}$";
+                        return TextUtils.matches(regex, name);
+                    }
+                });
+                Arrays.sort(oldCheckpointeds);
+                
+                for (int i = 1; i < oldCheckpointeds.length; i++) {
+                    FileUtils.appendTo(oldCheckpointeds[0], oldCheckpointeds[i]);
+                    oldCheckpointeds[i].delete();
+                }
+                
+                if (oldCheckpointeds.length > 0) {
+                    FileUtils.appendTo(oldCheckpointeds[0], this.gzipFile);
+                    this.gzipFile.delete();
+                    oldCheckpointeds[0].renameTo(newName);
+                } else {
+                    this.gzipFile.renameTo(newName);
+                }
+            } else {
+                this.gzipFile.renameTo(newName);
+            }
+            
             // Open new gzip file.
             this.out = initialize(this.gzipFile);
         } catch (IOException ioe) {