[HER-1546] Springify(5): Update checkpointing to work smoothly with spring-configured crawls

* PersistLogProcessor.java
    rotate logs on checkpoint
* CrawlerJournal.java, FrontierJournal.java
    rotate for checkpoint support without full Checkpointable
This commit is contained in:
gojomo
2009-11-20 00:47:54 +00:00
parent 8591002573
commit fbd3af35ac
3 changed files with 10 additions and 11 deletions
@@ -21,16 +21,16 @@ package org.archive.io;
import it.unimi.dsi.fastutil.io.FastBufferedOutputStream;
import it.unimi.dsi.mg4j.util.MutableString;
import java.io.Closeable;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.io.Writer;
import java.util.List;
import java.util.zip.GZIPOutputStream;
import org.archive.checkpointing.RecoverAction;
import org.archive.crawler.framework.Checkpoint;
import org.archive.util.ArchiveUtils;
import org.archive.util.FileUtils;
@@ -40,7 +40,7 @@ import org.archive.util.FileUtils;
*
* @author gojomo
*/
public class CrawlerJournal {
public class CrawlerJournal implements Closeable {
/** prefix for error lines*/
public static final String LOG_ERROR = "E ";
@@ -222,17 +222,17 @@ public class CrawlerJournal {
* @param checkpointDir
* @throws IOException
*/
public synchronized void checkpoint(final File checkpointDir,
List<RecoverAction> actions) throws IOException {
public synchronized void rotateForCheckpoint(Checkpoint checkpointInProgress) throws IOException {
if (this.out == null || !this.gzipFile.exists()) {
return;
}
close();
// Rename gzipFile with the checkpoint name as suffix.
this.gzipFile.renameTo(new File(this.gzipFile.getParentFile(),
this.gzipFile.getName() + "." + checkpointDir.getName()));
File newName = new File(this.gzipFile.getParentFile(),
this.gzipFile.getName() + "." + checkpointInProgress.getShortName());
FileUtils.moveAsideIfExists(newName);
this.gzipFile.renameTo(newName);
// Open new gzip file.
this.out = initialize(this.gzipFile);
}
}
@@ -27,7 +27,6 @@ import java.util.logging.Level;
import java.util.logging.Logger;
import org.apache.commons.httpclient.URIException;
import org.archive.checkpointing.Checkpointable;
import org.archive.crawler.framework.Frontier;
import org.archive.io.CrawlerJournal;
import org.archive.modules.CrawlURI;
@@ -46,7 +45,7 @@ import org.json.JSONObject;
*
* @author gojomo
*/
public class FrontierJournal extends CrawlerJournal implements Checkpointable {
public class FrontierJournal extends CrawlerJournal {
private static final Logger LOGGER = Logger.getLogger(
FrontierJournal.class.getName());
@@ -101,7 +101,7 @@ implements Checkpointable, Lifecycle {
public void doCheckpoint(Checkpoint checkpointInProgress) throws IOException {
// rotate log
log.doCheckpoint(checkpointInProgress);
log.rotateForCheckpoint(checkpointInProgress);
}
public void finishCheckpoint(Checkpoint checkpointInProgress) {}