mirror of
https://github.com/internetarchive/heritrix3.git
synced 2026-09-23 06:05:46 +00:00
[HER-1546] Springify(5): Update checkpointing to work smoothly with spring-configured crawls
* PersistLogProcessor.java
rotate logs on checkpoint
* CrawlerJournal.java, FrontierJournal.java
rotate for checkpoint support without full Checkpointable
This commit is contained in:
@@ -21,16 +21,16 @@ package org.archive.io;
|
||||
import it.unimi.dsi.fastutil.io.FastBufferedOutputStream;
|
||||
import it.unimi.dsi.mg4j.util.MutableString;
|
||||
|
||||
import java.io.Closeable;
|
||||
import java.io.File;
|
||||
import java.io.FileNotFoundException;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.OutputStreamWriter;
|
||||
import java.io.Writer;
|
||||
import java.util.List;
|
||||
import java.util.zip.GZIPOutputStream;
|
||||
|
||||
import org.archive.checkpointing.RecoverAction;
|
||||
import org.archive.crawler.framework.Checkpoint;
|
||||
import org.archive.util.ArchiveUtils;
|
||||
import org.archive.util.FileUtils;
|
||||
|
||||
@@ -40,7 +40,7 @@ import org.archive.util.FileUtils;
|
||||
*
|
||||
* @author gojomo
|
||||
*/
|
||||
public class CrawlerJournal {
|
||||
public class CrawlerJournal implements Closeable {
|
||||
|
||||
/** prefix for error lines*/
|
||||
public static final String LOG_ERROR = "E ";
|
||||
@@ -222,17 +222,17 @@ public class CrawlerJournal {
|
||||
* @param checkpointDir
|
||||
* @throws IOException
|
||||
*/
|
||||
public synchronized void checkpoint(final File checkpointDir,
|
||||
List<RecoverAction> actions) throws IOException {
|
||||
public synchronized void rotateForCheckpoint(Checkpoint checkpointInProgress) throws IOException {
|
||||
if (this.out == null || !this.gzipFile.exists()) {
|
||||
return;
|
||||
}
|
||||
close();
|
||||
// Rename gzipFile with the checkpoint name as suffix.
|
||||
this.gzipFile.renameTo(new File(this.gzipFile.getParentFile(),
|
||||
this.gzipFile.getName() + "." + checkpointDir.getName()));
|
||||
File newName = new File(this.gzipFile.getParentFile(),
|
||||
this.gzipFile.getName() + "." + checkpointInProgress.getShortName());
|
||||
FileUtils.moveAsideIfExists(newName);
|
||||
this.gzipFile.renameTo(newName);
|
||||
// Open new gzip file.
|
||||
this.out = initialize(this.gzipFile);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -27,7 +27,6 @@ import java.util.logging.Level;
|
||||
import java.util.logging.Logger;
|
||||
|
||||
import org.apache.commons.httpclient.URIException;
|
||||
import org.archive.checkpointing.Checkpointable;
|
||||
import org.archive.crawler.framework.Frontier;
|
||||
import org.archive.io.CrawlerJournal;
|
||||
import org.archive.modules.CrawlURI;
|
||||
@@ -46,7 +45,7 @@ import org.json.JSONObject;
|
||||
*
|
||||
* @author gojomo
|
||||
*/
|
||||
public class FrontierJournal extends CrawlerJournal implements Checkpointable {
|
||||
public class FrontierJournal extends CrawlerJournal {
|
||||
private static final Logger LOGGER = Logger.getLogger(
|
||||
FrontierJournal.class.getName());
|
||||
|
||||
|
||||
@@ -101,7 +101,7 @@ implements Checkpointable, Lifecycle {
|
||||
|
||||
public void doCheckpoint(Checkpoint checkpointInProgress) throws IOException {
|
||||
// rotate log
|
||||
log.doCheckpoint(checkpointInProgress);
|
||||
log.rotateForCheckpoint(checkpointInProgress);
|
||||
}
|
||||
|
||||
public void finishCheckpoint(Checkpoint checkpointInProgress) {}
|
||||
|
||||
Reference in New Issue
Block a user