lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-07-19 05:57:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

some progress on forgetting old checkpoints when checkpointing anew

Browse Source
This commit is contained in:
Noah Levitt
2013-08-27 18:39:25 -07:00
parent 77a12e63b8
commit 2fe7d8cd45
5 changed files with 143 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files
engine/src/main/java/org/archive/crawler/framework/CheckpointService.java
+76 -24
View File
@@ -20,6 +20,7 @@ package org.archive.crawler.framework;
import java.io.File;
import java.io.FileFilter;
import java.io.IOException;
import java.util.Arrays;
import java.util.Collections;
import java.util.Iterator;
@@ -31,6 +32,7 @@ import java.util.TimerTask;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.comparator.LastModifiedFileComparator;
import org.apache.commons.io.filefilter.FileFilterUtils;
import org.archive.checkpointing.Checkpoint;
@@ -68,31 +70,34 @@ public class CheckpointService implements Lifecycle, ApplicationContextAware, Ha
protected Checkpoint checkpointInProgress;
protected Checkpoint lastCheckpoint;
protected CrawlStatSnapshot lastCheckpointSnapshot = null;
/** service for auto-checkpoint tasks at an interval */
protected Timer timer = new Timer(true);
protected TimerTask checkpointTask = null;
/**
* Checkpoints directory
*/
protected ConfigPath checkpointsDir =
new ConfigPath("checkpoints subdirectory","checkpoints");
public ConfigPath getCheckpointsDir() {
return checkpointsDir;
}
/**
* Checkpoints directory
*/
public void setCheckpointsDir(ConfigPath checkpointsDir) {
this.checkpointsDir = checkpointsDir;
}
protected long checkpointIntervalMinutes = -1;
public long getCheckpointIntervalMinutes() {
return checkpointIntervalMinutes;
}
/**
* Period at which to create automatic checkpoints; -1 means
* no auto checkpointing.
*/
protected long checkpointIntervalMinutes = -1;
public long getCheckpointIntervalMinutes() {
return checkpointIntervalMinutes;
}
public void setCheckpointIntervalMinutes(long interval) {
long oldVal = checkpointIntervalMinutes;
this.checkpointIntervalMinutes = interval;
@@ -101,6 +106,23 @@ public class CheckpointService implements Lifecycle, ApplicationContextAware, Ha
}
}
protected boolean forgetAllButLatest = false;
public boolean getForgetAllButLatest() {
return forgetAllButLatest;
}
/**
* True to save only the latest checkpoint, false to save all of them.
* Default is false.
*/
public void setForgetAllButLatest(boolean forgetAllButLatest) {
boolean oldVal = this.forgetAllButLatest;
this.forgetAllButLatest = forgetAllButLatest;
if (this.forgetAllButLatest != oldVal) {
setupCheckpointTask();
}
}
protected Checkpoint recoveryCheckpoint;
@Autowired(required=false)
public void setRecoveryCheckpoint(Checkpoint checkpoint) {
@@ -185,7 +207,8 @@ public class CheckpointService implements Lifecycle, ApplicationContextAware, Ha
periodMs + " milliseconds.");
}
protected boolean isRunning = false;
protected boolean isRunning = false;
public synchronized boolean isRunning() {
return isRunning;
}
@@ -221,6 +244,7 @@ public class CheckpointService implements Lifecycle, ApplicationContextAware, Ha
}
}
long checkpointStart = System.currentTimeMillis();
Map<String,Checkpointable> toCheckpoint = appCtx.getBeansOfType(Checkpointable.class);
if (LOGGER.isLoggable(Level.FINE)) {
LOGGER.fine("checkpointing beans " + toCheckpoint);
@@ -228,25 +252,48 @@ public class CheckpointService implements Lifecycle, ApplicationContextAware, Ha
checkpointInProgress = new Checkpoint();
try {
checkpointInProgress.generateFrom(getCheckpointsDir(),getNextCheckpointNumber());
checkpointInProgress.setForgetAllButLatest(getForgetAllButLatest());
checkpointInProgress.generateFrom(getCheckpointsDir(),
getNextCheckpointNumber());
// pre (incl. acquire necessary locks)
// long startMs = System.currentTimeMillis();
for(Checkpointable c : toCheckpoint.values()) {
long startStart = System.currentTimeMillis();
for (Checkpointable c : toCheckpoint.values()) {
c.startCheckpoint(checkpointInProgress);
}
// long duration = System.currentTimeMillis() - startMs;
// System.err.println("all startCheckpoint() completed in "+duration+"ms");
LOGGER.info("all startCheckpoint() completed in "
+ (System.currentTimeMillis() - startStart) + "ms");
// flush/write
for(Checkpointable c : toCheckpoint.values()) {
// long doMs = System.currentTimeMillis();
long doStart = System.currentTimeMillis();
for (Checkpointable c : toCheckpoint.values()) {
long doMs = System.currentTimeMillis();
c.doCheckpoint(checkpointInProgress);
// long doDuration = System.currentTimeMillis() - doMs;
// System.err.println("doCheckpoint() "+c+" in "+doDuration+"ms");
long doDuration = System.currentTimeMillis() - doMs;
LOGGER.fine("doCheckpoint() " + c + " in " + doDuration + "ms");
}
checkpointInProgress.setSuccess(true);
appCtx.publishEvent(new CheckpointSuccessEvent(this,checkpointInProgress));
LOGGER.info("all doCheckpoint() completed in "
+ (System.currentTimeMillis() - doStart) + "ms");
if (getForgetAllButLatest() && lastCheckpoint != null) {
try {
long deleteStart = System.currentTimeMillis();
FileUtils.deleteDirectory(lastCheckpoint.getCheckpointDir().getFile());
lastCheckpoint = null;
LOGGER.info("deleted old checkpoint in "
+ (System.currentTimeMillis() - deleteStart) + "ms");
} catch (IOException e) {
LOGGER.log(Level.SEVERE,
"problem deleting last checkpoint directory "
+ lastCheckpoint.getCheckpointDir().getFile(),
e);
}
}
checkpointInProgress.setSuccess(true);
appCtx.publishEvent(new CheckpointSuccessEvent(this,
checkpointInProgress));
} catch (Exception e) {
checkpointFailed(e);
} finally {
@@ -254,14 +301,19 @@ public class CheckpointService implements Lifecycle, ApplicationContextAware, Ha
controller.getStatisticsTracker().getProgressStamp());
lastCheckpointSnapshot = controller.getStatisticsTracker().getSnapshot();
// close (incl. release locks)
for(Checkpointable c : toCheckpoint.values()) {
long finishStart = System.currentTimeMillis();
for (Checkpointable c : toCheckpoint.values()) {
c.finishCheckpoint(checkpointInProgress);
}
LOGGER.info("all finishCheckpoint() completed in "
+ (System.currentTimeMillis() - finishStart) + "ms");
}
LOGGER.info("completed checkpoint " + checkpointInProgress.getName()
+ " in " + (System.currentTimeMillis() - checkpointStart) + "ms");
this.nextCheckpointNumber++;
LOGGER.info("finished checkpoint "+checkpointInProgress.getName());
String nameToReport = checkpointInProgress.getSuccess() ? checkpointInProgress.getName() : null;
this.lastCheckpoint = this.checkpointInProgress;
this.checkpointInProgress = null;
return nameToReport;
}
engine/src/main/java/org/archive/crawler/reporting/CrawlerLoggerModule.java
+14 -11
View File
@@ -322,21 +322,26 @@ public class CrawlerLoggerModule
protected void rotateLogFiles(String generationSuffix)
throws IOException {
for (Logger l: fileHandlers.keySet()) {
GenerationFileHandler gfh =
(GenerationFileHandler)fileHandlers.get(l);
GenerationFileHandler newGfh =
gfh.rotate(generationSuffix, "");
rotateLogFiles(generationSuffix, false);
}
protected void rotateLogFiles(String generationSuffix, boolean mergeOld)
throws IOException {
for (Logger l : fileHandlers.keySet()) {
GenerationFileHandler gfh = (GenerationFileHandler) fileHandlers.get(l);
GenerationFileHandler newGfh = gfh.rotate(generationSuffix, "", mergeOld);
if (gfh.shouldManifest()) {
addToManifest((String) newGfh.getFilenameSeries().get(1),
MANIFEST_LOG_FILE, newGfh.shouldManifest());
MANIFEST_LOG_FILE, newGfh.shouldManifest());
}
l.removeHandler(gfh);
l.addHandler(newGfh);
fileHandlers.put(l, newGfh);
}
}
/**
* Close all log files and remove handlers from loggers.
*/
@@ -382,7 +387,8 @@ public class CrawlerLoggerModule
*/
public void doCheckpoint(Checkpoint checkpointInProgress) throws IOException {
// Rotate off crawler logs.
rotateLogFiles("." + checkpointInProgress.getName());
rotateLogFiles("." + checkpointInProgress.getName(),
checkpointInProgress.getForgetAllButLatest());
}
public void finishCheckpoint(Checkpoint checkpointInProgress) {}
@@ -411,11 +417,9 @@ public class CrawlerLoggerModule
return uriErrors;
}
public Logger getUriProcessing() {
return uriProcessing;
}
public int getAlertCount() {
if (atg != null) {
@@ -425,7 +429,6 @@ public class CrawlerLoggerModule
}
}
public void resetAlertCount() {
if (atg != null) {
atg.resetAlertCount();