diff --git a/commons/src/main/java/org/archive/checkpointing/Checkpoint.java b/commons/src/main/java/org/archive/checkpointing/Checkpoint.java index 3daf8a73..a5955147 100644 --- a/commons/src/main/java/org/archive/checkpointing/Checkpoint.java +++ b/commons/src/main/java/org/archive/checkpointing/Checkpoint.java @@ -173,4 +173,12 @@ public class Checkpoint implements InitializingBean { public static boolean hasValidStamp(File checkpointDirectory) { return (new File(checkpointDirectory,Checkpoint.VALIDITY_STAMP_FILENAME)).exists(); } + + private boolean forgetAllButLatest = false; + public void setForgetAllButLatest(boolean b) { + this.forgetAllButLatest = true; + } + public boolean getForgetAllButLatest() { + return forgetAllButLatest; + } } \ No newline at end of file diff --git a/commons/src/main/java/org/archive/io/GenerationFileHandler.java b/commons/src/main/java/org/archive/io/GenerationFileHandler.java index 747d9de0..c1ce8d79 100644 --- a/commons/src/main/java/org/archive/io/GenerationFileHandler.java +++ b/commons/src/main/java/org/archive/io/GenerationFileHandler.java @@ -91,29 +91,50 @@ public class GenerationFileHandler extends FileHandler { public GenerationFileHandler rotate(String storeSuffix, String activeSuffix) throws IOException { + return rotate(storeSuffix, activeSuffix, false); + } + + public GenerationFileHandler rotate(String storeSuffix, + String activeSuffix, boolean mergeOld) throws IOException { close(); - String filename = (String)filenameSeries.getFirst(); + String filename = (String) filenameSeries.getFirst(); if (!filename.endsWith(activeSuffix)) { - throw new FileNotFoundException("Active file does not have" + - " expected suffix"); + throw new FileNotFoundException("Active file does not have" + + " expected suffix"); } - String storeFilename = filename.substring(0, - filename.length() - activeSuffix.length()) + - storeSuffix; + String storeFilename = filename.substring(0, filename.length() + - activeSuffix.length()) + + storeSuffix; File activeFile = new File(filename); File storeFile = new File(storeFilename); FileUtils.moveAsideIfExists(storeFile); - if (!activeFile.renameTo(storeFile)) { - throw new IOException("Unable to move " + filename + " to " + - storeFilename); + + if (mergeOld) { + File fileToAppendTo = new File(filenameSeries.getLast()); + for (int i = filenameSeries.size() - 2; i >= 0; i--) { + File f = new File(filenameSeries.get(i)); + FileUtils.appendTo(fileToAppendTo, f); + f.delete(); + } + filenameSeries.clear(); + filenameSeries.add(filename); + if (!fileToAppendTo.renameTo(storeFile)) { + throw new IOException("Unable to move " + fileToAppendTo + " to " + + storeFilename); + } + } else { + if (!activeFile.renameTo(storeFile)) { + throw new IOException("Unable to move " + filename + " to " + + storeFilename); + } } filenameSeries.add(1, storeFilename); - GenerationFileHandler newGfh = - new GenerationFileHandler(filenameSeries, shouldManifest); + GenerationFileHandler newGfh = new GenerationFileHandler( + filenameSeries, shouldManifest); newGfh.setFormatter(this.getFormatter()); return newGfh; } - + /** * @return True if should manifest. */ diff --git a/commons/src/main/java/org/archive/util/FileUtils.java b/commons/src/main/java/org/archive/util/FileUtils.java index 3b5b93ad..e1629bd4 100644 --- a/commons/src/main/java/org/archive/util/FileUtils.java +++ b/commons/src/main/java/org/archive/util/FileUtils.java @@ -696,4 +696,16 @@ public class FileUtils { return file; } } + + public static void appendTo(File fileToAppendTo, File fileToAppendFrom) throws IOException { + byte[] buf = new byte[4096]; + FileOutputStream out = new FileOutputStream(fileToAppendTo, true); + FileInputStream in = new FileInputStream(fileToAppendFrom); + for (int n = in.read(buf); n > 0; n = in.read(buf)) { + out.write(buf, 0, n); + } + in.close(); + out.flush(); + out.close(); + } } \ No newline at end of file diff --git a/engine/src/main/java/org/archive/crawler/framework/CheckpointService.java b/engine/src/main/java/org/archive/crawler/framework/CheckpointService.java index 564c0916..4478f8c1 100644 --- a/engine/src/main/java/org/archive/crawler/framework/CheckpointService.java +++ b/engine/src/main/java/org/archive/crawler/framework/CheckpointService.java @@ -20,6 +20,7 @@ package org.archive.crawler.framework; import java.io.File; import java.io.FileFilter; +import java.io.IOException; import java.util.Arrays; import java.util.Collections; import java.util.Iterator; @@ -31,6 +32,7 @@ import java.util.TimerTask; import java.util.logging.Level; import java.util.logging.Logger; +import org.apache.commons.io.FileUtils; import org.apache.commons.io.comparator.LastModifiedFileComparator; import org.apache.commons.io.filefilter.FileFilterUtils; import org.archive.checkpointing.Checkpoint; @@ -68,31 +70,34 @@ public class CheckpointService implements Lifecycle, ApplicationContextAware, Ha protected Checkpoint checkpointInProgress; + protected Checkpoint lastCheckpoint; + protected CrawlStatSnapshot lastCheckpointSnapshot = null; /** service for auto-checkpoint tasks at an interval */ protected Timer timer = new Timer(true); protected TimerTask checkpointTask = null; - /** - * Checkpoints directory - */ protected ConfigPath checkpointsDir = new ConfigPath("checkpoints subdirectory","checkpoints"); public ConfigPath getCheckpointsDir() { return checkpointsDir; } + /** + * Checkpoints directory + */ public void setCheckpointsDir(ConfigPath checkpointsDir) { this.checkpointsDir = checkpointsDir; } + protected long checkpointIntervalMinutes = -1; + + public long getCheckpointIntervalMinutes() { + return checkpointIntervalMinutes; + } /** * Period at which to create automatic checkpoints; -1 means * no auto checkpointing. */ - protected long checkpointIntervalMinutes = -1; - public long getCheckpointIntervalMinutes() { - return checkpointIntervalMinutes; - } public void setCheckpointIntervalMinutes(long interval) { long oldVal = checkpointIntervalMinutes; this.checkpointIntervalMinutes = interval; @@ -101,6 +106,23 @@ public class CheckpointService implements Lifecycle, ApplicationContextAware, Ha } } + protected boolean forgetAllButLatest = false; + public boolean getForgetAllButLatest() { + return forgetAllButLatest; + } + + /** + * True to save only the latest checkpoint, false to save all of them. + * Default is false. + */ + public void setForgetAllButLatest(boolean forgetAllButLatest) { + boolean oldVal = this.forgetAllButLatest; + this.forgetAllButLatest = forgetAllButLatest; + if (this.forgetAllButLatest != oldVal) { + setupCheckpointTask(); + } + } + protected Checkpoint recoveryCheckpoint; @Autowired(required=false) public void setRecoveryCheckpoint(Checkpoint checkpoint) { @@ -185,7 +207,8 @@ public class CheckpointService implements Lifecycle, ApplicationContextAware, Ha periodMs + " milliseconds."); } - protected boolean isRunning = false; + protected boolean isRunning = false; + public synchronized boolean isRunning() { return isRunning; } @@ -221,6 +244,7 @@ public class CheckpointService implements Lifecycle, ApplicationContextAware, Ha } } + long checkpointStart = System.currentTimeMillis(); Map toCheckpoint = appCtx.getBeansOfType(Checkpointable.class); if (LOGGER.isLoggable(Level.FINE)) { LOGGER.fine("checkpointing beans " + toCheckpoint); @@ -228,25 +252,48 @@ public class CheckpointService implements Lifecycle, ApplicationContextAware, Ha checkpointInProgress = new Checkpoint(); try { - checkpointInProgress.generateFrom(getCheckpointsDir(),getNextCheckpointNumber()); - + checkpointInProgress.setForgetAllButLatest(getForgetAllButLatest()); + checkpointInProgress.generateFrom(getCheckpointsDir(), + getNextCheckpointNumber()); + // pre (incl. acquire necessary locks) -// long startMs = System.currentTimeMillis(); - for(Checkpointable c : toCheckpoint.values()) { + long startStart = System.currentTimeMillis(); + for (Checkpointable c : toCheckpoint.values()) { c.startCheckpoint(checkpointInProgress); } -// long duration = System.currentTimeMillis() - startMs; -// System.err.println("all startCheckpoint() completed in "+duration+"ms"); - + LOGGER.info("all startCheckpoint() completed in " + + (System.currentTimeMillis() - startStart) + "ms"); + // flush/write - for(Checkpointable c : toCheckpoint.values()) { -// long doMs = System.currentTimeMillis(); + long doStart = System.currentTimeMillis(); + for (Checkpointable c : toCheckpoint.values()) { + long doMs = System.currentTimeMillis(); c.doCheckpoint(checkpointInProgress); -// long doDuration = System.currentTimeMillis() - doMs; -// System.err.println("doCheckpoint() "+c+" in "+doDuration+"ms"); + long doDuration = System.currentTimeMillis() - doMs; + LOGGER.fine("doCheckpoint() " + c + " in " + doDuration + "ms"); } - checkpointInProgress.setSuccess(true); - appCtx.publishEvent(new CheckpointSuccessEvent(this,checkpointInProgress)); + LOGGER.info("all doCheckpoint() completed in " + + (System.currentTimeMillis() - doStart) + "ms"); + + if (getForgetAllButLatest() && lastCheckpoint != null) { + try { + long deleteStart = System.currentTimeMillis(); + FileUtils.deleteDirectory(lastCheckpoint.getCheckpointDir().getFile()); + lastCheckpoint = null; + LOGGER.info("deleted old checkpoint in " + + (System.currentTimeMillis() - deleteStart) + "ms"); + } catch (IOException e) { + LOGGER.log(Level.SEVERE, + "problem deleting last checkpoint directory " + + lastCheckpoint.getCheckpointDir().getFile(), + e); + } + } + + checkpointInProgress.setSuccess(true); + + appCtx.publishEvent(new CheckpointSuccessEvent(this, + checkpointInProgress)); } catch (Exception e) { checkpointFailed(e); } finally { @@ -254,14 +301,19 @@ public class CheckpointService implements Lifecycle, ApplicationContextAware, Ha controller.getStatisticsTracker().getProgressStamp()); lastCheckpointSnapshot = controller.getStatisticsTracker().getSnapshot(); // close (incl. release locks) - for(Checkpointable c : toCheckpoint.values()) { + long finishStart = System.currentTimeMillis(); + for (Checkpointable c : toCheckpoint.values()) { c.finishCheckpoint(checkpointInProgress); } + LOGGER.info("all finishCheckpoint() completed in " + + (System.currentTimeMillis() - finishStart) + "ms"); } - + LOGGER.info("completed checkpoint " + checkpointInProgress.getName() + + " in " + (System.currentTimeMillis() - checkpointStart) + "ms"); + this.nextCheckpointNumber++; - LOGGER.info("finished checkpoint "+checkpointInProgress.getName()); String nameToReport = checkpointInProgress.getSuccess() ? checkpointInProgress.getName() : null; + this.lastCheckpoint = this.checkpointInProgress; this.checkpointInProgress = null; return nameToReport; } diff --git a/engine/src/main/java/org/archive/crawler/reporting/CrawlerLoggerModule.java b/engine/src/main/java/org/archive/crawler/reporting/CrawlerLoggerModule.java index c635e580..21b9d265 100644 --- a/engine/src/main/java/org/archive/crawler/reporting/CrawlerLoggerModule.java +++ b/engine/src/main/java/org/archive/crawler/reporting/CrawlerLoggerModule.java @@ -322,21 +322,26 @@ public class CrawlerLoggerModule protected void rotateLogFiles(String generationSuffix) throws IOException { - for (Logger l: fileHandlers.keySet()) { - GenerationFileHandler gfh = - (GenerationFileHandler)fileHandlers.get(l); - GenerationFileHandler newGfh = - gfh.rotate(generationSuffix, ""); + rotateLogFiles(generationSuffix, false); + } + + protected void rotateLogFiles(String generationSuffix, boolean mergeOld) + throws IOException { + for (Logger l : fileHandlers.keySet()) { + GenerationFileHandler gfh = (GenerationFileHandler) fileHandlers.get(l); + GenerationFileHandler newGfh = gfh.rotate(generationSuffix, "", mergeOld); + if (gfh.shouldManifest()) { addToManifest((String) newGfh.getFilenameSeries().get(1), - MANIFEST_LOG_FILE, newGfh.shouldManifest()); + MANIFEST_LOG_FILE, newGfh.shouldManifest()); } + l.removeHandler(gfh); l.addHandler(newGfh); fileHandlers.put(l, newGfh); } } - + /** * Close all log files and remove handlers from loggers. */ @@ -382,7 +387,8 @@ public class CrawlerLoggerModule */ public void doCheckpoint(Checkpoint checkpointInProgress) throws IOException { // Rotate off crawler logs. - rotateLogFiles("." + checkpointInProgress.getName()); + rotateLogFiles("." + checkpointInProgress.getName(), + checkpointInProgress.getForgetAllButLatest()); } public void finishCheckpoint(Checkpoint checkpointInProgress) {} @@ -411,11 +417,9 @@ public class CrawlerLoggerModule return uriErrors; } - public Logger getUriProcessing() { return uriProcessing; } - public int getAlertCount() { if (atg != null) { @@ -425,7 +429,6 @@ public class CrawlerLoggerModule } } - public void resetAlertCount() { if (atg != null) { atg.resetAlertCount();