diff --git a/commons/src/main/java/org/archive/bdb/BdbModule.java b/commons/src/main/java/org/archive/bdb/BdbModule.java index 156ade88..0335afdc 100644 --- a/commons/src/main/java/org/archive/bdb/BdbModule.java +++ b/commons/src/main/java/org/archive/bdb/BdbModule.java @@ -172,14 +172,6 @@ public class BdbModule implements Lifecycle, Checkpointable, Closeable { this.useSharedCache = useSharedCache; } - boolean checkpointCopyLogs = true; - public boolean getCheckpointCopyLogs() { - return checkpointCopyLogs; - } - public void setCheckpointCopyLogs(boolean checkpointCopyLogs) { - this.checkpointCopyLogs = checkpointCopyLogs; - } - /** * Expected number of concurrent threads; used to tune nLockTables * according to JE FAQ diff --git a/engine/src/main/java/org/archive/crawler/framework/CheckpointService.java b/engine/src/main/java/org/archive/crawler/framework/CheckpointService.java index 127be310..9c517545 100644 --- a/engine/src/main/java/org/archive/crawler/framework/CheckpointService.java +++ b/engine/src/main/java/org/archive/crawler/framework/CheckpointService.java @@ -63,9 +63,9 @@ public class CheckpointService implements Lifecycle, ApplicationContextAware { CrawlStatSnapshot lastCheckpointSnapshot = null; - /**Setup in constructor or on a call to recovery */ - protected transient Timer timerThread = null; - + /** service for auto-checkpoint tasks at an interval */ + protected Timer timer = new Timer(true);; + protected TimerTask checkpointTask = null; /** * Checkpoints directory */ @@ -86,8 +86,12 @@ public class CheckpointService implements Lifecycle, ApplicationContextAware { public int getCheckpointIntervalMinutes() { return checkpointIntervalMinutes; } - public void setCheckpointIntervalMinutes(int checkpointIntervalMinutes) { - this.checkpointIntervalMinutes = checkpointIntervalMinutes; + public void setCheckpointIntervalMinutes(int interval) { + int oldVal = checkpointIntervalMinutes; + this.checkpointIntervalMinutes = interval; + if(checkpointIntervalMinutes!=oldVal) { + setupCheckpointTask(); + } } Checkpoint recoveryCheckpoint; @@ -121,17 +125,33 @@ public class CheckpointService implements Lifecycle, ApplicationContextAware { public CheckpointService() { } - public void start() { + public synchronized void start() { if (isRunning) { return; } this.isRunning = true; - // Convert period from hours to milliseconds. + setupCheckpointTask(); + } + + /** + * Setup checkpointTask according to current interval. (An already-scheduled + * task, if any, is canceled.) + */ + protected synchronized void setupCheckpointTask() { + if(checkpointTask!=null) { + checkpointTask.cancel(); + } + if(!isRunning) { + // don't setup before start (or after finish), even if + // triggered by interval change + return; + } + // Convert period from minutes to milliseconds. long periodMs = getCheckpointIntervalMinutes() * (60 * 1000); if(periodMs<=0) { return; } - TimerTask tt = new TimerTask() { + checkpointTask = new TimerTask() { public void run() { if (isCheckpointing()) { LOGGER.info("CheckpointTimerThread skipping checkpoint, " + @@ -143,24 +163,20 @@ public class CheckpointService implements Lifecycle, ApplicationContextAware { requestCrawlCheckpoint(); } }; - this.timerThread = new Timer(true); - this.timerThread.schedule(tt, periodMs, periodMs); - LOGGER.info("Installed Checkpoint TimerThread to checkpoint every " + + this.timer.schedule(checkpointTask, periodMs, periodMs); + LOGGER.info("Installed Checkpoint TimerTask to checkpoint every " + periodMs + " milliseconds."); } boolean isRunning = false; - public boolean isRunning() { + public synchronized boolean isRunning() { return isRunning; } - public void stop() { - if (this.timerThread != null) { - LOGGER.info("Cleaned up Checkpoint TimerThread."); - this.timerThread.cancel(); - this.timerThread = null; - } + public synchronized void stop() { + LOGGER.info("Cleaned up Checkpoint TimerThread."); + this.timer.cancel(); this.isRunning = false; } @@ -196,22 +212,22 @@ public class CheckpointService implements Lifecycle, ApplicationContextAware { try { // pre (incl. acquire necessary locks) - long startMs = System.currentTimeMillis(); +// long startMs = System.currentTimeMillis(); for(Checkpointable c : toCheckpoint.values()) { c.startCheckpoint(checkpointInProgress); } - long duration = System.currentTimeMillis() - startMs; +// long duration = System.currentTimeMillis() - startMs; // System.err.println("all startCheckpoint() completed in "+duration+"ms"); // flush/write for(Checkpointable c : toCheckpoint.values()) { - long doMs = System.currentTimeMillis(); +// long doMs = System.currentTimeMillis(); c.doCheckpoint(checkpointInProgress); - long doDuration = System.currentTimeMillis() - doMs; +// long doDuration = System.currentTimeMillis() - doMs; // System.err.println("doCheckpoint() "+c+" in "+doDuration+"ms"); } checkpointInProgress.setSuccess(true); - + appCtx.publishEvent(new CheckpointSuccessEvent(this,checkpointInProgress)); } catch (Exception e) { checkpointFailed(e); } finally { diff --git a/engine/src/main/java/org/archive/crawler/framework/CheckpointSuccessEvent.java b/engine/src/main/java/org/archive/crawler/framework/CheckpointSuccessEvent.java new file mode 100644 index 00000000..a2e22de9 --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/framework/CheckpointSuccessEvent.java @@ -0,0 +1,44 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.crawler.framework; + +import org.archive.checkpointing.Checkpoint; +import org.springframework.context.ApplicationEvent; + +/** + * Report success of a Checkpoint (so that it may be reported by the + * CrawlJOb to the job log). + * + * @contributor gojomo + */ +public class CheckpointSuccessEvent extends ApplicationEvent { + private static final long serialVersionUID = 1L; + Checkpoint checkpoint; + + public Checkpoint getCheckpoint() { + return checkpoint; + } + + public CheckpointSuccessEvent(CheckpointService source, Checkpoint cp) { + super(source); + checkpoint = cp; + } + +} diff --git a/engine/src/main/java/org/archive/crawler/framework/CrawlJob.java b/engine/src/main/java/org/archive/crawler/framework/CrawlJob.java index 4710cb7b..fc15f6ab 100644 --- a/engine/src/main/java/org/archive/crawler/framework/CrawlJob.java +++ b/engine/src/main/java/org/archive/crawler/framework/CrawlJob.java @@ -652,6 +652,9 @@ public class CrawlJob implements Comparable, ApplicationListener { if(event instanceof CrawlStateEvent) { getJobLogger().log(Level.INFO, ((CrawlStateEvent)event).getState().toString()); } + if(event instanceof CheckpointSuccessEvent) { + getJobLogger().log(Level.INFO, "CHECKPOINTED "+((CheckpointSuccessEvent)event).getCheckpoint().getName()); + } } /**