From d12d078a07e73e083e2b2955e88d531eda940a0e Mon Sep 17 00:00:00 2001 From: gojomo Date: Fri, 3 Sep 2010 01:03:01 +0000 Subject: [PATCH] [HER-1800] H3: Incomplete checkpoint listed in combo box * Checkpoint.java remove previous attempt to use Lifecycle to catch error * Checkpointable.java @Autowired(required=false) annotation for setRecoveryCheckpoint * CheckpointService.java HasValidator implementation for post-build validity check; also check at launch (probably redundant) * CheckpointValidator.java ensure validity stamp exists on any present Checkpoint --- .../org/archive/checkpointing/Checkpoint.java | 21 +------- .../archive/checkpointing/Checkpointable.java | 3 ++ .../crawler/framework/CheckpointService.java | 23 ++++++++- .../framework/CheckpointValidator.java | 48 +++++++++++++++++++ 4 files changed, 74 insertions(+), 21 deletions(-) create mode 100644 engine/src/main/java/org/archive/crawler/framework/CheckpointValidator.java diff --git a/commons/src/main/java/org/archive/checkpointing/Checkpoint.java b/commons/src/main/java/org/archive/checkpointing/Checkpoint.java index a61887d3..315368bd 100644 --- a/commons/src/main/java/org/archive/checkpointing/Checkpoint.java +++ b/commons/src/main/java/org/archive/checkpointing/Checkpoint.java @@ -31,14 +31,13 @@ import org.json.JSONException; import org.json.JSONObject; import org.springframework.beans.factory.InitializingBean; import org.springframework.beans.factory.annotation.Required; -import org.springframework.context.Lifecycle; /** * Represents a single checkpoint, by its name and main store directory. * * @contributor gojomo */ -public class Checkpoint implements InitializingBean, Lifecycle { +public class Checkpoint implements InitializingBean { private final static Logger LOGGER = Logger.getLogger(Checkpoint.class.getName()); @@ -92,24 +91,6 @@ public class Checkpoint implements InitializingBean, Lifecycle { shortName = name.substring(0, name.indexOf("-")); } - boolean isRunning = false; - public boolean isRunning() { - return isRunning; - } - public void start() { - if(isRunning) { - return; - } - isRunning = true; - if(!Checkpoint.hasValidStamp(checkpointDir.getFile())) { - throw new RuntimeException("checkpoint '"+checkpointDir.getFile().getAbsolutePath()+"' missing validity stamp file"); - } - } - public void stop() { - isRunning = false; - } - - public void setSuccess(boolean b) { success = b; } diff --git a/commons/src/main/java/org/archive/checkpointing/Checkpointable.java b/commons/src/main/java/org/archive/checkpointing/Checkpointable.java index 6d356f74..c95f0885 100644 --- a/commons/src/main/java/org/archive/checkpointing/Checkpointable.java +++ b/commons/src/main/java/org/archive/checkpointing/Checkpointable.java @@ -21,6 +21,8 @@ package org.archive.checkpointing; import java.io.IOException; +import org.springframework.beans.factory.annotation.Autowired; + /** * Interface for objects that can checkpoint their state, possibly @@ -72,5 +74,6 @@ public interface Checkpointable { * * @param recoveryCheckpoint Checkpoint */ + @Autowired(required=false) void setRecoveryCheckpoint(Checkpoint recoveryCheckpoint); } diff --git a/engine/src/main/java/org/archive/crawler/framework/CheckpointService.java b/engine/src/main/java/org/archive/crawler/framework/CheckpointService.java index f61ab9d2..6a8700e4 100644 --- a/engine/src/main/java/org/archive/crawler/framework/CheckpointService.java +++ b/engine/src/main/java/org/archive/crawler/framework/CheckpointService.java @@ -37,12 +37,14 @@ import org.archive.checkpointing.Checkpoint; import org.archive.checkpointing.Checkpointable; import org.archive.crawler.reporting.CrawlStatSnapshot; import org.archive.spring.ConfigPath; +import org.archive.spring.HasValidator; import org.springframework.beans.BeansException; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.ApplicationContext; import org.springframework.context.ApplicationContextAware; import org.springframework.context.Lifecycle; import org.springframework.context.support.AbstractApplicationContext; +import org.springframework.validation.Validator; /** * Executes checkpoints, and offers convenience methods for enumerating @@ -56,7 +58,7 @@ import org.springframework.context.support.AbstractApplicationContext; * @contributor gojomo * @contributor pjack */ -public class CheckpointService implements Lifecycle, ApplicationContextAware { +public class CheckpointService implements Lifecycle, ApplicationContextAware, HasValidator { private final static Logger LOGGER = Logger.getLogger(CheckpointService.class.getName()); @@ -133,6 +135,16 @@ public class CheckpointService implements Lifecycle, ApplicationContextAware { if (isRunning) { return; } + // report if checkpoint incomplete/invalid + if(getRecoveryCheckpoint()!=null) { + File cpDir = getRecoveryCheckpoint().getCheckpointDir().getFile(); + if(!Checkpoint.hasValidStamp(cpDir)) { + LOGGER.severe( + "checkpoint '"+cpDir.getAbsolutePath() + +"' missing validity stamp file; checkpoint data " + +"may be missing or otherwise corrupt."); + } + } this.isRunning = true; setupCheckpointTask(); } @@ -315,6 +327,9 @@ public class CheckpointService implements Lifecycle, ApplicationContextAware { */ @SuppressWarnings("unchecked") public void setRecoveryCheckpointByName(String selectedCheckpoint) { + if(isRunning) { + throw new RuntimeException("may not set recovery Checkpoint after launch"); + } Checkpoint recoveryCheckpoint = new Checkpoint(); recoveryCheckpoint.getCheckpointDir().setBase(getCheckpointsDir()); recoveryCheckpoint.getCheckpointDir().setPath(selectedCheckpoint); @@ -326,4 +341,10 @@ public class CheckpointService implements Lifecycle, ApplicationContextAware { c.setRecoveryCheckpoint(recoveryCheckpoint); } } + + static Validator VALIDATOR = new CheckpointValidator(); + @Override + public Validator getValidator() { + return VALIDATOR; + } } //EOC diff --git a/engine/src/main/java/org/archive/crawler/framework/CheckpointValidator.java b/engine/src/main/java/org/archive/crawler/framework/CheckpointValidator.java new file mode 100644 index 00000000..765b7bb5 --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/framework/CheckpointValidator.java @@ -0,0 +1,48 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.archive.crawler.framework; + +import org.archive.checkpointing.Checkpoint; +import org.springframework.validation.Errors; +import org.springframework.validation.Validator; + +public class CheckpointValidator implements Validator { + + @Override + @SuppressWarnings("unchecked") + public boolean supports(Class cls) { + return Checkpoint.class.isAssignableFrom(cls); + } + + @Override + public void validate(Object target, Errors errors) { + Checkpoint cp = ((CheckpointService)target).getRecoveryCheckpoint(); + if(cp==null) { + return; + } + if(!Checkpoint.hasValidStamp(cp.getCheckpointDir().getFile())) { + errors.rejectValue( + "recoveryCheckpoint.checkpointDir", + null, + "Configured recovery checkpoint "+cp.getName() + +" incomplete: lacks valid stamp file."); + } + } + +}