mirror of
https://github.com/internetarchive/heritrix3.git
synced 2026-09-13 01:05:47 +00:00
[HER-1800] H3: Incomplete checkpoint listed in combo box
* Checkpoint.java
remove previous attempt to use Lifecycle to catch error
* Checkpointable.java
@Autowired(required=false) annotation for setRecoveryCheckpoint
* CheckpointService.java
HasValidator implementation for post-build validity check; also check at launch (probably redundant)
* CheckpointValidator.java
ensure validity stamp exists on any present Checkpoint
This commit is contained in:
@@ -31,14 +31,13 @@ import org.json.JSONException;
|
||||
import org.json.JSONObject;
|
||||
import org.springframework.beans.factory.InitializingBean;
|
||||
import org.springframework.beans.factory.annotation.Required;
|
||||
import org.springframework.context.Lifecycle;
|
||||
|
||||
/**
|
||||
* Represents a single checkpoint, by its name and main store directory.
|
||||
*
|
||||
* @contributor gojomo
|
||||
*/
|
||||
public class Checkpoint implements InitializingBean, Lifecycle {
|
||||
public class Checkpoint implements InitializingBean {
|
||||
private final static Logger LOGGER =
|
||||
Logger.getLogger(Checkpoint.class.getName());
|
||||
|
||||
@@ -92,24 +91,6 @@ public class Checkpoint implements InitializingBean, Lifecycle {
|
||||
shortName = name.substring(0, name.indexOf("-"));
|
||||
}
|
||||
|
||||
boolean isRunning = false;
|
||||
public boolean isRunning() {
|
||||
return isRunning;
|
||||
}
|
||||
public void start() {
|
||||
if(isRunning) {
|
||||
return;
|
||||
}
|
||||
isRunning = true;
|
||||
if(!Checkpoint.hasValidStamp(checkpointDir.getFile())) {
|
||||
throw new RuntimeException("checkpoint '"+checkpointDir.getFile().getAbsolutePath()+"' missing validity stamp file");
|
||||
}
|
||||
}
|
||||
public void stop() {
|
||||
isRunning = false;
|
||||
}
|
||||
|
||||
|
||||
public void setSuccess(boolean b) {
|
||||
success = b;
|
||||
}
|
||||
|
||||
@@ -21,6 +21,8 @@ package org.archive.checkpointing;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
|
||||
|
||||
/**
|
||||
* Interface for objects that can checkpoint their state, possibly
|
||||
@@ -72,5 +74,6 @@ public interface Checkpointable {
|
||||
*
|
||||
* @param recoveryCheckpoint Checkpoint
|
||||
*/
|
||||
@Autowired(required=false)
|
||||
void setRecoveryCheckpoint(Checkpoint recoveryCheckpoint);
|
||||
}
|
||||
|
||||
@@ -37,12 +37,14 @@ import org.archive.checkpointing.Checkpoint;
|
||||
import org.archive.checkpointing.Checkpointable;
|
||||
import org.archive.crawler.reporting.CrawlStatSnapshot;
|
||||
import org.archive.spring.ConfigPath;
|
||||
import org.archive.spring.HasValidator;
|
||||
import org.springframework.beans.BeansException;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.context.ApplicationContext;
|
||||
import org.springframework.context.ApplicationContextAware;
|
||||
import org.springframework.context.Lifecycle;
|
||||
import org.springframework.context.support.AbstractApplicationContext;
|
||||
import org.springframework.validation.Validator;
|
||||
|
||||
/**
|
||||
* Executes checkpoints, and offers convenience methods for enumerating
|
||||
@@ -56,7 +58,7 @@ import org.springframework.context.support.AbstractApplicationContext;
|
||||
* @contributor gojomo
|
||||
* @contributor pjack
|
||||
*/
|
||||
public class CheckpointService implements Lifecycle, ApplicationContextAware {
|
||||
public class CheckpointService implements Lifecycle, ApplicationContextAware, HasValidator {
|
||||
private final static Logger LOGGER =
|
||||
Logger.getLogger(CheckpointService.class.getName());
|
||||
|
||||
@@ -133,6 +135,16 @@ public class CheckpointService implements Lifecycle, ApplicationContextAware {
|
||||
if (isRunning) {
|
||||
return;
|
||||
}
|
||||
// report if checkpoint incomplete/invalid
|
||||
if(getRecoveryCheckpoint()!=null) {
|
||||
File cpDir = getRecoveryCheckpoint().getCheckpointDir().getFile();
|
||||
if(!Checkpoint.hasValidStamp(cpDir)) {
|
||||
LOGGER.severe(
|
||||
"checkpoint '"+cpDir.getAbsolutePath()
|
||||
+"' missing validity stamp file; checkpoint data "
|
||||
+"may be missing or otherwise corrupt.");
|
||||
}
|
||||
}
|
||||
this.isRunning = true;
|
||||
setupCheckpointTask();
|
||||
}
|
||||
@@ -315,6 +327,9 @@ public class CheckpointService implements Lifecycle, ApplicationContextAware {
|
||||
*/
|
||||
@SuppressWarnings("unchecked")
|
||||
public void setRecoveryCheckpointByName(String selectedCheckpoint) {
|
||||
if(isRunning) {
|
||||
throw new RuntimeException("may not set recovery Checkpoint after launch");
|
||||
}
|
||||
Checkpoint recoveryCheckpoint = new Checkpoint();
|
||||
recoveryCheckpoint.getCheckpointDir().setBase(getCheckpointsDir());
|
||||
recoveryCheckpoint.getCheckpointDir().setPath(selectedCheckpoint);
|
||||
@@ -326,4 +341,10 @@ public class CheckpointService implements Lifecycle, ApplicationContextAware {
|
||||
c.setRecoveryCheckpoint(recoveryCheckpoint);
|
||||
}
|
||||
}
|
||||
|
||||
static Validator VALIDATOR = new CheckpointValidator();
|
||||
@Override
|
||||
public Validator getValidator() {
|
||||
return VALIDATOR;
|
||||
}
|
||||
} //EOC
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
/*
|
||||
* This file is part of the Heritrix web crawler (crawler.archive.org).
|
||||
*
|
||||
* Licensed to the Internet Archive (IA) by one or more individual
|
||||
* contributors.
|
||||
*
|
||||
* The IA licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.archive.crawler.framework;
|
||||
|
||||
import org.archive.checkpointing.Checkpoint;
|
||||
import org.springframework.validation.Errors;
|
||||
import org.springframework.validation.Validator;
|
||||
|
||||
public class CheckpointValidator implements Validator {
|
||||
|
||||
@Override
|
||||
@SuppressWarnings("unchecked")
|
||||
public boolean supports(Class cls) {
|
||||
return Checkpoint.class.isAssignableFrom(cls);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void validate(Object target, Errors errors) {
|
||||
Checkpoint cp = ((CheckpointService)target).getRecoveryCheckpoint();
|
||||
if(cp==null) {
|
||||
return;
|
||||
}
|
||||
if(!Checkpoint.hasValidStamp(cp.getCheckpointDir().getFile())) {
|
||||
errors.rejectValue(
|
||||
"recoveryCheckpoint.checkpointDir",
|
||||
null,
|
||||
"Configured recovery checkpoint "+cp.getName()
|
||||
+" incomplete: lacks valid stamp file.");
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
Reference in New Issue
Block a user