lee/heritrix3
1
0 Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-11-15 10:20:47 +00:00
Code Issues Packages Projects Releases Wiki Activity

[HER-1546] Springify(5): Update checkpointing to work smoothly with spring-configured crawls

Browse Source 
core checkpointing system classes:
* Checkpointable.java
    basic interface for checkpoint aware/capable beans
* Checkpoint.java
    one Checkpoint, either in progress or recovery -- a name + storage directory
* CheckpointService.java
    helper bean to enable, trigger checkpoints and recovery 
* Checkpointer.java(*2), CheckpointInputStream.java, CheckpointRecovery.java DefaultCheckpointRecovery.java, RecoverAction.java
    (deleted) no longer used
This commit is contained in:
gojomo committed 2009-11-19 22:45:07 +00:00
1 parent e42e2ed7f6
commit bdf4437ca2
9 files changed
+487 -719
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

commons/src/main/java/org/archive/checkpointing/CheckpointInputStream.java
-70
View File
@@ -1,70 +0,0 @@
/*
* This file is part of the Heritrix web crawler (crawler.archive.org).
*
* Licensed to the Internet Archive (IA) by one or more individual
* contributors.
*
* The IA licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.archive.checkpointing;
import java.io.IOException;
import java.io.InputStream;
import java.io.ObjectInputStream;
import java.net.URI;
/**
* Object input stream that provides information useful during checkpoint
* recovery.
*
* @author pjack
*/
public class CheckpointInputStream extends ObjectInputStream
implements CheckpointRecovery {
final private CheckpointRecovery recovery;
public CheckpointInputStream(InputStream input,
CheckpointRecovery recovery) throws IOException {
super(input);
this.recovery = recovery;
}
public String getRecoveredJobName() {
return recovery.getRecoveredJobName();
}
// public <T> void setState(Object module, Key<T> key, T value) {
// recovery.setState(module, key, value);
// }
public String translatePath(String path) {
return recovery.translatePath(path);
}
public URI translateURI(URI uri) {
return recovery.translateURI(uri);
}
// public void apply(SingleSheet global) {
// throw new UnsupportedOperationException();
// }
}
commons/src/main/java/org/archive/checkpointing/CheckpointRecovery.java
-40
View File
@@ -1,40 +0,0 @@
/*
* This file is part of the Heritrix web crawler (crawler.archive.org).
*
* Licensed to the Internet Archive (IA) by one or more individual
* contributors.
*
* The IA licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.archive.checkpointing;
import java.net.URI;
/**
* The information about a checkpoint recovery.
*
* @author pjack
*/
public interface CheckpointRecovery {
String getRecoveredJobName();
String translatePath(String path);
URI translateURI(URI uri);
// <T> void setState(Object module, Key<T> key, T value);
//
// void apply(SingleSheet global);
}
commons/src/main/java/org/archive/checkpointing/Checkpointable.java
+46 -6
View File
@@ -19,19 +19,59 @@
package org.archive.checkpointing;
import java.io.File;
import java.io.IOException;
import java.util.List;
import org.archive.crawler.framework.Checkpoint;
/**
* @author pjack
*
* Interface for objects that can checkpoint their state, possibly
* but not necessarily into the provided Checkpoint instance, on
* request.
*
* @contributor pjack
* @contributor gojomo
*/
public interface Checkpointable {
/**
* Note a checkpoint is about to begin. Most beans will ignore,
* but some can use this to wrap up tasks that shouldn't be
* half-done during a checkpoint (and hold a lock to prevent
* tasks from beginning during the checkpoint).
*
* @param checkpointInProgress Checkpoint
*/
void startCheckpoint(Checkpoint checkpointInProgress);
void checkpoint(File dir, List<RecoverAction> actions) throws IOException;
/**
* Do the actual checkpoint. Beans should ensure any state that
* they would need to recover gets saved in an appropriate place.
* A moderate amount of state may be saved as a JSONObject into
* the Checkpoint (which then keeps it in the checkpoint directory.)
* Larger amounts of state may be stored in a manner private to
* the bean, or via other collaborating beans (such as BdbModule)
* which checkpoint backgin database state.
*
* @param checkpointInProgress Checkpoint
* @throws IOException
*/
void doCheckpoint(Checkpoint checkpointInProgress) throws IOException;
//
/**
* Cleanup/unlock; need not complete for a checkpoint to be valid.
*
* @param checkpointInProgress Checkpoint
*/
void finishCheckpoint(Checkpoint checkpointInProgress);
/**
* Used to inform a bean that it should restore its state from
* the given Checkpoint when launched (Lifecycle start()). May be
* autowired or configured after build, but before launch, for
* example via CheckpointService.setRecoveryCheckpointByName().
*
* @param recoveryCheckpoint Checkpoint
*/
void setRecoveryCheckpoint(Checkpoint recoveryCheckpoint);
}
commons/src/main/java/org/archive/checkpointing/Checkpointer.java
-120
View File
@@ -1,120 +0,0 @@
/*
* This file is part of the Heritrix web crawler (crawler.archive.org).
*
* Licensed to the Internet Archive (IA) by one or more individual
* contributors.
*
* The IA licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.archive.checkpointing;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.util.ArrayList;
import java.util.List;
import org.archive.util.IoUtils;
/**
* Executes checkpoints and recovers.
*
* @author pjack
*/
public class Checkpointer {
final public static String ACTIONS_FILE = "actions.serialized";
final public static String OBJECT_GRAPH_FILE = "object_graph.serialized";
private Checkpointer() {
}
public static void checkpoint(/*SheetManager*/Object mgr, File dir)
throws IOException {
List<RecoverAction> actions = new ArrayList<RecoverAction>();
// for (Checkpointable c: mgr.getCheckpointables()) {
// c.checkpoint(dir, actions);
// }
writeObject(new File(dir, ACTIONS_FILE), actions);
writeObject(new File(dir, OBJECT_GRAPH_FILE), mgr);
}
private static void writeObject(File f, Object o)
throws IOException {
ObjectOutputStream oout = null;
try {
oout = new ObjectOutputStream(new FileOutputStream(f));
oout.writeObject(o);
} finally {
IoUtils.close(oout);
}
}
@SuppressWarnings("unused")
private static List<RecoverAction> readActions(File dir)
throws IOException {
File actionsFile = new File(dir, ACTIONS_FILE);
ObjectInputStream oinp = null;
try {
oinp = new ObjectInputStream(
new FileInputStream(actionsFile));
@SuppressWarnings("unchecked")
List<RecoverAction> actions = (List)oinp.readObject();
return actions;
} catch (ClassNotFoundException e) {
IOException io = new IOException();
io.initCause(e);
throw io;
} finally {
IoUtils.close(oinp);
}
}
// public static SheetManager recover(File dir, CheckpointRecovery recovery)
// throws IOException {
// List<RecoverAction> actions = readActions(dir);
// for (RecoverAction action: actions) try {
// action.recoverFrom(dir, recovery);
// } catch (Exception e) {
// IOException io = new IOException();
// io.initCause(e);
// throw io;
// }
//
// CheckpointInputStream cinp = null;
// try {
// File f = new File(dir, OBJECT_GRAPH_FILE);
// cinp = new CheckpointInputStream(new FileInputStream(f), recovery);
// SheetManager mgr = (SheetManager)cinp.readObject();
// recovery.apply(mgr.getGlobalSheet());
// return mgr;
// } catch (ClassNotFoundException e) {
// IOException io = new IOException();
// io.initCause(e);
// throw io;
// }finally {
// IoUtils.close(cinp);
// }
// }
}
commons/src/main/java/org/archive/checkpointing/DefaultCheckpointRecovery.java
-109
View File
@@ -1,109 +0,0 @@
/*
* This file is part of the Heritrix web crawler (crawler.archive.org).
*
* Licensed to the Internet Archive (IA) by one or more individual
* contributors.
*
* The IA licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.archive.checkpointing;
import java.net.URI;
import java.util.HashMap;
import java.util.Map;
/**
* Default implementation.
*
* @author pjack
*
*/
public class DefaultCheckpointRecovery implements CheckpointRecovery {
// final private Map<Object,Map<Key,Object>> newSettings =
// new IdentityHashMap<Object,Map<Key,Object>>();
final private Map<URI,URI> uriTranslations = new HashMap<URI,URI>();
final private Map<String,String> fileTranslations =
new HashMap<String,String>();
final private String name;
public DefaultCheckpointRecovery(String name) {
this.name = name;
}
public String getRecoveredJobName() {
return name;
}
public Map<String,String> getFileTranslations() {
return fileTranslations;
}
public Map<URI,URI> getURITranslations() {
return uriTranslations;
}
// public <T> void setState(Object module, Key<T> key, T value) {
// Map<Key,Object> map = newSettings.get(module);
// if (map == null) {
// map = new HashMap<Key,Object>();
// newSettings.put(module, map);
// }
//
// map.put(key, value);
// }
public String translatePath(String path) {
Map.Entry<String,String> match = null;
for (Map.Entry<String,String> me: fileTranslations.entrySet()) {
if (path.startsWith(me.getKey())) {
if ((match == null)
|| (match.getKey().length() < me.getKey().length())) {
match = me;
}
}
}
if (match == null) {
return path;
}
int size = match.getKey().length();
return match.getValue() + path.substring(size);
}
public URI translateURI(URI uri) {
URI r = uriTranslations.get(uri);
return r == null ? uri : r;
}
// public void apply(SingleSheet global) {
// for (Map.Entry<Object,Map<Key,Object>> mod: newSettings.entrySet()) {
// Object module = mod.getKey();
// for (Map.Entry<Key,Object> me: mod.getValue().entrySet()) {
// @SuppressWarnings("unchecked")
// Key<Object> k = me.getKey();
// global.set(module, k, me.getValue());
// }
// }
// }
}
commons/src/main/java/org/archive/checkpointing/RecoverAction.java
-38
View File
@@ -1,38 +0,0 @@
/*
* This file is part of the Heritrix web crawler (crawler.archive.org).
*
* Licensed to the Internet Archive (IA) by one or more individual
* contributors.
*
* The IA licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.archive.checkpointing;
import java.io.File;
import java.io.Serializable;
/**
* An action to be taken before a checkpoint's object graph is deserialized.
*
* @author pjack
*/
public interface RecoverAction extends Serializable {
void recoverFrom(File checkpointDir, CheckpointRecovery recovery)
throws Exception;
}