diff --git a/docs/operating.rst b/docs/operating.rst
index f034cdbe..92680fcf 100644
--- a/docs/operating.rst
+++ b/docs/operating.rst
@@ -27,6 +27,9 @@ Command-line Options
-b, --web-bind-hosts HOST
Specifies a comma-separated list of hostnames/IP-addresses to bind to the Web UI. You may use '/' as a
shorthand for 'all addresses'. **Default**: ``localhost/127.0.0.1``
+ -c,--checkpoint ARG
+ Recovers from the given checkpoint. May only be used with the --run-job option. The special value 'latest'
+ will recover the last checkpoint or if none exist will launch a new crawl.
-j, --job-dirs PATH
Sets the directory Heritrix stores jobs in. **Default:** ``$HERITRIX_HOME/jobs``
-l, --logging-properties PATH
@@ -685,11 +688,18 @@ To configure Heritrix to automatically run checkpoints periodically, set the
+
-
+
-By default only the latest checkpoint will be kept.
+When ``checkpointOnShutdown`` is enabled Heritrix will create a checkpoint if the job is running when the JVM is
+gracefully shutdown. Note that if Heritrix is killed, crashes or the server it is running on unexpectedly loses
+power the shutdown checkpoint will not be created. Consequently it may be ideal to enable both shutdown and interval
+checkpoints together.
+
+Setting ``forgetAllButLatest``` will ensure only the latest checkpoint is kept.
+
Restarting from a Checkpoint
~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -708,6 +718,10 @@ The web UI provides an option to restart a crawl from a checkpoint:
The job will now begin running from the chosen checkpoint.
+When running a job from the command-line with the ``--run-job`` CLI option you can use the ``--checkpoint`` to restart
+the job from a named checkpoint. The special name ``latest`` will restart from the latest checkpoint if any exist,
+otherwise it will launch a new crawl.
+
Crawl Recovery
--------------
diff --git a/engine/src/main/java/org/archive/crawler/Heritrix.java b/engine/src/main/java/org/archive/crawler/Heritrix.java
index 2c68174c..13adae33 100644
--- a/engine/src/main/java/org/archive/crawler/Heritrix.java
+++ b/engine/src/main/java/org/archive/crawler/Heritrix.java
@@ -128,6 +128,10 @@ public class Heritrix {
"\"password\" (which leaves username as the default 'admin'), " +
"\"username:password\", or \"@filename\" for a file that " +
"includes the single line \"username:password\". ");
+ options.addOption("c", "checkpoint", true,
+ "Recovers from the given checkpoint. May only be used with the " +
+ "--run-job option. The special value 'latest' will recover the " +
+ "last checkpoint or if none exist will launch a new crawl.");
options.addOption("j", "jobs-dir", true, "The jobs directory. " +
"Defaults to ./jobs");
options.addOption("l", "logging-properties", true,
@@ -265,6 +269,11 @@ public class Heritrix {
System.exit(1);
authPassword = ""; // suppresses uninitialized warning
}
+
+ if (cl.hasOption('c') && !cl.hasOption('r')) {
+ System.err.println("Cannot use --checkpoint without --run-job.");
+ System.exit(1);
+ }
File jobsDir = null;
if (cl.hasOption('j')) {
@@ -374,12 +383,21 @@ public class Heritrix {
}
if (cl.hasOption('r')) {
String jobName = cl.getOptionValue('r');
- engine.requestLaunch(jobName);
CrawlJob job = engine.getJob(jobName);
- if (job == null || job.getCrawlController() == null) {
+ if (job == null) {
+ System.err.println("Job not found: " + jobName);
+ System.exit(1);
+ }
+ job.validateConfiguration();
+ if (cl.hasOption('c')) {
+ job.getCheckpointService().setRecoveryCheckpointByName(cl.getOptionValue('c'));
+ }
+ job.launch();
+ if (job.getCrawlController() == null) {
System.err.println("Failed to launch job: " + jobName);
System.exit(1);
}
+
job.getCrawlController().requestCrawlResume();
engine.waitForNoRunningJobs(0);
engine.shutdown();
diff --git a/engine/src/main/java/org/archive/crawler/framework/CheckpointService.java b/engine/src/main/java/org/archive/crawler/framework/CheckpointService.java
index 76245ba9..cb92658e 100644
--- a/engine/src/main/java/org/archive/crawler/framework/CheckpointService.java
+++ b/engine/src/main/java/org/archive/crawler/framework/CheckpointService.java
@@ -56,7 +56,7 @@ import org.springframework.validation.Validator;
* build and before launch (setRecoveryCheckpointByName).
*
* Offers optional automatic checkpointing at a configurable interval
- * in minutes.
+ * in minutes.
*
* @author stack
* @author gojomo
@@ -80,6 +80,8 @@ public class CheckpointService implements Lifecycle, ApplicationContextAware, Ha
protected TimerTask checkpointTask = null;
protected ConfigPath checkpointsDir =
new ConfigPath("checkpoints subdirectory","checkpoints");
+ protected Thread shutdownHook;
+
public ConfigPath getCheckpointsDir() {
return checkpointsDir;
}
@@ -106,6 +108,21 @@ public class CheckpointService implements Lifecycle, ApplicationContextAware, Ha
setupCheckpointTask();
}
}
+
+ protected boolean checkpointOnShutdown = false;
+
+ public boolean getCheckpointOnShutdown() {
+ return checkpointOnShutdown;
+ }
+
+ /**
+ * Whether a checkpoint should be made when the JVM is shutdown.
+ * Default is false.
+ */
+ public void setCheckpointOnShutdown(boolean checkpointOnShutdown) {
+ this.checkpointOnShutdown = checkpointOnShutdown;
+ setupShutdownHook();
+ }
protected boolean forgetAllButLatest = false;
public boolean getForgetAllButLatest() {
@@ -179,6 +196,7 @@ public class CheckpointService implements Lifecycle, ApplicationContextAware, Ha
}
this.isRunning = true;
setupCheckpointTask();
+ setupShutdownHook();
}
/**
@@ -221,12 +239,31 @@ public class CheckpointService implements Lifecycle, ApplicationContextAware, Ha
public synchronized boolean isRunning() {
return isRunning;
}
-
+
+ protected synchronized void setupShutdownHook() {
+ if (!checkpointOnShutdown || shutdownHook != null || !isRunning) return;
+ shutdownHook = new Thread(() -> {
+ if (!checkpointOnShutdown) return;
+ LOGGER.info("Checkpointing on shutdown");
+ try {
+ // pause first to ensure no crawling occurs between the checkpoint and process termination
+ controller.requestCrawlPause();
+ requestCrawlCheckpoint();
+ } catch (Exception e) {
+ LOGGER.severe("Failed to checkpoint on shutdown: " + e.getMessage());
+ }
+ });
+ Runtime.getRuntime().addShutdownHook(shutdownHook);
+ }
public synchronized void stop() {
LOGGER.info("Cleaned up Checkpoint TimerThread.");
this.timer.cancel();
- this.isRunning = false;
+ this.isRunning = false;
+ if (shutdownHook != null) {
+ Runtime.getRuntime().removeShutdownHook(shutdownHook);
+ shutdownHook = null;
+ }
}
/**
diff --git a/engine/src/main/resources/org/archive/crawler/restlet/profile-crawler-beans.cxml b/engine/src/main/resources/org/archive/crawler/restlet/profile-crawler-beans.cxml
index 39defa3f..83e71c03 100644
--- a/engine/src/main/resources/org/archive/crawler/restlet/profile-crawler-beans.cxml
+++ b/engine/src/main/resources/org/archive/crawler/restlet/profile-crawler-beans.cxml
@@ -564,6 +564,7 @@ http://example.example/example
+