Add --checkpoint command-line option

This makes it possible to select a checkpoint from the command-line when using the --run-job option.
This commit is contained in:
Alex Osborne
2024-11-24 13:40:33 +09:00
parent 18c69def9a
commit 585fed8575
2 changed files with 23 additions and 2 deletions
+3
View File
@@ -27,6 +27,9 @@ Command-line Options
-b, --web-bind-hosts HOST
Specifies a comma-separated list of hostnames/IP-addresses to bind to the Web UI. You may use '/' as a
shorthand for 'all addresses'. **Default**: ``localhost/127.0.0.1``
-c,--checkpoint ARG
Recovers from the given checkpoint. May only be used with the --run-job option. The special value 'latest'
will recover the last checkpoint or if none exist will launch a new crawl.
-j, --job-dirs PATH
Sets the directory Heritrix stores jobs in. **Default:** ``$HERITRIX_HOME/jobs``
-l, --logging-properties PATH
@@ -128,6 +128,10 @@ public class Heritrix {
"\"password\" (which leaves username as the default 'admin'), " +
"\"username:password\", or \"@filename\" for a file that " +
"includes the single line \"username:password\". ");
options.addOption("c", "checkpoint", true,
"Recovers from the given checkpoint. May only be used with the " +
"--run-job option. The special value 'latest' will recover the " +
"last checkpoint or if none exist will launch a new crawl.");
options.addOption("j", "jobs-dir", true, "The jobs directory. " +
"Defaults to ./jobs");
options.addOption("l", "logging-properties", true,
@@ -265,6 +269,11 @@ public class Heritrix {
System.exit(1);
authPassword = ""; // suppresses uninitialized warning
}
if (cl.hasOption('c') && !cl.hasOption('r')) {
System.err.println("Cannot use --checkpoint without --run-job.");
System.exit(1);
}
File jobsDir = null;
if (cl.hasOption('j')) {
@@ -374,12 +383,21 @@ public class Heritrix {
}
if (cl.hasOption('r')) {
String jobName = cl.getOptionValue('r');
engine.requestLaunch(jobName);
CrawlJob job = engine.getJob(jobName);
if (job == null || job.getCrawlController() == null) {
if (job == null) {
System.err.println("Job not found: " + jobName);
System.exit(1);
}
job.validateConfiguration();
if (cl.hasOption('c')) {
job.getCheckpointService().setRecoveryCheckpointByName(cl.getOptionValue('c'));
}
job.launch();
if (job.getCrawlController() == null) {
System.err.println("Failed to launch job: " + jobName);
System.exit(1);
}
job.getCrawlController().requestCrawlResume();
engine.waitForNoRunningJobs(0);
engine.shutdown();