From 3b341f79dfe8ed70c0aa5538862fdd0dbb936bf2 Mon Sep 17 00:00:00 2001 From: Andrew Jackson Date: Mon, 8 Oct 2018 14:26:04 +0100 Subject: [PATCH 1/2] Provide a simple hook to restart from the latest checkpoint. --- .../crawler/framework/CheckpointService.java | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/engine/src/main/java/org/archive/crawler/framework/CheckpointService.java b/engine/src/main/java/org/archive/crawler/framework/CheckpointService.java index c9e97344..fdf9729d 100644 --- a/engine/src/main/java/org/archive/crawler/framework/CheckpointService.java +++ b/engine/src/main/java/org/archive/crawler/framework/CheckpointService.java @@ -393,6 +393,21 @@ public class CheckpointService implements Lifecycle, ApplicationContextAware, Ha if(isRunning) { throw new RuntimeException("may not set recovery Checkpoint after launch"); } + // If the selectedCheckpoint is 'latest', pick up the latest checkpoint + // and use that: + if ("latest".equalsIgnoreCase(selectedCheckpoint)) { + List cps = this.findAvailableCheckpointDirectories(); + if (cps == null || cps.size() == 0) { + throw new RuntimeException( + "Cannot find any checkpoints so cannot choose the latest one."); + } + // As per the API above, the most recent checkpoint is the first in + // the list: + File latestFile = cps.get(0); + // For the checkpoint we use the folder name: + selectedCheckpoint = latestFile.getName(); + } + // Now setup the checkpoint: Checkpoint recoveryCheckpoint = new Checkpoint(); recoveryCheckpoint.getCheckpointDir().setBase(getCheckpointsDir()); recoveryCheckpoint.getCheckpointDir().setPath(selectedCheckpoint); From 59ee9b2adc89d9a542b7ee17abbfb5978423a981 Mon Sep 17 00:00:00 2001 From: Andrew Jackson Date: Tue, 9 Oct 2018 10:28:34 +0100 Subject: [PATCH 2/2] Default to starting anew if no checkpoints are found. --- .../org/archive/crawler/framework/CheckpointService.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/engine/src/main/java/org/archive/crawler/framework/CheckpointService.java b/engine/src/main/java/org/archive/crawler/framework/CheckpointService.java index fdf9729d..2ab7e2eb 100644 --- a/engine/src/main/java/org/archive/crawler/framework/CheckpointService.java +++ b/engine/src/main/java/org/archive/crawler/framework/CheckpointService.java @@ -398,8 +398,9 @@ public class CheckpointService implements Lifecycle, ApplicationContextAware, Ha if ("latest".equalsIgnoreCase(selectedCheckpoint)) { List cps = this.findAvailableCheckpointDirectories(); if (cps == null || cps.size() == 0) { - throw new RuntimeException( - "Cannot find any checkpoints so cannot choose the latest one."); + LOGGER.warning( + "Cannot find any checkpoints so cannot choose the latest one! Assuming we should launch a new crawl."); + return; } // As per the API above, the most recent checkpoint is the first in // the list: