From 01153b0311935557ce82962af0e96f5df9f12d80 Mon Sep 17 00:00:00 2001 From: Noah Levitt Date: Fri, 4 Jan 2019 15:40:16 -0800 Subject: [PATCH] do not checkpoint if crawl job has not started Sometimes our crawl jobs get stuck in the NEW state (due to hbase problems). But the checkpoint service has started and sees fit to checkpoint every five minutes. Evidently, at this stage the seeds have not been queued yet, the frontier is empty. Thus, if we try to resume from one of these checkpoints, the crawl has an empty frontier and ends immediately. The fix is to avoid checkpointing in this NEW state before crawling has really started. --- .../org/archive/crawler/framework/CheckpointService.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/engine/src/main/java/org/archive/crawler/framework/CheckpointService.java b/engine/src/main/java/org/archive/crawler/framework/CheckpointService.java index 2ab7e2eb..73027b83 100644 --- a/engine/src/main/java/org/archive/crawler/framework/CheckpointService.java +++ b/engine/src/main/java/org/archive/crawler/framework/CheckpointService.java @@ -240,6 +240,11 @@ public class CheckpointService implements Lifecycle, ApplicationContextAware, Ha * Run a checkpoint of the crawler */ public synchronized String requestCrawlCheckpoint() throws IllegalStateException { + if (!controller.hasStarted()) { + LOGGER.info("crawl job has not started; ignoring"); + return null; + } + if (isCheckpointing()) { throw new IllegalStateException("Checkpoint already running."); }