diff --git a/modules/src/main/java/org/archive/modules/recrawl/AbstractPersistProcessor.java b/modules/src/main/java/org/archive/modules/recrawl/AbstractPersistProcessor.java new file mode 100644 index 00000000..5be7404e --- /dev/null +++ b/modules/src/main/java/org/archive/modules/recrawl/AbstractPersistProcessor.java @@ -0,0 +1,70 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.archive.modules.recrawl; + +import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_FETCH_HISTORY; +import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_WRITE_TAG; + +import java.util.Map; + +import org.archive.modules.CrawlURI; +import org.archive.modules.Processor; + +public abstract class AbstractPersistProcessor extends Processor { + + /** @see RecrawlAttributeConstants#A_WRITE_TAG */ + protected boolean onlyStoreIfWriteTagPresent = true; + public boolean getOnlyStoreIfWriteTagPresent() { + return onlyStoreIfWriteTagPresent; + } + public void setOnlyStoreIfWriteTagPresent(boolean onlyStoreIfWriteTagPresent) { + this.onlyStoreIfWriteTagPresent = onlyStoreIfWriteTagPresent; + } + + /** + * Whether the current CrawlURI's state should be persisted (to log or + * direct to database) + * + * @param curi + * CrawlURI + * @return true if state should be stored; false to skip persistence + */ + protected boolean shouldStore(CrawlURI curi) { + if (getOnlyStoreIfWriteTagPresent()) { + @SuppressWarnings("unchecked") + Map[] history = (Map[])curi.getData().get(A_FETCH_HISTORY); + return history != null && history[0] != null && history[0].containsKey(A_WRITE_TAG); + } else { + return curi.isSuccess(); + } + } + + /** + * Whether the current CrawlURI's state should be loaded + * + * @param curi CrawlURI + * @return true if state should be loaded; false to skip loading + */ + protected boolean shouldLoad(CrawlURI curi) { + // TODO: don't load some (prereqs?) + return true; + } + + +} diff --git a/modules/src/main/java/org/archive/modules/recrawl/PersistProcessor.java b/modules/src/main/java/org/archive/modules/recrawl/PersistProcessor.java index e05fe16f..9085afdd 100644 --- a/modules/src/main/java/org/archive/modules/recrawl/PersistProcessor.java +++ b/modules/src/main/java/org/archive/modules/recrawl/PersistProcessor.java @@ -19,8 +19,6 @@ package org.archive.modules.recrawl; -import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_FETCH_HISTORY; -import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_WRITE_TAG; import java.io.BufferedReader; import java.io.File; @@ -40,7 +38,6 @@ import org.apache.commons.io.IOUtils; import org.apache.commons.lang.SerializationUtils; import org.archive.bdb.BdbModule; import org.archive.modules.CrawlURI; -import org.archive.modules.Processor; import org.archive.util.ArchiveUtils; import org.archive.util.FileUtils; import org.archive.util.OneLineSimpleLogger; @@ -65,7 +62,7 @@ import com.sleepycat.je.EnvironmentConfig; * * @author gojomo */ -public abstract class PersistProcessor extends Processor { +public abstract class PersistProcessor extends AbstractPersistProcessor { private static final long serialVersionUID = 1L; @@ -84,15 +81,6 @@ public abstract class PersistProcessor extends Processor { HISTORY_DB_CONFIG = dbConfig; } - /** @see RecrawlAttributeConstants#A_WRITE_TAG */ - boolean onlyStoreIfWriteTagPresent = true; - public boolean getOnlyStoreIfWriteTagPresent() { - return onlyStoreIfWriteTagPresent; - } - public void setOnlyStoreIfWriteTagPresent(boolean onlyStoreIfWriteTagPresent) { - this.onlyStoreIfWriteTagPresent = onlyStoreIfWriteTagPresent; - } - public PersistProcessor() { } @@ -113,35 +101,6 @@ public abstract class PersistProcessor extends Processor { return SURT.fromURI(uri,true); } - /** - * Whether the current CrawlURI's state should be persisted (to log or - * direct to database) - * - * @param curi - * CrawlURI - * @return true if state should be stored; false to skip persistence - */ - protected boolean shouldStore(CrawlURI curi) { - if (getOnlyStoreIfWriteTagPresent()) { - @SuppressWarnings("unchecked") - Map[] history = (Map[])curi.getData().get(A_FETCH_HISTORY); - return history != null && history[0] != null && history[0].containsKey(A_WRITE_TAG); - } else { - return curi.isSuccess(); - } - } - - /** - * Whether the current CrawlURI's state should be loaded - * - * @param curi CrawlURI - * @return true if state should be loaded; false to skip loading - */ - protected boolean shouldLoad(CrawlURI curi) { - // TODO: don't load some (prereqs?) - return true; - } - /** * Copies entries from an existing environment db to a new one. If * historyMap is not provided, only logs the entries that would have been diff --git a/modules/src/main/java/org/archive/modules/recrawl/PersistStoreProcessor.java b/modules/src/main/java/org/archive/modules/recrawl/PersistStoreProcessor.java index 8065d6db..8977e344 100644 --- a/modules/src/main/java/org/archive/modules/recrawl/PersistStoreProcessor.java +++ b/modules/src/main/java/org/archive/modules/recrawl/PersistStoreProcessor.java @@ -37,7 +37,6 @@ public class PersistStoreProcessor extends PersistOnlineProcessor public PersistStoreProcessor() { } - @SuppressWarnings("unchecked") @Override protected void innerProcess(CrawlURI curi) throws InterruptedException { store.put(persistKeyFor(curi),curi.getPersistentDataMap());