diff --git a/contrib/src/main/java/org/archive/modules/recrawl/FetchHistoryHelper.java b/contrib/src/main/java/org/archive/modules/recrawl/FetchHistoryHelper.java index 4705a9f6..a4807a52 100644 --- a/contrib/src/main/java/org/archive/modules/recrawl/FetchHistoryHelper.java +++ b/contrib/src/main/java/org/archive/modules/recrawl/FetchHistoryHelper.java @@ -53,13 +53,12 @@ public class FetchHistoryHelper { */ @SuppressWarnings("unchecked") public static Map getFetchHistory(CrawlURI uri, long timestamp, int historyLength) { - Map data = uri.getData(); - Map[] history = (Map[])data.get(RecrawlAttributeConstants.A_FETCH_HISTORY); + Map[] history = uri.getFetchHistory(); if (history == null) { // there's no history records at all. // FetchHistoryProcessor assumes history is HashMap[], not Map[]. history = new HashMap[historyLength]; - data.put(RecrawlAttributeConstants.A_FETCH_HISTORY, history); + uri.setFetchHistory(history); } for (int i = 0; i < history.length; i++) { if (history[i] == null) { diff --git a/contrib/src/test/java/org/archive/modules/recrawl/wbm/WbmPersistLoadProcessorTest.java b/contrib/src/test/java/org/archive/modules/recrawl/wbm/WbmPersistLoadProcessorTest.java index 88d1c0cf..ab16e584 100644 --- a/contrib/src/test/java/org/archive/modules/recrawl/wbm/WbmPersistLoadProcessorTest.java +++ b/contrib/src/test/java/org/archive/modules/recrawl/wbm/WbmPersistLoadProcessorTest.java @@ -80,9 +80,7 @@ public class WbmPersistLoadProcessorTest extends TestCase { } protected Map getFetchHistory(CrawlURI curi, int idx) { - Map data = curi.getData(); - @SuppressWarnings("unchecked") - Map[] historyArray = (Map[])data.get(RecrawlAttributeConstants.A_FETCH_HISTORY); + Map[] historyArray = curi.getFetchHistory(); assertNotNull(historyArray); Map history = historyArray[idx]; return history; @@ -114,10 +112,10 @@ public class WbmPersistLoadProcessorTest extends TestCase { // put history entry newer than being loaded (i.e. loaded history entry will not be used for FetchHistoryProcessor // check below. long expected_ts = DateUtils.parse14DigitDate(TestNormalHttpResponse.EXPECTED_TS).getTime(); - Map[] fetchHistory = (Map[])curi.getData().get(RecrawlAttributeConstants.A_FETCH_HISTORY); + Map[] fetchHistory = curi.getFetchHistory(); if (fetchHistory == null) { fetchHistory = new HashMap[2]; - curi.getData().put(RecrawlAttributeConstants.A_FETCH_HISTORY, fetchHistory); + curi.setFetchHistory(fetchHistory); } final byte[] digestValue0 = sha1Digest("0"); final byte[] digestValue1 = sha1Digest("1"); diff --git a/modules/src/main/java/org/archive/modules/CrawlURI.java b/modules/src/main/java/org/archive/modules/CrawlURI.java index 92d2870f..82863698 100644 --- a/modules/src/main/java/org/archive/modules/CrawlURI.java +++ b/modules/src/main/java/org/archive/modules/CrawlURI.java @@ -57,7 +57,6 @@ import static org.archive.modules.fetcher.FetchStatusCodes.S_TOO_MANY_RETRIES; import static org.archive.modules.fetcher.FetchStatusCodes.S_UNATTEMPTED; import static org.archive.modules.fetcher.FetchStatusCodes.S_UNFETCHABLE_URI; import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_CONTENT_DIGEST_HISTORY; -import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_FETCH_HISTORY; import java.io.IOException; import java.io.ObjectInputStream; @@ -122,6 +121,8 @@ implements Reporter, Serializable, OverlayContext, Comparable { Logger.getLogger(CrawlURI.class.getName()); public static final int UNCALCULATED = -1; + /** fetch history array */ + public static final String A_FETCH_HISTORY = "fetch-history"; public static enum FetchType { HTTP_GET, HTTP_POST, UNKNOWN }; @@ -1826,6 +1827,10 @@ implements Reporter, Serializable, OverlayContext, Comparable { public HashMap[] getFetchHistory() { return (HashMap[]) getData().get(A_FETCH_HISTORY); } + + public void setFetchHistory(Map[] history) { + getData().put(A_FETCH_HISTORY, history); + } public HashMap getContentDigestHistory() { @SuppressWarnings("unchecked") diff --git a/modules/src/main/java/org/archive/modules/recrawl/FetchHistoryProcessor.java b/modules/src/main/java/org/archive/modules/recrawl/FetchHistoryProcessor.java index 889fa05c..2f4dce68 100644 --- a/modules/src/main/java/org/archive/modules/recrawl/FetchHistoryProcessor.java +++ b/modules/src/main/java/org/archive/modules/recrawl/FetchHistoryProcessor.java @@ -22,7 +22,6 @@ package org.archive.modules.recrawl; import static org.archive.modules.CoreAttributeConstants.A_FETCH_BEGAN_TIME; import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_CONTENT_DIGEST; import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_ETAG_HEADER; -import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_FETCH_HISTORY; import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_LAST_MODIFIED_HEADER; import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_REFERENCE_LENGTH; import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_STATUS; @@ -103,7 +102,7 @@ public class FetchHistoryProcessor extends Processor { } history[0] = latestFetch; - curi.getData().put(A_FETCH_HISTORY, history); + curi.setFetchHistory(history); if (curi.getFetchStatus() == 304) { if( history.length >= 2 && history[1] != null && history[1].containsKey(A_CONTENT_DIGEST)) { diff --git a/modules/src/main/java/org/archive/modules/recrawl/RecrawlAttributeConstants.java b/modules/src/main/java/org/archive/modules/recrawl/RecrawlAttributeConstants.java index e7c41dce..f284318d 100644 --- a/modules/src/main/java/org/archive/modules/recrawl/RecrawlAttributeConstants.java +++ b/modules/src/main/java/org/archive/modules/recrawl/RecrawlAttributeConstants.java @@ -19,6 +19,8 @@ package org.archive.modules.recrawl; +import org.archive.modules.CrawlURI; + /** * * @author pjack @@ -28,8 +30,11 @@ public interface RecrawlAttributeConstants { /* Duplication-reduction / recrawl / history constants */ - /** fetch history array */ - public static final String A_FETCH_HISTORY = "fetch-history"; + /** + * @deprecated Please use {@link org.archive.modules.CrawlURI#getFetchHistory()} and {@link org.archive.modules.CrawlURI#setFetchHistory(java.util.Map[])} + */ + @Deprecated + public static final String A_FETCH_HISTORY = CrawlURI.A_FETCH_HISTORY; /** content digest */ public static final String A_CONTENT_DIGEST = "content-digest"; /** header name (and AList key) for last-modified timestamp */