Replace constant with accessor methods

CrawlURI already had the accessor method, and the use of the constant
was a bit inconsistent. This change adds the corresponding mutator
method to make working with the CrawlURI history a bit simpler.
This commit is contained in:
Tim Hennekey
2020-01-24 12:28:11 -05:00
parent 3a8b58998c
commit 3003a8e751
5 changed files with 19 additions and 13 deletions
@@ -53,13 +53,12 @@ public class FetchHistoryHelper {
*/
@SuppressWarnings("unchecked")
public static Map<String, Object> getFetchHistory(CrawlURI uri, long timestamp, int historyLength) {
Map<String, Object> data = uri.getData();
Map<String, Object>[] history = (Map[])data.get(RecrawlAttributeConstants.A_FETCH_HISTORY);
Map<String, Object>[] history = uri.getFetchHistory();
if (history == null) {
// there's no history records at all.
// FetchHistoryProcessor assumes history is HashMap[], not Map[].
history = new HashMap[historyLength];
data.put(RecrawlAttributeConstants.A_FETCH_HISTORY, history);
uri.setFetchHistory(history);
}
for (int i = 0; i < history.length; i++) {
if (history[i] == null) {
@@ -80,9 +80,7 @@ public class WbmPersistLoadProcessorTest extends TestCase {
}
protected Map<String, Object> getFetchHistory(CrawlURI curi, int idx) {
Map<String, Object> data = curi.getData();
@SuppressWarnings("unchecked")
Map<String, Object>[] historyArray = (Map[])data.get(RecrawlAttributeConstants.A_FETCH_HISTORY);
Map<String, Object>[] historyArray = curi.getFetchHistory();
assertNotNull(historyArray);
Map<String, Object> history = historyArray[idx];
return history;
@@ -114,10 +112,10 @@ public class WbmPersistLoadProcessorTest extends TestCase {
// put history entry newer than being loaded (i.e. loaded history entry will not be used for FetchHistoryProcessor
// check below.
long expected_ts = DateUtils.parse14DigitDate(TestNormalHttpResponse.EXPECTED_TS).getTime();
Map<String, Object>[] fetchHistory = (Map[])curi.getData().get(RecrawlAttributeConstants.A_FETCH_HISTORY);
Map<String, Object>[] fetchHistory = curi.getFetchHistory();
if (fetchHistory == null) {
fetchHistory = new HashMap[2];
curi.getData().put(RecrawlAttributeConstants.A_FETCH_HISTORY, fetchHistory);
curi.setFetchHistory(fetchHistory);
}
final byte[] digestValue0 = sha1Digest("0");
final byte[] digestValue1 = sha1Digest("1");
@@ -57,7 +57,6 @@ import static org.archive.modules.fetcher.FetchStatusCodes.S_TOO_MANY_RETRIES;
import static org.archive.modules.fetcher.FetchStatusCodes.S_UNATTEMPTED;
import static org.archive.modules.fetcher.FetchStatusCodes.S_UNFETCHABLE_URI;
import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_CONTENT_DIGEST_HISTORY;
import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_FETCH_HISTORY;
import java.io.IOException;
import java.io.ObjectInputStream;
@@ -122,6 +121,8 @@ implements Reporter, Serializable, OverlayContext, Comparable<CrawlURI> {
Logger.getLogger(CrawlURI.class.getName());
public static final int UNCALCULATED = -1;
/** fetch history array */
public static final String A_FETCH_HISTORY = "fetch-history";
public static enum FetchType { HTTP_GET, HTTP_POST, UNKNOWN };
@@ -1826,6 +1827,10 @@ implements Reporter, Serializable, OverlayContext, Comparable<CrawlURI> {
public HashMap<String, Object>[] getFetchHistory() {
return (HashMap<String,Object>[]) getData().get(A_FETCH_HISTORY);
}
public void setFetchHistory(Map<String, Object>[] history) {
getData().put(A_FETCH_HISTORY, history);
}
public HashMap<String, Object> getContentDigestHistory() {
@SuppressWarnings("unchecked")
@@ -22,7 +22,6 @@ package org.archive.modules.recrawl;
import static org.archive.modules.CoreAttributeConstants.A_FETCH_BEGAN_TIME;
import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_CONTENT_DIGEST;
import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_ETAG_HEADER;
import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_FETCH_HISTORY;
import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_LAST_MODIFIED_HEADER;
import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_REFERENCE_LENGTH;
import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_STATUS;
@@ -103,7 +102,7 @@ public class FetchHistoryProcessor extends Processor {
}
history[0] = latestFetch;
curi.getData().put(A_FETCH_HISTORY, history);
curi.setFetchHistory(history);
if (curi.getFetchStatus() == 304) {
if( history.length >= 2 && history[1] != null && history[1].containsKey(A_CONTENT_DIGEST)) {
@@ -19,6 +19,8 @@
package org.archive.modules.recrawl;
import org.archive.modules.CrawlURI;
/**
*
* @author pjack
@@ -28,8 +30,11 @@ public interface RecrawlAttributeConstants {
/* Duplication-reduction / recrawl / history constants */
/** fetch history array */
public static final String A_FETCH_HISTORY = "fetch-history";
/**
* @deprecated Please use {@link org.archive.modules.CrawlURI#getFetchHistory()} and {@link org.archive.modules.CrawlURI#setFetchHistory(java.util.Map[])}
*/
@Deprecated
public static final String A_FETCH_HISTORY = CrawlURI.A_FETCH_HISTORY;
/** content digest */
public static final String A_CONTENT_DIGEST = "content-digest";
/** header name (and AList key) for last-modified timestamp */