mirror of
https://github.com/internetarchive/heritrix3.git
synced 2026-09-21 13:16:03 +00:00
Replace constant with accessor methods
CrawlURI already had the accessor method, and the use of the constant was a bit inconsistent. This change adds the corresponding mutator method to make working with the CrawlURI history a bit simpler.
This commit is contained in:
@@ -53,13 +53,12 @@ public class FetchHistoryHelper {
|
||||
*/
|
||||
@SuppressWarnings("unchecked")
|
||||
public static Map<String, Object> getFetchHistory(CrawlURI uri, long timestamp, int historyLength) {
|
||||
Map<String, Object> data = uri.getData();
|
||||
Map<String, Object>[] history = (Map[])data.get(RecrawlAttributeConstants.A_FETCH_HISTORY);
|
||||
Map<String, Object>[] history = uri.getFetchHistory();
|
||||
if (history == null) {
|
||||
// there's no history records at all.
|
||||
// FetchHistoryProcessor assumes history is HashMap[], not Map[].
|
||||
history = new HashMap[historyLength];
|
||||
data.put(RecrawlAttributeConstants.A_FETCH_HISTORY, history);
|
||||
uri.setFetchHistory(history);
|
||||
}
|
||||
for (int i = 0; i < history.length; i++) {
|
||||
if (history[i] == null) {
|
||||
|
||||
+3
-5
@@ -80,9 +80,7 @@ public class WbmPersistLoadProcessorTest extends TestCase {
|
||||
}
|
||||
|
||||
protected Map<String, Object> getFetchHistory(CrawlURI curi, int idx) {
|
||||
Map<String, Object> data = curi.getData();
|
||||
@SuppressWarnings("unchecked")
|
||||
Map<String, Object>[] historyArray = (Map[])data.get(RecrawlAttributeConstants.A_FETCH_HISTORY);
|
||||
Map<String, Object>[] historyArray = curi.getFetchHistory();
|
||||
assertNotNull(historyArray);
|
||||
Map<String, Object> history = historyArray[idx];
|
||||
return history;
|
||||
@@ -114,10 +112,10 @@ public class WbmPersistLoadProcessorTest extends TestCase {
|
||||
// put history entry newer than being loaded (i.e. loaded history entry will not be used for FetchHistoryProcessor
|
||||
// check below.
|
||||
long expected_ts = DateUtils.parse14DigitDate(TestNormalHttpResponse.EXPECTED_TS).getTime();
|
||||
Map<String, Object>[] fetchHistory = (Map[])curi.getData().get(RecrawlAttributeConstants.A_FETCH_HISTORY);
|
||||
Map<String, Object>[] fetchHistory = curi.getFetchHistory();
|
||||
if (fetchHistory == null) {
|
||||
fetchHistory = new HashMap[2];
|
||||
curi.getData().put(RecrawlAttributeConstants.A_FETCH_HISTORY, fetchHistory);
|
||||
curi.setFetchHistory(fetchHistory);
|
||||
}
|
||||
final byte[] digestValue0 = sha1Digest("0");
|
||||
final byte[] digestValue1 = sha1Digest("1");
|
||||
|
||||
@@ -57,7 +57,6 @@ import static org.archive.modules.fetcher.FetchStatusCodes.S_TOO_MANY_RETRIES;
|
||||
import static org.archive.modules.fetcher.FetchStatusCodes.S_UNATTEMPTED;
|
||||
import static org.archive.modules.fetcher.FetchStatusCodes.S_UNFETCHABLE_URI;
|
||||
import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_CONTENT_DIGEST_HISTORY;
|
||||
import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_FETCH_HISTORY;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.ObjectInputStream;
|
||||
@@ -122,6 +121,8 @@ implements Reporter, Serializable, OverlayContext, Comparable<CrawlURI> {
|
||||
Logger.getLogger(CrawlURI.class.getName());
|
||||
|
||||
public static final int UNCALCULATED = -1;
|
||||
/** fetch history array */
|
||||
public static final String A_FETCH_HISTORY = "fetch-history";
|
||||
|
||||
public static enum FetchType { HTTP_GET, HTTP_POST, UNKNOWN };
|
||||
|
||||
@@ -1826,6 +1827,10 @@ implements Reporter, Serializable, OverlayContext, Comparable<CrawlURI> {
|
||||
public HashMap<String, Object>[] getFetchHistory() {
|
||||
return (HashMap<String,Object>[]) getData().get(A_FETCH_HISTORY);
|
||||
}
|
||||
|
||||
public void setFetchHistory(Map<String, Object>[] history) {
|
||||
getData().put(A_FETCH_HISTORY, history);
|
||||
}
|
||||
|
||||
public HashMap<String, Object> getContentDigestHistory() {
|
||||
@SuppressWarnings("unchecked")
|
||||
|
||||
@@ -22,7 +22,6 @@ package org.archive.modules.recrawl;
|
||||
import static org.archive.modules.CoreAttributeConstants.A_FETCH_BEGAN_TIME;
|
||||
import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_CONTENT_DIGEST;
|
||||
import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_ETAG_HEADER;
|
||||
import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_FETCH_HISTORY;
|
||||
import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_LAST_MODIFIED_HEADER;
|
||||
import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_REFERENCE_LENGTH;
|
||||
import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_STATUS;
|
||||
@@ -103,7 +102,7 @@ public class FetchHistoryProcessor extends Processor {
|
||||
}
|
||||
history[0] = latestFetch;
|
||||
|
||||
curi.getData().put(A_FETCH_HISTORY, history);
|
||||
curi.setFetchHistory(history);
|
||||
|
||||
if (curi.getFetchStatus() == 304) {
|
||||
if( history.length >= 2 && history[1] != null && history[1].containsKey(A_CONTENT_DIGEST)) {
|
||||
|
||||
@@ -19,6 +19,8 @@
|
||||
|
||||
package org.archive.modules.recrawl;
|
||||
|
||||
import org.archive.modules.CrawlURI;
|
||||
|
||||
/**
|
||||
*
|
||||
* @author pjack
|
||||
@@ -28,8 +30,11 @@ public interface RecrawlAttributeConstants {
|
||||
|
||||
/* Duplication-reduction / recrawl / history constants */
|
||||
|
||||
/** fetch history array */
|
||||
public static final String A_FETCH_HISTORY = "fetch-history";
|
||||
/**
|
||||
* @deprecated Please use {@link org.archive.modules.CrawlURI#getFetchHistory()} and {@link org.archive.modules.CrawlURI#setFetchHistory(java.util.Map[])}
|
||||
*/
|
||||
@Deprecated
|
||||
public static final String A_FETCH_HISTORY = CrawlURI.A_FETCH_HISTORY;
|
||||
/** content digest */
|
||||
public static final String A_CONTENT_DIGEST = "content-digest";
|
||||
/** header name (and AList key) for last-modified timestamp */
|
||||
|
||||
Reference in New Issue
Block a user