lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-09-23 14:15:47 +00:00
Code Issues Packages Projects Releases Wiki Activity

Replace constant with accessor methods

Browse Source 
CrawlURI already had the accessor method, and the use of the constant
was a bit inconsistent. This change adds the corresponding mutator
method to make working with the CrawlURI history a bit simpler.
This commit is contained in:
Tim Hennekey
2020-01-24 12:28:11 -05:00
parent 3a8b58998c
commit 3003a8e751
5 changed files with 19 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/src/main/java/org/archive/modules/CrawlURI.java
+6 -1
View File
@@ -57,7 +57,6 @@ import static org.archive.modules.fetcher.FetchStatusCodes.S_TOO_MANY_RETRIES;
import static org.archive.modules.fetcher.FetchStatusCodes.S_UNATTEMPTED;
import static org.archive.modules.fetcher.FetchStatusCodes.S_UNFETCHABLE_URI;
import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_CONTENT_DIGEST_HISTORY;
import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_FETCH_HISTORY;
import java.io.IOException;
import java.io.ObjectInputStream;
@@ -122,6 +121,8 @@ implements Reporter, Serializable, OverlayContext, Comparable<CrawlURI> {
Logger.getLogger(CrawlURI.class.getName());
public static final int UNCALCULATED = -1;
/** fetch history array */
public static final String A_FETCH_HISTORY = "fetch-history";
public static enum FetchType { HTTP_GET, HTTP_POST, UNKNOWN };
@@ -1826,6 +1827,10 @@ implements Reporter, Serializable, OverlayContext, Comparable<CrawlURI> {
public HashMap<String, Object>[] getFetchHistory() {
return (HashMap<String,Object>[]) getData().get(A_FETCH_HISTORY);
}
public void setFetchHistory(Map<String, Object>[] history) {
getData().put(A_FETCH_HISTORY, history);
}
public HashMap<String, Object> getContentDigestHistory() {
@SuppressWarnings("unchecked")
modules/src/main/java/org/archive/modules/recrawl/FetchHistoryProcessor.java
+1 -2
View File
@@ -22,7 +22,6 @@ package org.archive.modules.recrawl;
import static org.archive.modules.CoreAttributeConstants.A_FETCH_BEGAN_TIME;
import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_CONTENT_DIGEST;
import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_ETAG_HEADER;
import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_FETCH_HISTORY;
import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_LAST_MODIFIED_HEADER;
import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_REFERENCE_LENGTH;
import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_STATUS;
@@ -103,7 +102,7 @@ public class FetchHistoryProcessor extends Processor {
}
history[0] = latestFetch;
curi.getData().put(A_FETCH_HISTORY, history);
curi.setFetchHistory(history);
if (curi.getFetchStatus() == 304) {
if( history.length >= 2 && history[1] != null && history[1].containsKey(A_CONTENT_DIGEST)) {
modules/src/main/java/org/archive/modules/recrawl/RecrawlAttributeConstants.java
+7 -2
View File
@@ -19,6 +19,8 @@
package org.archive.modules.recrawl;
import org.archive.modules.CrawlURI;
/**
*
* @author pjack
@@ -28,8 +30,11 @@ public interface RecrawlAttributeConstants {
/* Duplication-reduction / recrawl / history constants */
/** fetch history array */
public static final String A_FETCH_HISTORY = "fetch-history";
/**
* @deprecated Please use {@link org.archive.modules.CrawlURI#getFetchHistory()} and {@link org.archive.modules.CrawlURI#setFetchHistory(java.util.Map[])}
*/
@Deprecated
public static final String A_FETCH_HISTORY = CrawlURI.A_FETCH_HISTORY;
/** content digest */
public static final String A_CONTENT_DIGEST = "content-digest";
/** header name (and AList key) for last-modified timestamp */