lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-09-26 15:46:31 +00:00
Code Issues Packages Projects Releases Wiki Activity

convenience method CrawlURI.getFetchHistory()

Browse Source
This commit is contained in:
Noah Levitt
2012-09-02 12:25:19 -07:00
parent 1fe4781cf5
commit f3963d5f89
9 changed files with 70 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/src/main/java/org/archive/modules/CrawlURI.java
+7
View File
@@ -56,6 +56,7 @@ import static org.archive.modules.fetcher.FetchStatusCodes.S_TOO_MANY_LINK_HOPS;
import static org.archive.modules.fetcher.FetchStatusCodes.S_TOO_MANY_RETRIES;
import static org.archive.modules.fetcher.FetchStatusCodes.S_UNATTEMPTED;
import static org.archive.modules.fetcher.FetchStatusCodes.S_UNFETCHABLE_URI;
import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_FETCH_HISTORY;
import java.io.IOException;
import java.io.ObjectInputStream;
@@ -1887,4 +1888,10 @@ implements Reporter, Serializable, OverlayContext {
public void setHttpAuthChallenges(Map<String, String> httpAuthChallenges) {
getData().put(A_HTTP_AUTH_CHALLENGES, httpAuthChallenges);
}
@SuppressWarnings("unchecked")
public HashMap<String, Object>[] getFetchHistory() {
return (HashMap<String,Object>[]) getData().get(A_FETCH_HISTORY);
}
}
modules/src/main/java/org/archive/modules/deciderules/recrawl/IdenticalDigestDecideRule.java
+8 -14
View File
@@ -20,7 +20,6 @@
package org.archive.modules.deciderules.recrawl;
import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_CONTENT_DIGEST;
import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_FETCH_HISTORY;
import java.util.Map;
@@ -69,20 +68,15 @@ public class IdenticalDigestDecideRule extends PredicatedDecideRule {
* @return true if last two history entries have identical digests,
* otherwise false
*/
@SuppressWarnings("unchecked")
public static boolean hasIdenticalDigest(CrawlURI curi) {
if(curi.containsDataKey(A_FETCH_HISTORY)) {
Map<String,Object>[] history =
(Map<String,Object>[])curi.getData().get(A_FETCH_HISTORY);
return history[0] != null
&& history[0].containsKey(A_CONTENT_DIGEST)
&& history[1] != null
&& history[1].containsKey(A_CONTENT_DIGEST)
&& history[0].get(A_CONTENT_DIGEST).equals(
history[1].get(A_CONTENT_DIGEST));
} else {
return false;
}
Map<String,Object>[] history = curi.getFetchHistory();
return history != null
&& history[0] != null
&& history[0].containsKey(A_CONTENT_DIGEST)
&& history[1] != null
&& history[1].containsKey(A_CONTENT_DIGEST)
&& history[0].get(A_CONTENT_DIGEST).equals(history[1].get(A_CONTENT_DIGEST));
}
}
modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java
+9 -11
View File
@@ -43,6 +43,7 @@ import java.security.KeyStoreException;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
@@ -747,11 +748,9 @@ public class FetchHTTP extends AbstractFetchHTTP implements Lifecycle {
// special handling for 304-not modified
if (curi.getFetchStatus() == HttpStatus.SC_NOT_MODIFIED
&& curi.containsDataKey(A_FETCH_HISTORY)) {
@SuppressWarnings("unchecked")
Map<String, ?> history[] = (Map[])curi.getData().get(A_FETCH_HISTORY);
Map<String, Object>[] history = curi.getFetchHistory();
if (history[0] != null
&& history[0]
.containsKey(A_REFERENCE_LENGTH)) {
&& history[0].containsKey(A_REFERENCE_LENGTH)) {
long referenceLength = (Long) history[0].get(A_REFERENCE_LENGTH);
// carry-forward previous 'reference-length' for future
curi.getData().put(A_REFERENCE_LENGTH, referenceLength);
@@ -1029,19 +1028,18 @@ public class FetchHTTP extends AbstractFetchHTTP implements Lifecycle {
* @param sourceHeader header to consult in URI history
* @param targetHeader header to set if possible
*/
protected void setConditionalGetHeader(CrawlURI curi, HttpMethod method,
protected void setConditionalGetHeader(CrawlURI curi, HttpMethod method,
boolean conditional, String sourceHeader, String targetHeader) {
if (conditional) {
try {
@SuppressWarnings("unchecked")
Map<String, ?>[] history = (Map[])curi.getData().get(A_FETCH_HISTORY);
HashMap<String, Object>[] history = curi.getFetchHistory();
int previousStatus = (Integer) history[0].get(A_STATUS);
if(previousStatus<=0) {
if (previousStatus <= 0) {
// do not reuse headers from any broken fetch
return;
return;
}
String previousValue = (String) history[0].get(sourceHeader);
if(previousValue!=null) {
if (previousValue != null) {
method.setRequestHeader(targetHeader, previousValue);
}
} catch (RuntimeException e) {
@@ -1049,7 +1047,7 @@ public class FetchHTTP extends AbstractFetchHTTP implements Lifecycle {
}
}
}
/**
* Setup proxy, based on attributes in CrawlURI and settings,
* in given HostConfiguration
modules/src/main/java/org/archive/modules/fetcher/FetchHTTP2.java
+3 -6
View File
@@ -24,7 +24,6 @@ import static org.archive.modules.fetcher.FetchErrors.TIMER_TRUNC;
import static org.archive.modules.fetcher.FetchStatusCodes.S_CONNECT_FAILED;
import static org.archive.modules.fetcher.FetchStatusCodes.S_CONNECT_LOST;
import static org.archive.modules.fetcher.FetchStatusCodes.S_DOMAIN_PREREQUISITE_FAILURE;
import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_FETCH_HISTORY;
import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_REFERENCE_LENGTH;
import java.io.IOException;
@@ -1112,11 +1111,9 @@ public class FetchHTTP2 extends AbstractFetchHTTP implements Lifecycle {
curi.setContentSize(rec.getRecordedInput().getSize());
// special handling for 304-not modified
if (curi.getFetchStatus() == HttpStatus.SC_NOT_MODIFIED
&& curi.containsDataKey(A_FETCH_HISTORY)) {
@SuppressWarnings("unchecked")
Map<String, ?> history[] = (Map<String,?>[])curi.getData().get(A_FETCH_HISTORY);
if (history[0] != null
&& history[0].containsKey(A_REFERENCE_LENGTH)) {
&& curi.getFetchHistory() != null) {
Map<String, Object>[] history = curi.getFetchHistory();
if (history[0] != null && history[0].containsKey(A_REFERENCE_LENGTH)) {
long referenceLength = (Long) history[0].get(A_REFERENCE_LENGTH);
// carry-forward previous 'reference-length' for future
curi.getData().put(A_REFERENCE_LENGTH, referenceLength);
modules/src/main/java/org/archive/modules/recrawl/AbstractPersistProcessor.java
+3 -6
View File
@@ -18,11 +18,8 @@
*/
package org.archive.modules.recrawl;
import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_FETCH_HISTORY;
import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_WRITE_TAG;
import java.util.Map;
import org.archive.modules.CrawlURI;
import org.archive.modules.Processor;
@@ -68,10 +65,10 @@ public abstract class AbstractPersistProcessor extends Processor {
* @param curi
* @return true if {@code curi} has WRITE_TAG in the latest fetch history (i.e. this crawl).
*/
@SuppressWarnings("unchecked")
protected boolean hasWriteTag(CrawlURI uri) {
Map<String,Object>[] history = (Map<String,Object>[])uri.getData().get(A_FETCH_HISTORY);
return history != null && history[0] != null && history[0].containsKey(A_WRITE_TAG);
return uri.getFetchHistory() != null
&& uri.getFetchHistory()[0] != null
&& uri.getFetchHistory()[0].containsKey(A_WRITE_TAG);
}
/**
modules/src/main/java/org/archive/modules/recrawl/FetchHistoryProcessor.java
+37 -34
View File
@@ -64,60 +64,63 @@ public class FetchHistoryProcessor extends Processor {
@Override
protected void innerProcess(CrawlURI puri) throws InterruptedException {
CrawlURI curi = (CrawlURI) puri;
CrawlURI curi = (CrawlURI) puri;
curi.addPersistentDataMapKey(A_FETCH_HISTORY);
HashMap<String, Object> latestFetch = new HashMap<String,Object>();
HashMap<String, Object> latestFetch = new HashMap<String, Object>();
// save status
latestFetch.put(A_STATUS,curi.getFetchStatus());
latestFetch.put(A_STATUS, curi.getFetchStatus());
// save fetch start time
latestFetch.put(A_FETCH_BEGAN_TIME,curi.getData().get(A_FETCH_BEGAN_TIME));
latestFetch.put(A_FETCH_BEGAN_TIME,
curi.getData().get(A_FETCH_BEGAN_TIME));
// save digest
String digest = curi.getContentDigestSchemeString();
if(digest!=null) {
latestFetch.put(A_CONTENT_DIGEST,digest);
if (digest != null) {
latestFetch.put(A_CONTENT_DIGEST, digest);
}
// save relevant HTTP headers, if available
if(curi.isHttpTransaction()) {
if (curi.isHttpTransaction()) {
saveHeader(curi, latestFetch, A_ETAG_HEADER);
saveHeader(curi, latestFetch, A_LAST_MODIFIED_HEADER);
// save reference length (real or virtual)
long referenceLength;
if(curi.containsDataKey(A_REFERENCE_LENGTH) ) {
// reuse previous length if available (see FetchHTTP#setSizes).
long referenceLength;
if (curi.containsDataKey(A_REFERENCE_LENGTH)) {
// reuse previous length if available (see FetchHTTP#setSizes).
referenceLength = (Long) curi.getData().get(A_REFERENCE_LENGTH);
} else {
// normally, use content-length
referenceLength = curi.getContentLength();
}
latestFetch.put(A_REFERENCE_LENGTH,referenceLength);
latestFetch.put(A_REFERENCE_LENGTH, referenceLength);
}
// get or create proper-sized history array
int targetHistoryLength = getHistoryLength();
@SuppressWarnings("unchecked")
HashMap<String, ?>[] history =
(HashMap<String, ?>[]) (curi.containsDataKey(A_FETCH_HISTORY)
? curi.getData().get(A_FETCH_HISTORY)
: new HashMap[targetHistoryLength]);
if(history.length != targetHistoryLength) {
@SuppressWarnings("unchecked")
HashMap<String, ?>[] newHistory = new HashMap[targetHistoryLength];
System.arraycopy(
history,0,
newHistory,0,
Math.min(history.length,newHistory.length));
history = newHistory;
}
HashMap<String, Object>[] history = historyRealloc(curi);
// rotate all history entries up one slot, insert new at [0]
for(int i = history.length-1; i >0; i--) {
history[i] = history[i-1];
for (int i = history.length - 1; i > 0; i--) {
history[i] = history[i - 1];
}
history[0]=latestFetch;
curi.getData().put(A_FETCH_HISTORY,history);
history[0] = latestFetch;
curi.getData().put(A_FETCH_HISTORY, history);
}
/** Get or create proper-sized history array */
@SuppressWarnings("unchecked")
protected HashMap<String, Object>[] historyRealloc(CrawlURI curi) {
int targetHistoryLength = getHistoryLength();
HashMap<String, Object>[] history = curi.getFetchHistory();
if (history == null) {
history = new HashMap[targetHistoryLength];
}
if (history.length != targetHistoryLength) {
HashMap<String, Object>[] newHistory = new HashMap[targetHistoryLength];
System.arraycopy(history, 0, newHistory, 0,
Math.min(history.length, newHistory.length));
history = newHistory;
}
return history;
}
/** Save a header from the given HTTP operation into the Map. */
modules/src/main/java/org/archive/modules/writer/ARCWriterProcessor.java
+1 -3
View File
@@ -19,7 +19,6 @@
package org.archive.modules.writer;
import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_FETCH_HISTORY;
import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_WRITE_TAG;
import java.io.IOException;
@@ -168,8 +167,7 @@ public class ARCWriterProcessor extends WriterPoolProcessor {
}
curi.addExtraInfo("arcFilename", filename);
@SuppressWarnings("unchecked")
Map<String,Object>[] history = (Map<String,Object>[])curi.getData().get(A_FETCH_HISTORY);
Map<String,Object>[] history = curi.getFetchHistory();
if (history != null && history[0] != null) {
history[0].put(A_WRITE_TAG, filename);
}
modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java
+1 -5
View File
@@ -45,7 +45,6 @@ import static org.archive.modules.CoreAttributeConstants.HEADER_TRUNC;
import static org.archive.modules.CoreAttributeConstants.LENGTH_TRUNC;
import static org.archive.modules.CoreAttributeConstants.TIMER_TRUNC;
import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_ETAG_HEADER;
import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_FETCH_HISTORY;
import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_LAST_MODIFIED_HEADER;
import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_WRITE_TAG;
@@ -67,8 +66,6 @@ import java.util.concurrent.atomic.AtomicLong;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.apache.commons.httpclient.Header;
import org.apache.commons.httpclient.HttpMethod;
import org.apache.commons.httpclient.HttpStatus;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
@@ -288,8 +285,7 @@ public class WARCWriterProcessor extends WriterPoolProcessor implements WARCWrit
}
curi.addExtraInfo("warcFilename", filename);
@SuppressWarnings("unchecked")
Map<String,Object>[] history = (Map<String,Object>[])curi.getData().get(A_FETCH_HISTORY);
Map<String,Object>[] history = curi.getFetchHistory();
if (history != null && history[0] != null) {
history[0].put(A_WRITE_TAG, filename);
}
modules/src/main/java/org/archive/modules/writer/WriterPoolProcessor.java
+1 -3
View File
@@ -22,7 +22,6 @@ package org.archive.modules.writer;
import static org.archive.modules.CoreAttributeConstants.A_DNS_SERVER_IP_LABEL;
import static org.archive.modules.fetcher.FetchStatusCodes.S_DNS_SUCCESS;
import static org.archive.modules.fetcher.FetchStatusCodes.S_WHOIS_SUCCESS;
import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_FETCH_HISTORY;
import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_WRITE_TAG;
import java.io.File;
@@ -488,8 +487,7 @@ implements Lifecycle, Checkpointable, WriterPoolSettings {
*/
protected void copyForwardWriteTagIfDupe(CrawlURI curi) {
if (IdenticalDigestDecideRule.hasIdenticalDigest(curi)) {
@SuppressWarnings("unchecked")
Map<String,Object>[] history = (Map<String,Object>[])curi.getData().get(A_FETCH_HISTORY);
Map<String,Object>[] history = curi.getFetchHistory();
if (history[1].containsKey(A_WRITE_TAG)) {
history[0].put(A_WRITE_TAG, history[1].get(A_WRITE_TAG));
}