From f3963d5f895d62b01a4171f5dead85fe4820b005 Mon Sep 17 00:00:00 2001 From: Noah Levitt Date: Sun, 2 Sep 2012 12:25:19 -0700 Subject: [PATCH] convenience method CrawlURI.getFetchHistory() --- .../java/org/archive/modules/CrawlURI.java | 7 ++ .../recrawl/IdenticalDigestDecideRule.java | 22 +++--- .../archive/modules/fetcher/FetchHTTP.java | 20 +++--- .../archive/modules/fetcher/FetchHTTP2.java | 9 +-- .../recrawl/AbstractPersistProcessor.java | 9 +-- .../recrawl/FetchHistoryProcessor.java | 71 ++++++++++--------- .../modules/writer/ARCWriterProcessor.java | 4 +- .../modules/writer/WARCWriterProcessor.java | 6 +- .../modules/writer/WriterPoolProcessor.java | 4 +- 9 files changed, 70 insertions(+), 82 deletions(-) diff --git a/modules/src/main/java/org/archive/modules/CrawlURI.java b/modules/src/main/java/org/archive/modules/CrawlURI.java index 66bfe7b7..86c9799d 100644 --- a/modules/src/main/java/org/archive/modules/CrawlURI.java +++ b/modules/src/main/java/org/archive/modules/CrawlURI.java @@ -56,6 +56,7 @@ import static org.archive.modules.fetcher.FetchStatusCodes.S_TOO_MANY_LINK_HOPS; import static org.archive.modules.fetcher.FetchStatusCodes.S_TOO_MANY_RETRIES; import static org.archive.modules.fetcher.FetchStatusCodes.S_UNATTEMPTED; import static org.archive.modules.fetcher.FetchStatusCodes.S_UNFETCHABLE_URI; +import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_FETCH_HISTORY; import java.io.IOException; import java.io.ObjectInputStream; @@ -1887,4 +1888,10 @@ implements Reporter, Serializable, OverlayContext { public void setHttpAuthChallenges(Map httpAuthChallenges) { getData().put(A_HTTP_AUTH_CHALLENGES, httpAuthChallenges); } + + @SuppressWarnings("unchecked") + public HashMap[] getFetchHistory() { + return (HashMap[]) getData().get(A_FETCH_HISTORY); + } + } diff --git a/modules/src/main/java/org/archive/modules/deciderules/recrawl/IdenticalDigestDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/recrawl/IdenticalDigestDecideRule.java index 9a0e6cf2..38d272ba 100644 --- a/modules/src/main/java/org/archive/modules/deciderules/recrawl/IdenticalDigestDecideRule.java +++ b/modules/src/main/java/org/archive/modules/deciderules/recrawl/IdenticalDigestDecideRule.java @@ -20,7 +20,6 @@ package org.archive.modules.deciderules.recrawl; import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_CONTENT_DIGEST; -import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_FETCH_HISTORY; import java.util.Map; @@ -69,20 +68,15 @@ public class IdenticalDigestDecideRule extends PredicatedDecideRule { * @return true if last two history entries have identical digests, * otherwise false */ - @SuppressWarnings("unchecked") public static boolean hasIdenticalDigest(CrawlURI curi) { - if(curi.containsDataKey(A_FETCH_HISTORY)) { - Map[] history = - (Map[])curi.getData().get(A_FETCH_HISTORY); - return history[0] != null - && history[0].containsKey(A_CONTENT_DIGEST) - && history[1] != null - && history[1].containsKey(A_CONTENT_DIGEST) - && history[0].get(A_CONTENT_DIGEST).equals( - history[1].get(A_CONTENT_DIGEST)); - } else { - return false; - } + Map[] history = curi.getFetchHistory(); + + return history != null + && history[0] != null + && history[0].containsKey(A_CONTENT_DIGEST) + && history[1] != null + && history[1].containsKey(A_CONTENT_DIGEST) + && history[0].get(A_CONTENT_DIGEST).equals(history[1].get(A_CONTENT_DIGEST)); } } diff --git a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java index 7c27181d..43419aa7 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java +++ b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java @@ -43,6 +43,7 @@ import java.security.KeyStoreException; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.util.Arrays; +import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.List; @@ -747,11 +748,9 @@ public class FetchHTTP extends AbstractFetchHTTP implements Lifecycle { // special handling for 304-not modified if (curi.getFetchStatus() == HttpStatus.SC_NOT_MODIFIED && curi.containsDataKey(A_FETCH_HISTORY)) { - @SuppressWarnings("unchecked") - Map history[] = (Map[])curi.getData().get(A_FETCH_HISTORY); + Map[] history = curi.getFetchHistory(); if (history[0] != null - && history[0] - .containsKey(A_REFERENCE_LENGTH)) { + && history[0].containsKey(A_REFERENCE_LENGTH)) { long referenceLength = (Long) history[0].get(A_REFERENCE_LENGTH); // carry-forward previous 'reference-length' for future curi.getData().put(A_REFERENCE_LENGTH, referenceLength); @@ -1029,19 +1028,18 @@ public class FetchHTTP extends AbstractFetchHTTP implements Lifecycle { * @param sourceHeader header to consult in URI history * @param targetHeader header to set if possible */ - protected void setConditionalGetHeader(CrawlURI curi, HttpMethod method, + protected void setConditionalGetHeader(CrawlURI curi, HttpMethod method, boolean conditional, String sourceHeader, String targetHeader) { if (conditional) { try { - @SuppressWarnings("unchecked") - Map[] history = (Map[])curi.getData().get(A_FETCH_HISTORY); + HashMap[] history = curi.getFetchHistory(); int previousStatus = (Integer) history[0].get(A_STATUS); - if(previousStatus<=0) { + if (previousStatus <= 0) { // do not reuse headers from any broken fetch - return; + return; } String previousValue = (String) history[0].get(sourceHeader); - if(previousValue!=null) { + if (previousValue != null) { method.setRequestHeader(targetHeader, previousValue); } } catch (RuntimeException e) { @@ -1049,7 +1047,7 @@ public class FetchHTTP extends AbstractFetchHTTP implements Lifecycle { } } } - + /** * Setup proxy, based on attributes in CrawlURI and settings, * in given HostConfiguration diff --git a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP2.java b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP2.java index f75d2555..101f2b13 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP2.java +++ b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP2.java @@ -24,7 +24,6 @@ import static org.archive.modules.fetcher.FetchErrors.TIMER_TRUNC; import static org.archive.modules.fetcher.FetchStatusCodes.S_CONNECT_FAILED; import static org.archive.modules.fetcher.FetchStatusCodes.S_CONNECT_LOST; import static org.archive.modules.fetcher.FetchStatusCodes.S_DOMAIN_PREREQUISITE_FAILURE; -import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_FETCH_HISTORY; import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_REFERENCE_LENGTH; import java.io.IOException; @@ -1112,11 +1111,9 @@ public class FetchHTTP2 extends AbstractFetchHTTP implements Lifecycle { curi.setContentSize(rec.getRecordedInput().getSize()); // special handling for 304-not modified if (curi.getFetchStatus() == HttpStatus.SC_NOT_MODIFIED - && curi.containsDataKey(A_FETCH_HISTORY)) { - @SuppressWarnings("unchecked") - Map history[] = (Map[])curi.getData().get(A_FETCH_HISTORY); - if (history[0] != null - && history[0].containsKey(A_REFERENCE_LENGTH)) { + && curi.getFetchHistory() != null) { + Map[] history = curi.getFetchHistory(); + if (history[0] != null && history[0].containsKey(A_REFERENCE_LENGTH)) { long referenceLength = (Long) history[0].get(A_REFERENCE_LENGTH); // carry-forward previous 'reference-length' for future curi.getData().put(A_REFERENCE_LENGTH, referenceLength); diff --git a/modules/src/main/java/org/archive/modules/recrawl/AbstractPersistProcessor.java b/modules/src/main/java/org/archive/modules/recrawl/AbstractPersistProcessor.java index c7a5b571..361b4205 100644 --- a/modules/src/main/java/org/archive/modules/recrawl/AbstractPersistProcessor.java +++ b/modules/src/main/java/org/archive/modules/recrawl/AbstractPersistProcessor.java @@ -18,11 +18,8 @@ */ package org.archive.modules.recrawl; -import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_FETCH_HISTORY; import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_WRITE_TAG; -import java.util.Map; - import org.archive.modules.CrawlURI; import org.archive.modules.Processor; @@ -68,10 +65,10 @@ public abstract class AbstractPersistProcessor extends Processor { * @param curi * @return true if {@code curi} has WRITE_TAG in the latest fetch history (i.e. this crawl). */ - @SuppressWarnings("unchecked") protected boolean hasWriteTag(CrawlURI uri) { - Map[] history = (Map[])uri.getData().get(A_FETCH_HISTORY); - return history != null && history[0] != null && history[0].containsKey(A_WRITE_TAG); + return uri.getFetchHistory() != null + && uri.getFetchHistory()[0] != null + && uri.getFetchHistory()[0].containsKey(A_WRITE_TAG); } /** diff --git a/modules/src/main/java/org/archive/modules/recrawl/FetchHistoryProcessor.java b/modules/src/main/java/org/archive/modules/recrawl/FetchHistoryProcessor.java index 972ad196..ce4abb8f 100644 --- a/modules/src/main/java/org/archive/modules/recrawl/FetchHistoryProcessor.java +++ b/modules/src/main/java/org/archive/modules/recrawl/FetchHistoryProcessor.java @@ -64,60 +64,63 @@ public class FetchHistoryProcessor extends Processor { @Override protected void innerProcess(CrawlURI puri) throws InterruptedException { - CrawlURI curi = (CrawlURI) puri; + CrawlURI curi = (CrawlURI) puri; curi.addPersistentDataMapKey(A_FETCH_HISTORY); - HashMap latestFetch = new HashMap(); + HashMap latestFetch = new HashMap(); // save status - latestFetch.put(A_STATUS,curi.getFetchStatus()); + latestFetch.put(A_STATUS, curi.getFetchStatus()); // save fetch start time - latestFetch.put(A_FETCH_BEGAN_TIME,curi.getData().get(A_FETCH_BEGAN_TIME)); + latestFetch.put(A_FETCH_BEGAN_TIME, + curi.getData().get(A_FETCH_BEGAN_TIME)); // save digest String digest = curi.getContentDigestSchemeString(); - if(digest!=null) { - latestFetch.put(A_CONTENT_DIGEST,digest); + if (digest != null) { + latestFetch.put(A_CONTENT_DIGEST, digest); } // save relevant HTTP headers, if available - if(curi.isHttpTransaction()) { + if (curi.isHttpTransaction()) { saveHeader(curi, latestFetch, A_ETAG_HEADER); saveHeader(curi, latestFetch, A_LAST_MODIFIED_HEADER); // save reference length (real or virtual) - long referenceLength; - if(curi.containsDataKey(A_REFERENCE_LENGTH) ) { - // reuse previous length if available (see FetchHTTP#setSizes). + long referenceLength; + if (curi.containsDataKey(A_REFERENCE_LENGTH)) { + // reuse previous length if available (see FetchHTTP#setSizes). referenceLength = (Long) curi.getData().get(A_REFERENCE_LENGTH); } else { // normally, use content-length referenceLength = curi.getContentLength(); } - latestFetch.put(A_REFERENCE_LENGTH,referenceLength); + latestFetch.put(A_REFERENCE_LENGTH, referenceLength); } - - // get or create proper-sized history array - int targetHistoryLength = getHistoryLength(); - @SuppressWarnings("unchecked") - HashMap[] history = - (HashMap[]) (curi.containsDataKey(A_FETCH_HISTORY) - ? curi.getData().get(A_FETCH_HISTORY) - : new HashMap[targetHistoryLength]); - if(history.length != targetHistoryLength) { - @SuppressWarnings("unchecked") - HashMap[] newHistory = new HashMap[targetHistoryLength]; - System.arraycopy( - history,0, - newHistory,0, - Math.min(history.length,newHistory.length)); - history = newHistory; - } - + + HashMap[] history = historyRealloc(curi); + // rotate all history entries up one slot, insert new at [0] - for(int i = history.length-1; i >0; i--) { - history[i] = history[i-1]; + for (int i = history.length - 1; i > 0; i--) { + history[i] = history[i - 1]; } - history[0]=latestFetch; - - curi.getData().put(A_FETCH_HISTORY,history); + history[0] = latestFetch; + + curi.getData().put(A_FETCH_HISTORY, history); + } + + /** Get or create proper-sized history array */ + @SuppressWarnings("unchecked") + protected HashMap[] historyRealloc(CrawlURI curi) { + int targetHistoryLength = getHistoryLength(); + HashMap[] history = curi.getFetchHistory(); + if (history == null) { + history = new HashMap[targetHistoryLength]; + } + if (history.length != targetHistoryLength) { + HashMap[] newHistory = new HashMap[targetHistoryLength]; + System.arraycopy(history, 0, newHistory, 0, + Math.min(history.length, newHistory.length)); + history = newHistory; + } + return history; } /** Save a header from the given HTTP operation into the Map. */ diff --git a/modules/src/main/java/org/archive/modules/writer/ARCWriterProcessor.java b/modules/src/main/java/org/archive/modules/writer/ARCWriterProcessor.java index d0beea34..87978e7e 100644 --- a/modules/src/main/java/org/archive/modules/writer/ARCWriterProcessor.java +++ b/modules/src/main/java/org/archive/modules/writer/ARCWriterProcessor.java @@ -19,7 +19,6 @@ package org.archive.modules.writer; -import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_FETCH_HISTORY; import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_WRITE_TAG; import java.io.IOException; @@ -168,8 +167,7 @@ public class ARCWriterProcessor extends WriterPoolProcessor { } curi.addExtraInfo("arcFilename", filename); - @SuppressWarnings("unchecked") - Map[] history = (Map[])curi.getData().get(A_FETCH_HISTORY); + Map[] history = curi.getFetchHistory(); if (history != null && history[0] != null) { history[0].put(A_WRITE_TAG, filename); } diff --git a/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java b/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java index b33053c0..293d560c 100644 --- a/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java +++ b/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java @@ -45,7 +45,6 @@ import static org.archive.modules.CoreAttributeConstants.HEADER_TRUNC; import static org.archive.modules.CoreAttributeConstants.LENGTH_TRUNC; import static org.archive.modules.CoreAttributeConstants.TIMER_TRUNC; import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_ETAG_HEADER; -import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_FETCH_HISTORY; import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_LAST_MODIFIED_HEADER; import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_WRITE_TAG; @@ -67,8 +66,6 @@ import java.util.concurrent.atomic.AtomicLong; import java.util.logging.Level; import java.util.logging.Logger; -import org.apache.commons.httpclient.Header; -import org.apache.commons.httpclient.HttpMethod; import org.apache.commons.httpclient.HttpStatus; import org.apache.commons.io.IOUtils; import org.apache.commons.lang.StringUtils; @@ -288,8 +285,7 @@ public class WARCWriterProcessor extends WriterPoolProcessor implements WARCWrit } curi.addExtraInfo("warcFilename", filename); - @SuppressWarnings("unchecked") - Map[] history = (Map[])curi.getData().get(A_FETCH_HISTORY); + Map[] history = curi.getFetchHistory(); if (history != null && history[0] != null) { history[0].put(A_WRITE_TAG, filename); } diff --git a/modules/src/main/java/org/archive/modules/writer/WriterPoolProcessor.java b/modules/src/main/java/org/archive/modules/writer/WriterPoolProcessor.java index 7e2fb785..6b3e5f98 100644 --- a/modules/src/main/java/org/archive/modules/writer/WriterPoolProcessor.java +++ b/modules/src/main/java/org/archive/modules/writer/WriterPoolProcessor.java @@ -22,7 +22,6 @@ package org.archive.modules.writer; import static org.archive.modules.CoreAttributeConstants.A_DNS_SERVER_IP_LABEL; import static org.archive.modules.fetcher.FetchStatusCodes.S_DNS_SUCCESS; import static org.archive.modules.fetcher.FetchStatusCodes.S_WHOIS_SUCCESS; -import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_FETCH_HISTORY; import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_WRITE_TAG; import java.io.File; @@ -488,8 +487,7 @@ implements Lifecycle, Checkpointable, WriterPoolSettings { */ protected void copyForwardWriteTagIfDupe(CrawlURI curi) { if (IdenticalDigestDecideRule.hasIdenticalDigest(curi)) { - @SuppressWarnings("unchecked") - Map[] history = (Map[])curi.getData().get(A_FETCH_HISTORY); + Map[] history = curi.getFetchHistory(); if (history[1].containsKey(A_WRITE_TAG)) { history[0].put(A_WRITE_TAG, history[1].get(A_WRITE_TAG)); }