From ca5802f7ba819f4db8ddd9a209040ac0d19eecc8 Mon Sep 17 00:00:00 2001 From: Kenji Nagahashi Date: Wed, 26 Sep 2012 12:31:55 -0700 Subject: [PATCH 01/14] separated hq from local hertirix3 repo, fixed up layout From 6531733972a52c29931ae1907cd9088eb39ad167 Mon Sep 17 00:00:00 2001 From: Kenji Nagahashi Date: Mon, 10 Dec 2012 14:37:18 -0800 Subject: [PATCH 02/14] H3 module for deduplicating with WBM index (with test case). Added EasyMock to project dependency. --- .../recrawl/wbm/WbmPersistLoadProcessor.java | 329 ++++++++++++++++++ .../wbm/WbmPersistLoadProcessorTest.java | 163 +++++++++ 2 files changed, 492 insertions(+) create mode 100644 main/java/org/archive/modules/recrawl/wbm/WbmPersistLoadProcessor.java create mode 100644 test/java/org/archive/modules/recrawl/wbm/WbmPersistLoadProcessorTest.java diff --git a/main/java/org/archive/modules/recrawl/wbm/WbmPersistLoadProcessor.java b/main/java/org/archive/modules/recrawl/wbm/WbmPersistLoadProcessor.java new file mode 100644 index 00000000..2cbd8ec0 --- /dev/null +++ b/main/java/org/archive/modules/recrawl/wbm/WbmPersistLoadProcessor.java @@ -0,0 +1,329 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.archive.modules.recrawl.wbm; + +import java.io.IOException; +import java.io.InputStream; +import java.io.UnsupportedEncodingException; +import java.net.URLEncoder; +import java.nio.ByteBuffer; +import java.util.Date; +import java.util.Map; +import java.util.concurrent.atomic.AtomicLong; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.http.Header; +import org.apache.http.HeaderElement; +import org.apache.http.HttpEntity; +import org.apache.http.HttpException; +import org.apache.http.HttpRequest; +import org.apache.http.HttpRequestInterceptor; +import org.apache.http.HttpResponse; +import org.apache.http.HttpResponseInterceptor; +import org.apache.http.StatusLine; +import org.apache.http.client.HttpClient; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.impl.client.DefaultHttpClient; +import org.apache.http.impl.conn.tsccm.ThreadSafeClientConnManager; +import org.apache.http.params.CoreConnectionPNames; +import org.apache.http.params.HttpParams; +import org.apache.http.protocol.HttpContext; +import org.archive.modules.CrawlURI; +import org.archive.modules.ProcessResult; +import org.archive.modules.Processor; +import org.archive.modules.hq.GzipInflatingHttpEntityWrapper; +import org.archive.modules.hq.recrawl.RecrawlDataSchemaBase; +import org.archive.modules.recrawl.RecrawlAttributeConstants; +import org.archive.util.ArchiveUtils; + +/** + * A {@link Processor} for retrieving recrawl info from remote Wayback Machine index. + * This is currently in the early stage of experiment. Both low-level protocol and WBM API + * semantics will certainly undergo several revisions. + *

Current interface:

+ *

http://web-beta.archive.org/cdx/search?url=archive.org&startDate=1999 will return raw + * CDX lines for archive.org, since 1999-01-01 00:00:00. + *

+ *

As index is updated in a separate batch processing job, there's no "Store" counterpart.

+ * @contributor Kenji Nagahashi. + */ +public class WbmPersistLoadProcessor extends Processor { + private static final Log log = LogFactory.getLog(WbmPersistLoadProcessor.class); + + private HttpClient client; + + private String queryURL = "http://web-beta.archive.org/cdx/search"; + public void setQueryURL(String queryURL) { + this.queryURL = queryURL; + } + public String getQueryURL() { + return queryURL; + } + + private String contentDigestScheme = "sha1:"; + /** + * set Content-Digest scheme string to prepend to the hash string found in CDX. + * Heritrix's Content-Digest comparison including this part. + * {@code "sha1:"} by default. + * @param contentDigestScheme + */ + public void setContentDigestScheme(String contentDigestScheme) { + this.contentDigestScheme = contentDigestScheme; + } + public String getContentDigestScheme() { + return contentDigestScheme; + } + private int socketTimeout = 10000; + /** + * socket timeout (SO_TIMEOUT) for HTTP cient in milliseconds. + */ + public void setSocketTimeout(int socketTimeout) { + this.socketTimeout = socketTimeout; + } + public int getSocketTimeout() { + return socketTimeout; + } + + private int connectionTimeout = 10000; + /** + * connection timeout for HTTP client in milliseconds. + * @param connectionTimeout + */ + public void setConnectionTimeout(int connectionTimeout) { + this.connectionTimeout = connectionTimeout; + } + public int getConnectionTimeout() { + return connectionTimeout; + } + + private int maxConnections = 10; + public int getMaxConnections() { + return maxConnections; + } + public void setMaxConnections(int maxConnections) { + this.maxConnections = maxConnections; + if (client != null) { + ((ThreadSafeClientConnManager)client.getConnectionManager()) + .setDefaultMaxPerRoute(this.maxConnections); + } + } + + // statistics + private AtomicLong loadedCount = new AtomicLong(); + public long getLoadedCount() { + return loadedCount.get(); + } + private AtomicLong failedCount = new AtomicLong(); + public long getFailedCount() { + return failedCount.get(); + } + + public void setHttpClient(HttpClient client) { + this.client = client; + } + + // XXX HttpHeadquarterAdapter has the same code. move to common library. + private static boolean contains(HeaderElement[] elements, String value) { + for (int i = 0; i < elements.length; i++) { + if (elements[i].getName().equalsIgnoreCase(value)) { + return true; + } + } + return false; + } + public synchronized HttpClient getHttpClient() { + if (client == null) { + ThreadSafeClientConnManager conman = new ThreadSafeClientConnManager(); + conman.setDefaultMaxPerRoute(maxConnections); + final DefaultHttpClient client = new DefaultHttpClient(conman); + HttpParams params = client.getParams(); + params.setIntParameter(CoreConnectionPNames.SO_TIMEOUT, socketTimeout); + params.setIntParameter(CoreConnectionPNames.CONNECTION_TIMEOUT, connectionTimeout); + // setup request/response intercepter for handling gzip-compressed response. + client.addRequestInterceptor(new HttpRequestInterceptor() { + @Override + public void process(final HttpRequest request, final HttpContext context) + throws HttpException, IOException { + if (!request.containsHeader("Accept-Encoding")) { + request.addHeader("Accept-Encoding", "gzip"); + } + } + }); + client.addResponseInterceptor(new HttpResponseInterceptor() { + @Override + public void process(final HttpResponse response, final HttpContext context) + throws HttpException, IOException { + HttpEntity entity = response.getEntity(); + Header ceheader = entity.getContentEncoding(); + if (ceheader != null && contains(ceheader.getElements(), "gzip")) { + response.setEntity(new GzipInflatingHttpEntityWrapper(response.getEntity())); + } + } + }); + this.client = client; + } + return client; + } + + private long queryRangeSecs = 6L*30*24*3600; + /** + * + * @param queryRangeSecs + */ + public void setQueryRangeSecs(long queryRangeSecs) { + this.queryRangeSecs = queryRangeSecs; + } + public long getQueryRangeSecs() { + return queryRangeSecs; + } + + protected String buildURL(CrawlURI curi) { + // we don't need to pass scheme part, but no problem passing it. + StringBuilder sb = new StringBuilder(queryURL); + sb.append("?url="); + try { + sb.append(URLEncoder.encode(curi.toString(), "UTF-8")); + } catch (UnsupportedEncodingException ex) { + // expecting it's never thrown + } + long range = queryRangeSecs; + if (range > 0) { + Date now = new Date(); + Date startDate = new Date(now.getTime() - range*1000); + sb.append("&startDate="); + sb.append(ArchiveUtils.get14DigitDate(startDate)); + } + sb.append("&limit=1&last=true"); + return sb.toString(); + } + + @Override + protected ProcessResult innerProcessResult(CrawlURI curi) throws InterruptedException { + final String url = buildURL(curi); + HttpGet m = new HttpGet(url); + HttpEntity entity = null; + int attempts = 0; + do { + if (Thread.interrupted()) + throw new InterruptedException("interrupted while GET " + url); + if (attempts > 0) { + Thread.sleep(5000); + } + try { + HttpResponse resp = getHttpClient().execute(m); + StatusLine sl = resp.getStatusLine(); + if (sl.getStatusCode() != 200) { + log.error("GET " + url + " failed with status=" + sl.getStatusCode() + " " + sl.getReasonPhrase()); + entity = resp.getEntity(); + entity.getContent().close(); + entity = null; + continue; + } + entity = resp.getEntity(); + } catch (Exception ex) { + log.error("GET " + url + " failed with error " + ex.getMessage()); + } + } while (entity == null && ++attempts < 3); + if (entity == null) { + log.error("giving up on GET " + url + " after " + attempts + " attempts"); + failedCount.incrementAndGet(); + return ProcessResult.PROCEED; + } + InputStream is = null; + String hash = null; + try { + hash = getLastHash(is = entity.getContent()); + } catch (IOException ex) { + } finally { + if (is != null) + ArchiveUtils.closeQuietly(is); + } + if (hash != null) { + Map history = RecrawlDataSchemaBase.getFetchHistory(curi); + history.put(RecrawlAttributeConstants.A_CONTENT_DIGEST, contentDigestScheme + hash); + loadedCount.incrementAndGet(); + } else { + failedCount.incrementAndGet(); + } + return ProcessResult.PROCEED; + } + + protected String getLastHash(InputStream is) throws IOException { + // read CDX lines, save most recent (at the end) hash. + ByteBuffer buffer = ByteBuffer.allocate(32); + int field = 0; + int c; + do { + if (field == 5) { + buffer.clear(); + while (buffer.remaining() > 0) { + c = is.read(); + if (c >= 'A' && c <= 'Z' || c >= '0' && c <= '9') { + buffer.put((byte)c); + } else { + break; + } + } + if (buffer.remaining() == 0) { + c = is.read(); + if (c == ' ') { + field++; + continue; + } + } else { + buffer.clear(); + } + } + while (true) { + c = is.read(); + if (c == -1) { + break; + } else if (c == '\n') { + field = 0; + break; + } else if (c == ' ') { + field++; + break; + } + } + } while (c != -1); + if (buffer.remaining() == 0) { + return new String(buffer.array()); + } else { + return null; + } + } + + /** + * unused. + */ + @Override + protected void innerProcess(CrawlURI uri) throws InterruptedException { + } + + @Override + protected boolean shouldProcess(CrawlURI uri) { + // TODO: we want deduplicate robots.txt, too. + //if (uri.isPrerequisite()) return false; + String scheme = uri.getUURI().getScheme(); + if (!(scheme.equals("http") || scheme.equals("https"))) return false; + return true; + } +} diff --git a/test/java/org/archive/modules/recrawl/wbm/WbmPersistLoadProcessorTest.java b/test/java/org/archive/modules/recrawl/wbm/WbmPersistLoadProcessorTest.java new file mode 100644 index 00000000..aa4750b0 --- /dev/null +++ b/test/java/org/archive/modules/recrawl/wbm/WbmPersistLoadProcessorTest.java @@ -0,0 +1,163 @@ +package org.archive.modules.recrawl.wbm; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.net.URLEncoder; +import java.util.Map; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.regex.Pattern; + +import org.apache.http.HttpResponse; +import org.apache.http.ProtocolVersion; +import org.apache.http.client.HttpClient; +import org.apache.http.client.methods.HttpUriRequest; +import org.apache.http.entity.ByteArrayEntity; +import org.apache.http.message.BasicHttpResponse; +import org.apache.http.message.BasicStatusLine; +import org.archive.modules.CrawlURI; +import org.archive.modules.ProcessResult; +import org.archive.modules.hq.recrawl.RecrawlDataSchemaBase; +import org.archive.modules.recrawl.RecrawlAttributeConstants; +import org.archive.net.UURIFactory; +import org.easymock.EasyMock; +import org.junit.Test; + +import com.google.common.util.concurrent.ExecutionList; + +/** + * unit test for {@link WbmPersistLoadProcessor}. + * depends on WBM index query API. + * + * TODO: + *
    + *
  • test if connection is properly released back to the pool for all possible cases. + *
+ * @author kenji + * + */ +public class WbmPersistLoadProcessorTest { + @Test + public void testBuildURL() throws Exception { + WbmPersistLoadProcessor t = new WbmPersistLoadProcessor(); + final String URL = "http://archive.org/"; + CrawlURI curi = new CrawlURI(UURIFactory.getInstance(URL)); + String url = t.buildURL(curi); + System.err.println(url); + assertTrue("has encode URL", Pattern.matches(".*[&?]url="+URLEncoder.encode(URL, "UTF-8")+"([&].*)?", url)); + assertTrue("has startDate", Pattern.matches(".*[&?]startDate=\\d{14}([&].*)?", url)); + assertTrue("has limit", Pattern.matches(".*[&?]limit=\\d+([&].*)?", url)); + assertTrue("has last=true", Pattern.matches(".*[&?]last=true([&].*)?", url)); + } + + /** + * stub HttpResponse for normal case + * @author kenji + * + */ + public static class TestNormalHttpResponse extends BasicHttpResponse { + public static final String EXPECTED_HASH = "GHN5VKF3TBKNSEZTASOM23BJRTKFFNJK"; + public TestNormalHttpResponse() { + super(new BasicStatusLine(new ProtocolVersion("HTTP", 1, 0), 200, "OK")); + setEntity(new ByteArrayEntity( + ("org,archive)/ 20121101155310 http://archive.org/ text/html 200 "+ + EXPECTED_HASH+" - - 6908 982548871 "+ + "google.es-20121101-155506/IA-FOC-google.es-20121101073708-00001.warc.gz\n" + ).getBytes() + )); + } + } + + @Test + public void testInnerProcessResultSingleShotWithMock() throws Exception { + // because AbstractHttpClient marks most execute(...) methods final, it is very tiresome to implement + // a stub by implementing HttpClient interface. So I use EasyMock. + HttpClient client = EasyMock.createMock(HttpClient.class); + HttpResponse testResponse = new TestNormalHttpResponse(); + EasyMock.expect(client.execute((HttpUriRequest)EasyMock.notNull())).andReturn(testResponse); + EasyMock.replay(client); + + final String CONTENT_DIGEST_SCHEME = "sha1:"; + WbmPersistLoadProcessor t = new WbmPersistLoadProcessor(); + t.setHttpClient(client); + t.setContentDigestScheme(CONTENT_DIGEST_SCHEME); + CrawlURI curi = new CrawlURI(UURIFactory.getInstance("http://archive.org/")); + ProcessResult result = t.innerProcessResult(curi); + assertEquals("result is PROCEED", ProcessResult.PROCEED, result); + + Map history = RecrawlDataSchemaBase.getFetchHistory(curi); + assertNotNull("history", history); + String hash = (String)history.get(RecrawlAttributeConstants.A_CONTENT_DIGEST); + assertEquals("CONTENT_DIGEST", CONTENT_DIGEST_SCHEME+TestNormalHttpResponse.EXPECTED_HASH, hash); + } + + @Test + public void testInnerProcessResultSingleShotWithRealServer() throws Exception { + WbmPersistLoadProcessor t = new WbmPersistLoadProcessor(); + //CrawlURI curi = new CrawlURI(UURIFactory.getInstance("http://archive.org/")); + CrawlURI curi = new CrawlURI(UURIFactory.getInstance("http://www.mext.go.jp/null.gif")); + ProcessResult result = t.innerProcessResult(curi); + Map history = RecrawlDataSchemaBase.getFetchHistory(curi); + assertNotNull("getFetchHistory returns non-null", history); + String hash = (String)history.get(RecrawlAttributeConstants.A_CONTENT_DIGEST); + assertNotNull("CONTENT_DIGEST is non-null", hash); + assertTrue("CONTENT_DIGEST starts with scheme", hash.startsWith(t.getContentDigestScheme())); + assertEquals("CONTENT_DIGEST is a String of length 32", 32, hash.substring(t.getContentDigestScheme().length()).length()); + + assertEquals("should always return PROCEED", ProcessResult.PROCEED, result); + } + + public static class LoadTask implements Runnable { + private WbmPersistLoadProcessor p; + private String uri; + public LoadTask(WbmPersistLoadProcessor p, String uri) { + this.p = p; + this.uri = uri; + } + @Override + public void run() { + try { + CrawlURI curi = new CrawlURI(UURIFactory.getInstance(this.uri)); + p.innerProcessResult(curi); + //System.err.println(curi.toString()); + } catch (Exception ex) { + ex.printStackTrace(); + } + } + } + + /** + * test for performance. + * not annotated as test case. run this through main(). + * @throws Exception + */ + //@Test + public void testInnerProcessResultMany() throws Exception { + final WbmPersistLoadProcessor t = new WbmPersistLoadProcessor(); + InputStream is = getClass().getResourceAsStream("/test-url-list.txt"); + BufferedReader br = new BufferedReader(new InputStreamReader(is)); + String line; + int nurls = 0; + ExecutorService executor = Executors.newFixedThreadPool(100); + ExecutionList tasks = new ExecutionList(); + while ((line = br.readLine()) != null) { + tasks.add(new LoadTask(t, line), executor); + nurls++; + } + long t0 = System.currentTimeMillis(); + tasks.run(); + executor.awaitTermination(30, TimeUnit.SECONDS); + long el = System.currentTimeMillis() - t0; + System.err.println(nurls + " urls, time=" + el + "ms (" + (nurls / (el / 1000.0)) + " URI/s"); + } + + public static void main() throws Exception { + (new WbmPersistLoadProcessorTest()).testInnerProcessResultMany(); + } +} From 1364b0f5713572d65cf3fa5c5aa761a74f7a0567 Mon Sep 17 00:00:00 2001 From: Kenji Nagahashi Date: Wed, 9 Jan 2013 12:44:30 -0800 Subject: [PATCH 03/14] updated queryURL to production WBM. --- .../modules/recrawl/wbm/WbmPersistLoadProcessor.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/main/java/org/archive/modules/recrawl/wbm/WbmPersistLoadProcessor.java b/main/java/org/archive/modules/recrawl/wbm/WbmPersistLoadProcessor.java index 2cbd8ec0..7fed5288 100644 --- a/main/java/org/archive/modules/recrawl/wbm/WbmPersistLoadProcessor.java +++ b/main/java/org/archive/modules/recrawl/wbm/WbmPersistLoadProcessor.java @@ -69,7 +69,9 @@ public class WbmPersistLoadProcessor extends Processor { private HttpClient client; - private String queryURL = "http://web-beta.archive.org/cdx/search"; + // ~Jan 2, 2013 + //private String queryURL = "http://web-beta.archive.org/cdx/search"; + private String queryURL = "http://web.archive.org/cdx/search/cdx"; public void setQueryURL(String queryURL) { this.queryURL = queryURL; } @@ -92,7 +94,7 @@ public class WbmPersistLoadProcessor extends Processor { } private int socketTimeout = 10000; /** - * socket timeout (SO_TIMEOUT) for HTTP cient in milliseconds. + * socket timeout (SO_TIMEOUT) for HTTP client in milliseconds. */ public void setSocketTimeout(int socketTimeout) { this.socketTimeout = socketTimeout; From 01072b4b462fd56c2cce4ea04658a62f0fe12231 Mon Sep 17 00:00:00 2001 From: Kenji Nagahashi Date: Wed, 6 Mar 2013 17:28:38 -0800 Subject: [PATCH 04/14] support mixing fetch histories from multiple sources (hbase, WBM). --- .../modules/recrawl/FetchHistoryHelper.java | 99 +++++++++++++++++++ .../recrawl/wbm/WbmPersistLoadProcessor.java | 79 ++++++++++----- .../wbm/WbmPersistLoadProcessorTest.java | 41 +++++++- 3 files changed, 187 insertions(+), 32 deletions(-) create mode 100644 main/java/org/archive/modules/recrawl/FetchHistoryHelper.java diff --git a/main/java/org/archive/modules/recrawl/FetchHistoryHelper.java b/main/java/org/archive/modules/recrawl/FetchHistoryHelper.java new file mode 100644 index 00000000..bc659a79 --- /dev/null +++ b/main/java/org/archive/modules/recrawl/FetchHistoryHelper.java @@ -0,0 +1,99 @@ +package org.archive.modules.recrawl; + +import java.text.DateFormat; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.HashMap; +import java.util.Map; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.archive.modules.CrawlURI; + +public class FetchHistoryHelper { + private static final Log logger = LogFactory.getLog(FetchHistoryHelper.class); + /** + * key for storing timestamp in crawl history map. + */ + public static final String A_TIMESTAMP = ".ts"; + + public static String getHeaderValue(org.apache.commons.httpclient.HttpMethod method, String name) { + org.apache.commons.httpclient.Header header = method.getResponseHeader(name); + return header != null ? header.getValue() : null; + } + + /** + * returns a Map to store recrawl data, positioned property in CrawlURI's + * fetch history array property, according to {@code timestamp}. this makes it possible + * to import crawl history data from multiple sources. + * @param uri target {@link CrawlURI} + * @param timestamp timestamp (in ms) of crawl history to be added. + * @return Map object to store recrawl data, or null if {@code timestamp} is older + * than existing crawl history entry and there's no room for it. + * @see #setHistoryLength(int) + */ + @SuppressWarnings("unchecked") + public static Map getFetchHistory(CrawlURI uri, long timestamp, int historyLength) { + Map data = uri.getData(); + Map[] history = (Map[])data.get(RecrawlAttributeConstants.A_FETCH_HISTORY); + if (history == null) { + // there's no history records at all. + // FetchHistoryProcessor assumes history is HashMap[], not Map[]. + history = new HashMap[historyLength]; + data.put(RecrawlAttributeConstants.A_FETCH_HISTORY, history); + } + for (int i = 0; i < history.length; i++) { + if (history[i] == null) { + history[i] = new HashMap(); + history[i].put(A_TIMESTAMP, timestamp); + return history[i]; + } + Object ts = history[i].get(A_TIMESTAMP); + // no timestamp value is regarded as older than anything. + if (!(ts instanceof Long) || timestamp > (Long)ts) { + if (i < history.length - 2) { + System.arraycopy(history, i, history, i + 1, history.length - i - 1); + } else if (i == history.length - 2) { + history[i + 1] = history[i]; + } + history[i] = new HashMap(); + history[i].put(A_TIMESTAMP, timestamp); + return history[i]; + } + } + return null; + } + + protected static final DateFormat HTTP_DATE_FORMAT = new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss zzz"); + + protected FetchHistoryHelper() { + } + + /** + * converts time in HTTP Date format {@code dateStr} to seconds + * since epoch. + * @param dateStr time in HTTP Date format. + * @return seconds since epoch + */ + public static long parseHttpDate(String dateStr) { + synchronized (HTTP_DATE_FORMAT) { + try { + Date d = HTTP_DATE_FORMAT.parse(dateStr); + return d.getTime() / 1000; + } catch (ParseException ex) { + if (logger.isDebugEnabled()) + logger.debug("bad HTTP DATE: " + dateStr); + return 0; + } + } + } + + public static String formatHttpDate(long time) { + synchronized (HTTP_DATE_FORMAT) { + // format is not thread safe either + return HTTP_DATE_FORMAT.format(new Date(time * 1000)); + } + } + +} \ No newline at end of file diff --git a/main/java/org/archive/modules/recrawl/wbm/WbmPersistLoadProcessor.java b/main/java/org/archive/modules/recrawl/wbm/WbmPersistLoadProcessor.java index 7fed5288..f58409ba 100644 --- a/main/java/org/archive/modules/recrawl/wbm/WbmPersistLoadProcessor.java +++ b/main/java/org/archive/modules/recrawl/wbm/WbmPersistLoadProcessor.java @@ -23,7 +23,9 @@ import java.io.InputStream; import java.io.UnsupportedEncodingException; import java.net.URLEncoder; import java.nio.ByteBuffer; +import java.text.ParseException; import java.util.Date; +import java.util.HashMap; import java.util.Map; import java.util.concurrent.atomic.AtomicLong; @@ -49,16 +51,17 @@ import org.archive.modules.CrawlURI; import org.archive.modules.ProcessResult; import org.archive.modules.Processor; import org.archive.modules.hq.GzipInflatingHttpEntityWrapper; -import org.archive.modules.hq.recrawl.RecrawlDataSchemaBase; +import org.archive.modules.recrawl.FetchHistoryHelper; import org.archive.modules.recrawl.RecrawlAttributeConstants; import org.archive.util.ArchiveUtils; +import org.archive.util.DateUtils; /** * A {@link Processor} for retrieving recrawl info from remote Wayback Machine index. * This is currently in the early stage of experiment. Both low-level protocol and WBM API * semantics will certainly undergo several revisions. *

Current interface:

- *

http://web-beta.archive.org/cdx/search?url=archive.org&startDate=1999 will return raw + *

http://web-beta.archive.org/cdx/search/cdx?url=archive.org&startDate=1999 will return raw * CDX lines for archive.org, since 1999-01-01 00:00:00. *

*

As index is updated in a separate batch processing job, there's no "Store" counterpart.

@@ -69,6 +72,15 @@ public class WbmPersistLoadProcessor extends Processor { private HttpClient client; + private int historyLength = 2; + + public void setHistoryLength(int historyLength) { + this.historyLength = historyLength; + } + public int getHistoryLength() { + return historyLength; + } + // ~Jan 2, 2013 //private String queryURL = "http://web-beta.archive.org/cdx/search"; private String queryURL = "http://web.archive.org/cdx/search/cdx"; @@ -249,17 +261,19 @@ public class WbmPersistLoadProcessor extends Processor { return ProcessResult.PROCEED; } InputStream is = null; - String hash = null; + Map info = null; try { - hash = getLastHash(is = entity.getContent()); + info = getLastCrawl(is = entity.getContent()); } catch (IOException ex) { } finally { if (is != null) ArchiveUtils.closeQuietly(is); } - if (hash != null) { - Map history = RecrawlDataSchemaBase.getFetchHistory(curi); - history.put(RecrawlAttributeConstants.A_CONTENT_DIGEST, contentDigestScheme + hash); + if (info != null) { + Map history = FetchHistoryHelper.getFetchHistory(curi, + (Long)info.get(FetchHistoryHelper.A_TIMESTAMP), historyLength); + if (history != null) + history.putAll(info); loadedCount.incrementAndGet(); } else { failedCount.incrementAndGet(); @@ -267,34 +281,37 @@ public class WbmPersistLoadProcessor extends Processor { return ProcessResult.PROCEED; } - protected String getLastHash(InputStream is) throws IOException { + protected HashMap getLastCrawl(InputStream is) throws IOException { // read CDX lines, save most recent (at the end) hash. ByteBuffer buffer = ByteBuffer.allocate(32); + ByteBuffer tsbuffer = ByteBuffer.allocate(14); int field = 0; int c; do { - if (field == 5) { - buffer.clear(); - while (buffer.remaining() > 0) { + c = is.read(); + if (field == 1) { + // 14-digits timestamp + tsbuffer.clear(); + while (Character.isDigit(c) && tsbuffer.remaining() > 0) { + tsbuffer.put((byte)c); c = is.read(); - if (c >= 'A' && c <= 'Z' || c >= '0' && c <= '9') { - buffer.put((byte)c); - } else { - break; - } } - if (buffer.remaining() == 0) { + if (c != ' ' || tsbuffer.position() != 14) { + tsbuffer.clear(); + } + // fall through to skip the rest + } else if (field == 5) { + buffer.clear(); + while ((c >= 'A' && c <= 'Z' || c >= '0' && c <= '9') && buffer.remaining() > 0) { + buffer.put((byte)c); c = is.read(); - if (c == ' ') { - field++; - continue; - } - } else { + } + if (c != ' ' || buffer.position() != 32) { buffer.clear(); } + // fall through to skip the rest } - while (true) { - c = is.read(); + while (true) { if (c == -1) { break; } else if (c == '\n') { @@ -304,13 +321,21 @@ public class WbmPersistLoadProcessor extends Processor { field++; break; } + c = is.read(); } } while (c != -1); + + HashMap info = new HashMap(); if (buffer.remaining() == 0) { - return new String(buffer.array()); - } else { - return null; + info.put(RecrawlAttributeConstants.A_CONTENT_DIGEST, contentDigestScheme + new String(buffer.array())); } + if (tsbuffer.remaining() == 0) { + try { + info.put(FetchHistoryHelper.A_TIMESTAMP, DateUtils.parse14DigitDate(new String(tsbuffer.array())).getTime()); + } catch (ParseException ex) { + } + } + return info.isEmpty() ? null : info; } /** diff --git a/test/java/org/archive/modules/recrawl/wbm/WbmPersistLoadProcessorTest.java b/test/java/org/archive/modules/recrawl/wbm/WbmPersistLoadProcessorTest.java index aa4750b0..55a5aa8e 100644 --- a/test/java/org/archive/modules/recrawl/wbm/WbmPersistLoadProcessorTest.java +++ b/test/java/org/archive/modules/recrawl/wbm/WbmPersistLoadProcessorTest.java @@ -8,6 +8,7 @@ import java.io.BufferedReader; import java.io.InputStream; import java.io.InputStreamReader; import java.net.URLEncoder; +import java.util.HashMap; import java.util.Map; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -23,9 +24,10 @@ import org.apache.http.message.BasicHttpResponse; import org.apache.http.message.BasicStatusLine; import org.archive.modules.CrawlURI; import org.archive.modules.ProcessResult; -import org.archive.modules.hq.recrawl.RecrawlDataSchemaBase; +import org.archive.modules.recrawl.FetchHistoryHelper; import org.archive.modules.recrawl.RecrawlAttributeConstants; import org.archive.net.UURIFactory; +import org.archive.util.DateUtils; import org.easymock.EasyMock; import org.junit.Test; @@ -37,6 +39,7 @@ import com.google.common.util.concurrent.ExecutionList; * * TODO: *
    + *
  • test for pathological cases: illegal chars, incorrect length, etc. *
  • test if connection is properly released back to the pool for all possible cases. *
* @author kenji @@ -57,16 +60,17 @@ public class WbmPersistLoadProcessorTest { } /** - * stub HttpResponse for normal case + * stub HttpResponse for normal case. * @author kenji * */ public static class TestNormalHttpResponse extends BasicHttpResponse { + public static final String EXPECTED_TS = "20121101155310"; public static final String EXPECTED_HASH = "GHN5VKF3TBKNSEZTASOM23BJRTKFFNJK"; public TestNormalHttpResponse() { super(new BasicStatusLine(new ProtocolVersion("HTTP", 1, 0), 200, "OK")); setEntity(new ByteArrayEntity( - ("org,archive)/ 20121101155310 http://archive.org/ text/html 200 "+ + ("org,archive)/ "+EXPECTED_TS+" http://archive.org/ text/html 200 "+ EXPECTED_HASH+" - - 6908 982548871 "+ "google.es-20121101-155506/IA-FOC-google.es-20121101073708-00001.warc.gz\n" ).getBytes() @@ -74,6 +78,15 @@ public class WbmPersistLoadProcessorTest { } } + protected Map getFetchHistory(CrawlURI curi, int idx) { + Map data = curi.getData(); + @SuppressWarnings("unchecked") + Map[] historyArray = (Map[])data.get(RecrawlAttributeConstants.A_FETCH_HISTORY); + assertNotNull(historyArray); + Map history = historyArray[idx]; + return history; + } + @Test public void testInnerProcessResultSingleShotWithMock() throws Exception { // because AbstractHttpClient marks most execute(...) methods final, it is very tiresome to implement @@ -88,13 +101,31 @@ public class WbmPersistLoadProcessorTest { t.setHttpClient(client); t.setContentDigestScheme(CONTENT_DIGEST_SCHEME); CrawlURI curi = new CrawlURI(UURIFactory.getInstance("http://archive.org/")); + + // put history entry newer than being loaded + long expected_ts = DateUtils.parse14DigitDate(TestNormalHttpResponse.EXPECTED_TS).getTime(); + Map[] fetchHistory = (Map[])curi.getData().get(RecrawlAttributeConstants.A_FETCH_HISTORY); + if (fetchHistory == null) { + fetchHistory = new HashMap[2]; + curi.getData().put(RecrawlAttributeConstants.A_FETCH_HISTORY, fetchHistory); + } + fetchHistory[0] = new HashMap(); + fetchHistory[0].put(FetchHistoryHelper.A_TIMESTAMP, expected_ts + 2000); + fetchHistory[1] = new HashMap(); + fetchHistory[1].put(FetchHistoryHelper.A_TIMESTAMP, expected_ts - 2000); + ProcessResult result = t.innerProcessResult(curi); assertEquals("result is PROCEED", ProcessResult.PROCEED, result); - Map history = RecrawlDataSchemaBase.getFetchHistory(curi); + // newly loaded history entry should fall in between two existing entries (index=1) + Map history = getFetchHistory(curi, 1); assertNotNull("history", history); String hash = (String)history.get(RecrawlAttributeConstants.A_CONTENT_DIGEST); assertEquals("CONTENT_DIGEST", CONTENT_DIGEST_SCHEME+TestNormalHttpResponse.EXPECTED_HASH, hash); + + Long ts = (Long)history.get(FetchHistoryHelper.A_TIMESTAMP); + assertNotNull("ts is non-null", ts); + assertEquals("'ts' has expected timestamp", expected_ts, ts.longValue()); } @Test @@ -103,7 +134,7 @@ public class WbmPersistLoadProcessorTest { //CrawlURI curi = new CrawlURI(UURIFactory.getInstance("http://archive.org/")); CrawlURI curi = new CrawlURI(UURIFactory.getInstance("http://www.mext.go.jp/null.gif")); ProcessResult result = t.innerProcessResult(curi); - Map history = RecrawlDataSchemaBase.getFetchHistory(curi); + Map history = getFetchHistory(curi, 0); assertNotNull("getFetchHistory returns non-null", history); String hash = (String)history.get(RecrawlAttributeConstants.A_CONTENT_DIGEST); assertNotNull("CONTENT_DIGEST is non-null", hash); From ad70b3abd3d528a0a402f52344612d44a91ab9f5 Mon Sep 17 00:00:00 2001 From: Kenji Nagahashi Date: Thu, 13 Jun 2013 20:54:24 -0700 Subject: [PATCH 05/14] moved hbaes de-duplication code to heritrix-contrib. chnaged groupId to address duplicate groupId warning. --- .../modules/recrawl/FetchHistoryHelper.java | 99 ------------------- 1 file changed, 99 deletions(-) delete mode 100644 main/java/org/archive/modules/recrawl/FetchHistoryHelper.java diff --git a/main/java/org/archive/modules/recrawl/FetchHistoryHelper.java b/main/java/org/archive/modules/recrawl/FetchHistoryHelper.java deleted file mode 100644 index bc659a79..00000000 --- a/main/java/org/archive/modules/recrawl/FetchHistoryHelper.java +++ /dev/null @@ -1,99 +0,0 @@ -package org.archive.modules.recrawl; - -import java.text.DateFormat; -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.Date; -import java.util.HashMap; -import java.util.Map; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.archive.modules.CrawlURI; - -public class FetchHistoryHelper { - private static final Log logger = LogFactory.getLog(FetchHistoryHelper.class); - /** - * key for storing timestamp in crawl history map. - */ - public static final String A_TIMESTAMP = ".ts"; - - public static String getHeaderValue(org.apache.commons.httpclient.HttpMethod method, String name) { - org.apache.commons.httpclient.Header header = method.getResponseHeader(name); - return header != null ? header.getValue() : null; - } - - /** - * returns a Map to store recrawl data, positioned property in CrawlURI's - * fetch history array property, according to {@code timestamp}. this makes it possible - * to import crawl history data from multiple sources. - * @param uri target {@link CrawlURI} - * @param timestamp timestamp (in ms) of crawl history to be added. - * @return Map object to store recrawl data, or null if {@code timestamp} is older - * than existing crawl history entry and there's no room for it. - * @see #setHistoryLength(int) - */ - @SuppressWarnings("unchecked") - public static Map getFetchHistory(CrawlURI uri, long timestamp, int historyLength) { - Map data = uri.getData(); - Map[] history = (Map[])data.get(RecrawlAttributeConstants.A_FETCH_HISTORY); - if (history == null) { - // there's no history records at all. - // FetchHistoryProcessor assumes history is HashMap[], not Map[]. - history = new HashMap[historyLength]; - data.put(RecrawlAttributeConstants.A_FETCH_HISTORY, history); - } - for (int i = 0; i < history.length; i++) { - if (history[i] == null) { - history[i] = new HashMap(); - history[i].put(A_TIMESTAMP, timestamp); - return history[i]; - } - Object ts = history[i].get(A_TIMESTAMP); - // no timestamp value is regarded as older than anything. - if (!(ts instanceof Long) || timestamp > (Long)ts) { - if (i < history.length - 2) { - System.arraycopy(history, i, history, i + 1, history.length - i - 1); - } else if (i == history.length - 2) { - history[i + 1] = history[i]; - } - history[i] = new HashMap(); - history[i].put(A_TIMESTAMP, timestamp); - return history[i]; - } - } - return null; - } - - protected static final DateFormat HTTP_DATE_FORMAT = new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss zzz"); - - protected FetchHistoryHelper() { - } - - /** - * converts time in HTTP Date format {@code dateStr} to seconds - * since epoch. - * @param dateStr time in HTTP Date format. - * @return seconds since epoch - */ - public static long parseHttpDate(String dateStr) { - synchronized (HTTP_DATE_FORMAT) { - try { - Date d = HTTP_DATE_FORMAT.parse(dateStr); - return d.getTime() / 1000; - } catch (ParseException ex) { - if (logger.isDebugEnabled()) - logger.debug("bad HTTP DATE: " + dateStr); - return 0; - } - } - } - - public static String formatHttpDate(long time) { - synchronized (HTTP_DATE_FORMAT) { - // format is not thread safe either - return HTTP_DATE_FORMAT.format(new Date(time * 1000)); - } - } - -} \ No newline at end of file From 4d559ec81d208ae982af2f8c5ded38f1d49a12a0 Mon Sep 17 00:00:00 2001 From: Kenji Nagahashi Date: Fri, 27 Sep 2013 10:21:51 -0700 Subject: [PATCH 06/14] WbmPersistLoadProcessor.java: fixed indentation. --- .../recrawl/wbm/WbmPersistLoadProcessor.java | 386 +++++++++--------- 1 file changed, 193 insertions(+), 193 deletions(-) diff --git a/main/java/org/archive/modules/recrawl/wbm/WbmPersistLoadProcessor.java b/main/java/org/archive/modules/recrawl/wbm/WbmPersistLoadProcessor.java index f58409ba..620eb250 100644 --- a/main/java/org/archive/modules/recrawl/wbm/WbmPersistLoadProcessor.java +++ b/main/java/org/archive/modules/recrawl/wbm/WbmPersistLoadProcessor.java @@ -68,29 +68,29 @@ import org.archive.util.DateUtils; * @contributor Kenji Nagahashi. */ public class WbmPersistLoadProcessor extends Processor { - private static final Log log = LogFactory.getLog(WbmPersistLoadProcessor.class); + private static final Log log = LogFactory.getLog(WbmPersistLoadProcessor.class); private HttpClient client; - + private int historyLength = 2; - + public void setHistoryLength(int historyLength) { - this.historyLength = historyLength; + this.historyLength = historyLength; } public int getHistoryLength() { - return historyLength; + return historyLength; } - + // ~Jan 2, 2013 //private String queryURL = "http://web-beta.archive.org/cdx/search"; private String queryURL = "http://web.archive.org/cdx/search/cdx"; public void setQueryURL(String queryURL) { - this.queryURL = queryURL; + this.queryURL = queryURL; } public String getQueryURL() { - return queryURL; + return queryURL; } - + private String contentDigestScheme = "sha1:"; /** * set Content-Digest scheme string to prepend to the hash string found in CDX. @@ -99,243 +99,243 @@ public class WbmPersistLoadProcessor extends Processor { * @param contentDigestScheme */ public void setContentDigestScheme(String contentDigestScheme) { - this.contentDigestScheme = contentDigestScheme; + this.contentDigestScheme = contentDigestScheme; } public String getContentDigestScheme() { - return contentDigestScheme; + return contentDigestScheme; } private int socketTimeout = 10000; /** * socket timeout (SO_TIMEOUT) for HTTP client in milliseconds. */ public void setSocketTimeout(int socketTimeout) { - this.socketTimeout = socketTimeout; + this.socketTimeout = socketTimeout; } public int getSocketTimeout() { - return socketTimeout; + return socketTimeout; } - + private int connectionTimeout = 10000; /** * connection timeout for HTTP client in milliseconds. * @param connectionTimeout */ public void setConnectionTimeout(int connectionTimeout) { - this.connectionTimeout = connectionTimeout; + this.connectionTimeout = connectionTimeout; } public int getConnectionTimeout() { - return connectionTimeout; + return connectionTimeout; } - + private int maxConnections = 10; public int getMaxConnections() { - return maxConnections; + return maxConnections; } public void setMaxConnections(int maxConnections) { - this.maxConnections = maxConnections; - if (client != null) { - ((ThreadSafeClientConnManager)client.getConnectionManager()) - .setDefaultMaxPerRoute(this.maxConnections); - } + this.maxConnections = maxConnections; + if (client != null) { + ((ThreadSafeClientConnManager)client.getConnectionManager()) + .setDefaultMaxPerRoute(this.maxConnections); + } } - + // statistics private AtomicLong loadedCount = new AtomicLong(); public long getLoadedCount() { - return loadedCount.get(); + return loadedCount.get(); } private AtomicLong failedCount = new AtomicLong(); public long getFailedCount() { - return failedCount.get(); + return failedCount.get(); } - + public void setHttpClient(HttpClient client) { - this.client = client; + this.client = client; } - + // XXX HttpHeadquarterAdapter has the same code. move to common library. private static boolean contains(HeaderElement[] elements, String value) { - for (int i = 0; i < elements.length; i++) { - if (elements[i].getName().equalsIgnoreCase(value)) { - return true; - } - } - return false; + for (int i = 0; i < elements.length; i++) { + if (elements[i].getName().equalsIgnoreCase(value)) { + return true; + } + } + return false; } public synchronized HttpClient getHttpClient() { - if (client == null) { - ThreadSafeClientConnManager conman = new ThreadSafeClientConnManager(); - conman.setDefaultMaxPerRoute(maxConnections); - final DefaultHttpClient client = new DefaultHttpClient(conman); - HttpParams params = client.getParams(); - params.setIntParameter(CoreConnectionPNames.SO_TIMEOUT, socketTimeout); - params.setIntParameter(CoreConnectionPNames.CONNECTION_TIMEOUT, connectionTimeout); - // setup request/response intercepter for handling gzip-compressed response. - client.addRequestInterceptor(new HttpRequestInterceptor() { - @Override - public void process(final HttpRequest request, final HttpContext context) - throws HttpException, IOException { - if (!request.containsHeader("Accept-Encoding")) { - request.addHeader("Accept-Encoding", "gzip"); - } - } - }); - client.addResponseInterceptor(new HttpResponseInterceptor() { - @Override - public void process(final HttpResponse response, final HttpContext context) - throws HttpException, IOException { - HttpEntity entity = response.getEntity(); - Header ceheader = entity.getContentEncoding(); - if (ceheader != null && contains(ceheader.getElements(), "gzip")) { - response.setEntity(new GzipInflatingHttpEntityWrapper(response.getEntity())); - } - } - }); - this.client = client; - } - return client; + if (client == null) { + ThreadSafeClientConnManager conman = new ThreadSafeClientConnManager(); + conman.setDefaultMaxPerRoute(maxConnections); + final DefaultHttpClient client = new DefaultHttpClient(conman); + HttpParams params = client.getParams(); + params.setIntParameter(CoreConnectionPNames.SO_TIMEOUT, socketTimeout); + params.setIntParameter(CoreConnectionPNames.CONNECTION_TIMEOUT, connectionTimeout); + // setup request/response intercepter for handling gzip-compressed response. + client.addRequestInterceptor(new HttpRequestInterceptor() { + @Override + public void process(final HttpRequest request, final HttpContext context) + throws HttpException, IOException { + if (!request.containsHeader("Accept-Encoding")) { + request.addHeader("Accept-Encoding", "gzip"); + } + } + }); + client.addResponseInterceptor(new HttpResponseInterceptor() { + @Override + public void process(final HttpResponse response, final HttpContext context) + throws HttpException, IOException { + HttpEntity entity = response.getEntity(); + Header ceheader = entity.getContentEncoding(); + if (ceheader != null && contains(ceheader.getElements(), "gzip")) { + response.setEntity(new GzipInflatingHttpEntityWrapper(response.getEntity())); + } + } + }); + this.client = client; + } + return client; } - + private long queryRangeSecs = 6L*30*24*3600; /** * * @param queryRangeSecs */ public void setQueryRangeSecs(long queryRangeSecs) { - this.queryRangeSecs = queryRangeSecs; + this.queryRangeSecs = queryRangeSecs; } public long getQueryRangeSecs() { - return queryRangeSecs; + return queryRangeSecs; } - + protected String buildURL(CrawlURI curi) { - // we don't need to pass scheme part, but no problem passing it. - StringBuilder sb = new StringBuilder(queryURL); - sb.append("?url="); - try { - sb.append(URLEncoder.encode(curi.toString(), "UTF-8")); - } catch (UnsupportedEncodingException ex) { - // expecting it's never thrown - } - long range = queryRangeSecs; - if (range > 0) { - Date now = new Date(); - Date startDate = new Date(now.getTime() - range*1000); - sb.append("&startDate="); - sb.append(ArchiveUtils.get14DigitDate(startDate)); - } - sb.append("&limit=1&last=true"); - return sb.toString(); + // we don't need to pass scheme part, but no problem passing it. + StringBuilder sb = new StringBuilder(queryURL); + sb.append("?url="); + try { + sb.append(URLEncoder.encode(curi.toString(), "UTF-8")); + } catch (UnsupportedEncodingException ex) { + // expecting it's never thrown + } + long range = queryRangeSecs; + if (range > 0) { + Date now = new Date(); + Date startDate = new Date(now.getTime() - range*1000); + sb.append("&startDate="); + sb.append(ArchiveUtils.get14DigitDate(startDate)); + } + sb.append("&limit=1&last=true"); + return sb.toString(); } - + @Override protected ProcessResult innerProcessResult(CrawlURI curi) throws InterruptedException { - final String url = buildURL(curi); - HttpGet m = new HttpGet(url); - HttpEntity entity = null; - int attempts = 0; - do { - if (Thread.interrupted()) - throw new InterruptedException("interrupted while GET " + url); - if (attempts > 0) { - Thread.sleep(5000); - } - try { - HttpResponse resp = getHttpClient().execute(m); - StatusLine sl = resp.getStatusLine(); - if (sl.getStatusCode() != 200) { - log.error("GET " + url + " failed with status=" + sl.getStatusCode() + " " + sl.getReasonPhrase()); - entity = resp.getEntity(); - entity.getContent().close(); - entity = null; - continue; - } - entity = resp.getEntity(); - } catch (Exception ex) { - log.error("GET " + url + " failed with error " + ex.getMessage()); - } - } while (entity == null && ++attempts < 3); - if (entity == null) { - log.error("giving up on GET " + url + " after " + attempts + " attempts"); - failedCount.incrementAndGet(); - return ProcessResult.PROCEED; - } - InputStream is = null; - Map info = null; - try { - info = getLastCrawl(is = entity.getContent()); - } catch (IOException ex) { - } finally { - if (is != null) - ArchiveUtils.closeQuietly(is); - } - if (info != null) { - Map history = FetchHistoryHelper.getFetchHistory(curi, - (Long)info.get(FetchHistoryHelper.A_TIMESTAMP), historyLength); - if (history != null) - history.putAll(info); - loadedCount.incrementAndGet(); - } else { - failedCount.incrementAndGet(); - } - return ProcessResult.PROCEED; + final String url = buildURL(curi); + HttpGet m = new HttpGet(url); + HttpEntity entity = null; + int attempts = 0; + do { + if (Thread.interrupted()) + throw new InterruptedException("interrupted while GET " + url); + if (attempts > 0) { + Thread.sleep(5000); + } + try { + HttpResponse resp = getHttpClient().execute(m); + StatusLine sl = resp.getStatusLine(); + if (sl.getStatusCode() != 200) { + log.error("GET " + url + " failed with status=" + sl.getStatusCode() + " " + sl.getReasonPhrase()); + entity = resp.getEntity(); + entity.getContent().close(); + entity = null; + continue; + } + entity = resp.getEntity(); + } catch (Exception ex) { + log.error("GET " + url + " failed with error " + ex.getMessage()); + } + } while (entity == null && ++attempts < 3); + if (entity == null) { + log.error("giving up on GET " + url + " after " + attempts + " attempts"); + failedCount.incrementAndGet(); + return ProcessResult.PROCEED; + } + InputStream is = null; + Map info = null; + try { + info = getLastCrawl(is = entity.getContent()); + } catch (IOException ex) { + } finally { + if (is != null) + ArchiveUtils.closeQuietly(is); + } + if (info != null) { + Map history = FetchHistoryHelper.getFetchHistory(curi, + (Long)info.get(FetchHistoryHelper.A_TIMESTAMP), historyLength); + if (history != null) + history.putAll(info); + loadedCount.incrementAndGet(); + } else { + failedCount.incrementAndGet(); + } + return ProcessResult.PROCEED; } - + protected HashMap getLastCrawl(InputStream is) throws IOException { - // read CDX lines, save most recent (at the end) hash. - ByteBuffer buffer = ByteBuffer.allocate(32); - ByteBuffer tsbuffer = ByteBuffer.allocate(14); - int field = 0; - int c; - do { - c = is.read(); - if (field == 1) { - // 14-digits timestamp - tsbuffer.clear(); - while (Character.isDigit(c) && tsbuffer.remaining() > 0) { - tsbuffer.put((byte)c); - c = is.read(); - } - if (c != ' ' || tsbuffer.position() != 14) { - tsbuffer.clear(); - } - // fall through to skip the rest - } else if (field == 5) { - buffer.clear(); - while ((c >= 'A' && c <= 'Z' || c >= '0' && c <= '9') && buffer.remaining() > 0) { - buffer.put((byte)c); - c = is.read(); - } - if (c != ' ' || buffer.position() != 32) { - buffer.clear(); - } - // fall through to skip the rest - } - while (true) { - if (c == -1) { - break; - } else if (c == '\n') { - field = 0; - break; - } else if (c == ' ') { - field++; - break; - } - c = is.read(); - } - } while (c != -1); - - HashMap info = new HashMap(); - if (buffer.remaining() == 0) { - info.put(RecrawlAttributeConstants.A_CONTENT_DIGEST, contentDigestScheme + new String(buffer.array())); - } - if (tsbuffer.remaining() == 0) { - try { - info.put(FetchHistoryHelper.A_TIMESTAMP, DateUtils.parse14DigitDate(new String(tsbuffer.array())).getTime()); - } catch (ParseException ex) { - } - } - return info.isEmpty() ? null : info; + // read CDX lines, save most recent (at the end) hash. + ByteBuffer buffer = ByteBuffer.allocate(32); + ByteBuffer tsbuffer = ByteBuffer.allocate(14); + int field = 0; + int c; + do { + c = is.read(); + if (field == 1) { + // 14-digits timestamp + tsbuffer.clear(); + while (Character.isDigit(c) && tsbuffer.remaining() > 0) { + tsbuffer.put((byte)c); + c = is.read(); + } + if (c != ' ' || tsbuffer.position() != 14) { + tsbuffer.clear(); + } + // fall through to skip the rest + } else if (field == 5) { + buffer.clear(); + while ((c >= 'A' && c <= 'Z' || c >= '0' && c <= '9') && buffer.remaining() > 0) { + buffer.put((byte)c); + c = is.read(); + } + if (c != ' ' || buffer.position() != 32) { + buffer.clear(); + } + // fall through to skip the rest + } + while (true) { + if (c == -1) { + break; + } else if (c == '\n') { + field = 0; + break; + } else if (c == ' ') { + field++; + break; + } + c = is.read(); + } + } while (c != -1); + + HashMap info = new HashMap(); + if (buffer.remaining() == 0) { + info.put(RecrawlAttributeConstants.A_CONTENT_DIGEST, contentDigestScheme + new String(buffer.array())); + } + if (tsbuffer.remaining() == 0) { + try { + info.put(FetchHistoryHelper.A_TIMESTAMP, DateUtils.parse14DigitDate(new String(tsbuffer.array())).getTime()); + } catch (ParseException ex) { + } + } + return info.isEmpty() ? null : info; } /** @@ -344,7 +344,7 @@ public class WbmPersistLoadProcessor extends Processor { @Override protected void innerProcess(CrawlURI uri) throws InterruptedException { } - + @Override protected boolean shouldProcess(CrawlURI uri) { // TODO: we want deduplicate robots.txt, too. From df03249a65144ed76721c991300ffe1a4338a0c3 Mon Sep 17 00:00:00 2001 From: Kenji Nagahashi Date: Fri, 27 Sep 2013 11:58:41 -0700 Subject: [PATCH 07/14] WbmPersistLoadProcessor: make buildURL() more flexible. now uses template string with placeholders. no hard-coded parameters. --- .../recrawl/wbm/WbmPersistLoadProcessor.java | 99 ++++++++++++++++--- .../wbm/WbmPersistLoadProcessorTest.java | 3 +- 2 files changed, 89 insertions(+), 13 deletions(-) diff --git a/main/java/org/archive/modules/recrawl/wbm/WbmPersistLoadProcessor.java b/main/java/org/archive/modules/recrawl/wbm/WbmPersistLoadProcessor.java index 620eb250..4a436375 100644 --- a/main/java/org/archive/modules/recrawl/wbm/WbmPersistLoadProcessor.java +++ b/main/java/org/archive/modules/recrawl/wbm/WbmPersistLoadProcessor.java @@ -24,8 +24,10 @@ import java.io.UnsupportedEncodingException; import java.net.URLEncoder; import java.nio.ByteBuffer; import java.text.ParseException; +import java.util.ArrayList; import java.util.Date; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicLong; @@ -83,13 +85,73 @@ public class WbmPersistLoadProcessor extends Processor { // ~Jan 2, 2013 //private String queryURL = "http://web-beta.archive.org/cdx/search"; - private String queryURL = "http://web.archive.org/cdx/search/cdx"; + //private String queryURL = "http://web.archive.org/cdx/search/cdx"; + // ~Sep 26, 2013 + private String queryURL = "http://wwwb-front2.us.archive.org:8083/web/timemap/cdx?url=$u&limit=-1"; public void setQueryURL(String queryURL) { this.queryURL = queryURL; + prepareQueryURL(); } public String getQueryURL() { return queryURL; } + + public interface FormatSegment { + void print(StringBuilder sb, String[] args); + } + private static class StaticSegment implements FormatSegment { + String s; + public StaticSegment(String s) { + this.s = s; + } + public void print(StringBuilder sb, String[] args) { + sb.append(s); + } + } + private static class InterpolateSegment implements FormatSegment { + int aidx; + public InterpolateSegment(int aidx) { + this.aidx = aidx; + } + public void print(StringBuilder sb, String[] args) { + sb.append(args[aidx]); + } + } + private FormatSegment[] preparedQueryURL; + + /** + * pre-scan queryURL template so that actual queryURL can be built + * with minimal processing. + */ + private void prepareQueryURL() { + List segments = new ArrayList(); + final int l = queryURL.length(); + int p = 0; + int q; + while (p < l && (q = queryURL.indexOf('$', p)) >= 0) { + if (q + 2 > l) { + // '$' at the end. keep it as if it were '$$' + break; + } + if (q > p) { + segments.add(new StaticSegment(queryURL.substring(p, q))); + } + char c = queryURL.charAt(q + 1); + if (c == 'u') { + segments.add(new InterpolateSegment(0)); + } else if (c == 's') { + segments.add(new InterpolateSegment(1)); + } else { + // copy '$'-sequence so that it's easy to spot errors + segments.add(new StaticSegment(queryURL.substring(q, q + 2))); + } + p = q + 2; + } + if (p < l) { + segments.add(new StaticSegment(queryURL.substring(p))); + } + preparedQueryURL = segments.toArray(new FormatSegment[segments.size()]); + } private String contentDigestScheme = "sha1:"; /** @@ -208,26 +270,39 @@ public class WbmPersistLoadProcessor extends Processor { return queryRangeSecs; } + private String buildStartDate() { + final long range = queryRangeSecs; + if (range <= 0) + return ArchiveUtils.get14DigitDate(new Date(0)); + Date now = new Date(); + Date startDate = new Date(now.getTime() - range*1000); + return ArchiveUtils.get14DigitDate(startDate); + } + protected String buildURL(CrawlURI curi) { // we don't need to pass scheme part, but no problem passing it. - StringBuilder sb = new StringBuilder(queryURL); - sb.append("?url="); + StringBuilder sb = new StringBuilder(); + final FormatSegment[] segments = preparedQueryURL; + String encodedURL; try { - sb.append(URLEncoder.encode(curi.toString(), "UTF-8")); + encodedURL = URLEncoder.encode(curi.toString(), "UTF-8"); } catch (UnsupportedEncodingException ex) { - // expecting it's never thrown + encodedURL = curi.toString(); } - long range = queryRangeSecs; - if (range > 0) { - Date now = new Date(); - Date startDate = new Date(now.getTime() - range*1000); - sb.append("&startDate="); - sb.append(ArchiveUtils.get14DigitDate(startDate)); + final String[] args = new String[] { + encodedURL, + buildStartDate() + }; + for (FormatSegment fs : segments) { + fs.print(sb, args); } - sb.append("&limit=1&last=true"); return sb.toString(); } + public WbmPersistLoadProcessor() { + prepareQueryURL(); + } + @Override protected ProcessResult innerProcessResult(CrawlURI curi) throws InterruptedException { final String url = buildURL(curi); diff --git a/test/java/org/archive/modules/recrawl/wbm/WbmPersistLoadProcessorTest.java b/test/java/org/archive/modules/recrawl/wbm/WbmPersistLoadProcessorTest.java index 55a5aa8e..b6ec9614 100644 --- a/test/java/org/archive/modules/recrawl/wbm/WbmPersistLoadProcessorTest.java +++ b/test/java/org/archive/modules/recrawl/wbm/WbmPersistLoadProcessorTest.java @@ -49,6 +49,7 @@ public class WbmPersistLoadProcessorTest { @Test public void testBuildURL() throws Exception { WbmPersistLoadProcessor t = new WbmPersistLoadProcessor(); + t.setQueryURL("http://web.archive.org/cdx/search/cdx?url=$u&startDate=$s&limit=1"); final String URL = "http://archive.org/"; CrawlURI curi = new CrawlURI(UURIFactory.getInstance(URL)); String url = t.buildURL(curi); @@ -56,7 +57,7 @@ public class WbmPersistLoadProcessorTest { assertTrue("has encode URL", Pattern.matches(".*[&?]url="+URLEncoder.encode(URL, "UTF-8")+"([&].*)?", url)); assertTrue("has startDate", Pattern.matches(".*[&?]startDate=\\d{14}([&].*)?", url)); assertTrue("has limit", Pattern.matches(".*[&?]limit=\\d+([&].*)?", url)); - assertTrue("has last=true", Pattern.matches(".*[&?]last=true([&].*)?", url)); + //assertTrue("has last=true", Pattern.matches(".*[&?]last=true([&].*)?", url)); } /** From aa9208f5e6a41a2d842ed9bbde1f8bd2d63effb7 Mon Sep 17 00:00:00 2001 From: Kenji Nagahashi Date: Tue, 1 Oct 2013 12:10:25 -0700 Subject: [PATCH 08/14] updated WbmPersistLoadProcessor for cookie-based authentication. allows turning gzip compression on/off. main for quick test. --- .../recrawl/wbm/WbmPersistLoadProcessor.java | 100 ++++++++++++++++-- .../wbm/WbmPersistLoadProcessorTest.java | 3 +- 2 files changed, 91 insertions(+), 12 deletions(-) diff --git a/main/java/org/archive/modules/recrawl/wbm/WbmPersistLoadProcessor.java b/main/java/org/archive/modules/recrawl/wbm/WbmPersistLoadProcessor.java index 4a436375..40d2eb68 100644 --- a/main/java/org/archive/modules/recrawl/wbm/WbmPersistLoadProcessor.java +++ b/main/java/org/archive/modules/recrawl/wbm/WbmPersistLoadProcessor.java @@ -25,10 +25,13 @@ import java.net.URLEncoder; import java.nio.ByteBuffer; import java.text.ParseException; import java.util.ArrayList; +import java.util.Collections; import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Map.Entry; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicLong; import org.apache.commons.logging.Log; @@ -200,6 +203,47 @@ public class WbmPersistLoadProcessor extends Processor { .setDefaultMaxPerRoute(this.maxConnections); } } + private boolean gzipAccepted = false; + public boolean isGzipAccepted() { + return gzipAccepted; + } + /** + * if set to true, {@link WbmPersistLoadProcessor} adds a header + * {@code Accept-Encoding: gzip} to HTTP requests. New CDX server + * see this header to decide whether to compress the response. it is also + * possible to override gzipAccepted=true setting with gzip=false + * request parameter. + * It is off by default, as it should make little sense to compress single + * line of CDX. + * @param gzipAccepted true to allow gzip compression. + */ + public void setGzipAccepted(boolean gzipAccepted) { + this.gzipAccepted = gzipAccepted; + } + + private Map requestHeaders = new ConcurrentHashMap(1, 0.75f, 2); + public Map getRequestHeaders() { + return requestHeaders; + } + /** + * all key-value pairs in this map will be added as HTTP headers. + * typically used for providing authentication cookies. this method + * makes a copy of {@requestHeaders}. + * note: this property may be dropped in the future if + * I come up with better interface. + * @param requestHeaders map of <header-name, header-value>. + */ + public void setRequestHeaders(Map requestHeaders) { + if (requestHeaders == null) { + this.requestHeaders.clear(); + } else { + // TODO: mmm, ConcurrentHashMap may be overkill. simple synchronized Hashtable would work + // just okay? + ConcurrentHashMap m = new ConcurrentHashMap(1, 0.75f, 2); + m.putAll(requestHeaders); + this.requestHeaders = m; + } + } // statistics private AtomicLong loadedCount = new AtomicLong(); @@ -237,9 +281,15 @@ public class WbmPersistLoadProcessor extends Processor { @Override public void process(final HttpRequest request, final HttpContext context) throws HttpException, IOException { - if (!request.containsHeader("Accept-Encoding")) { + if (gzipAccepted && !request.containsHeader("Accept-Encoding")) { request.addHeader("Accept-Encoding", "gzip"); } + // add extra headers configured. + if (requestHeaders != null) { + for (Entry ent : requestHeaders.entrySet()) { + request.addHeader(ent.getKey(), ent.getValue()); + } + } } }); client.addResponseInterceptor(new HttpResponseInterceptor() { @@ -279,15 +329,15 @@ public class WbmPersistLoadProcessor extends Processor { return ArchiveUtils.get14DigitDate(startDate); } - protected String buildURL(CrawlURI curi) { + protected String buildURL(String url) { // we don't need to pass scheme part, but no problem passing it. StringBuilder sb = new StringBuilder(); final FormatSegment[] segments = preparedQueryURL; String encodedURL; try { - encodedURL = URLEncoder.encode(curi.toString(), "UTF-8"); + encodedURL = URLEncoder.encode(url, "UTF-8"); } catch (UnsupportedEncodingException ex) { - encodedURL = curi.toString(); + encodedURL = url; } final String[] args = new String[] { encodedURL, @@ -303,9 +353,8 @@ public class WbmPersistLoadProcessor extends Processor { prepareQueryURL(); } - @Override - protected ProcessResult innerProcessResult(CrawlURI curi) throws InterruptedException { - final String url = buildURL(curi); + protected InputStream getCDX(String qurl) throws InterruptedException, IOException { + final String url = buildURL(qurl); HttpGet m = new HttpGet(url); HttpEntity entity = null; int attempts = 0; @@ -331,15 +380,26 @@ public class WbmPersistLoadProcessor extends Processor { } } while (entity == null && ++attempts < 3); if (entity == null) { - log.error("giving up on GET " + url + " after " + attempts + " attempts"); + throw new IOException("giving up on GET " + url + " after " + attempts + " attempts"); + } + return entity.getContent(); + } + + @Override + protected ProcessResult innerProcessResult(CrawlURI curi) throws InterruptedException { + InputStream is; + try { + is = getCDX(curi.toString()); + } catch (IOException ex) { + log.error(ex.getMessage()); failedCount.incrementAndGet(); return ProcessResult.PROCEED; } - InputStream is = null; Map info = null; try { - info = getLastCrawl(is = entity.getContent()); + info = getLastCrawl(is); } catch (IOException ex) { + log.error("error parsing response", ex); } finally { if (is != null) ArchiveUtils.closeQuietly(is); @@ -428,4 +488,24 @@ public class WbmPersistLoadProcessor extends Processor { if (!(scheme.equals("http") || scheme.equals("https"))) return false; return true; } + + /** + * main entry point for quick test. + * @param args + */ + public static void main(String[] args) throws Exception { + String url = args[0]; + String cookie = args.length > 1 ? args[1] : null; + WbmPersistLoadProcessor wp = new WbmPersistLoadProcessor(); + if (cookie != null) { + wp.setRequestHeaders(Collections.singletonMap("Cookie", cookie)); + } + InputStream is = wp.getCDX(url); + byte[] b = new byte[1024]; + int n; + while ((n = is.read(b)) > 0) { + System.out.write(b, 0, n); + } + is.close(); + } } diff --git a/test/java/org/archive/modules/recrawl/wbm/WbmPersistLoadProcessorTest.java b/test/java/org/archive/modules/recrawl/wbm/WbmPersistLoadProcessorTest.java index b6ec9614..6c095f38 100644 --- a/test/java/org/archive/modules/recrawl/wbm/WbmPersistLoadProcessorTest.java +++ b/test/java/org/archive/modules/recrawl/wbm/WbmPersistLoadProcessorTest.java @@ -51,8 +51,7 @@ public class WbmPersistLoadProcessorTest { WbmPersistLoadProcessor t = new WbmPersistLoadProcessor(); t.setQueryURL("http://web.archive.org/cdx/search/cdx?url=$u&startDate=$s&limit=1"); final String URL = "http://archive.org/"; - CrawlURI curi = new CrawlURI(UURIFactory.getInstance(URL)); - String url = t.buildURL(curi); + String url = t.buildURL(URL); System.err.println(url); assertTrue("has encode URL", Pattern.matches(".*[&?]url="+URLEncoder.encode(URL, "UTF-8")+"([&].*)?", url)); assertTrue("has startDate", Pattern.matches(".*[&?]startDate=\\d{14}([&].*)?", url)); From 87e8e6b31d89ff1e9a05643d9f5c7de0dc1c5594 Mon Sep 17 00:00:00 2001 From: Kenji Nagahashi Date: Thu, 3 Oct 2013 12:18:35 -0700 Subject: [PATCH 09/14] WbmPersistLoadProcessor: add simple performance metric. drop confusing metric failedCount, add missedCount and errorCount. --- .../recrawl/wbm/WbmPersistLoadProcessor.java | 51 ++++++++++++++++--- 1 file changed, 43 insertions(+), 8 deletions(-) diff --git a/main/java/org/archive/modules/recrawl/wbm/WbmPersistLoadProcessor.java b/main/java/org/archive/modules/recrawl/wbm/WbmPersistLoadProcessor.java index 40d2eb68..5ed5e7b2 100644 --- a/main/java/org/archive/modules/recrawl/wbm/WbmPersistLoadProcessor.java +++ b/main/java/org/archive/modules/recrawl/wbm/WbmPersistLoadProcessor.java @@ -199,8 +199,11 @@ public class WbmPersistLoadProcessor extends Processor { public void setMaxConnections(int maxConnections) { this.maxConnections = maxConnections; if (client != null) { - ((ThreadSafeClientConnManager)client.getConnectionManager()) - .setDefaultMaxPerRoute(this.maxConnections); + ThreadSafeClientConnManager conman = + (ThreadSafeClientConnManager)client.getConnectionManager(); + if (conman.getMaxTotal() < this.maxConnections) + conman.setMaxTotal(this.maxConnections); + conman.setDefaultMaxPerRoute(this.maxConnections); } } private boolean gzipAccepted = false; @@ -247,12 +250,39 @@ public class WbmPersistLoadProcessor extends Processor { // statistics private AtomicLong loadedCount = new AtomicLong(); + /** + * number of times successfully loaded recrawl info. + * @return long + */ public long getLoadedCount() { return loadedCount.get(); } - private AtomicLong failedCount = new AtomicLong(); - public long getFailedCount() { - return failedCount.get(); + private AtomicLong missedCount = new AtomicLong(); + /** + * number of times getting no recrawl info. + * @return long + */ + public long getMissedCount() { + return missedCount.get(); + } + private AtomicLong errorCount = new AtomicLong(); + /** + * number of times cdx-server API call failed. + * @return long + */ + public long getErrorCount() { + return errorCount.get(); + } + + private AtomicLong cumulativeFetchTime = new AtomicLong(); + /** + * total milliseconds spent in API call. + * it is a sum of time waited for next available connection, + * and actual HTTP request-response round-trip, across all threads. + * @return + */ + public long getCumulativeFetchTime() { + return cumulativeFetchTime.get(); } public void setHttpClient(HttpClient client) { @@ -272,6 +302,7 @@ public class WbmPersistLoadProcessor extends Processor { if (client == null) { ThreadSafeClientConnManager conman = new ThreadSafeClientConnManager(); conman.setDefaultMaxPerRoute(maxConnections); + conman.setMaxTotal(Math.max(conman.getMaxTotal(), maxConnections)); final DefaultHttpClient client = new DefaultHttpClient(conman); HttpParams params = client.getParams(); params.setIntParameter(CoreConnectionPNames.SO_TIMEOUT, socketTimeout); @@ -365,7 +396,9 @@ public class WbmPersistLoadProcessor extends Processor { Thread.sleep(5000); } try { + long t0 = System.currentTimeMillis(); HttpResponse resp = getHttpClient().execute(m); + cumulativeFetchTime.addAndGet(System.currentTimeMillis() - t0); StatusLine sl = resp.getStatusLine(); if (sl.getStatusCode() != 200) { log.error("GET " + url + " failed with status=" + sl.getStatusCode() + " " + sl.getReasonPhrase()); @@ -375,8 +408,10 @@ public class WbmPersistLoadProcessor extends Processor { continue; } entity = resp.getEntity(); + } catch (IOException ex) { + log.error("GEt " + url + " failed with error " + ex.getMessage()); } catch (Exception ex) { - log.error("GET " + url + " failed with error " + ex.getMessage()); + log.error("GET " + url + " failed with error ", ex); } } while (entity == null && ++attempts < 3); if (entity == null) { @@ -392,7 +427,7 @@ public class WbmPersistLoadProcessor extends Processor { is = getCDX(curi.toString()); } catch (IOException ex) { log.error(ex.getMessage()); - failedCount.incrementAndGet(); + errorCount.incrementAndGet(); return ProcessResult.PROCEED; } Map info = null; @@ -411,7 +446,7 @@ public class WbmPersistLoadProcessor extends Processor { history.putAll(info); loadedCount.incrementAndGet(); } else { - failedCount.incrementAndGet(); + missedCount.incrementAndGet(); } return ProcessResult.PROCEED; } From 3450cf37e42fc60b8c3f9f37583436106727b261 Mon Sep 17 00:00:00 2001 From: Kenji Nagahashi Date: Sun, 24 Aug 2014 00:45:04 -0700 Subject: [PATCH 10/14] Update HQ modules and WBM-dedup module to match Heritrix 3.3.0. HQ modules: for method removed in Heritrix, API deprecated in httpclient WBM-dedup: fix NPE in FetchHistoryProcessor due to changed expectation. --- .../recrawl/wbm/WbmPersistLoadProcessor.java | 118 ++++++++++-------- .../wbm/WbmPersistLoadProcessorTest.java | 42 ++++++- 2 files changed, 102 insertions(+), 58 deletions(-) diff --git a/main/java/org/archive/modules/recrawl/wbm/WbmPersistLoadProcessor.java b/main/java/org/archive/modules/recrawl/wbm/WbmPersistLoadProcessor.java index 5ed5e7b2..4b55ee31 100644 --- a/main/java/org/archive/modules/recrawl/wbm/WbmPersistLoadProcessor.java +++ b/main/java/org/archive/modules/recrawl/wbm/WbmPersistLoadProcessor.java @@ -30,32 +30,24 @@ import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Map.Entry; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicLong; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.http.Header; import org.apache.http.HeaderElement; import org.apache.http.HttpEntity; -import org.apache.http.HttpException; -import org.apache.http.HttpRequest; -import org.apache.http.HttpRequestInterceptor; import org.apache.http.HttpResponse; -import org.apache.http.HttpResponseInterceptor; import org.apache.http.StatusLine; import org.apache.http.client.HttpClient; +import org.apache.http.client.config.RequestConfig; import org.apache.http.client.methods.HttpGet; -import org.apache.http.impl.client.DefaultHttpClient; -import org.apache.http.impl.conn.tsccm.ThreadSafeClientConnManager; -import org.apache.http.params.CoreConnectionPNames; -import org.apache.http.params.HttpParams; -import org.apache.http.protocol.HttpContext; +import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; +import org.archive.modules.CoreAttributeConstants; import org.archive.modules.CrawlURI; import org.archive.modules.ProcessResult; import org.archive.modules.Processor; -import org.archive.modules.hq.GzipInflatingHttpEntityWrapper; import org.archive.modules.recrawl.FetchHistoryHelper; import org.archive.modules.recrawl.RecrawlAttributeConstants; import org.archive.util.ArchiveUtils; @@ -76,6 +68,7 @@ public class WbmPersistLoadProcessor extends Processor { private static final Log log = LogFactory.getLog(WbmPersistLoadProcessor.class); private HttpClient client; + private PoolingHttpClientConnectionManager conman; private int historyLength = 2; @@ -90,7 +83,7 @@ public class WbmPersistLoadProcessor extends Processor { //private String queryURL = "http://web-beta.archive.org/cdx/search"; //private String queryURL = "http://web.archive.org/cdx/search/cdx"; // ~Sep 26, 2013 - private String queryURL = "http://wwwb-front2.us.archive.org:8083/web/timemap/cdx?url=$u&limit=-1"; + private String queryURL = "http://wwwb-dedup.us.archive.org:8083/web/timemap/cdx?url=$u&limit=-1"; public void setQueryURL(String queryURL) { this.queryURL = queryURL; prepareQueryURL(); @@ -196,11 +189,9 @@ public class WbmPersistLoadProcessor extends Processor { public int getMaxConnections() { return maxConnections; } - public void setMaxConnections(int maxConnections) { + public synchronized void setMaxConnections(int maxConnections) { this.maxConnections = maxConnections; - if (client != null) { - ThreadSafeClientConnManager conman = - (ThreadSafeClientConnManager)client.getConnectionManager(); + if (conman != null) { if (conman.getMaxTotal() < this.maxConnections) conman.setMaxTotal(this.maxConnections); conman.setDefaultMaxPerRoute(this.maxConnections); @@ -300,41 +291,57 @@ public class WbmPersistLoadProcessor extends Processor { } public synchronized HttpClient getHttpClient() { if (client == null) { - ThreadSafeClientConnManager conman = new ThreadSafeClientConnManager(); - conman.setDefaultMaxPerRoute(maxConnections); - conman.setMaxTotal(Math.max(conman.getMaxTotal(), maxConnections)); - final DefaultHttpClient client = new DefaultHttpClient(conman); - HttpParams params = client.getParams(); - params.setIntParameter(CoreConnectionPNames.SO_TIMEOUT, socketTimeout); - params.setIntParameter(CoreConnectionPNames.CONNECTION_TIMEOUT, connectionTimeout); + if (conman == null) { + conman = new PoolingHttpClientConnectionManager(); + conman.setDefaultMaxPerRoute(maxConnections); + conman.setMaxTotal(Math.max(conman.getMaxTotal(), maxConnections)); + } + HttpClientBuilder builder = HttpClientBuilder.create() + .disableCookieManagement() + .setConnectionManager(conman); + builder.useSystemProperties(); + // config code for older version of httpclient. +// builder.setDefaultSocketConfig(SocketConfig.custom().setSoTimeout(socketTimeout).build()); +// HttpParams params = client.getParams(); +// params.setIntParameter(CoreConnectionPNames.SO_TIMEOUT, socketTimeout); +// params.setIntParameter(CoreConnectionPNames.CONNECTION_TIMEOUT, connectionTimeout); // setup request/response intercepter for handling gzip-compressed response. - client.addRequestInterceptor(new HttpRequestInterceptor() { - @Override - public void process(final HttpRequest request, final HttpContext context) - throws HttpException, IOException { - if (gzipAccepted && !request.containsHeader("Accept-Encoding")) { - request.addHeader("Accept-Encoding", "gzip"); - } - // add extra headers configured. - if (requestHeaders != null) { - for (Entry ent : requestHeaders.entrySet()) { - request.addHeader(ent.getKey(), ent.getValue()); - } - } - } - }); - client.addResponseInterceptor(new HttpResponseInterceptor() { - @Override - public void process(final HttpResponse response, final HttpContext context) - throws HttpException, IOException { - HttpEntity entity = response.getEntity(); - Header ceheader = entity.getContentEncoding(); - if (ceheader != null && contains(ceheader.getElements(), "gzip")) { - response.setEntity(new GzipInflatingHttpEntityWrapper(response.getEntity())); - } - } - }); - this.client = client; + // Disabled because httpclient 4.3.3 sends "Accept-Encoding: gzip,deflate" by + // default. Response parsing will fail If gzip-decompression ResponseInterceptor + // is installed. +// builder.addInterceptorLast(new HttpRequestInterceptor() { +// @Override +// public void process(final HttpRequest request, final HttpContext context) +// throws HttpException, IOException { +// System.err.println("RequestInterceptor"); +// if (request.containsHeader("Accept-Encoding")) { +// System.err.println("already has Accept-Encoding: " + request.getHeaders("Accept-Encoding")[0]); +// } +// if (gzipAccepted) { +// if (!request.containsHeader("Accept-Encoding")) { +// request.addHeader("Accept-Encoding", "gzip"); +// } +// } +// // add extra headers configured. +// if (requestHeaders != null) { +// for (Entry ent : requestHeaders.entrySet()) { +// request.addHeader(ent.getKey(), ent.getValue()); +// } +// } +// } +// }); +// builder.addInterceptorFirst(new HttpResponseInterceptor() { +// @Override +// public void process(final HttpResponse response, final HttpContext context) +// throws HttpException, IOException { +// HttpEntity entity = response.getEntity(); +// Header ceheader = entity.getContentEncoding(); +// if (ceheader != null && contains(ceheader.getElements(), "gzip")) { +// response.setEntity(new GzipInflatingHttpEntityWrapper(response.getEntity())); +// } +// } +// }); + this.client = builder.build(); } return client; } @@ -387,6 +394,8 @@ public class WbmPersistLoadProcessor extends Processor { protected InputStream getCDX(String qurl) throws InterruptedException, IOException { final String url = buildURL(qurl); HttpGet m = new HttpGet(url); + m.setConfig(RequestConfig.custom().setConnectTimeout(connectionTimeout) + .setSocketTimeout(socketTimeout).build()); HttpEntity entity = null; int attempts = 0; do { @@ -501,7 +510,12 @@ public class WbmPersistLoadProcessor extends Processor { } if (tsbuffer.remaining() == 0) { try { - info.put(FetchHistoryHelper.A_TIMESTAMP, DateUtils.parse14DigitDate(new String(tsbuffer.array())).getTime()); + long ts = DateUtils.parse14DigitDate(new String(tsbuffer.array())).getTime(); + // A_TIMESTAMP has been used for sorting history long before A_FETCH_BEGAN_TIME + // field was introduced. Now FetchHistoryProcessor fails if A_FETCH_BEGAN_TIME is + // not set. We could stop storing A_TIMESTAMP and sort by A_FETCH_BEGAN_TIME. + info.put(FetchHistoryHelper.A_TIMESTAMP, ts); + info.put(CoreAttributeConstants.A_FETCH_BEGAN_TIME, ts); } catch (ParseException ex) { } } diff --git a/test/java/org/archive/modules/recrawl/wbm/WbmPersistLoadProcessorTest.java b/test/java/org/archive/modules/recrawl/wbm/WbmPersistLoadProcessorTest.java index 6c095f38..36d6b900 100644 --- a/test/java/org/archive/modules/recrawl/wbm/WbmPersistLoadProcessorTest.java +++ b/test/java/org/archive/modules/recrawl/wbm/WbmPersistLoadProcessorTest.java @@ -8,6 +8,8 @@ import java.io.BufferedReader; import java.io.InputStream; import java.io.InputStreamReader; import java.net.URLEncoder; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; import java.util.HashMap; import java.util.Map; import java.util.concurrent.ExecutorService; @@ -22,11 +24,14 @@ import org.apache.http.client.methods.HttpUriRequest; import org.apache.http.entity.ByteArrayEntity; import org.apache.http.message.BasicHttpResponse; import org.apache.http.message.BasicStatusLine; +import org.archive.modules.CoreAttributeConstants; import org.archive.modules.CrawlURI; import org.archive.modules.ProcessResult; import org.archive.modules.recrawl.FetchHistoryHelper; +import org.archive.modules.recrawl.FetchHistoryProcessor; import org.archive.modules.recrawl.RecrawlAttributeConstants; import org.archive.net.UURIFactory; +import org.archive.util.Base32; import org.archive.util.DateUtils; import org.easymock.EasyMock; import org.junit.Test; @@ -87,6 +92,11 @@ public class WbmPersistLoadProcessorTest { return history; } + private byte[] sha1Digest(String text) throws NoSuchAlgorithmException { + MessageDigest md = MessageDigest.getInstance("sha1"); + return md.digest(text.getBytes()); + } + @Test public void testInnerProcessResultSingleShotWithMock() throws Exception { // because AbstractHttpClient marks most execute(...) methods final, it is very tiresome to implement @@ -102,17 +112,25 @@ public class WbmPersistLoadProcessorTest { t.setContentDigestScheme(CONTENT_DIGEST_SCHEME); CrawlURI curi = new CrawlURI(UURIFactory.getInstance("http://archive.org/")); - // put history entry newer than being loaded + // put history entry newer than being loaded (i.e. loaded history entry will not be used for FetchHistoryProcessor + // check below. long expected_ts = DateUtils.parse14DigitDate(TestNormalHttpResponse.EXPECTED_TS).getTime(); Map[] fetchHistory = (Map[])curi.getData().get(RecrawlAttributeConstants.A_FETCH_HISTORY); if (fetchHistory == null) { fetchHistory = new HashMap[2]; curi.getData().put(RecrawlAttributeConstants.A_FETCH_HISTORY, fetchHistory); } + final byte[] digestValue0 = sha1Digest("0"); + final byte[] digestValue1 = sha1Digest("1"); fetchHistory[0] = new HashMap(); fetchHistory[0].put(FetchHistoryHelper.A_TIMESTAMP, expected_ts + 2000); + fetchHistory[0].put(CoreAttributeConstants.A_FETCH_BEGAN_TIME, expected_ts + 2000); + fetchHistory[0].put(RecrawlAttributeConstants.A_CONTENT_DIGEST, + CONTENT_DIGEST_SCHEME + Base32.encode(digestValue0)); fetchHistory[1] = new HashMap(); fetchHistory[1].put(FetchHistoryHelper.A_TIMESTAMP, expected_ts - 2000); + fetchHistory[1].put(RecrawlAttributeConstants.A_CONTENT_DIGEST, + CONTENT_DIGEST_SCHEME + Base32.encode(digestValue1)); ProcessResult result = t.innerProcessResult(curi); assertEquals("result is PROCEED", ProcessResult.PROCEED, result); @@ -126,6 +144,18 @@ public class WbmPersistLoadProcessorTest { Long ts = (Long)history.get(FetchHistoryHelper.A_TIMESTAMP); assertNotNull("ts is non-null", ts); assertEquals("'ts' has expected timestamp", expected_ts, ts.longValue()); + + // Check compatibility with FetchHistoryProcessor. + // TODO: This is not testing WbmPersistLoadProcessor - only testing stub fetchHistory + // setup above (OK as long as it matches WbmPersistLoadProcessor). We need a separate + // test method. + curi.setFetchStatus(200); + curi.setFetchBeginTime(System.currentTimeMillis()); + // FetchHistoryProcessor once failed for a revisit case. We'd need to test other cases + // too (TODO). + curi.setContentDigest("sha1", digestValue0); + FetchHistoryProcessor fhp = new FetchHistoryProcessor(); + fhp.process(curi); } @Test @@ -154,11 +184,11 @@ public class WbmPersistLoadProcessorTest { @Override public void run() { try { - CrawlURI curi = new CrawlURI(UURIFactory.getInstance(this.uri)); - p.innerProcessResult(curi); - //System.err.println(curi.toString()); + CrawlURI curi = new CrawlURI(UURIFactory.getInstance(this.uri)); + p.innerProcessResult(curi); + //System.err.println(curi.toString()); } catch (Exception ex) { - ex.printStackTrace(); + ex.printStackTrace(); } } } @@ -182,7 +212,7 @@ public class WbmPersistLoadProcessorTest { nurls++; } long t0 = System.currentTimeMillis(); - tasks.run(); + tasks.execute(); executor.awaitTermination(30, TimeUnit.SECONDS); long el = System.currentTimeMillis() - t0; System.err.println(nurls + " urls, time=" + el + "ms (" + (nurls / (el / 1000.0)) + " URI/s"); From d88e19716b1acdd62baa4be5caf55aea0718ad7e Mon Sep 17 00:00:00 2001 From: Kenji Nagahashi Date: Mon, 25 Aug 2014 15:37:22 -0700 Subject: [PATCH 11/14] move WbmPersistLoadProcessor to final location in the tree --- .../org/archive/modules/recrawl/wbm/WbmPersistLoadProcessor.java | 0 .../archive/modules/recrawl/wbm/WbmPersistLoadProcessorTest.java | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename {main => contrib/src/main}/java/org/archive/modules/recrawl/wbm/WbmPersistLoadProcessor.java (100%) rename {test => contrib/src/test}/java/org/archive/modules/recrawl/wbm/WbmPersistLoadProcessorTest.java (100%) diff --git a/main/java/org/archive/modules/recrawl/wbm/WbmPersistLoadProcessor.java b/contrib/src/main/java/org/archive/modules/recrawl/wbm/WbmPersistLoadProcessor.java similarity index 100% rename from main/java/org/archive/modules/recrawl/wbm/WbmPersistLoadProcessor.java rename to contrib/src/main/java/org/archive/modules/recrawl/wbm/WbmPersistLoadProcessor.java diff --git a/test/java/org/archive/modules/recrawl/wbm/WbmPersistLoadProcessorTest.java b/contrib/src/test/java/org/archive/modules/recrawl/wbm/WbmPersistLoadProcessorTest.java similarity index 100% rename from test/java/org/archive/modules/recrawl/wbm/WbmPersistLoadProcessorTest.java rename to contrib/src/test/java/org/archive/modules/recrawl/wbm/WbmPersistLoadProcessorTest.java From bb511796ec658ed07f69ca9895ce2e1994569826 Mon Sep 17 00:00:00 2001 From: Kenji Nagahashi Date: Mon, 25 Aug 2014 16:44:48 -0700 Subject: [PATCH 12/14] =?UTF-8?q?Update=20POMs=20for=20newly=20imported=20?= =?UTF-8?q?Wayback-dedup=20module.=20=09lock=20httpclient=20version=20to?= =?UTF-8?q?=204.3.3=20stated=20by=20=09heritrix-commons.=20have=20to=20exc?= =?UTF-8?q?lude=20hadoop=E2=80=99s=20=09dependency=20on=20jets3t=200.9.0?= =?UTF-8?q?=20as=20it=20pulls=20in=09=09httpcore=204.1.2.=20=09add=20easy?= =?UTF-8?q?=20mock=203.1=20to=20heritrix-contrib.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- commons/pom.xml | 1 - contrib/pom.xml | 12 ++++++++++++ pom.xml | 6 ++++++ 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/commons/pom.xml b/commons/pom.xml index 66b60e88..b4d2244c 100644 --- a/commons/pom.xml +++ b/commons/pom.xml @@ -26,7 +26,6 @@ org.apache.httpcomponents httpclient - 4.3.3 com.sleepycat diff --git a/contrib/pom.xml b/contrib/pom.xml index 23e02519..9ecbdffd 100644 --- a/contrib/pom.xml +++ b/contrib/pom.xml @@ -18,6 +18,12 @@ org.apache.hbase hbase-client 0.96.1.1-cdh5.0.2 + + + jets3t + net.java.dev.jets3t + + org.archive.heritrix @@ -41,6 +47,12 @@ itextpdf 5.5.0 + + org.easymock + easymock + 3.1 + test + diff --git a/pom.xml b/pom.xml index e67d61b1..ddb2f121 100644 --- a/pom.xml +++ b/pom.xml @@ -118,6 +118,12 @@ http://maven.apache.org/guides/mini/guide-m1-m2.html engine ${project.version} + + + org.apache.httpcomponents + httpclient + 4.3.3 + From fc3817518268a885f439840025971cfddd94c69d Mon Sep 17 00:00:00 2001 From: Kenji Nagahashi Date: Mon, 25 Aug 2014 17:05:50 -0700 Subject: [PATCH 13/14] FIX: WbmPersistLoadProcessor.requestHeaders are ignored. (code commented out during update for httpclient 4.3.3) --- .../recrawl/wbm/WbmPersistLoadProcessor.java | 30 +++++++++++-------- 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/contrib/src/main/java/org/archive/modules/recrawl/wbm/WbmPersistLoadProcessor.java b/contrib/src/main/java/org/archive/modules/recrawl/wbm/WbmPersistLoadProcessor.java index 4b55ee31..ad6cf7ab 100644 --- a/contrib/src/main/java/org/archive/modules/recrawl/wbm/WbmPersistLoadProcessor.java +++ b/contrib/src/main/java/org/archive/modules/recrawl/wbm/WbmPersistLoadProcessor.java @@ -30,6 +30,7 @@ import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicLong; @@ -37,6 +38,8 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.http.HeaderElement; import org.apache.http.HttpEntity; +import org.apache.http.HttpRequest; +import org.apache.http.HttpRequestInterceptor; import org.apache.http.HttpResponse; import org.apache.http.StatusLine; import org.apache.http.client.HttpClient; @@ -44,6 +47,7 @@ import org.apache.http.client.config.RequestConfig; import org.apache.http.client.methods.HttpGet; import org.apache.http.impl.client.HttpClientBuilder; import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; +import org.apache.http.protocol.HttpContext; import org.archive.modules.CoreAttributeConstants; import org.archive.modules.CrawlURI; import org.archive.modules.ProcessResult; @@ -300,6 +304,9 @@ public class WbmPersistLoadProcessor extends Processor { .disableCookieManagement() .setConnectionManager(conman); builder.useSystemProperties(); + // TODO: use setDefaultHeaders for adding requestHeaders? It's a bit painful + // because we need to convert it to a Collection of Header objects. + // config code for older version of httpclient. // builder.setDefaultSocketConfig(SocketConfig.custom().setSoTimeout(socketTimeout).build()); // HttpParams params = client.getParams(); @@ -309,10 +316,9 @@ public class WbmPersistLoadProcessor extends Processor { // Disabled because httpclient 4.3.3 sends "Accept-Encoding: gzip,deflate" by // default. Response parsing will fail If gzip-decompression ResponseInterceptor // is installed. -// builder.addInterceptorLast(new HttpRequestInterceptor() { -// @Override -// public void process(final HttpRequest request, final HttpContext context) -// throws HttpException, IOException { + builder.addInterceptorLast(new HttpRequestInterceptor() { + @Override + public void process(final HttpRequest request, final HttpContext context) { // System.err.println("RequestInterceptor"); // if (request.containsHeader("Accept-Encoding")) { // System.err.println("already has Accept-Encoding: " + request.getHeaders("Accept-Encoding")[0]); @@ -322,14 +328,14 @@ public class WbmPersistLoadProcessor extends Processor { // request.addHeader("Accept-Encoding", "gzip"); // } // } -// // add extra headers configured. -// if (requestHeaders != null) { -// for (Entry ent : requestHeaders.entrySet()) { -// request.addHeader(ent.getKey(), ent.getValue()); -// } -// } -// } -// }); + // add extra headers configured. + if (requestHeaders != null) { + for (Entry ent : requestHeaders.entrySet()) { + request.addHeader(ent.getKey(), ent.getValue()); + } + } + } + }); // builder.addInterceptorFirst(new HttpResponseInterceptor() { // @Override // public void process(final HttpResponse response, final HttpContext context) From f343ce2ae1124cd8563772836301932e221a20ee Mon Sep 17 00:00:00 2001 From: Kenji Nagahashi Date: Tue, 26 Aug 2014 10:56:35 -0700 Subject: [PATCH 14/14] suppress JUnit 4 dependency through hbase-client. rewrite WbmPersistLoadProcessorTest for JUnit 3. (also disable a test that depends on production CDX server) --- contrib/pom.xml | 4 +++ .../wbm/WbmPersistLoadProcessorTest.java | 27 +++++++++---------- 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/contrib/pom.xml b/contrib/pom.xml index 9ecbdffd..41d36ae0 100644 --- a/contrib/pom.xml +++ b/contrib/pom.xml @@ -23,6 +23,10 @@ jets3t net.java.dev.jets3t + + junit + junit + diff --git a/contrib/src/test/java/org/archive/modules/recrawl/wbm/WbmPersistLoadProcessorTest.java b/contrib/src/test/java/org/archive/modules/recrawl/wbm/WbmPersistLoadProcessorTest.java index 36d6b900..88d1c0cf 100644 --- a/contrib/src/test/java/org/archive/modules/recrawl/wbm/WbmPersistLoadProcessorTest.java +++ b/contrib/src/test/java/org/archive/modules/recrawl/wbm/WbmPersistLoadProcessorTest.java @@ -1,9 +1,5 @@ package org.archive.modules.recrawl.wbm; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; - import java.io.BufferedReader; import java.io.InputStream; import java.io.InputStreamReader; @@ -17,6 +13,8 @@ import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.regex.Pattern; +import junit.framework.TestCase; + import org.apache.http.HttpResponse; import org.apache.http.ProtocolVersion; import org.apache.http.client.HttpClient; @@ -34,13 +32,11 @@ import org.archive.net.UURIFactory; import org.archive.util.Base32; import org.archive.util.DateUtils; import org.easymock.EasyMock; -import org.junit.Test; import com.google.common.util.concurrent.ExecutionList; /** * unit test for {@link WbmPersistLoadProcessor}. - * depends on WBM index query API. * * TODO: *
    @@ -50,8 +46,8 @@ import com.google.common.util.concurrent.ExecutionList; * @author kenji * */ -public class WbmPersistLoadProcessorTest { - @Test +public class WbmPersistLoadProcessorTest extends TestCase { + public void testBuildURL() throws Exception { WbmPersistLoadProcessor t = new WbmPersistLoadProcessor(); t.setQueryURL("http://web.archive.org/cdx/search/cdx?url=$u&startDate=$s&limit=1"); @@ -97,8 +93,11 @@ public class WbmPersistLoadProcessorTest { return md.digest(text.getBytes()); } - @Test - public void testInnerProcessResultSingleShotWithMock() throws Exception { + /** + * this test is disabled because it talks to production CDX server. + * @throws Exception + */ + public void _testInnerProcessResultSingleShotWithMock() throws Exception { // because AbstractHttpClient marks most execute(...) methods final, it is very tiresome to implement // a stub by implementing HttpClient interface. So I use EasyMock. HttpClient client = EasyMock.createMock(HttpClient.class); @@ -158,7 +157,6 @@ public class WbmPersistLoadProcessorTest { fhp.process(curi); } - @Test public void testInnerProcessResultSingleShotWithRealServer() throws Exception { WbmPersistLoadProcessor t = new WbmPersistLoadProcessor(); //CrawlURI curi = new CrawlURI(UURIFactory.getInstance("http://archive.org/")); @@ -195,11 +193,10 @@ public class WbmPersistLoadProcessorTest { /** * test for performance. - * not annotated as test case. run this through main(). + * not named as test case. run this through main(). * @throws Exception */ - //@Test - public void testInnerProcessResultMany() throws Exception { + public void measureInnerProcessResultMany() throws Exception { final WbmPersistLoadProcessor t = new WbmPersistLoadProcessor(); InputStream is = getClass().getResourceAsStream("/test-url-list.txt"); BufferedReader br = new BufferedReader(new InputStreamReader(is)); @@ -219,6 +216,6 @@ public class WbmPersistLoadProcessorTest { } public static void main() throws Exception { - (new WbmPersistLoadProcessorTest()).testInnerProcessResultMany(); + (new WbmPersistLoadProcessorTest()).measureInnerProcessResultMany(); } }