diff --git a/commons/pom.xml b/commons/pom.xml index 66b60e88..b4d2244c 100644 --- a/commons/pom.xml +++ b/commons/pom.xml @@ -26,7 +26,6 @@ org.apache.httpcomponents httpclient - 4.3.3 com.sleepycat diff --git a/commons/src/main/java/org/archive/util/Transform.java b/commons/src/main/java/org/archive/util/Transform.java deleted file mode 100644 index c066dbf5..00000000 --- a/commons/src/main/java/org/archive/util/Transform.java +++ /dev/null @@ -1,154 +0,0 @@ -/* - * This file is part of the Heritrix web crawler (crawler.archive.org). - * - * Licensed to the Internet Archive (IA) by one or more individual - * contributors. - * - * The IA licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.archive.util; - -import java.util.AbstractCollection; -import java.util.Collection; -import java.util.Iterator; - - -/** - * A transformation of a collection. The elements in the transform are based - * on the elements of some other collection; the original collection's - * elements are transformed using a specified transformer. Changes to the - * original collection are automatically reflected in the transform and - * vice-versa. - * - *

If the transformer returns null for a given original object, then that - * object will not be included in the transform. Thus the transform might - * be smaller than the original collection. Note that Transform instances - * can never contain the null element. - * - *

This collection implementation does not support the optional add - * operation. - * - * @author pjack - * - * @param the type of the original elements in the collection - * @param the type of the tranformed elements - */ -public class Transform -extends AbstractCollection { - - /** The original collection. */ - final private Collection delegate; - - /** Transforms the original objects. */ - final private Transformer transformer; - - /** - * Constructor. - * - * @param delegate The collection whose elements to transform. - * @param transformer Transforms the elements - */ - public Transform(Collection delegate, - Transformer transformer) { - this.delegate = delegate; - this.transformer = transformer; - } - - public int size() { - int count = 0; - Iterator iter = iterator(); - while (iter.hasNext()) { - iter.next(); - count++; - } - return count; - } - - public Iterator iterator() { - return new TransformIterator( - delegate.iterator(), transformer); - } - - - /** - * Returns a transform containing only objects of a given class. - * - * @param the target class - * @param c the collection to transform - * @param cls the class of objects to return - * @return a collection containing only objects of class cls - */ - public static Collection subclasses( - Collection c, - final Class cls) { - Transformer t = new Transformer() { - public Target transform(Object s) { - if (cls.isInstance(s)) { - return cls.cast(s); - } else { - return null; - } - } - }; - return new Transform(c, t); - } -} - - -class TransformIterator implements Iterator { - - final private Iterator iterator; - final private Transformer transformer; - private Transformed next; - - public TransformIterator(Iterator iterator, - Transformer transformer) { - this.iterator = iterator; - this.transformer = transformer; - } - - public boolean hasNext() { - if (next != null) { - return true; - } - while (iterator.hasNext()) { - Original o = iterator.next(); - next = transformer.transform(o); - if (next != null) { - return true; - } - } - return false; - } - - public Transformed next() { - if (!hasNext()) { - throw new IllegalStateException(); - } - Transformed r = next; - next = null; - return r; - } - - // FIXME: this can break standard Iterator contract, for example - // transformIterator.next(); - // if(transformIterator.hasNext()) { - // transformIterator.remove(); - // } - // usual iterator contract is to remove the last object returned - // by next; in this case the subsequent - public void remove() { - iterator.remove(); - } - -} diff --git a/commons/src/main/java/org/archive/util/Transformer.java b/commons/src/main/java/org/archive/util/Transformer.java deleted file mode 100644 index 4c36b649..00000000 --- a/commons/src/main/java/org/archive/util/Transformer.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * This file is part of the Heritrix web crawler (crawler.archive.org). - * - * Licensed to the Internet Archive (IA) by one or more individual - * contributors. - * - * The IA licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.archive.util; - - -/** - * Transforms objects from one thing into another. - * - * @author pjack - * - * @param the type of the original objects - * @param the type of the transformed objects - */ -public interface Transformer { - - /** - * Transforms the given object. - * - * @param o the object to transform - * @return the transformed object - */ - Transformed transform(Original o); - -} diff --git a/commons/src/test/java/org/archive/util/TransformTest.java b/commons/src/test/java/org/archive/util/TransformTest.java deleted file mode 100644 index 51641fcf..00000000 --- a/commons/src/test/java/org/archive/util/TransformTest.java +++ /dev/null @@ -1,126 +0,0 @@ -package org.archive.util; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.HashSet; -import java.util.List; -import java.util.Random; -import java.util.Set; - -import junit.framework.TestCase; - -/** - * Tests the {@link dex.misc.Transform} class. - */ -public class TransformTest extends TestCase { - - // Convert integers to strings, eliminating negative numbers - private static class PositiveToString - implements Transformer { - public String transform(Integer i) { - if (i < 0) { - return null; - } - return i.toString(); - } - } - - /** - * Tests using a simple Transformer. The Transformer changes - * positive integers into strings. The test sets up a - * list of random integers, remembering which ones are - * positive. The Transform is created, and the Transform's - * contents are compared against the list of remembered positive - * integers. - */ - public void testTransform() { - Transformer transformer = new PositiveToString(); - - // Transform of an empty collection should be empty. - List empty = new ArrayList(); - assertTrue(new Transform(empty, transformer).isEmpty()); - - // Some simple test data. - Integer[] testData = new Integer[] { -5, 3, 2, -11, 0, 111, -161 }; - String[] expected = new String[] { "3", "2", "0", "111" }; - List list = Arrays.asList(testData); - Transform c = new Transform(list, - transformer); - List expectedList = Arrays.asList(expected); - assertEquals(new ArrayList(c), expectedList); - - // Same test as above, with random data - for (int i = 0; i < 100; i++) { - randomTest(); - } - } - - private void randomTest() { - Transformer transformer = new PositiveToString(); - Random random = new Random(); - int max = random.nextInt(1024) + 10; - List testData = new ArrayList(max); - List expected = new ArrayList(max); - for (int i = 0; i < max; i++) { - int e = random.nextInt(); - testData.add(e); - if (e >= 0) { - expected.add(Integer.toString(e)); - } - } - - Transform c = new Transform(testData, - transformer); - List results = new ArrayList(c); - assertEquals(expected, results); - } - - /** - * Tests the static subclasses method. The test sets up a list of - * Number instances that may contain random Double, Float, Integer - * or Long values. The Long values are remembered. The subclasses - * method is used to create a Transform containing only the Long - * values. The Transform is compared against the list of remembered - * Long values. - */ - public void testSubclasses() { - Random random = new Random(); - for (int i = 0; i < 100; i++) { - int max = random.nextInt(1024) + 10; - List testData = new ArrayList(max); - List expected = new ArrayList(max); - for (int j = 0; j < max; j++) { - int v = random.nextInt(4); - switch (v) { - case 0: - long l = random.nextLong(); - testData.add(l); - expected.add(l); - break; - case 1: - testData.add(random.nextInt()); - break; - case 2: - testData.add(random.nextDouble()); - break; - case 3: - testData.add(random.nextFloat()); - break; - } - } - Collection c = Transform.subclasses(testData, Long.class); - List results = new ArrayList(c); - assertEquals(expected, results); - } - } - - public void testSingleton() { - Set set = new HashSet(); - set.add(3); - Collection c = Transform.subclasses(set, Integer.class); - for (Integer i : c) { - System.out.println(i); - } - } -} diff --git a/contrib/pom.xml b/contrib/pom.xml index 23e02519..41d36ae0 100644 --- a/contrib/pom.xml +++ b/contrib/pom.xml @@ -18,6 +18,16 @@ org.apache.hbase hbase-client 0.96.1.1-cdh5.0.2 + + + jets3t + net.java.dev.jets3t + + + junit + junit + + org.archive.heritrix @@ -41,6 +51,12 @@ itextpdf 5.5.0 + + org.easymock + easymock + 3.1 + test + diff --git a/contrib/src/main/java/org/archive/crawler/frontier/AMQPUrlReceiver.java b/contrib/src/main/java/org/archive/crawler/frontier/AMQPUrlReceiver.java index c25d78d1..e8c23f41 100644 --- a/contrib/src/main/java/org/archive/crawler/frontier/AMQPUrlReceiver.java +++ b/contrib/src/main/java/org/archive/crawler/frontier/AMQPUrlReceiver.java @@ -320,7 +320,7 @@ public class AMQPUrlReceiver implements Lifecycle, ApplicationListenerCurrent interface:

+ *

http://web-beta.archive.org/cdx/search/cdx?url=archive.org&startDate=1999 will return raw + * CDX lines for archive.org, since 1999-01-01 00:00:00. + *

+ *

As index is updated in a separate batch processing job, there's no "Store" counterpart.

+ * @contributor Kenji Nagahashi. + */ +public class WbmPersistLoadProcessor extends Processor { + private static final Log log = LogFactory.getLog(WbmPersistLoadProcessor.class); + + private HttpClient client; + private PoolingHttpClientConnectionManager conman; + + private int historyLength = 2; + + public void setHistoryLength(int historyLength) { + this.historyLength = historyLength; + } + public int getHistoryLength() { + return historyLength; + } + + // ~Jan 2, 2013 + //private String queryURL = "http://web-beta.archive.org/cdx/search"; + //private String queryURL = "http://web.archive.org/cdx/search/cdx"; + // ~Sep 26, 2013 + private String queryURL = "http://wwwb-dedup.us.archive.org:8083/web/timemap/cdx?url=$u&limit=-1"; + public void setQueryURL(String queryURL) { + this.queryURL = queryURL; + prepareQueryURL(); + } + public String getQueryURL() { + return queryURL; + } + + public interface FormatSegment { + void print(StringBuilder sb, String[] args); + } + private static class StaticSegment implements FormatSegment { + String s; + public StaticSegment(String s) { + this.s = s; + } + public void print(StringBuilder sb, String[] args) { + sb.append(s); + } + } + private static class InterpolateSegment implements FormatSegment { + int aidx; + public InterpolateSegment(int aidx) { + this.aidx = aidx; + } + public void print(StringBuilder sb, String[] args) { + sb.append(args[aidx]); + } + } + private FormatSegment[] preparedQueryURL; + + /** + * pre-scan queryURL template so that actual queryURL can be built + * with minimal processing. + */ + private void prepareQueryURL() { + List segments = new ArrayList(); + final int l = queryURL.length(); + int p = 0; + int q; + while (p < l && (q = queryURL.indexOf('$', p)) >= 0) { + if (q + 2 > l) { + // '$' at the end. keep it as if it were '$$' + break; + } + if (q > p) { + segments.add(new StaticSegment(queryURL.substring(p, q))); + } + char c = queryURL.charAt(q + 1); + if (c == 'u') { + segments.add(new InterpolateSegment(0)); + } else if (c == 's') { + segments.add(new InterpolateSegment(1)); + } else { + // copy '$'-sequence so that it's easy to spot errors + segments.add(new StaticSegment(queryURL.substring(q, q + 2))); + } + p = q + 2; + } + if (p < l) { + segments.add(new StaticSegment(queryURL.substring(p))); + } + preparedQueryURL = segments.toArray(new FormatSegment[segments.size()]); + } + + private String contentDigestScheme = "sha1:"; + /** + * set Content-Digest scheme string to prepend to the hash string found in CDX. + * Heritrix's Content-Digest comparison including this part. + * {@code "sha1:"} by default. + * @param contentDigestScheme + */ + public void setContentDigestScheme(String contentDigestScheme) { + this.contentDigestScheme = contentDigestScheme; + } + public String getContentDigestScheme() { + return contentDigestScheme; + } + private int socketTimeout = 10000; + /** + * socket timeout (SO_TIMEOUT) for HTTP client in milliseconds. + */ + public void setSocketTimeout(int socketTimeout) { + this.socketTimeout = socketTimeout; + } + public int getSocketTimeout() { + return socketTimeout; + } + + private int connectionTimeout = 10000; + /** + * connection timeout for HTTP client in milliseconds. + * @param connectionTimeout + */ + public void setConnectionTimeout(int connectionTimeout) { + this.connectionTimeout = connectionTimeout; + } + public int getConnectionTimeout() { + return connectionTimeout; + } + + private int maxConnections = 10; + public int getMaxConnections() { + return maxConnections; + } + public synchronized void setMaxConnections(int maxConnections) { + this.maxConnections = maxConnections; + if (conman != null) { + if (conman.getMaxTotal() < this.maxConnections) + conman.setMaxTotal(this.maxConnections); + conman.setDefaultMaxPerRoute(this.maxConnections); + } + } + private boolean gzipAccepted = false; + public boolean isGzipAccepted() { + return gzipAccepted; + } + /** + * if set to true, {@link WbmPersistLoadProcessor} adds a header + * {@code Accept-Encoding: gzip} to HTTP requests. New CDX server + * see this header to decide whether to compress the response. it is also + * possible to override gzipAccepted=true setting with gzip=false + * request parameter. + * It is off by default, as it should make little sense to compress single + * line of CDX. + * @param gzipAccepted true to allow gzip compression. + */ + public void setGzipAccepted(boolean gzipAccepted) { + this.gzipAccepted = gzipAccepted; + } + + private Map requestHeaders = new ConcurrentHashMap(1, 0.75f, 2); + public Map getRequestHeaders() { + return requestHeaders; + } + /** + * all key-value pairs in this map will be added as HTTP headers. + * typically used for providing authentication cookies. this method + * makes a copy of {@requestHeaders}. + * note: this property may be dropped in the future if + * I come up with better interface. + * @param requestHeaders map of <header-name, header-value>. + */ + public void setRequestHeaders(Map requestHeaders) { + if (requestHeaders == null) { + this.requestHeaders.clear(); + } else { + // TODO: mmm, ConcurrentHashMap may be overkill. simple synchronized Hashtable would work + // just okay? + ConcurrentHashMap m = new ConcurrentHashMap(1, 0.75f, 2); + m.putAll(requestHeaders); + this.requestHeaders = m; + } + } + + // statistics + private AtomicLong loadedCount = new AtomicLong(); + /** + * number of times successfully loaded recrawl info. + * @return long + */ + public long getLoadedCount() { + return loadedCount.get(); + } + private AtomicLong missedCount = new AtomicLong(); + /** + * number of times getting no recrawl info. + * @return long + */ + public long getMissedCount() { + return missedCount.get(); + } + private AtomicLong errorCount = new AtomicLong(); + /** + * number of times cdx-server API call failed. + * @return long + */ + public long getErrorCount() { + return errorCount.get(); + } + + private AtomicLong cumulativeFetchTime = new AtomicLong(); + /** + * total milliseconds spent in API call. + * it is a sum of time waited for next available connection, + * and actual HTTP request-response round-trip, across all threads. + * @return + */ + public long getCumulativeFetchTime() { + return cumulativeFetchTime.get(); + } + + public void setHttpClient(HttpClient client) { + this.client = client; + } + + // XXX HttpHeadquarterAdapter has the same code. move to common library. + private static boolean contains(HeaderElement[] elements, String value) { + for (int i = 0; i < elements.length; i++) { + if (elements[i].getName().equalsIgnoreCase(value)) { + return true; + } + } + return false; + } + public synchronized HttpClient getHttpClient() { + if (client == null) { + if (conman == null) { + conman = new PoolingHttpClientConnectionManager(); + conman.setDefaultMaxPerRoute(maxConnections); + conman.setMaxTotal(Math.max(conman.getMaxTotal(), maxConnections)); + } + HttpClientBuilder builder = HttpClientBuilder.create() + .disableCookieManagement() + .setConnectionManager(conman); + builder.useSystemProperties(); + // TODO: use setDefaultHeaders for adding requestHeaders? It's a bit painful + // because we need to convert it to a Collection of Header objects. + + // config code for older version of httpclient. +// builder.setDefaultSocketConfig(SocketConfig.custom().setSoTimeout(socketTimeout).build()); +// HttpParams params = client.getParams(); +// params.setIntParameter(CoreConnectionPNames.SO_TIMEOUT, socketTimeout); +// params.setIntParameter(CoreConnectionPNames.CONNECTION_TIMEOUT, connectionTimeout); + // setup request/response intercepter for handling gzip-compressed response. + // Disabled because httpclient 4.3.3 sends "Accept-Encoding: gzip,deflate" by + // default. Response parsing will fail If gzip-decompression ResponseInterceptor + // is installed. + builder.addInterceptorLast(new HttpRequestInterceptor() { + @Override + public void process(final HttpRequest request, final HttpContext context) { +// System.err.println("RequestInterceptor"); +// if (request.containsHeader("Accept-Encoding")) { +// System.err.println("already has Accept-Encoding: " + request.getHeaders("Accept-Encoding")[0]); +// } +// if (gzipAccepted) { +// if (!request.containsHeader("Accept-Encoding")) { +// request.addHeader("Accept-Encoding", "gzip"); +// } +// } + // add extra headers configured. + if (requestHeaders != null) { + for (Entry ent : requestHeaders.entrySet()) { + request.addHeader(ent.getKey(), ent.getValue()); + } + } + } + }); +// builder.addInterceptorFirst(new HttpResponseInterceptor() { +// @Override +// public void process(final HttpResponse response, final HttpContext context) +// throws HttpException, IOException { +// HttpEntity entity = response.getEntity(); +// Header ceheader = entity.getContentEncoding(); +// if (ceheader != null && contains(ceheader.getElements(), "gzip")) { +// response.setEntity(new GzipInflatingHttpEntityWrapper(response.getEntity())); +// } +// } +// }); + this.client = builder.build(); + } + return client; + } + + private long queryRangeSecs = 6L*30*24*3600; + /** + * + * @param queryRangeSecs + */ + public void setQueryRangeSecs(long queryRangeSecs) { + this.queryRangeSecs = queryRangeSecs; + } + public long getQueryRangeSecs() { + return queryRangeSecs; + } + + private String buildStartDate() { + final long range = queryRangeSecs; + if (range <= 0) + return ArchiveUtils.get14DigitDate(new Date(0)); + Date now = new Date(); + Date startDate = new Date(now.getTime() - range*1000); + return ArchiveUtils.get14DigitDate(startDate); + } + + protected String buildURL(String url) { + // we don't need to pass scheme part, but no problem passing it. + StringBuilder sb = new StringBuilder(); + final FormatSegment[] segments = preparedQueryURL; + String encodedURL; + try { + encodedURL = URLEncoder.encode(url, "UTF-8"); + } catch (UnsupportedEncodingException ex) { + encodedURL = url; + } + final String[] args = new String[] { + encodedURL, + buildStartDate() + }; + for (FormatSegment fs : segments) { + fs.print(sb, args); + } + return sb.toString(); + } + + public WbmPersistLoadProcessor() { + prepareQueryURL(); + } + + protected InputStream getCDX(String qurl) throws InterruptedException, IOException { + final String url = buildURL(qurl); + HttpGet m = new HttpGet(url); + m.setConfig(RequestConfig.custom().setConnectTimeout(connectionTimeout) + .setSocketTimeout(socketTimeout).build()); + HttpEntity entity = null; + int attempts = 0; + do { + if (Thread.interrupted()) + throw new InterruptedException("interrupted while GET " + url); + if (attempts > 0) { + Thread.sleep(5000); + } + try { + long t0 = System.currentTimeMillis(); + HttpResponse resp = getHttpClient().execute(m); + cumulativeFetchTime.addAndGet(System.currentTimeMillis() - t0); + StatusLine sl = resp.getStatusLine(); + if (sl.getStatusCode() != 200) { + log.error("GET " + url + " failed with status=" + sl.getStatusCode() + " " + sl.getReasonPhrase()); + entity = resp.getEntity(); + entity.getContent().close(); + entity = null; + continue; + } + entity = resp.getEntity(); + } catch (IOException ex) { + log.error("GEt " + url + " failed with error " + ex.getMessage()); + } catch (Exception ex) { + log.error("GET " + url + " failed with error ", ex); + } + } while (entity == null && ++attempts < 3); + if (entity == null) { + throw new IOException("giving up on GET " + url + " after " + attempts + " attempts"); + } + return entity.getContent(); + } + + @Override + protected ProcessResult innerProcessResult(CrawlURI curi) throws InterruptedException { + InputStream is; + try { + is = getCDX(curi.toString()); + } catch (IOException ex) { + log.error(ex.getMessage()); + errorCount.incrementAndGet(); + return ProcessResult.PROCEED; + } + Map info = null; + try { + info = getLastCrawl(is); + } catch (IOException ex) { + log.error("error parsing response", ex); + } finally { + if (is != null) + ArchiveUtils.closeQuietly(is); + } + if (info != null) { + Map history = FetchHistoryHelper.getFetchHistory(curi, + (Long)info.get(FetchHistoryHelper.A_TIMESTAMP), historyLength); + if (history != null) + history.putAll(info); + loadedCount.incrementAndGet(); + } else { + missedCount.incrementAndGet(); + } + return ProcessResult.PROCEED; + } + + protected HashMap getLastCrawl(InputStream is) throws IOException { + // read CDX lines, save most recent (at the end) hash. + ByteBuffer buffer = ByteBuffer.allocate(32); + ByteBuffer tsbuffer = ByteBuffer.allocate(14); + int field = 0; + int c; + do { + c = is.read(); + if (field == 1) { + // 14-digits timestamp + tsbuffer.clear(); + while (Character.isDigit(c) && tsbuffer.remaining() > 0) { + tsbuffer.put((byte)c); + c = is.read(); + } + if (c != ' ' || tsbuffer.position() != 14) { + tsbuffer.clear(); + } + // fall through to skip the rest + } else if (field == 5) { + buffer.clear(); + while ((c >= 'A' && c <= 'Z' || c >= '0' && c <= '9') && buffer.remaining() > 0) { + buffer.put((byte)c); + c = is.read(); + } + if (c != ' ' || buffer.position() != 32) { + buffer.clear(); + } + // fall through to skip the rest + } + while (true) { + if (c == -1) { + break; + } else if (c == '\n') { + field = 0; + break; + } else if (c == ' ') { + field++; + break; + } + c = is.read(); + } + } while (c != -1); + + HashMap info = new HashMap(); + if (buffer.remaining() == 0) { + info.put(RecrawlAttributeConstants.A_CONTENT_DIGEST, contentDigestScheme + new String(buffer.array())); + } + if (tsbuffer.remaining() == 0) { + try { + long ts = DateUtils.parse14DigitDate(new String(tsbuffer.array())).getTime(); + // A_TIMESTAMP has been used for sorting history long before A_FETCH_BEGAN_TIME + // field was introduced. Now FetchHistoryProcessor fails if A_FETCH_BEGAN_TIME is + // not set. We could stop storing A_TIMESTAMP and sort by A_FETCH_BEGAN_TIME. + info.put(FetchHistoryHelper.A_TIMESTAMP, ts); + info.put(CoreAttributeConstants.A_FETCH_BEGAN_TIME, ts); + } catch (ParseException ex) { + } + } + return info.isEmpty() ? null : info; + } + + /** + * unused. + */ + @Override + protected void innerProcess(CrawlURI uri) throws InterruptedException { + } + + @Override + protected boolean shouldProcess(CrawlURI uri) { + // TODO: we want deduplicate robots.txt, too. + //if (uri.isPrerequisite()) return false; + String scheme = uri.getUURI().getScheme(); + if (!(scheme.equals("http") || scheme.equals("https"))) return false; + return true; + } + + /** + * main entry point for quick test. + * @param args + */ + public static void main(String[] args) throws Exception { + String url = args[0]; + String cookie = args.length > 1 ? args[1] : null; + WbmPersistLoadProcessor wp = new WbmPersistLoadProcessor(); + if (cookie != null) { + wp.setRequestHeaders(Collections.singletonMap("Cookie", cookie)); + } + InputStream is = wp.getCDX(url); + byte[] b = new byte[1024]; + int n; + while ((n = is.read(b)) > 0) { + System.out.write(b, 0, n); + } + is.close(); + } +} diff --git a/contrib/src/test/java/org/archive/modules/recrawl/wbm/WbmPersistLoadProcessorTest.java b/contrib/src/test/java/org/archive/modules/recrawl/wbm/WbmPersistLoadProcessorTest.java new file mode 100644 index 00000000..88d1c0cf --- /dev/null +++ b/contrib/src/test/java/org/archive/modules/recrawl/wbm/WbmPersistLoadProcessorTest.java @@ -0,0 +1,221 @@ +package org.archive.modules.recrawl.wbm; + +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.net.URLEncoder; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.regex.Pattern; + +import junit.framework.TestCase; + +import org.apache.http.HttpResponse; +import org.apache.http.ProtocolVersion; +import org.apache.http.client.HttpClient; +import org.apache.http.client.methods.HttpUriRequest; +import org.apache.http.entity.ByteArrayEntity; +import org.apache.http.message.BasicHttpResponse; +import org.apache.http.message.BasicStatusLine; +import org.archive.modules.CoreAttributeConstants; +import org.archive.modules.CrawlURI; +import org.archive.modules.ProcessResult; +import org.archive.modules.recrawl.FetchHistoryHelper; +import org.archive.modules.recrawl.FetchHistoryProcessor; +import org.archive.modules.recrawl.RecrawlAttributeConstants; +import org.archive.net.UURIFactory; +import org.archive.util.Base32; +import org.archive.util.DateUtils; +import org.easymock.EasyMock; + +import com.google.common.util.concurrent.ExecutionList; + +/** + * unit test for {@link WbmPersistLoadProcessor}. + * + * TODO: + *
    + *
  • test for pathological cases: illegal chars, incorrect length, etc. + *
  • test if connection is properly released back to the pool for all possible cases. + *
+ * @author kenji + * + */ +public class WbmPersistLoadProcessorTest extends TestCase { + + public void testBuildURL() throws Exception { + WbmPersistLoadProcessor t = new WbmPersistLoadProcessor(); + t.setQueryURL("http://web.archive.org/cdx/search/cdx?url=$u&startDate=$s&limit=1"); + final String URL = "http://archive.org/"; + String url = t.buildURL(URL); + System.err.println(url); + assertTrue("has encode URL", Pattern.matches(".*[&?]url="+URLEncoder.encode(URL, "UTF-8")+"([&].*)?", url)); + assertTrue("has startDate", Pattern.matches(".*[&?]startDate=\\d{14}([&].*)?", url)); + assertTrue("has limit", Pattern.matches(".*[&?]limit=\\d+([&].*)?", url)); + //assertTrue("has last=true", Pattern.matches(".*[&?]last=true([&].*)?", url)); + } + + /** + * stub HttpResponse for normal case. + * @author kenji + * + */ + public static class TestNormalHttpResponse extends BasicHttpResponse { + public static final String EXPECTED_TS = "20121101155310"; + public static final String EXPECTED_HASH = "GHN5VKF3TBKNSEZTASOM23BJRTKFFNJK"; + public TestNormalHttpResponse() { + super(new BasicStatusLine(new ProtocolVersion("HTTP", 1, 0), 200, "OK")); + setEntity(new ByteArrayEntity( + ("org,archive)/ "+EXPECTED_TS+" http://archive.org/ text/html 200 "+ + EXPECTED_HASH+" - - 6908 982548871 "+ + "google.es-20121101-155506/IA-FOC-google.es-20121101073708-00001.warc.gz\n" + ).getBytes() + )); + } + } + + protected Map getFetchHistory(CrawlURI curi, int idx) { + Map data = curi.getData(); + @SuppressWarnings("unchecked") + Map[] historyArray = (Map[])data.get(RecrawlAttributeConstants.A_FETCH_HISTORY); + assertNotNull(historyArray); + Map history = historyArray[idx]; + return history; + } + + private byte[] sha1Digest(String text) throws NoSuchAlgorithmException { + MessageDigest md = MessageDigest.getInstance("sha1"); + return md.digest(text.getBytes()); + } + + /** + * this test is disabled because it talks to production CDX server. + * @throws Exception + */ + public void _testInnerProcessResultSingleShotWithMock() throws Exception { + // because AbstractHttpClient marks most execute(...) methods final, it is very tiresome to implement + // a stub by implementing HttpClient interface. So I use EasyMock. + HttpClient client = EasyMock.createMock(HttpClient.class); + HttpResponse testResponse = new TestNormalHttpResponse(); + EasyMock.expect(client.execute((HttpUriRequest)EasyMock.notNull())).andReturn(testResponse); + EasyMock.replay(client); + + final String CONTENT_DIGEST_SCHEME = "sha1:"; + WbmPersistLoadProcessor t = new WbmPersistLoadProcessor(); + t.setHttpClient(client); + t.setContentDigestScheme(CONTENT_DIGEST_SCHEME); + CrawlURI curi = new CrawlURI(UURIFactory.getInstance("http://archive.org/")); + + // put history entry newer than being loaded (i.e. loaded history entry will not be used for FetchHistoryProcessor + // check below. + long expected_ts = DateUtils.parse14DigitDate(TestNormalHttpResponse.EXPECTED_TS).getTime(); + Map[] fetchHistory = (Map[])curi.getData().get(RecrawlAttributeConstants.A_FETCH_HISTORY); + if (fetchHistory == null) { + fetchHistory = new HashMap[2]; + curi.getData().put(RecrawlAttributeConstants.A_FETCH_HISTORY, fetchHistory); + } + final byte[] digestValue0 = sha1Digest("0"); + final byte[] digestValue1 = sha1Digest("1"); + fetchHistory[0] = new HashMap(); + fetchHistory[0].put(FetchHistoryHelper.A_TIMESTAMP, expected_ts + 2000); + fetchHistory[0].put(CoreAttributeConstants.A_FETCH_BEGAN_TIME, expected_ts + 2000); + fetchHistory[0].put(RecrawlAttributeConstants.A_CONTENT_DIGEST, + CONTENT_DIGEST_SCHEME + Base32.encode(digestValue0)); + fetchHistory[1] = new HashMap(); + fetchHistory[1].put(FetchHistoryHelper.A_TIMESTAMP, expected_ts - 2000); + fetchHistory[1].put(RecrawlAttributeConstants.A_CONTENT_DIGEST, + CONTENT_DIGEST_SCHEME + Base32.encode(digestValue1)); + + ProcessResult result = t.innerProcessResult(curi); + assertEquals("result is PROCEED", ProcessResult.PROCEED, result); + + // newly loaded history entry should fall in between two existing entries (index=1) + Map history = getFetchHistory(curi, 1); + assertNotNull("history", history); + String hash = (String)history.get(RecrawlAttributeConstants.A_CONTENT_DIGEST); + assertEquals("CONTENT_DIGEST", CONTENT_DIGEST_SCHEME+TestNormalHttpResponse.EXPECTED_HASH, hash); + + Long ts = (Long)history.get(FetchHistoryHelper.A_TIMESTAMP); + assertNotNull("ts is non-null", ts); + assertEquals("'ts' has expected timestamp", expected_ts, ts.longValue()); + + // Check compatibility with FetchHistoryProcessor. + // TODO: This is not testing WbmPersistLoadProcessor - only testing stub fetchHistory + // setup above (OK as long as it matches WbmPersistLoadProcessor). We need a separate + // test method. + curi.setFetchStatus(200); + curi.setFetchBeginTime(System.currentTimeMillis()); + // FetchHistoryProcessor once failed for a revisit case. We'd need to test other cases + // too (TODO). + curi.setContentDigest("sha1", digestValue0); + FetchHistoryProcessor fhp = new FetchHistoryProcessor(); + fhp.process(curi); + } + + public void testInnerProcessResultSingleShotWithRealServer() throws Exception { + WbmPersistLoadProcessor t = new WbmPersistLoadProcessor(); + //CrawlURI curi = new CrawlURI(UURIFactory.getInstance("http://archive.org/")); + CrawlURI curi = new CrawlURI(UURIFactory.getInstance("http://www.mext.go.jp/null.gif")); + ProcessResult result = t.innerProcessResult(curi); + Map history = getFetchHistory(curi, 0); + assertNotNull("getFetchHistory returns non-null", history); + String hash = (String)history.get(RecrawlAttributeConstants.A_CONTENT_DIGEST); + assertNotNull("CONTENT_DIGEST is non-null", hash); + assertTrue("CONTENT_DIGEST starts with scheme", hash.startsWith(t.getContentDigestScheme())); + assertEquals("CONTENT_DIGEST is a String of length 32", 32, hash.substring(t.getContentDigestScheme().length()).length()); + + assertEquals("should always return PROCEED", ProcessResult.PROCEED, result); + } + + public static class LoadTask implements Runnable { + private WbmPersistLoadProcessor p; + private String uri; + public LoadTask(WbmPersistLoadProcessor p, String uri) { + this.p = p; + this.uri = uri; + } + @Override + public void run() { + try { + CrawlURI curi = new CrawlURI(UURIFactory.getInstance(this.uri)); + p.innerProcessResult(curi); + //System.err.println(curi.toString()); + } catch (Exception ex) { + ex.printStackTrace(); + } + } + } + + /** + * test for performance. + * not named as test case. run this through main(). + * @throws Exception + */ + public void measureInnerProcessResultMany() throws Exception { + final WbmPersistLoadProcessor t = new WbmPersistLoadProcessor(); + InputStream is = getClass().getResourceAsStream("/test-url-list.txt"); + BufferedReader br = new BufferedReader(new InputStreamReader(is)); + String line; + int nurls = 0; + ExecutorService executor = Executors.newFixedThreadPool(100); + ExecutionList tasks = new ExecutionList(); + while ((line = br.readLine()) != null) { + tasks.add(new LoadTask(t, line), executor); + nurls++; + } + long t0 = System.currentTimeMillis(); + tasks.execute(); + executor.awaitTermination(30, TimeUnit.SECONDS); + long el = System.currentTimeMillis() - t0; + System.err.println(nurls + " urls, time=" + el + "ms (" + (nurls / (el / 1000.0)) + " URI/s"); + } + + public static void main() throws Exception { + (new WbmPersistLoadProcessorTest()).measureInnerProcessResultMany(); + } +} diff --git a/engine/src/main/java/org/archive/crawler/restlet/JobRelatedResource.java b/engine/src/main/java/org/archive/crawler/restlet/JobRelatedResource.java index 911a86e8..f033395d 100644 --- a/engine/src/main/java/org/archive/crawler/restlet/JobRelatedResource.java +++ b/engine/src/main/java/org/archive/crawler/restlet/JobRelatedResource.java @@ -44,6 +44,7 @@ import org.restlet.data.Response; import org.restlet.resource.ResourceException; import org.springframework.beans.BeanUtils; import org.springframework.beans.BeanWrapperImpl; +import org.springframework.beans.BeansException; /** * Shared superclass for resources that represent functional aspects @@ -118,8 +119,13 @@ public abstract class JobRelatedResource extends BaseResource { for (PropertyDescriptor pd : getPropertyDescriptors(bwrap)) { if (pd.getReadMethod() != null) { String propName = pd.getName(); - Object propValue = bwrap.getPropertyValue(propName); - addPresentableNestedNames(namedBeans, propValue, alreadyWritten); + try { + Object propValue = bwrap.getPropertyValue(propName); + addPresentableNestedNames(namedBeans, propValue, alreadyWritten); + } catch (BeansException ex) { + // getter may throw UnsupportedOperationException, for ex. + // ignore such properties. + } } } if (obj.getClass().isArray()) { diff --git a/engine/src/test/java/org/archive/crawler/datamodel/CrawlURITest.java b/engine/src/test/java/org/archive/crawler/datamodel/CrawlURITest.java index 642c5bf7..3ea4771d 100644 --- a/engine/src/test/java/org/archive/crawler/datamodel/CrawlURITest.java +++ b/engine/src/test/java/org/archive/crawler/datamodel/CrawlURITest.java @@ -1,8 +1,8 @@ /* * This file is part of the Heritrix web crawler (crawler.archive.org). * - * Licensed to the Internet Archive (IA) by one or more individual - * contributors. + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. * * The IA licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with @@ -28,21 +28,22 @@ import java.io.ObjectOutputStream; import org.apache.commons.httpclient.URIException; import org.archive.modules.CrawlURI; import org.archive.modules.SchedulingConstants; +import org.archive.modules.extractor.LinkContext.SimpleLinkContext; import org.archive.net.UURI; import org.archive.net.UURIFactory; import org.archive.util.TmpDirTestCase; /** * Tests related to CrawlURI - * + * * @contributor stack * @contributor gojomo * @version $Revision$, $Date$ */ public class CrawlURITest extends TmpDirTestCase { - + CrawlURI seed = null; - + protected void setUp() throws Exception { super.setUp(); final String url = "http://www.dh.gov.uk/Home/fs/en"; @@ -57,14 +58,14 @@ public class CrawlURITest extends TmpDirTestCase { /** * Test serialization/deserialization works. - * + * * @throws IOException * @throws ClassNotFoundException */ final public void testSerialization() - throws IOException, ClassNotFoundException { - File serialize = new File(getTmpDir(), - this.getClass().getName() + ".serialize"); + throws IOException, ClassNotFoundException { + File serialize = new File(getTmpDir(), + this.getClass().getName() + ".serialize"); try { FileOutputStream fos = new FileOutputStream(serialize); ObjectOutputStream oos = new ObjectOutputStream(fos); @@ -81,43 +82,137 @@ public class CrawlURITest extends TmpDirTestCase { deserializedCuri = (CrawlURI)ois.readObject(); deserializedCuri = (CrawlURI)ois.readObject(); assertEquals("Deserialized not equal to original", - this.seed.toString(), deserializedCuri.toString()); + this.seed.toString(), deserializedCuri.toString()); String host = this.seed.getUURI().getHost(); assertTrue("Deserialized host not null", - host != null && host.length() >= 0); + host != null && host.length() >= 0); } finally { serialize.delete(); } } - + public void testCandidateURIWithLoadedAList() - throws URIException { + throws URIException { UURI uuri = UURIFactory.getInstance("http://www.archive.org"); CrawlURI curi = new CrawlURI(uuri); curi.setSeed(true); curi.getData().put("key", "value"); assertTrue("Didn't find AList item", - curi.getData().get("key").equals("value")); + curi.getData().get("key").equals("value")); } - + public void testExtendHopsPath() { assertEquals("from empty","L",CrawlURI.extendHopsPath("",'L')); - + assertEquals("from one","LX",CrawlURI.extendHopsPath("L",'X')); - + assertEquals( - "from fortynine", - "LLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLX", - CrawlURI.extendHopsPath("LLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLL",'X')); - + "from fortynine", + "LLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLX", + CrawlURI.extendHopsPath("LLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLL",'X')); + assertEquals( "from fifty", "1+LLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLX", CrawlURI.extendHopsPath("LLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLL",'X')); - + assertEquals( "from 149", "100+LLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLX", CrawlURI.extendHopsPath("99+LLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLL",'X')); } + + + public void testNullPathFromSeed() throws URIException { + // check comparing with null + CrawlURI a = new CrawlURI( + UURIFactory.getInstance("http://example.com/1"), // a == b + null, // a < b + UURIFactory.getInstance("http://example.com/via/1"), // a == b + new SimpleLinkContext("1")); // a == b + assertEquals("", a.getPathFromSeed()); + + CrawlURI b = new CrawlURI( + UURIFactory.getInstance("http://example.com/1"), // a == b + "", // a < b + UURIFactory.getInstance("http://example.com/via/1"), // a == b + new SimpleLinkContext("1")); // a == b + assertEquals("", b.getPathFromSeed()); + + assertEquals(0, a.compareTo(b)); + assertEquals(0, b.compareTo(a)); + + } + + public void testOrdering() throws URIException { + // check that via is highest precedence + CrawlURI a = new CrawlURI( + UURIFactory.getInstance("http://example.com/2"), // a > b + "2", // a > b + UURIFactory.getInstance("http://example.com/via/1"), // a < b + new SimpleLinkContext("2")); // a > b + CrawlURI b = new CrawlURI( + UURIFactory.getInstance("http://example.com/1"), // a > b + "1", // a > b + UURIFactory.getInstance("http://example.com/via/2"), // a < b + new SimpleLinkContext("1")); // a > b + assertEquals(-1, a.compareTo(b)); + assertEquals(1, b.compareTo(a)); + + // check that uri is next highest + a = new CrawlURI( + UURIFactory.getInstance("http://example.com/1"), // a < b + "2", // a > b + UURIFactory.getInstance("http://example.com/via/1"), // a == b + new SimpleLinkContext("2")); // a > b + b = new CrawlURI( + UURIFactory.getInstance("http://example.com/2"), // a < b + "1", // a > b + UURIFactory.getInstance("http://example.com/via/1"), // a == b + new SimpleLinkContext("1")); // a > b + assertEquals(-1, a.compareTo(b)); + assertEquals(1, b.compareTo(a)); + + // check that via context is next + a = new CrawlURI( + UURIFactory.getInstance("http://example.com/1"), // a == b + "2", // a > b + UURIFactory.getInstance("http://example.com/via/1"), // a == b + new SimpleLinkContext("1")); // a < b + b = new CrawlURI( + UURIFactory.getInstance("http://example.com/1"), // a == b + "1", // a > b + UURIFactory.getInstance("http://example.com/via/1"), // a == b + new SimpleLinkContext("2")); // a < b + assertEquals(-1, a.compareTo(b)); + assertEquals(1, b.compareTo(a)); + + // check that pathFromSeed is next + a = new CrawlURI( + UURIFactory.getInstance("http://example.com/1"), // a == b + "1", // a < b + UURIFactory.getInstance("http://example.com/via/1"), // a == b + new SimpleLinkContext("1")); // a == b + b = new CrawlURI( + UURIFactory.getInstance("http://example.com/1"), // a == b + "2", // a < b + UURIFactory.getInstance("http://example.com/via/1"), // a == b + new SimpleLinkContext("1")); // a == b + assertEquals(-1, a.compareTo(b)); + assertEquals(1, b.compareTo(a)); + + // check equality + a = new CrawlURI( + UURIFactory.getInstance("http://example.com/1"), // a == b + "1", // a == b + UURIFactory.getInstance("http://example.com/via/1"), // a == b + new SimpleLinkContext("1")); // a == b + b = new CrawlURI( + UURIFactory.getInstance("http://example.com/1"), // a == b + "1", // a == b + UURIFactory.getInstance("http://example.com/via/1"), // a == b + new SimpleLinkContext("1")); // a == b + assertEquals(0, a.compareTo(b)); + assertEquals(0, b.compareTo(a)); + } } diff --git a/modules/src/main/java/org/archive/modules/CrawlURI.java b/modules/src/main/java/org/archive/modules/CrawlURI.java index d5eefb7e..dc666061 100644 --- a/modules/src/main/java/org/archive/modules/CrawlURI.java +++ b/modules/src/main/java/org/archive/modules/CrawlURI.java @@ -1978,7 +1978,7 @@ implements Reporter, Serializable, OverlayContext, Comparable { return 0; } else if (a == null && b != null) { return -1; - } else if (a != null && b != null) { + } else if (a != null && b == null) { return 1; } else { return a.compareTo(b); diff --git a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java index 1f78a7e9..368cc650 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java +++ b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java @@ -669,7 +669,7 @@ public class FetchHTTP extends Processor implements Lifecycle { long contentLength = -1l; Header h = response.getLastHeader("content-length"); - if (h != null) { + if (h != null && h.getValue().trim().length()>0) { contentLength = Long.parseLong(h.getValue()); } try { diff --git a/pom.xml b/pom.xml index e67d61b1..ddb2f121 100644 --- a/pom.xml +++ b/pom.xml @@ -118,6 +118,12 @@ http://maven.apache.org/guides/mini/guide-m1-m2.html engine ${project.version} + + + org.apache.httpcomponents + httpclient + 4.3.3 +