lee/heritrix3
1
0 Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-11-08 15:04:51 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'internetarchive/master' into ari-3977

Browse Source
This commit is contained in:
Hunter Stern committed 2014-09-01 09:23:26 -07:00
2 parents 7f1700400b 25c3b3e680
commit 2705bd3179
13 files changed
+937 -348
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

commons/pom.xml
-1
View File
@@ -26,7 +26,6 @@
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<version>4.3.3</version>
</dependency>
<dependency>
<groupId>com.sleepycat</groupId>
commons/src/main/java/org/archive/util/Transform.java
-154
View File
@@ -1,154 +0,0 @@
/*
* This file is part of the Heritrix web crawler (crawler.archive.org).
*
* Licensed to the Internet Archive (IA) by one or more individual
* contributors.
*
* The IA licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.archive.util;
import java.util.AbstractCollection;
import java.util.Collection;
import java.util.Iterator;
/**
* A transformation of a collection. The elements in the transform are based
* on the elements of some other collection; the original collection's
* elements are transformed using a specified transformer. Changes to the
* original collection are automatically reflected in the transform and
* vice-versa.
*
* <p>If the transformer returns null for a given original object, then that
* object will not be included in the transform. Thus the transform might
* be smaller than the original collection. Note that Transform instances
* can never contain the null element.
*
* <p>This collection implementation does not support the optional add
* operation.
*
* @author pjack
*
* @param <Original> the type of the original elements in the collection
* @param <Transformed> the type of the tranformed elements
*/
public class Transform<Original,Transformed>
extends AbstractCollection<Transformed> {
/** The original collection. */
final private Collection<? extends Original> delegate;
/** Transforms the original objects. */
final private Transformer<Original,Transformed> transformer;
/**
* Constructor.
*
* @param delegate The collection whose elements to transform.
* @param transformer Transforms the elements
*/
public Transform(Collection<? extends Original> delegate,
Transformer<Original,Transformed> transformer) {
this.delegate = delegate;
this.transformer = transformer;
}
public int size() {
int count = 0;
Iterator<Transformed> iter = iterator();
while (iter.hasNext()) {
iter.next();
count++;
}
return count;
}
public Iterator<Transformed> iterator() {
return new TransformIterator<Original,Transformed>(
delegate.iterator(), transformer);
}
/**
* Returns a transform containing only objects of a given class.
*
* @param <Target> the target class
* @param c the collection to transform
* @param cls the class of objects to return
* @return a collection containing only objects of class cls
*/
public static <Target> Collection<Target> subclasses(
Collection<? extends Object> c,
final Class<Target> cls) {
Transformer<Object,Target> t = new Transformer<Object,Target>() {
public Target transform(Object s) {
if (cls.isInstance(s)) {
return cls.cast(s);
} else {
return null;
}
}
};
return new Transform<Object,Target>(c, t);
}
}
class TransformIterator<Original,Transformed> implements Iterator<Transformed> {
final private Iterator<? extends Original> iterator;
final private Transformer<Original,Transformed> transformer;
private Transformed next;
public TransformIterator(Iterator<? extends Original> iterator,
Transformer<Original,Transformed> transformer) {
this.iterator = iterator;
this.transformer = transformer;
}
public boolean hasNext() {
if (next != null) {
return true;
}
while (iterator.hasNext()) {
Original o = iterator.next();
next = transformer.transform(o);
if (next != null) {
return true;
}
}
return false;
}
public Transformed next() {
if (!hasNext()) {
throw new IllegalStateException();
}
Transformed r = next;
next = null;
return r;
}
// FIXME: this can break standard Iterator contract, for example
// transformIterator.next();
// if(transformIterator.hasNext()) {
// transformIterator.remove();
// }
// usual iterator contract is to remove the last object returned
// by next; in this case the subsequent
public void remove() {
iterator.remove();
}
}
commons/src/main/java/org/archive/util/Transformer.java
-40
View File
@@ -1,40 +0,0 @@
/*
* This file is part of the Heritrix web crawler (crawler.archive.org).
*
* Licensed to the Internet Archive (IA) by one or more individual
* contributors.
*
* The IA licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.archive.util;
/**
* Transforms objects from one thing into another.
*
* @author pjack
*
* @param <Original> the type of the original objects
* @param <Transformed> the type of the transformed objects
*/
public interface Transformer<Original,Transformed> {
/**
* Transforms the given object.
*
* @param o the object to transform
* @return the transformed object
*/
Transformed transform(Original o);
}
commons/src/test/java/org/archive/util/TransformTest.java
-126
View File
@@ -1,126 +0,0 @@
package org.archive.util;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashSet;
import java.util.List;
import java.util.Random;
import java.util.Set;
import junit.framework.TestCase;
/**
* Tests the {@link dex.misc.Transform} class.
*/
public class TransformTest extends TestCase {
// Convert integers to strings, eliminating negative numbers
private static class PositiveToString
implements Transformer<Integer, String> {
public String transform(Integer i) {
if (i < 0) {
return null;
}
return i.toString();
}
}
/**
* Tests using a simple Transformer. The Transformer changes
* positive integers into strings. The test sets up a
* list of random integers, remembering which ones are
* positive. The Transform is created, and the Transform's
* contents are compared against the list of remembered positive
* integers.
*/
public void testTransform() {
Transformer<Integer,String> transformer = new PositiveToString();
// Transform of an empty collection should be empty.
List<Integer> empty = new ArrayList<Integer>();
assertTrue(new Transform<Integer,String>(empty, transformer).isEmpty());
// Some simple test data.
Integer[] testData = new Integer[] { -5, 3, 2, -11, 0, 111, -161 };
String[] expected = new String[] { "3", "2", "0", "111" };
List<Integer> list = Arrays.asList(testData);
Transform<Integer,String> c = new Transform<Integer,String>(list,
transformer);
List<String> expectedList = Arrays.asList(expected);
assertEquals(new ArrayList<String>(c), expectedList);
// Same test as above, with random data
for (int i = 0; i < 100; i++) {
randomTest();
}
}
private void randomTest() {
Transformer<Integer, String> transformer = new PositiveToString();
Random random = new Random();
int max = random.nextInt(1024) + 10;
List<Integer> testData = new ArrayList<Integer>(max);
List<String> expected = new ArrayList<String>(max);
for (int i = 0; i < max; i++) {
int e = random.nextInt();
testData.add(e);
if (e >= 0) {
expected.add(Integer.toString(e));
}
}
Transform<Integer,String> c = new Transform<Integer,String>(testData,
transformer);
List<String> results = new ArrayList<String>(c);
assertEquals(expected, results);
}
/**
* Tests the static subclasses method. The test sets up a list of
* Number instances that may contain random Double, Float, Integer
* or Long values. The Long values are remembered. The subclasses
* method is used to create a Transform containing only the Long
* values. The Transform is compared against the list of remembered
* Long values.
*/
public void testSubclasses() {
Random random = new Random();
for (int i = 0; i < 100; i++) {
int max = random.nextInt(1024) + 10;
List<Number> testData = new ArrayList<Number>(max);
List<Long> expected = new ArrayList<Long>(max);
for (int j = 0; j < max; j++) {
int v = random.nextInt(4);
switch (v) {
case 0:
long l = random.nextLong();
testData.add(l);
expected.add(l);
break;
case 1:
testData.add(random.nextInt());
break;
case 2:
testData.add(random.nextDouble());
break;
case 3:
testData.add(random.nextFloat());
break;
}
}
Collection<Long> c = Transform.subclasses(testData, Long.class);
List<Long> results = new ArrayList<Long>(c);
assertEquals(expected, results);
}
}
public void testSingleton() {
Set<Number> set = new HashSet<Number>();
set.add(3);
Collection<Integer> c = Transform.subclasses(set, Integer.class);
for (Integer i : c) {
System.out.println(i);
}
}
}
contrib/pom.xml
+16
View File
@@ -18,6 +18,16 @@
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-client</artifactId>
<version>0.96.1.1-cdh5.0.2</version>
<exclusions>
<exclusion>
<artifactId>jets3t</artifactId>
<groupId>net.java.dev.jets3t</groupId>
</exclusion>
<exclusion>
<artifactId>junit</artifactId>
<groupId>junit</groupId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.archive.heritrix</groupId>
@@ -41,6 +51,12 @@
<artifactId>itextpdf</artifactId>
<version>5.5.0</version>
</dependency>
<dependency>
<groupId>org.easymock</groupId>
<artifactId>easymock</artifactId>
<version>3.1</version>
<scope>test</scope>
</dependency>
</dependencies>
<repositories>
<repository>
contrib/src/main/java/org/archive/crawler/frontier/AMQPUrlReceiver.java
+1 -1
View File
@@ -320,7 +320,7 @@ public class AMQPUrlReceiver implements Lifecycle, ApplicationListener<CrawlStat
curi.setSchedulingDirective(SchedulingConstants.HIGH);
curi.setPrecedence(1);
// XXX? curi.setForceFetch(true);
curi.setForceFetch(true);
curi.getAnnotations().add(A_RECEIVED_FROM_AMQP);
contrib/src/main/java/org/archive/modules/recrawl/wbm/WbmPersistLoadProcessor.java
+566
View File
@@ -0,0 +1,566 @@
/*
* This file is part of the Heritrix web crawler (crawler.archive.org).
*
* Licensed to the Internet Archive (IA) by one or more individual
* contributors.
*
* The IA licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.archive.modules.recrawl.wbm;
import java.io.IOException;
import java.io.InputStream;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.nio.ByteBuffer;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.http.HeaderElement;
import org.apache.http.HttpEntity;
import org.apache.http.HttpRequest;
import org.apache.http.HttpRequestInterceptor;
import org.apache.http.HttpResponse;
import org.apache.http.StatusLine;
import org.apache.http.client.HttpClient;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.apache.http.protocol.HttpContext;
import org.archive.modules.CoreAttributeConstants;
import org.archive.modules.CrawlURI;
import org.archive.modules.ProcessResult;
import org.archive.modules.Processor;
import org.archive.modules.recrawl.FetchHistoryHelper;
import org.archive.modules.recrawl.RecrawlAttributeConstants;
import org.archive.util.ArchiveUtils;
import org.archive.util.DateUtils;
/**
* A {@link Processor} for retrieving recrawl info from remote Wayback Machine index.
* This is currently in the early stage of experiment. Both low-level protocol and WBM API
* semantics will certainly undergo several revisions.
* <p>Current interface:</p>
* <p>http://web-beta.archive.org/cdx/search/cdx?url=archive.org&startDate=1999 will return raw
* CDX lines for archive.org, since 1999-01-01 00:00:00.
* </p>
* <p>As index is updated in a separate batch processing job, there's no "Store" counterpart.</p>
* @contributor Kenji Nagahashi.
*/
public class WbmPersistLoadProcessor extends Processor {
private static final Log log = LogFactory.getLog(WbmPersistLoadProcessor.class);
private HttpClient client;
private PoolingHttpClientConnectionManager conman;
private int historyLength = 2;
public void setHistoryLength(int historyLength) {
this.historyLength = historyLength;
}
public int getHistoryLength() {
return historyLength;
}
// ~Jan 2, 2013
//private String queryURL = "http://web-beta.archive.org/cdx/search";
//private String queryURL = "http://web.archive.org/cdx/search/cdx";
// ~Sep 26, 2013
private String queryURL = "http://wwwb-dedup.us.archive.org:8083/web/timemap/cdx?url=$u&limit=-1";
public void setQueryURL(String queryURL) {
this.queryURL = queryURL;
prepareQueryURL();
}
public String getQueryURL() {
return queryURL;
}
public interface FormatSegment {
void print(StringBuilder sb, String[] args);
}
private static class StaticSegment implements FormatSegment {
String s;
public StaticSegment(String s) {
this.s = s;
}
public void print(StringBuilder sb, String[] args) {
sb.append(s);
}
}
private static class InterpolateSegment implements FormatSegment {
int aidx;
public InterpolateSegment(int aidx) {
this.aidx = aidx;
}
public void print(StringBuilder sb, String[] args) {
sb.append(args[aidx]);
}
}
private FormatSegment[] preparedQueryURL;
/**
* pre-scan queryURL template so that actual queryURL can be built
* with minimal processing.
*/
private void prepareQueryURL() {
List<FormatSegment> segments = new ArrayList<FormatSegment>();
final int l = queryURL.length();
int p = 0;
int q;
while (p < l && (q = queryURL.indexOf('$', p)) >= 0) {
if (q + 2 > l) {
// '$' at the end. keep it as if it were '$$'
break;
}
if (q > p) {
segments.add(new StaticSegment(queryURL.substring(p, q)));
}
char c = queryURL.charAt(q + 1);
if (c == 'u') {
segments.add(new InterpolateSegment(0));
} else if (c == 's') {
segments.add(new InterpolateSegment(1));
} else {
// copy '$'-sequence so that it's easy to spot errors
segments.add(new StaticSegment(queryURL.substring(q, q + 2)));
}
p = q + 2;
}
if (p < l) {
segments.add(new StaticSegment(queryURL.substring(p)));
}
preparedQueryURL = segments.toArray(new FormatSegment[segments.size()]);
}
private String contentDigestScheme = "sha1:";
/**
* set Content-Digest scheme string to prepend to the hash string found in CDX.
* Heritrix's Content-Digest comparison including this part.
* {@code "sha1:"} by default.
* @param contentDigestScheme
*/
public void setContentDigestScheme(String contentDigestScheme) {
this.contentDigestScheme = contentDigestScheme;
}
public String getContentDigestScheme() {
return contentDigestScheme;
}
private int socketTimeout = 10000;
/**
* socket timeout (SO_TIMEOUT) for HTTP client in milliseconds.
*/
public void setSocketTimeout(int socketTimeout) {
this.socketTimeout = socketTimeout;
}
public int getSocketTimeout() {
return socketTimeout;
}
private int connectionTimeout = 10000;
/**
* connection timeout for HTTP client in milliseconds.
* @param connectionTimeout
*/
public void setConnectionTimeout(int connectionTimeout) {
this.connectionTimeout = connectionTimeout;
}
public int getConnectionTimeout() {
return connectionTimeout;
}
private int maxConnections = 10;
public int getMaxConnections() {
return maxConnections;
}
public synchronized void setMaxConnections(int maxConnections) {
this.maxConnections = maxConnections;
if (conman != null) {
if (conman.getMaxTotal() < this.maxConnections)
conman.setMaxTotal(this.maxConnections);
conman.setDefaultMaxPerRoute(this.maxConnections);
}
}
private boolean gzipAccepted = false;
public boolean isGzipAccepted() {
return gzipAccepted;
}
/**
* if set to true, {@link WbmPersistLoadProcessor} adds a header
* {@code Accept-Encoding: gzip} to HTTP requests. New CDX server
* see this header to decide whether to compress the response. it is also
* possible to override gzipAccepted=true setting with gzip=false
* request parameter.
* It is off by default, as it should make little sense to compress single
* line of CDX.
* @param gzipAccepted true to allow gzip compression.
*/
public void setGzipAccepted(boolean gzipAccepted) {
this.gzipAccepted = gzipAccepted;
}
private Map<String, String> requestHeaders = new ConcurrentHashMap<String, String>(1, 0.75f, 2);
public Map<String, String> getRequestHeaders() {
return requestHeaders;
}
/**
* all key-value pairs in this map will be added as HTTP headers.
* typically used for providing authentication cookies. this method
* makes a copy of {@requestHeaders}.
* <em>note:</em> this property may be dropped in the future if
* I come up with better interface.
* @param requestHeaders map of &lt;header-name, header-value>.
*/
public void setRequestHeaders(Map<String, String> requestHeaders) {
if (requestHeaders == null) {
this.requestHeaders.clear();
} else {
// TODO: mmm, ConcurrentHashMap may be overkill. simple synchronized Hashtable would work
// just okay?
ConcurrentHashMap<String, String> m = new ConcurrentHashMap<String, String>(1, 0.75f, 2);
m.putAll(requestHeaders);
this.requestHeaders = m;
}
}
// statistics
private AtomicLong loadedCount = new AtomicLong();
/**
* number of times successfully loaded recrawl info.
* @return long
*/
public long getLoadedCount() {
return loadedCount.get();
}
private AtomicLong missedCount = new AtomicLong();
/**
* number of times getting no recrawl info.
* @return long
*/
public long getMissedCount() {
return missedCount.get();
}
private AtomicLong errorCount = new AtomicLong();
/**
* number of times cdx-server API call failed.
* @return long
*/
public long getErrorCount() {
return errorCount.get();
}
private AtomicLong cumulativeFetchTime = new AtomicLong();
/**
* total milliseconds spent in API call.
* it is a sum of time waited for next available connection,
* and actual HTTP request-response round-trip, across all threads.
* @return
*/
public long getCumulativeFetchTime() {
return cumulativeFetchTime.get();
}
public void setHttpClient(HttpClient client) {
this.client = client;
}
// XXX HttpHeadquarterAdapter has the same code. move to common library.
private static boolean contains(HeaderElement[] elements, String value) {
for (int i = 0; i < elements.length; i++) {
if (elements[i].getName().equalsIgnoreCase(value)) {
return true;
}
}
return false;
}
public synchronized HttpClient getHttpClient() {
if (client == null) {
if (conman == null) {
conman = new PoolingHttpClientConnectionManager();
conman.setDefaultMaxPerRoute(maxConnections);
conman.setMaxTotal(Math.max(conman.getMaxTotal(), maxConnections));
}
HttpClientBuilder builder = HttpClientBuilder.create()
.disableCookieManagement()
.setConnectionManager(conman);
builder.useSystemProperties();
// TODO: use setDefaultHeaders for adding requestHeaders? It's a bit painful
// because we need to convert it to a Collection of Header objects.
// config code for older version of httpclient.
// builder.setDefaultSocketConfig(SocketConfig.custom().setSoTimeout(socketTimeout).build());
// HttpParams params = client.getParams();
// params.setIntParameter(CoreConnectionPNames.SO_TIMEOUT, socketTimeout);
// params.setIntParameter(CoreConnectionPNames.CONNECTION_TIMEOUT, connectionTimeout);
// setup request/response intercepter for handling gzip-compressed response.
// Disabled because httpclient 4.3.3 sends "Accept-Encoding: gzip,deflate" by
// default. Response parsing will fail If gzip-decompression ResponseInterceptor
// is installed.
builder.addInterceptorLast(new HttpRequestInterceptor() {
@Override
public void process(final HttpRequest request, final HttpContext context) {
// System.err.println("RequestInterceptor");
// if (request.containsHeader("Accept-Encoding")) {
// System.err.println("already has Accept-Encoding: " + request.getHeaders("Accept-Encoding")[0]);
// }
// if (gzipAccepted) {
// if (!request.containsHeader("Accept-Encoding")) {
// request.addHeader("Accept-Encoding", "gzip");
// }
// }
// add extra headers configured.
if (requestHeaders != null) {
for (Entry<String, String> ent : requestHeaders.entrySet()) {
request.addHeader(ent.getKey(), ent.getValue());
}
}
}
});
// builder.addInterceptorFirst(new HttpResponseInterceptor() {
// @Override
// public void process(final HttpResponse response, final HttpContext context)
// throws HttpException, IOException {
// HttpEntity entity = response.getEntity();
// Header ceheader = entity.getContentEncoding();
// if (ceheader != null && contains(ceheader.getElements(), "gzip")) {
// response.setEntity(new GzipInflatingHttpEntityWrapper(response.getEntity()));
// }
// }
// });
this.client = builder.build();
}
return client;
}
private long queryRangeSecs = 6L*30*24*3600;
/**
*
* @param queryRangeSecs
*/
public void setQueryRangeSecs(long queryRangeSecs) {
this.queryRangeSecs = queryRangeSecs;
}
public long getQueryRangeSecs() {
return queryRangeSecs;
}
private String buildStartDate() {
final long range = queryRangeSecs;
if (range <= 0)
return ArchiveUtils.get14DigitDate(new Date(0));
Date now = new Date();
Date startDate = new Date(now.getTime() - range*1000);
return ArchiveUtils.get14DigitDate(startDate);
}
protected String buildURL(String url) {
// we don't need to pass scheme part, but no problem passing it.
StringBuilder sb = new StringBuilder();
final FormatSegment[] segments = preparedQueryURL;
String encodedURL;
try {
encodedURL = URLEncoder.encode(url, "UTF-8");
} catch (UnsupportedEncodingException ex) {
encodedURL = url;
}
final String[] args = new String[] {
encodedURL,
buildStartDate()
};
for (FormatSegment fs : segments) {
fs.print(sb, args);
}
return sb.toString();
}
public WbmPersistLoadProcessor() {
prepareQueryURL();
}
protected InputStream getCDX(String qurl) throws InterruptedException, IOException {
final String url = buildURL(qurl);
HttpGet m = new HttpGet(url);
m.setConfig(RequestConfig.custom().setConnectTimeout(connectionTimeout)
.setSocketTimeout(socketTimeout).build());
HttpEntity entity = null;
int attempts = 0;
do {
if (Thread.interrupted())
throw new InterruptedException("interrupted while GET " + url);
if (attempts > 0) {
Thread.sleep(5000);
}
try {
long t0 = System.currentTimeMillis();
HttpResponse resp = getHttpClient().execute(m);
cumulativeFetchTime.addAndGet(System.currentTimeMillis() - t0);
StatusLine sl = resp.getStatusLine();
if (sl.getStatusCode() != 200) {
log.error("GET " + url + " failed with status=" + sl.getStatusCode() + " " + sl.getReasonPhrase());
entity = resp.getEntity();
entity.getContent().close();
entity = null;
continue;
}
entity = resp.getEntity();
} catch (IOException ex) {
log.error("GEt " + url + " failed with error " + ex.getMessage());
} catch (Exception ex) {
log.error("GET " + url + " failed with error ", ex);
}
} while (entity == null && ++attempts < 3);
if (entity == null) {
throw new IOException("giving up on GET " + url + " after " + attempts + " attempts");
}
return entity.getContent();
}
@Override
protected ProcessResult innerProcessResult(CrawlURI curi) throws InterruptedException {
InputStream is;
try {
is = getCDX(curi.toString());
} catch (IOException ex) {
log.error(ex.getMessage());
errorCount.incrementAndGet();
return ProcessResult.PROCEED;
}
Map<String, Object> info = null;
try {
info = getLastCrawl(is);
} catch (IOException ex) {
log.error("error parsing response", ex);
} finally {
if (is != null)
ArchiveUtils.closeQuietly(is);
}
if (info != null) {
Map<String, Object> history = FetchHistoryHelper.getFetchHistory(curi,
(Long)info.get(FetchHistoryHelper.A_TIMESTAMP), historyLength);
if (history != null)
history.putAll(info);
loadedCount.incrementAndGet();
} else {
missedCount.incrementAndGet();
}
return ProcessResult.PROCEED;
}
protected HashMap<String, Object> getLastCrawl(InputStream is) throws IOException {
// read CDX lines, save most recent (at the end) hash.
ByteBuffer buffer = ByteBuffer.allocate(32);
ByteBuffer tsbuffer = ByteBuffer.allocate(14);
int field = 0;
int c;
do {
c = is.read();
if (field == 1) {
// 14-digits timestamp
tsbuffer.clear();
while (Character.isDigit(c) && tsbuffer.remaining() > 0) {
tsbuffer.put((byte)c);
c = is.read();
}
if (c != ' ' || tsbuffer.position() != 14) {
tsbuffer.clear();
}
// fall through to skip the rest
} else if (field == 5) {
buffer.clear();
while ((c >= 'A' && c <= 'Z' || c >= '0' && c <= '9') && buffer.remaining() > 0) {
buffer.put((byte)c);
c = is.read();
}
if (c != ' ' || buffer.position() != 32) {
buffer.clear();
}
// fall through to skip the rest
}
while (true) {
if (c == -1) {
break;
} else if (c == '\n') {
field = 0;
break;
} else if (c == ' ') {
field++;
break;
}
c = is.read();
}
} while (c != -1);
HashMap<String, Object> info = new HashMap<String, Object>();
if (buffer.remaining() == 0) {
info.put(RecrawlAttributeConstants.A_CONTENT_DIGEST, contentDigestScheme + new String(buffer.array()));
}
if (tsbuffer.remaining() == 0) {
try {
long ts = DateUtils.parse14DigitDate(new String(tsbuffer.array())).getTime();
// A_TIMESTAMP has been used for sorting history long before A_FETCH_BEGAN_TIME
// field was introduced. Now FetchHistoryProcessor fails if A_FETCH_BEGAN_TIME is
// not set. We could stop storing A_TIMESTAMP and sort by A_FETCH_BEGAN_TIME.
info.put(FetchHistoryHelper.A_TIMESTAMP, ts);
info.put(CoreAttributeConstants.A_FETCH_BEGAN_TIME, ts);
} catch (ParseException ex) {
}
}
return info.isEmpty() ? null : info;
}
/**
* unused.
*/
@Override
protected void innerProcess(CrawlURI uri) throws InterruptedException {
}
@Override
protected boolean shouldProcess(CrawlURI uri) {
// TODO: we want deduplicate robots.txt, too.
//if (uri.isPrerequisite()) return false;
String scheme = uri.getUURI().getScheme();
if (!(scheme.equals("http") || scheme.equals("https"))) return false;
return true;
}
/**
* main entry point for quick test.
* @param args
*/
public static void main(String[] args) throws Exception {
String url = args[0];
String cookie = args.length > 1 ? args[1] : null;
WbmPersistLoadProcessor wp = new WbmPersistLoadProcessor();
if (cookie != null) {
wp.setRequestHeaders(Collections.singletonMap("Cookie", cookie));
}
InputStream is = wp.getCDX(url);
byte[] b = new byte[1024];
int n;
while ((n = is.read(b)) > 0) {
System.out.write(b, 0, n);
}
is.close();
}
}
contrib/src/test/java/org/archive/modules/recrawl/wbm/WbmPersistLoadProcessorTest.java
+221
View File
@@ -0,0 +1,221 @@
package org.archive.modules.recrawl.wbm;
import java.io.BufferedReader;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.URLEncoder;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.regex.Pattern;
import junit.framework.TestCase;
import org.apache.http.HttpResponse;
import org.apache.http.ProtocolVersion;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.entity.ByteArrayEntity;
import org.apache.http.message.BasicHttpResponse;
import org.apache.http.message.BasicStatusLine;
import org.archive.modules.CoreAttributeConstants;
import org.archive.modules.CrawlURI;
import org.archive.modules.ProcessResult;
import org.archive.modules.recrawl.FetchHistoryHelper;
import org.archive.modules.recrawl.FetchHistoryProcessor;
import org.archive.modules.recrawl.RecrawlAttributeConstants;
import org.archive.net.UURIFactory;
import org.archive.util.Base32;
import org.archive.util.DateUtils;
import org.easymock.EasyMock;
import com.google.common.util.concurrent.ExecutionList;
/**
* unit test for {@link WbmPersistLoadProcessor}.
*
* TODO:
* <ul>
* <li>test for pathological cases: illegal chars, incorrect length, etc.
* <li>test if connection is properly released back to the pool for all possible cases.
* </ul>
* @author kenji
*
*/
public class WbmPersistLoadProcessorTest extends TestCase {
public void testBuildURL() throws Exception {
WbmPersistLoadProcessor t = new WbmPersistLoadProcessor();
t.setQueryURL("http://web.archive.org/cdx/search/cdx?url=$u&startDate=$s&limit=1");
final String URL = "http://archive.org/";
String url = t.buildURL(URL);
System.err.println(url);
assertTrue("has encode URL", Pattern.matches(".*[&?]url="+URLEncoder.encode(URL, "UTF-8")+"([&].*)?", url));
assertTrue("has startDate", Pattern.matches(".*[&?]startDate=\\d{14}([&].*)?", url));
assertTrue("has limit", Pattern.matches(".*[&?]limit=\\d+([&].*)?", url));
//assertTrue("has last=true", Pattern.matches(".*[&?]last=true([&].*)?", url));
}
/**
* stub HttpResponse for normal case.
* @author kenji
*
*/
public static class TestNormalHttpResponse extends BasicHttpResponse {
public static final String EXPECTED_TS = "20121101155310";
public static final String EXPECTED_HASH = "GHN5VKF3TBKNSEZTASOM23BJRTKFFNJK";
public TestNormalHttpResponse() {
super(new BasicStatusLine(new ProtocolVersion("HTTP", 1, 0), 200, "OK"));
setEntity(new ByteArrayEntity(
("org,archive)/ "+EXPECTED_TS+" http://archive.org/ text/html 200 "+
EXPECTED_HASH+" - - 6908 982548871 "+
"google.es-20121101-155506/IA-FOC-google.es-20121101073708-00001.warc.gz\n"
).getBytes()
));
}
}
protected Map<String, Object> getFetchHistory(CrawlURI curi, int idx) {
Map<String, Object> data = curi.getData();
@SuppressWarnings("unchecked")
Map<String, Object>[] historyArray = (Map[])data.get(RecrawlAttributeConstants.A_FETCH_HISTORY);
assertNotNull(historyArray);
Map<String, Object> history = historyArray[idx];
return history;
}
private byte[] sha1Digest(String text) throws NoSuchAlgorithmException {
MessageDigest md = MessageDigest.getInstance("sha1");
return md.digest(text.getBytes());
}
/**
* this test is disabled because it talks to production CDX server.
* @throws Exception
*/
public void _testInnerProcessResultSingleShotWithMock() throws Exception {
// because AbstractHttpClient marks most execute(...) methods final, it is very tiresome to implement
// a stub by implementing HttpClient interface. So I use EasyMock.
HttpClient client = EasyMock.createMock(HttpClient.class);
HttpResponse testResponse = new TestNormalHttpResponse();
EasyMock.expect(client.execute((HttpUriRequest)EasyMock.notNull())).andReturn(testResponse);
EasyMock.replay(client);
final String CONTENT_DIGEST_SCHEME = "sha1:";
WbmPersistLoadProcessor t = new WbmPersistLoadProcessor();
t.setHttpClient(client);
t.setContentDigestScheme(CONTENT_DIGEST_SCHEME);
CrawlURI curi = new CrawlURI(UURIFactory.getInstance("http://archive.org/"));
// put history entry newer than being loaded (i.e. loaded history entry will not be used for FetchHistoryProcessor
// check below.
long expected_ts = DateUtils.parse14DigitDate(TestNormalHttpResponse.EXPECTED_TS).getTime();
Map<String, Object>[] fetchHistory = (Map[])curi.getData().get(RecrawlAttributeConstants.A_FETCH_HISTORY);
if (fetchHistory == null) {
fetchHistory = new HashMap[2];
curi.getData().put(RecrawlAttributeConstants.A_FETCH_HISTORY, fetchHistory);
}
final byte[] digestValue0 = sha1Digest("0");
final byte[] digestValue1 = sha1Digest("1");
fetchHistory[0] = new HashMap<String, Object>();
fetchHistory[0].put(FetchHistoryHelper.A_TIMESTAMP, expected_ts + 2000);
fetchHistory[0].put(CoreAttributeConstants.A_FETCH_BEGAN_TIME, expected_ts + 2000);
fetchHistory[0].put(RecrawlAttributeConstants.A_CONTENT_DIGEST,
CONTENT_DIGEST_SCHEME + Base32.encode(digestValue0));
fetchHistory[1] = new HashMap<String, Object>();
fetchHistory[1].put(FetchHistoryHelper.A_TIMESTAMP, expected_ts - 2000);
fetchHistory[1].put(RecrawlAttributeConstants.A_CONTENT_DIGEST,
CONTENT_DIGEST_SCHEME + Base32.encode(digestValue1));
ProcessResult result = t.innerProcessResult(curi);
assertEquals("result is PROCEED", ProcessResult.PROCEED, result);
// newly loaded history entry should fall in between two existing entries (index=1)
Map<String, Object> history = getFetchHistory(curi, 1);
assertNotNull("history", history);
String hash = (String)history.get(RecrawlAttributeConstants.A_CONTENT_DIGEST);
assertEquals("CONTENT_DIGEST", CONTENT_DIGEST_SCHEME+TestNormalHttpResponse.EXPECTED_HASH, hash);
Long ts = (Long)history.get(FetchHistoryHelper.A_TIMESTAMP);
assertNotNull("ts is non-null", ts);
assertEquals("'ts' has expected timestamp", expected_ts, ts.longValue());
// Check compatibility with FetchHistoryProcessor.
// TODO: This is not testing WbmPersistLoadProcessor - only testing stub fetchHistory
// setup above (OK as long as it matches WbmPersistLoadProcessor). We need a separate
// test method.
curi.setFetchStatus(200);
curi.setFetchBeginTime(System.currentTimeMillis());
// FetchHistoryProcessor once failed for a revisit case. We'd need to test other cases
// too (TODO).
curi.setContentDigest("sha1", digestValue0);
FetchHistoryProcessor fhp = new FetchHistoryProcessor();
fhp.process(curi);
}
public void testInnerProcessResultSingleShotWithRealServer() throws Exception {
WbmPersistLoadProcessor t = new WbmPersistLoadProcessor();
//CrawlURI curi = new CrawlURI(UURIFactory.getInstance("http://archive.org/"));
CrawlURI curi = new CrawlURI(UURIFactory.getInstance("http://www.mext.go.jp/null.gif"));
ProcessResult result = t.innerProcessResult(curi);
Map<String, Object> history = getFetchHistory(curi, 0);
assertNotNull("getFetchHistory returns non-null", history);
String hash = (String)history.get(RecrawlAttributeConstants.A_CONTENT_DIGEST);
assertNotNull("CONTENT_DIGEST is non-null", hash);
assertTrue("CONTENT_DIGEST starts with scheme", hash.startsWith(t.getContentDigestScheme()));
assertEquals("CONTENT_DIGEST is a String of length 32", 32, hash.substring(t.getContentDigestScheme().length()).length());
assertEquals("should always return PROCEED", ProcessResult.PROCEED, result);
}
public static class LoadTask implements Runnable {
private WbmPersistLoadProcessor p;
private String uri;
public LoadTask(WbmPersistLoadProcessor p, String uri) {
this.p = p;
this.uri = uri;
}
@Override
public void run() {
try {
CrawlURI curi = new CrawlURI(UURIFactory.getInstance(this.uri));
p.innerProcessResult(curi);
//System.err.println(curi.toString());
} catch (Exception ex) {
ex.printStackTrace();
}
}
}
/**
* test for performance.
* not named as test case. run this through main().
* @throws Exception
*/
public void measureInnerProcessResultMany() throws Exception {
final WbmPersistLoadProcessor t = new WbmPersistLoadProcessor();
InputStream is = getClass().getResourceAsStream("/test-url-list.txt");
BufferedReader br = new BufferedReader(new InputStreamReader(is));
String line;
int nurls = 0;
ExecutorService executor = Executors.newFixedThreadPool(100);
ExecutionList tasks = new ExecutionList();
while ((line = br.readLine()) != null) {
tasks.add(new LoadTask(t, line), executor);
nurls++;
}
long t0 = System.currentTimeMillis();
tasks.execute();
executor.awaitTermination(30, TimeUnit.SECONDS);
long el = System.currentTimeMillis() - t0;
System.err.println(nurls + " urls, time=" + el + "ms (" + (nurls / (el / 1000.0)) + " URI/s");
}
public static void main() throws Exception {
(new WbmPersistLoadProcessorTest()).measureInnerProcessResultMany();
}
}
engine/src/main/java/org/archive/crawler/restlet/JobRelatedResource.java
+8 -2
View File
@@ -44,6 +44,7 @@ import org.restlet.data.Response;
import org.restlet.resource.ResourceException;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.BeanWrapperImpl;
import org.springframework.beans.BeansException;
/**
* Shared superclass for resources that represent functional aspects
@@ -118,8 +119,13 @@ public abstract class JobRelatedResource extends BaseResource {
for (PropertyDescriptor pd : getPropertyDescriptors(bwrap)) {
if (pd.getReadMethod() != null) {
String propName = pd.getName();
Object propValue = bwrap.getPropertyValue(propName);
addPresentableNestedNames(namedBeans, propValue, alreadyWritten);
try {
Object propValue = bwrap.getPropertyValue(propName);
addPresentableNestedNames(namedBeans, propValue, alreadyWritten);
} catch (BeansException ex) {
// getter may throw UnsupportedOperationException, for ex.
// ignore such properties.
}
}
}
if (obj.getClass().isArray()) {
engine/src/test/java/org/archive/crawler/datamodel/CrawlURITest.java
+117 -22
View File
@@ -1,8 +1,8 @@
/*
* This file is part of the Heritrix web crawler (crawler.archive.org).
*
* Licensed to the Internet Archive (IA) by one or more individual
* contributors.
* Licensed to the Internet Archive (IA) by one or more individual
* contributors.
*
* The IA licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
@@ -28,21 +28,22 @@ import java.io.ObjectOutputStream;
import org.apache.commons.httpclient.URIException;
import org.archive.modules.CrawlURI;
import org.archive.modules.SchedulingConstants;
import org.archive.modules.extractor.LinkContext.SimpleLinkContext;
import org.archive.net.UURI;
import org.archive.net.UURIFactory;
import org.archive.util.TmpDirTestCase;
/**
* Tests related to CrawlURI
*
*
* @contributor stack
* @contributor gojomo
* @version $Revision$, $Date$
*/
public class CrawlURITest extends TmpDirTestCase {
CrawlURI seed = null;
protected void setUp() throws Exception {
super.setUp();
final String url = "http://www.dh.gov.uk/Home/fs/en";
@@ -57,14 +58,14 @@ public class CrawlURITest extends TmpDirTestCase {
/**
* Test serialization/deserialization works.
*
*
* @throws IOException
* @throws ClassNotFoundException
*/
final public void testSerialization()
throws IOException, ClassNotFoundException {
File serialize = new File(getTmpDir(),
this.getClass().getName() + ".serialize");
throws IOException, ClassNotFoundException {
File serialize = new File(getTmpDir(),
this.getClass().getName() + ".serialize");
try {
FileOutputStream fos = new FileOutputStream(serialize);
ObjectOutputStream oos = new ObjectOutputStream(fos);
@@ -81,43 +82,137 @@ public class CrawlURITest extends TmpDirTestCase {
deserializedCuri = (CrawlURI)ois.readObject();
deserializedCuri = (CrawlURI)ois.readObject();
assertEquals("Deserialized not equal to original",
this.seed.toString(), deserializedCuri.toString());
this.seed.toString(), deserializedCuri.toString());
String host = this.seed.getUURI().getHost();
assertTrue("Deserialized host not null",
host != null && host.length() >= 0);
host != null && host.length() >= 0);
} finally {
serialize.delete();
}
}
public void testCandidateURIWithLoadedAList()
throws URIException {
throws URIException {
UURI uuri = UURIFactory.getInstance("http://www.archive.org");
CrawlURI curi = new CrawlURI(uuri);
curi.setSeed(true);
curi.getData().put("key", "value");
assertTrue("Didn't find AList item",
curi.getData().get("key").equals("value"));
curi.getData().get("key").equals("value"));
}
public void testExtendHopsPath() {
assertEquals("from empty","L",CrawlURI.extendHopsPath("",'L'));
assertEquals("from one","LX",CrawlURI.extendHopsPath("L",'X'));
assertEquals(
"from fortynine",
"LLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLX",
CrawlURI.extendHopsPath("LLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLL",'X'));
"from fortynine",
"LLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLX",
CrawlURI.extendHopsPath("LLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLL",'X'));
assertEquals(
"from fifty",
"1+LLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLX",
CrawlURI.extendHopsPath("LLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLL",'X'));
assertEquals(
"from 149",
"100+LLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLX",
CrawlURI.extendHopsPath("99+LLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLL",'X'));
}
public void testNullPathFromSeed() throws URIException {
// check comparing with null
CrawlURI a = new CrawlURI(
UURIFactory.getInstance("http://example.com/1"), // a == b
null, // a < b
UURIFactory.getInstance("http://example.com/via/1"), // a == b
new SimpleLinkContext("1")); // a == b
assertEquals("", a.getPathFromSeed());
CrawlURI b = new CrawlURI(
UURIFactory.getInstance("http://example.com/1"), // a == b
"", // a < b
UURIFactory.getInstance("http://example.com/via/1"), // a == b
new SimpleLinkContext("1")); // a == b
assertEquals("", b.getPathFromSeed());
assertEquals(0, a.compareTo(b));
assertEquals(0, b.compareTo(a));
}
public void testOrdering() throws URIException {
// check that via is highest precedence
CrawlURI a = new CrawlURI(
UURIFactory.getInstance("http://example.com/2"), // a > b
"2", // a > b
UURIFactory.getInstance("http://example.com/via/1"), // a < b
new SimpleLinkContext("2")); // a > b
CrawlURI b = new CrawlURI(
UURIFactory.getInstance("http://example.com/1"), // a > b
"1", // a > b
UURIFactory.getInstance("http://example.com/via/2"), // a < b
new SimpleLinkContext("1")); // a > b
assertEquals(-1, a.compareTo(b));
assertEquals(1, b.compareTo(a));
// check that uri is next highest
a = new CrawlURI(
UURIFactory.getInstance("http://example.com/1"), // a < b
"2", // a > b
UURIFactory.getInstance("http://example.com/via/1"), // a == b
new SimpleLinkContext("2")); // a > b
b = new CrawlURI(
UURIFactory.getInstance("http://example.com/2"), // a < b
"1", // a > b
UURIFactory.getInstance("http://example.com/via/1"), // a == b
new SimpleLinkContext("1")); // a > b
assertEquals(-1, a.compareTo(b));
assertEquals(1, b.compareTo(a));
// check that via context is next
a = new CrawlURI(
UURIFactory.getInstance("http://example.com/1"), // a == b
"2", // a > b
UURIFactory.getInstance("http://example.com/via/1"), // a == b
new SimpleLinkContext("1")); // a < b
b = new CrawlURI(
UURIFactory.getInstance("http://example.com/1"), // a == b
"1", // a > b
UURIFactory.getInstance("http://example.com/via/1"), // a == b
new SimpleLinkContext("2")); // a < b
assertEquals(-1, a.compareTo(b));
assertEquals(1, b.compareTo(a));
// check that pathFromSeed is next
a = new CrawlURI(
UURIFactory.getInstance("http://example.com/1"), // a == b
"1", // a < b
UURIFactory.getInstance("http://example.com/via/1"), // a == b
new SimpleLinkContext("1")); // a == b
b = new CrawlURI(
UURIFactory.getInstance("http://example.com/1"), // a == b
"2", // a < b
UURIFactory.getInstance("http://example.com/via/1"), // a == b
new SimpleLinkContext("1")); // a == b
assertEquals(-1, a.compareTo(b));
assertEquals(1, b.compareTo(a));
// check equality
a = new CrawlURI(
UURIFactory.getInstance("http://example.com/1"), // a == b
"1", // a == b
UURIFactory.getInstance("http://example.com/via/1"), // a == b
new SimpleLinkContext("1")); // a == b
b = new CrawlURI(
UURIFactory.getInstance("http://example.com/1"), // a == b
"1", // a == b
UURIFactory.getInstance("http://example.com/via/1"), // a == b
new SimpleLinkContext("1")); // a == b
assertEquals(0, a.compareTo(b));
assertEquals(0, b.compareTo(a));
}
}
modules/src/main/java/org/archive/modules/CrawlURI.java
+1 -1
View File
@@ -1978,7 +1978,7 @@ implements Reporter, Serializable, OverlayContext, Comparable<CrawlURI> {
return 0;
} else if (a == null && b != null) {
return -1;
} else if (a != null && b != null) {
} else if (a != null && b == null) {
return 1;
} else {
return a.compareTo(b);
modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java
+1 -1
View File
@@ -669,7 +669,7 @@ public class FetchHTTP extends Processor implements Lifecycle {
long contentLength = -1l;
Header h = response.getLastHeader("content-length");
if (h != null) {
if (h != null && h.getValue().trim().length()>0) {
contentLength = Long.parseLong(h.getValue());
}
try {
pom.xml
+6
View File
@@ -118,6 +118,12 @@ http://maven.apache.org/guides/mini/guide-m1-m2.html
<artifactId>engine</artifactId>
<version>${project.version}</version>
</dependency>
<!-- lock httpclient version -->
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<version>4.3.3</version>
</dependency>
</dependencies>
</dependencyManagement>