lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-09-25 07:05:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

remove LegacyFetchHTTP and its classes

Browse Source
This commit is contained in:
Noah Levitt
2012-09-08 00:49:30 -07:00
parent 71761a72a7
commit 33df8faaf0
16 changed files with 1 additions and 3376 deletions
Download Patch File Download Diff File Expand all files Collapse all files
commons/src/main/java/org/archive/httpclient/HttpRecorderGetMethod.java
-120
View File
@@ -1,120 +0,0 @@
/*
* This file is part of the Heritrix web crawler (crawler.archive.org).
*
* Licensed to the Internet Archive (IA) by one or more individual
* contributors.
*
* The IA licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.archive.httpclient;
import java.io.IOException;
import java.util.logging.Logger;
import org.apache.commons.httpclient.HttpConnection;
import org.apache.commons.httpclient.HttpException;
import org.apache.commons.httpclient.HttpState;
import org.apache.commons.httpclient.methods.GetMethod;
import org.archive.util.Recorder;
/**
* Override of GetMethod that marks the passed HttpRecorder w/ the transition
* from HTTP head to body and that forces a close on the http connection.
*
* The actions done in this subclass used to be done by copying
* org.apache.commons.HttpMethodBase, overlaying our version in place of the
* one that came w/ httpclient. Here is the patch of the difference between
* shipped httpclient code and our mods:
* <pre>
* -- -1338,6 +1346,12 --
*
* public void releaseConnection() {
*
* + // HERITRIX always ants the streams closed.
* + if (responseConnection != null)
* + {
* + responseConnection.close();
* + }
* +
* if (responseStream != null) {
* try {
* // FYI - this may indirectly invoke responseBodyConsumed.
* -- -1959,6 +1973,11 --
* this.statusLine = null;
* }
* }
* + // HERITRIX mark transition from header to content.
* + if (this.httpRecorder != null)
* + {
* + this.httpRecorder.markContentBegin();
* + }
* readResponseBody(state, conn);
* processResponseBody(state, conn);
* } catch (IOException e) {
* </pre>
*
* <p>We're not supposed to have access to the underlying connection object;
* am only violating contract because see cases where httpclient is skipping
* out w/o cleaning up after itself.
*
* @author stack
* @version $Revision$, $Date$
*/
public class HttpRecorderGetMethod extends GetMethod {
protected static Logger logger =
Logger.getLogger(HttpRecorderGetMethod.class.getName());
/**
* Instance of http recorder method.
*/
protected HttpRecorderMethod httpRecorderMethod = null;
public HttpRecorderGetMethod(String uri, Recorder recorder) {
super(uri);
this.httpRecorderMethod = new HttpRecorderMethod(recorder);
}
protected void readResponseBody(HttpState state, HttpConnection connection)
throws IOException, HttpException {
// We're about to read the body. Mark transition in http recorder.
this.httpRecorderMethod.markContentBegin(connection);
super.readResponseBody(state, connection);
}
protected boolean shouldCloseConnection(HttpConnection conn) {
// Always close connection after each request. As best I can tell, this
// is superfluous -- we've set our client to be HTTP/1.0. Doing this
// out of paranoia.
return true;
}
public int execute(HttpState state, HttpConnection conn)
throws HttpException, IOException {
// Save off the connection so we can close it on our way out in case
// httpclient fails to (We're not supposed to have access to the
// underlying connection object; am only violating contract because
// see cases where httpclient is skipping out w/o cleaning up
// after itself).
this.httpRecorderMethod.setConnection(conn);
return super.execute(state, conn);
}
protected void addProxyConnectionHeader(HttpState state, HttpConnection conn)
throws IOException, HttpException {
super.addProxyConnectionHeader(state, conn);
this.httpRecorderMethod.handleAddProxyConnectionHeader(this);
}
}
commons/src/main/java/org/archive/httpclient/HttpRecorderMethod.java
-107
View File
@@ -1,107 +0,0 @@
/*
* This file is part of the Heritrix web crawler (crawler.archive.org).
*
* Licensed to the Internet Archive (IA) by one or more individual
* contributors.
*
* The IA licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.archive.httpclient;
import java.util.logging.Logger;
import org.apache.commons.httpclient.Header;
import org.apache.commons.httpclient.HttpConnection;
import org.apache.commons.httpclient.HttpMethod;
import org.archive.util.Recorder;
/**
* This class encapsulates the specializations supplied by the
* overrides {@link HttpRecorderGetMethod} and {@link HttpRecorderPostMethod}.
*
* It keeps instance of HttpRecorder and HttpConnection.
*
* @author stack
* @version $Revision$, $Date$
*/
public class HttpRecorderMethod {
protected static Logger logger =
Logger.getLogger(HttpRecorderMethod.class.getName());
/**
* Instance of http recorder we're using recording this http get.
*/
private Recorder httpRecorder = null;
/**
* Save around so can force close.
*
* See [ 922080 ] IllegalArgumentException (size is wrong).
* https://sourceforge.net/tracker/?func=detail&aid=922080&group_id=73833&atid=539099
*/
private HttpConnection connection = null;
public HttpRecorderMethod(Recorder recorder) {
this.httpRecorder = recorder;
}
public void markContentBegin(HttpConnection c) {
if (c != this.connection) {
// We're checking that we're not being asked to work on
// a connection that is other than the one we started
// this method#execute with.
throw new IllegalArgumentException("Connections differ: " +
this.connection + " " + c + " " +
Thread.currentThread().getName());
}
this.httpRecorder.markContentBegin();
}
/**
* @return Returns the connection.
*/
public HttpConnection getConnection() {
return this.connection;
}
/**
* @param connection The connection to set.
*/
public void setConnection(HttpConnection connection) {
this.connection = connection;
}
/**
* @return Returns the httpRecorder.
*/
public Recorder getHttpRecorder() {
return httpRecorder;
}
/**
* If a 'Proxy-Connection' header has been added to the request,
* it'll be of a 'keep-alive' type. Until we support 'keep-alives',
* override the Proxy-Connection setting and instead pass a 'close'
* (Otherwise every request has to timeout before we notice
* end-of-document).
* @param method Method to find proxy-connection header in.
*/
public void handleAddProxyConnectionHeader(HttpMethod method) {
Header h = method.getRequestHeader("Proxy-Connection");
if (h != null) {
h.setValue("close");
method.setRequestHeader(h);
}
}
}
commons/src/main/java/org/archive/httpclient/HttpRecorderPostMethod.java
-82
View File
@@ -1,82 +0,0 @@
/*
* This file is part of the Heritrix web crawler (crawler.archive.org).
*
* Licensed to the Internet Archive (IA) by one or more individual
* contributors.
*
* The IA licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.archive.httpclient;
import java.io.IOException;
import org.apache.commons.httpclient.HttpConnection;
import org.apache.commons.httpclient.HttpException;
import org.apache.commons.httpclient.HttpState;
import org.apache.commons.httpclient.methods.PostMethod;
import org.archive.util.Recorder;
/**
* Override of PostMethod that marks the passed HttpRecorder w/ the transition
* from HTTP head to body and that forces a close on the responseConnection.
*
* This is a copy of {@link HttpRecorderGetMethod}. Only difference is the
* parent subclass.
*
* @author stack
* @version $Date$ $Revision$
*/
public class HttpRecorderPostMethod extends PostMethod {
/**
* Instance of http recorder method.
*/
protected HttpRecorderMethod httpRecorderMethod = null;
public HttpRecorderPostMethod(String uri, Recorder recorder) {
super(uri);
this.httpRecorderMethod = new HttpRecorderMethod(recorder);
}
protected void readResponseBody(HttpState state, HttpConnection connection)
throws IOException, HttpException {
// We're about to read the body. Mark transition in http recorder.
this.httpRecorderMethod.markContentBegin(connection);
super.readResponseBody(state, connection);
}
protected boolean shouldCloseConnection(HttpConnection conn) {
// Always close connection after each request. As best I can tell, this
// is superfluous -- we've set our client to be HTTP/1.0. Doing this
// out of paranoia.
return true;
}
public int execute(HttpState state, HttpConnection conn)
throws HttpException, IOException {
// Save off the connection so we can close it on our way out in case
// httpclient fails to (We're not supposed to have access to the
// underlying connection object; am only violating contract because
// see cases where httpclient is skipping out w/o cleaning up
// after itself).
this.httpRecorderMethod.setConnection(conn);
return super.execute(state, conn);
}
protected void addProxyConnectionHeader(HttpState state, HttpConnection conn)
throws IOException, HttpException {
super.addProxyConnectionHeader(state, conn);
this.httpRecorderMethod.handleAddProxyConnectionHeader(this);
}
}
commons/src/main/java/org/archive/httpclient/SingleHttpConnectionManager.java
-70
View File
@@ -1,70 +0,0 @@
/*
* This file is part of the Heritrix web crawler (crawler.archive.org).
*
* Licensed to the Internet Archive (IA) by one or more individual
* contributors.
*
* The IA licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.archive.httpclient;
import java.io.IOException;
import java.io.InputStream;
import org.apache.commons.httpclient.HostConfiguration;
import org.apache.commons.httpclient.HttpConnection;
import org.apache.commons.httpclient.SimpleHttpConnectionManager;
/**
* An HttpClient-compatible HttpConnection "manager" that actually
* just gives out a new connection each time -- skipping the overhead
* of connection management, since we already throttle our crawler
* with external mechanisms.
*
* @author gojomo
*/
public class SingleHttpConnectionManager extends SimpleHttpConnectionManager {
public SingleHttpConnectionManager() {
super();
}
public HttpConnection getConnectionWithTimeout(
HostConfiguration hostConfiguration, long timeout) {
HttpConnection conn = new HttpConnection(hostConfiguration);
conn.setHttpConnectionManager(this);
conn.getParams().setDefaults(this.getParams());
return conn;
}
public void releaseConnection(HttpConnection conn) {
// ensure connection is closed
conn.close();
finishLast(conn);
}
protected static void finishLast(HttpConnection conn) {
// copied from superclass because it wasn't made available to subclasses
InputStream lastResponse = conn.getLastResponseInputStream();
if (lastResponse != null) {
conn.setLastResponseInputStream(null);
try {
lastResponse.close();
} catch (IOException ioe) {
//FIXME: badness - close to force reconnect.
conn.close();
}
}
}
}
commons/src/main/java/org/archive/httpclient/ThreadLocalHttpConnectionManager.java
-291
View File
@@ -1,291 +0,0 @@
/**
* ====================================================================
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* ====================================================================
*
*/
package org.archive.httpclient;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.apache.commons.httpclient.HostConfiguration;
import org.apache.commons.httpclient.HttpConnection;
import org.apache.commons.httpclient.HttpConnectionManager;
import org.apache.commons.httpclient.params.HttpConnectionManagerParams;
/**
* A simple, but thread-safe HttpClient {@link HttpConnectionManager}.
* Based on {@link org.apache.commons.httpclient.SimpleHttpConnectionManager}.
*
* <b>Java &gt;= 1.4 is recommended.</b>
*
* @author Christian Kohlschuetter
*/
public final class ThreadLocalHttpConnectionManager implements
HttpConnectionManager {
private static final CloserThread closer = new CloserThread();
private static final Logger logger = Logger
.getLogger(ThreadLocalHttpConnectionManager.class.getName());
private final ThreadLocal<ConnectionInfo> tl = new ThreadLocal<ConnectionInfo>() {
protected synchronized ConnectionInfo initialValue() {
return new ConnectionInfo();
}
};
private ConnectionInfo getConnectionInfo() {
return (ConnectionInfo) tl.get();
}
private static final class ConnectionInfo {
/** The http connection */
private HttpConnection conn = null;
/**
* The time the connection was made idle.
*/
private long idleStartTime = Long.MAX_VALUE;
}
public ThreadLocalHttpConnectionManager() {
}
/**
* Since the same connection is about to be reused, make sure the
* previous request was completely processed, and if not
* consume it now.
* @param conn The connection
* @return true, if the connection is reusable
*/
private static boolean finishLastResponse(final HttpConnection conn) {
InputStream lastResponse = conn.getLastResponseInputStream();
if(lastResponse != null) {
conn.setLastResponseInputStream(null);
try {
lastResponse.close();
return true;
} catch (IOException ioe) {
// force reconnect.
return false;
}
} else {
return false;
}
}
/**
* Collection of parameters associated with this connection manager.
*/
private HttpConnectionManagerParams params = new HttpConnectionManagerParams();
/**
* @see HttpConnectionManager#getConnection(HostConfiguration)
*/
public HttpConnection getConnection(
final HostConfiguration hostConfiguration) {
return getConnection(hostConfiguration, 0);
}
/**
* Gets the staleCheckingEnabled value to be set on HttpConnections that are created.
*
* @return <code>true</code> if stale checking will be enabled on HttpConections
*
* @see HttpConnection#isStaleCheckingEnabled()
*
* @deprecated Use {@link HttpConnectionManagerParams#isStaleCheckingEnabled()},
* {@link HttpConnectionManager#getParams()}.
*/
public boolean isConnectionStaleCheckingEnabled() {
return this.params.isStaleCheckingEnabled();
}
/**
* Sets the staleCheckingEnabled value to be set on HttpConnections that are created.
*
* @param connectionStaleCheckingEnabled <code>true</code> if stale checking will be enabled
* on HttpConections
*
* @see HttpConnection#setStaleCheckingEnabled(boolean)
*
* @deprecated Use {@link HttpConnectionManagerParams#setStaleCheckingEnabled(boolean)},
* {@link HttpConnectionManager#getParams()}.
*/
public void setConnectionStaleCheckingEnabled(
final boolean connectionStaleCheckingEnabled) {
this.params.setStaleCheckingEnabled(connectionStaleCheckingEnabled);
}
/**
* @see HttpConnectionManager#getConnectionWithTimeout(HostConfiguration, long)
*
* @since 3.0
*/
public HttpConnection getConnectionWithTimeout(
final HostConfiguration hostConfiguration, final long timeout) {
final ConnectionInfo ci = getConnectionInfo();
HttpConnection httpConnection = ci.conn;
// make sure the host and proxy are correct for this connection
// close it and set the values if they are not
if(httpConnection == null || !finishLastResponse(httpConnection)
|| !hostConfiguration.hostEquals(httpConnection)
|| !hostConfiguration.proxyEquals(httpConnection)) {
if(httpConnection != null && httpConnection.isOpen()) {
closer.closeConnection(httpConnection);
}
httpConnection = new HttpConnection(hostConfiguration);
httpConnection.setHttpConnectionManager(this);
httpConnection.getParams().setDefaults(this.params);
ci.conn = httpConnection;
httpConnection.setHost(hostConfiguration.getHost());
httpConnection.setPort(hostConfiguration.getPort());
httpConnection.setProtocol(hostConfiguration.getProtocol());
httpConnection.setLocalAddress(hostConfiguration.getLocalAddress());
httpConnection.setProxyHost(hostConfiguration.getProxyHost());
httpConnection.setProxyPort(hostConfiguration.getProxyPort());
}
// remove the connection from the timeout handler
ci.idleStartTime = Long.MAX_VALUE;
return httpConnection;
}
/**
* @see HttpConnectionManager#getConnection(HostConfiguration, long)
*
* @deprecated Use #getConnectionWithTimeout(HostConfiguration, long)
*/
public HttpConnection getConnection(
final HostConfiguration hostConfiguration, final long timeout) {
return getConnectionWithTimeout(hostConfiguration, timeout);
}
/**
* @see HttpConnectionManager#releaseConnection(org.apache.commons.httpclient.HttpConnection)
*/
public void releaseConnection(final HttpConnection conn) {
final ConnectionInfo ci = getConnectionInfo();
HttpConnection httpConnection = ci.conn;
if(conn != httpConnection) {
throw new IllegalStateException(
"Unexpected release of an unknown connection.");
}
finishLastResponse(httpConnection);
// track the time the connection was made idle
ci.idleStartTime = System.currentTimeMillis();
}
/**
* Returns {@link HttpConnectionManagerParams parameters} associated
* with this connection manager.
*
* @since 2.1
*
* @see HttpConnectionManagerParams
*/
public HttpConnectionManagerParams getParams() {
return this.params;
}
/**
* Assigns {@link HttpConnectionManagerParams parameters} for this
* connection manager.
*
* @since 2.1
*
* @see HttpConnectionManagerParams
*/
public void setParams(final HttpConnectionManagerParams p) {
if(p == null) {
throw new IllegalArgumentException("Parameters may not be null");
}
this.params = p;
}
/**
* @since 3.0
*/
public void closeIdleConnections(final long idleTimeout) {
long maxIdleTime = System.currentTimeMillis() - idleTimeout;
final ConnectionInfo ci = getConnectionInfo();
if(ci.idleStartTime <= maxIdleTime) {
ci.conn.close();
}
}
private static final class CloserThread extends Thread {
private List<HttpConnection> connections
= new ArrayList<HttpConnection>();
private static final int SLEEP_INTERVAL = 5000;
public CloserThread() {
super("HttpConnection closer");
// Make this a daemon thread so it can't be responsible for the JVM
// not shutting down.
setDaemon(true);
start();
}
public void closeConnection(final HttpConnection conn) {
synchronized (connections) {
connections.add(conn);
}
}
public void run() {
try {
while (!Thread.interrupted()) {
Thread.sleep(SLEEP_INTERVAL);
List<HttpConnection> s;
synchronized (connections) {
s = connections;
connections = new ArrayList<HttpConnection>();
}
logger.log(Level.INFO, "Closing " + s.size()
+ " HttpConnections");
for(final Iterator<HttpConnection> it = s.iterator();
it.hasNext();) {
HttpConnection conn = it.next();
conn.close();
conn.setHttpConnectionManager(null);
it.remove();
}
}
} catch (InterruptedException e) {
return;
}
}
}
}
engine/src/test/java/org/archive/modules/fetcher/FetchHTTPTest.java
-54
View File
@@ -1,54 +0,0 @@
/*
* This file is part of the Heritrix web crawler (crawler.archive.org).
*
* Licensed to the Internet Archive (IA) by one or more individual
* contributors.
*
* The IA licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.archive.modules.fetcher;
import java.io.IOException;
import org.archive.modules.CrawlMetadata;
/**
*
*/
public class FetchHTTPTest extends FetchHTTPTestBase {
@Override
protected AbstractFetchHTTP makeModule() throws IOException {
LegacyFetchHTTP fetchHttp = new LegacyFetchHTTP();
fetchHttp.setCookieStorage(new SimpleCookieStorage());
fetchHttp.setServerCache(new DefaultServerCache());
CrawlMetadata uap = new CrawlMetadata();
uap.setUserAgentTemplate(getUserAgentString());
fetchHttp.setUserAgentProvider(uap);
fetchHttp.start();
return fetchHttp;
}
@Override
public void testHttpProxyAuth() throws Exception {
// XXX skip cuz it's slow in FetchHTTP for some reason
}
@Override
public void testConnectionTimeout() throws Exception {
// XXX skip cuz it's slow cuz you can't change the connection timeout after FetchHTTP.start() has run
}
}
modules/src/main/java/org/archive/modules/credential/CommonsHttpCredentialUtil.java
-147
View File
@@ -1,147 +0,0 @@
/*
* This file is part of the Heritrix web crawler (crawler.archive.org).
*
* Licensed to the Internet Archive (IA) by one or more individual
* contributors.
*
* The IA licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.archive.modules.credential;
import java.util.Iterator;
import java.util.Map;
import java.util.logging.Logger;
import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpMethod;
import org.apache.commons.httpclient.HttpMethodBase;
import org.apache.commons.httpclient.NameValuePair;
import org.apache.commons.httpclient.URIException;
import org.apache.commons.httpclient.UsernamePasswordCredentials;
import org.apache.commons.httpclient.auth.AuthChallengeProcessor;
import org.apache.commons.httpclient.auth.AuthScheme;
import org.apache.commons.httpclient.auth.AuthScope;
import org.apache.commons.httpclient.auth.AuthenticationException;
import org.apache.commons.httpclient.auth.MalformedChallengeException;
import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.httpclient.methods.PostMethod;
import org.apache.commons.lang.StringUtils;
import org.archive.modules.CrawlURI;
public class CommonsHttpCredentialUtil {
private static Logger logger = Logger.getLogger(CommonsHttpCredentialUtil.class.getName());
public static boolean populate(CrawlURI curi, HttpClient http,
HttpMethod method, Credential cred, Map<String, String> httpAuthChallenges) {
if (cred instanceof HttpAuthenticationCredential) {
return populate(curi, http, method, (HttpAuthenticationCredential) cred, httpAuthChallenges);
} else if (cred instanceof HtmlFormCredential) {
return populate(curi, http, method, (HtmlFormCredential) cred);
} else {
throw new RuntimeException("not implemented for Credential subtype " + cred.getClass());
}
}
public static boolean populate(CrawlURI curi, HttpClient http,
HttpMethod method, HtmlFormCredential cred) {
// http is not used
boolean result = false;
Map<String,String> formItems = cred.getFormItems();
if (formItems == null || formItems.size() <= 0) {
try {
logger.severe("No form items for " + method.getURI());
}
catch (URIException e) {
logger.severe("No form items and exception getting uri: " +
e.getMessage());
}
return result;
}
NameValuePair[] data = new NameValuePair[formItems.size()];
int index = 0;
String key = null;
for (Iterator<String> i = formItems.keySet().iterator(); i.hasNext();) {
key = i.next();
data[index++] = new NameValuePair(key, (String)formItems.get(key));
}
if (method instanceof PostMethod) {
((PostMethod)method).setRequestBody(data);
result = true;
} else if (method instanceof GetMethod) {
// Append these values to the query string.
// Get current query string, then add data, then get it again
// only this time its our data only... then append.
HttpMethodBase hmb = (HttpMethodBase)method;
String currentQuery = hmb.getQueryString();
hmb.setQueryString(data);
String newQuery = hmb.getQueryString();
hmb.setQueryString(
((StringUtils.isNotEmpty(currentQuery))
? currentQuery + "&"
: "")
+ newQuery);
result = true;
} else {
logger.severe("Unknown method type: " + method);
}
return result;
}
public static boolean populate(CrawlURI curi, HttpClient http,
HttpMethod method, HttpAuthenticationCredential cred, Map<String,String> httpAuthChallenges) {
boolean result = false;
AuthChallengeProcessor authChallengeProcessor = new AuthChallengeProcessor(http.getParams());
try {
AuthScheme authScheme = authChallengeProcessor.processChallenge(method.getHostAuthState(), httpAuthChallenges);
method.getHostAuthState().setAuthScheme(authScheme);
} catch (MalformedChallengeException e) {
return result;
} catch (AuthenticationException e) {
return result;
}
// Always add the credential to HttpState. Doing this because no way of
// removing the credential once added AND there is a bug in the
// credentials management system in that it always sets URI root to
// null: it means the key used to find a credential is NOT realm + root
// URI but just the realm. Unless I set it everytime, there is
// possibility that as this thread progresses, it might come across a
// realm already loaded but the login and password are from another
// server. We'll get a failed authentication that'd be difficult to
// explain.
//
// Have to make a UsernamePasswordCredentials. The httpclient auth code
// does an instanceof down in its guts.
UsernamePasswordCredentials upc = null;
try {
upc = new UsernamePasswordCredentials(cred.getLogin(),
cred.getPassword());
http.getState().setCredentials(new AuthScope(curi.getUURI().getHost(),
curi.getUURI().getPort(), cred.getRealm()), upc);
logger.fine("Credentials for realm " + cred.getRealm() +
" for CrawlURI " + curi.toString() + " added to request");
result = true;
} catch (URIException e) {
logger.severe("Failed to parse host from " + curi + ": " +
e.getMessage());
}
return result;
}
}
modules/src/main/java/org/archive/modules/fetcher/AbstractCookieStorage.java
-236
View File
@@ -1,236 +0,0 @@
/*
* This file is part of the Heritrix web crawler (crawler.archive.org).
*
* Licensed to the Internet Archive (IA) by one or more individual
* contributors.
*
* The IA licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.archive.modules.fetcher;
import it.unimi.dsi.mg4j.util.MutableString;
import java.io.BufferedReader;
import java.io.Closeable;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.IOException;
import java.io.Reader;
import java.util.Date;
import java.util.Map;
import java.util.SortedMap;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.apache.commons.httpclient.Cookie;
import org.apache.commons.io.IOUtils;
import org.archive.spring.ConfigFile;
import org.archive.spring.ConfigPath;
import org.springframework.context.Lifecycle;
/**
* @author pjack
*
*/
public abstract class AbstractCookieStorage
implements CookieStorage,
Lifecycle, // InitializingBean,
Closeable {
final private static Logger LOGGER =
Logger.getLogger(AbstractCookieStorage.class.getName());
protected ConfigFile cookiesLoadFile = null;
public ConfigFile getCookiesLoadFile() {
return cookiesLoadFile;
}
public void setCookiesLoadFile(ConfigFile cookiesLoadFile) {
this.cookiesLoadFile = cookiesLoadFile;
}
protected ConfigPath cookiesSaveFile = null;
public ConfigPath getCookiesSaveFile() {
return cookiesSaveFile;
}
public void setCookiesSaveFile(ConfigPath cookiesSaveFile) {
this.cookiesSaveFile = cookiesSaveFile;
}
protected boolean isRunning = false;
public void start() {
if(isRunning()) {
return;
}
SortedMap<String,Cookie> cookies = prepareMap();
if (getCookiesLoadFile()!=null) {
loadCookies(getCookiesLoadFile(), cookies);
}
isRunning = true;
}
public boolean isRunning() {
return isRunning;
}
public void stop() {
isRunning = false;
}
protected abstract SortedMap<String,Cookie> prepareMap();
/**
* Load cookies. The input is text in the Netscape's 'cookies.txt' file
* format. Example entry of cookies.txt file:
* <p>
* www.archive.org FALSE / FALSE 1311699995 details-visit texts-cralond
* </p>
* <p>
* Each line has 7 tab-separated fields:
* </p>
* <ol>
* <li>DOMAIN: The domain that created and have access to the cookie value.</li>
* <li>FLAG: A TRUE or FALSE value indicating if hosts within the given
* domain can access the cookie value.</li>
* <li>PATH: The path within the domain that the cookie value is valid for.</li>
* <li>SECURE: A TRUE or FALSE value indicating if to use a secure
* connection to access the cookie value.</li>
* <li>EXPIRATION: The expiration time of the cookie value, or -1 for no
* expiration</li>
* <li>NAME: The name of the cookie value</li>
* <li>VALUE: The cookie value</li>
* </ol>
*
* @param reader
* input
* @param cookiesFile
* file in the Netscape's 'cookies.txt' format.
*/
public static void loadCookies(Reader reader,
SortedMap<String, Cookie> cookies) {
BufferedReader br = new BufferedReader(reader);
try {
String line;
int lineNo = 1;
while ((line = br.readLine()) != null) {
if (!line.matches("\\s*(?:#.*)?")) { // skip blank links and comments
String[] tokens = line.split("\\t");
if (tokens.length == 7) {
long epochSeconds = Long.parseLong(tokens[4]);
Date expirationDate = (epochSeconds >= 0 ? new Date(epochSeconds * 1000) : null);
Cookie cookie = new Cookie(tokens[0], tokens[5],
tokens[6], tokens[2], expirationDate,
Boolean.valueOf(tokens[3]).booleanValue());
cookie.setDomainAttributeSpecified(Boolean.valueOf(tokens[1]).booleanValue());
LOGGER.fine("Adding cookie: domain " + cookie.getDomain() + " cookie " + cookie.toExternalForm());
cookies.put(cookie.getSortKey(), cookie);
} else {
LOGGER.warning("cookies input line " + lineNo + " invalid, expected 7 tab-delimited tokens");
}
}
lineNo++;
}
} catch (IOException e) {
LOGGER.log(Level.WARNING,e.getMessage(), e);
}
}
protected static void loadCookies(ConfigFile file,
SortedMap<String, Cookie> cookies) {
Reader reader = null;
try {
reader = file.obtainReader();
loadCookies(reader, cookies);
} finally {
IOUtils.closeQuietly(reader);
}
}
public static void loadCookies(String cookiesFile,
SortedMap<String,Cookie> result) {
// Do nothing if cookiesFile is not specified.
if (cookiesFile == null || cookiesFile.length() <= 0) {
return;
}
FileReader reader = null;
try {
reader = new FileReader(cookiesFile);
loadCookies(reader, result);
} catch (FileNotFoundException e) {
LOGGER.log(Level.WARNING,"Could not find file: " + cookiesFile, e);
} finally {
IOUtils.closeQuietly(reader);
}
}
public static void saveCookies(String saveCookiesFile, Map<String,Cookie> cookies) {
// Do nothing if cookiesFile is not specified.
if (saveCookiesFile == null || saveCookiesFile.length() <= 0) {
return;
}
FileOutputStream out = null;
try {
out = new FileOutputStream(new File(saveCookiesFile));
String tab ="\t";
out.write("# Heritrix Cookie File\n".getBytes());
out.write("# This file is the Netscape cookies.txt format\n\n".getBytes());
for (Cookie cookie: cookies.values()) {
// Guess an initial size
MutableString line = new MutableString(1024 * 2);
line.append(cookie.getDomain());
line.append(tab);
line.append(cookie.isDomainAttributeSpecified() ? "TRUE" : "FALSE");
line.append(tab);
line.append(cookie.getPath());
line.append(tab);
line.append(cookie.getSecure() ? "TRUE" : "FALSE");
line.append(tab);
line.append(cookie.getExpiryDate() != null ? cookie.getExpiryDate().getTime() / 1000 : -1);
line.append(tab);
line.append(cookie.getName());
line.append(tab);
line.append(cookie.getValue() != null ? cookie.getValue() : "");
line.append("\n");
out.write(line.toString().getBytes());
}
} catch (IOException e) {
LOGGER.log(Level.SEVERE, "Unable to write " + saveCookiesFile, e);
} finally {
IOUtils.closeQuietly(out);
}
}
public abstract SortedMap<String,Cookie> getCookiesMap();
public void saveCookiesMap(Map<String, Cookie> map) {
innerSaveCookiesMap(map);
if (getCookiesSaveFile()!=null) {
saveCookies(getCookiesSaveFile().getFile().getAbsolutePath(), map);
}
}
protected abstract void innerSaveCookiesMap(Map<String,Cookie> map);
public void close() throws IOException {
}
}
modules/src/main/java/org/archive/modules/fetcher/AbstractCookieStore.java
+1 -1
View File
@@ -44,7 +44,7 @@ public abstract class AbstractCookieStore implements CookieStore, Lifecycle, Clo
Checkpointable {
private static final Logger logger =
Logger.getLogger(AbstractCookieStorage.class.getName());
Logger.getLogger(AbstractCookieStore.class.getName());
protected ConfigFile cookiesLoadFile = null;
public ConfigFile getCookiesLoadFile() {
modules/src/main/java/org/archive/modules/fetcher/BdbCookieStorage.java
-116
View File
@@ -1,116 +0,0 @@
/*
* This file is part of the Heritrix web crawler (crawler.archive.org).
*
* Licensed to the Internet Archive (IA) by one or more individual
* contributors.
*
* The IA licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.archive.modules.fetcher;
import java.io.IOException;
import java.util.Map;
import java.util.SortedMap;
import org.apache.commons.httpclient.Cookie;
import org.archive.bdb.BdbModule;
import org.archive.checkpointing.Checkpoint;
import org.archive.checkpointing.Checkpointable;
import org.springframework.beans.factory.annotation.Autowired;
import com.sleepycat.bind.serial.SerialBinding;
import com.sleepycat.bind.serial.StoredClassCatalog;
import com.sleepycat.bind.tuple.StringBinding;
import com.sleepycat.collections.StoredSortedMap;
import com.sleepycat.je.Database;
import com.sleepycat.je.DatabaseException;
/**
* CookieStorage using BDB, so that cookies accumulated in large crawls
* do not outgrow RAM.
*
* @author pjack
*/
public class BdbCookieStorage extends AbstractCookieStorage implements Checkpointable {
@SuppressWarnings("unused")
private static final long serialVersionUID = 1L;
protected BdbModule bdb;
@Autowired
public void setBdbModule(BdbModule bdb) {
this.bdb = bdb;
}
/** are we a checkpoint recovery? (in which case, reuse stored cookie data?) */
protected boolean isCheckpointRecovery = false;
public static String COOKIEDB_NAME = "http_cookies";
private transient Database cookieDb;
private transient StoredSortedMap<String,Cookie> cookies;
public BdbCookieStorage() {
}
protected SortedMap<String,Cookie> prepareMap() {
try {
StoredClassCatalog classCatalog = bdb.getClassCatalog();
BdbModule.BdbConfig dbConfig = new BdbModule.BdbConfig();
dbConfig.setTransactional(false);
dbConfig.setAllowCreate(true);
cookieDb = bdb.openDatabase(COOKIEDB_NAME, dbConfig, isCheckpointRecovery);
cookies =
new StoredSortedMap<String,Cookie>(
cookieDb,
new StringBinding(),
new SerialBinding<Cookie>(classCatalog,Cookie.class),
true);
return cookies;
} catch (DatabaseException e) {
throw new RuntimeException(e);
}
}
public SortedMap<String, Cookie> getCookiesMap() {
// assert cookies != null : "cookie map not set up";
return cookies;
}
protected void innerSaveCookiesMap(Map<String, Cookie> map) {
}
@Override
public void startCheckpoint(Checkpoint checkpointInProgress) {
// do nothing; handled by map checkpoint via BdbModule
}
@Override
public void doCheckpoint(Checkpoint checkpointInProgress)
throws IOException {
// do nothing; handled by map checkpoint via BdbModule
}
@Override
public void finishCheckpoint(Checkpoint checkpointInProgress) {
// do nothing; handled by map checkpoint via BdbModule
}
@Override
public void setRecoveryCheckpoint(Checkpoint recoveryCheckpoint) {
// just remember that we are doing checkpoint-recovery;
// actual state recovery happens via BdbModule
isCheckpointRecovery = true;
}
}
modules/src/main/java/org/archive/modules/fetcher/CookieStorage.java
-33
View File
@@ -1,33 +0,0 @@
/*
* This file is part of the Heritrix web crawler (crawler.archive.org).
*
* Licensed to the Internet Archive (IA) by one or more individual
* contributors.
*
* The IA licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.archive.modules.fetcher;
import java.util.Map;
import java.util.SortedMap;
import org.apache.commons.httpclient.Cookie;
import org.springframework.context.Lifecycle;
public interface CookieStorage extends Lifecycle {
SortedMap<String,Cookie> getCookiesMap();
void saveCookiesMap(Map<String,Cookie> map);
}
modules/src/main/java/org/archive/modules/fetcher/HeritrixHttpMethodRetryHandler.java
-84
View File
@@ -1,84 +0,0 @@
/*
* This file is part of the Heritrix web crawler (crawler.archive.org).
*
* Licensed to the Internet Archive (IA) by one or more individual
* contributors.
*
* The IA licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.archive.modules.fetcher;
import java.io.IOException;
import java.net.SocketTimeoutException;
import org.apache.commons.httpclient.HttpMethod;
import org.apache.commons.httpclient.HttpMethodRetryHandler;
import org.apache.commons.httpclient.NoHttpResponseException;
import org.apache.commons.httpclient.methods.PostMethod;
/**
* Retry handler that tries ten times to establish connection and then once
* established, if a GET method, tries ten times to get response (If POST,
* it tries once only).
*
* Its unsafe retrying POSTs. See 'Rule of Thumb' under 'Method Recovery'
* here: <a href="http://jakarta.apache.org/commons/httpclient/tutorial.html">
* HttpClient Tutorial</a>.
*
* @author stack
* @version $Date$, $Revision$
*/
public class HeritrixHttpMethodRetryHandler implements HttpMethodRetryHandler {
private static final int DEFAULT_RETRY_COUNT = 10;
private final int maxRetryCount;
/**
* Constructor.
*/
public HeritrixHttpMethodRetryHandler() {
this(DEFAULT_RETRY_COUNT);
}
/**
* Constructor.
* @param maxRetryCount Maximum amount of times to retry.
*/
public HeritrixHttpMethodRetryHandler(int maxRetryCount) {
this.maxRetryCount = maxRetryCount;
}
public boolean retryMethod(HttpMethod method, IOException exception,
int executionCount) {
if(exception instanceof SocketTimeoutException) {
// already waited for the configured amount of time with no reply;
// do not retry further until next go round
return false;
}
if (executionCount >= this.maxRetryCount) {
// Do not retry if over max retry count
return false;
}
if (exception instanceof NoHttpResponseException) {
// Retry if the server dropped connection on us
return true;
}
if (!method.isRequestSent() && (!(method instanceof PostMethod))) {
// Retry if the request has not been sent fully or
// if it's OK to retry methods that have been sent
return true;
}
// otherwise do not retry
return false;
}
}
modules/src/main/java/org/archive/modules/fetcher/HeritrixProtocolSocketFactory.java
-196
View File
@@ -1,196 +0,0 @@
/*
* This file is part of the Heritrix web crawler (crawler.archive.org).
*
* Licensed to the Internet Archive (IA) by one or more individual
* contributors.
*
* The IA licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.archive.modules.fetcher;
import java.io.IOException;
import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.net.Socket;
import java.net.SocketTimeoutException;
import java.net.UnknownHostException;
import org.apache.commons.httpclient.ConnectTimeoutException;
import org.apache.commons.httpclient.params.HttpConnectionParams;
import org.apache.commons.httpclient.protocol.ProtocolSocketFactory;
/**
* Version of protocol socket factory that tries to get IP from heritrix IP
* cache -- if its been set into the HttpConnectionParameters.
*
* Copied the guts of DefaultProtocolSocketFactory. This factory gets
* setup by {@link LegacyFetchHTTP}.
*
* @author stack
* @version $Date$, $Revision$
*/
public class HeritrixProtocolSocketFactory
implements ProtocolSocketFactory {
/**
* Constructor.
*/
public HeritrixProtocolSocketFactory() {
super();
}
/**
* @see #createSocket(java.lang.String,int,java.net.InetAddress,int)
*/
public Socket createSocket(
String host,
int port,
InetAddress localAddress,
int localPort
) throws IOException, UnknownHostException {
return new Socket(host, port, localAddress, localPort);
}
/**
* Attempts to get a new socket connection to the given host within the
* given time limit.
* <p>
* This method employs several techniques to circumvent the limitations
* of older JREs that do not support connect timeout. When running in
* JRE 1.4 or above reflection is used to call
* Socket#connect(SocketAddress endpoint, int timeout) method. When
* executing in older JREs a controller thread is executed. The
* controller thread attempts to create a new socket within the given
* limit of time. If socket constructor does not return until the
* timeout expires, the controller terminates and throws an
* {@link ConnectTimeoutException}
* </p>
*
* @param host the host name/IP
* @param port the port on the host
* @param localAddress the local host name/IP to bind the socket to
* @param localPort the port on the local machine
* @param params {@link HttpConnectionParams Http connection parameters}
*
* @return Socket a new socket
*
* @throws IOException if an I/O error occurs while creating the socket
* @throws UnknownHostException if the IP address of the host cannot be
* @throws IOException if an I/O error occurs while creating the socket
* @throws UnknownHostException if the IP address of the host cannot be
* determined
* @throws ConnectTimeoutException if socket cannot be connected within the
* given time limit
*
* @since 3.0
*/
public Socket createSocket(
final String host,
final int port,
final InetAddress localAddress,
final int localPort,
final HttpConnectionParams params)
throws IOException, UnknownHostException, ConnectTimeoutException {
// Below code is from the DefaultSSLProtocolSocketFactory#createSocket
// method only it has workarounds to deal with pre-1.4 JVMs. I've
// cut these out.
if (params == null) {
throw new IllegalArgumentException("Parameters may not be null");
}
Socket socket = null;
int timeout = params.getConnectionTimeout();
if (timeout == 0) {
socket = createSocket(host, port, localAddress, localPort);
} else {
socket = new Socket();
InetAddress hostAddress;
Thread current = Thread.currentThread();
if (current instanceof HostResolver) {
HostResolver resolver = (HostResolver)current;
hostAddress = resolver.resolve(host);
} else {
hostAddress = null;
}
InetSocketAddress address = (hostAddress != null)?
new InetSocketAddress(hostAddress, port):
new InetSocketAddress(host, port);
socket.bind(new InetSocketAddress(localAddress, localPort));
try {
socket.connect(address, timeout);
} catch (SocketTimeoutException e) {
// Add timeout info. to the exception.
throw new SocketTimeoutException(e.getMessage() +
": timeout set at " + Integer.toString(timeout) + "ms.");
}
assert socket.isConnected(): "Socket not connected " + host;
}
return socket;
}
/**
* Get host address using first the heritrix cache of addresses, then,
* failing that, go to the dnsjava cache.
*
* Default access and static so can be used by other classes in this
* package.
*
* @param host Host whose address we're to fetch.
* @return an IP address for this host or null if one can't be found
* in caches.
* @exception IOException If we fail to get host IP from ServerCache.
*/
/*
static InetAddress getHostAddress(final ServerCache cache,
final String host) throws IOException {
InetAddress result = null;
if (cache != null) {
CrawlHost ch = cache.getHostFor(host);
if (ch != null) {
result = ch.getIP();
}
}
if (result == null) {
throw new IOException("Failed to get host " + host +
" address from ServerCache");
}
return result;
}
*/
/**
* @see ProtocolSocketFactory#createSocket(java.lang.String,int)
*/
public Socket createSocket(String host, int port)
throws IOException, UnknownHostException {
return new Socket(host, port);
}
/**
* All instances of DefaultProtocolSocketFactory are the same.
* @param obj Object to compare.
* @return True if equal
*/
public boolean equals(Object obj) {
return ((obj != null) &&
obj.getClass().equals(HeritrixProtocolSocketFactory.class));
}
/**
* All instances of DefaultProtocolSocketFactory have the same hash code.
* @return Hash code for this object.
*/
public int hashCode() {
return HeritrixProtocolSocketFactory.class.hashCode();
}
}
modules/src/main/java/org/archive/modules/fetcher/HeritrixSSLProtocolSocketFactory.java
-151
View File
@@ -1,151 +0,0 @@
/*
* This file is part of the Heritrix web crawler (crawler.archive.org).
*
* Licensed to the Internet Archive (IA) by one or more individual
* contributors.
*
* The IA licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.archive.modules.fetcher;
import java.io.IOException;
import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.net.Socket;
import java.net.SocketTimeoutException;
import java.net.UnknownHostException;
import java.security.KeyManagementException;
import java.security.KeyStoreException;
import java.security.NoSuchAlgorithmException;
import javax.net.ssl.SSLContext;
import javax.net.ssl.SSLSocketFactory;
import javax.net.ssl.TrustManager;
import org.apache.commons.httpclient.params.HttpConnectionParams;
import org.apache.commons.httpclient.protocol.SecureProtocolSocketFactory;
import org.archive.httpclient.ConfigurableX509TrustManager;
/**
* Implementation of the commons-httpclient SSLProtocolSocketFactory so we
* can return SSLSockets whose trust manager is
* {@link org.archive.httpclient.ConfigurableX509TrustManager}.
*
* We also go to the heritrix cache to get IPs to use making connection.
* To this, we have dependency on {@link HeritrixProtocolSocketFactory};
* its assumed this class and it are used together.
* See {@link HeritrixProtocolSocketFactory#getHostAddress(ServerCache,String)}.
*
* @author stack
* @version $Id$
* @see org.archive.httpclient.ConfigurableX509TrustManager
*/
public class HeritrixSSLProtocolSocketFactory
implements SecureProtocolSocketFactory {
/***
* Socket factory with default trust manager installed.
*/
private SSLSocketFactory sslDefaultFactory = null;
/**
* Shutdown constructor.
* @throws KeyManagementException
* @throws KeyStoreException
* @throws NoSuchAlgorithmException
*/
public HeritrixSSLProtocolSocketFactory()
throws KeyManagementException, KeyStoreException, NoSuchAlgorithmException{
// Get an SSL context and initialize it.
SSLContext context = SSLContext.getInstance("SSL");
// I tried to get the default KeyManagers but doesn't work unless you
// point at a physical keystore. Passing null seems to do the right
// thing so we'll go w/ that.
context.init(null, new TrustManager[] {
new ConfigurableX509TrustManager(
ConfigurableX509TrustManager.DEFAULT)}, null);
this.sslDefaultFactory = context.getSocketFactory();
}
public Socket createSocket(String host, int port, InetAddress clientHost,
int clientPort)
throws IOException, UnknownHostException {
return this.sslDefaultFactory.createSocket(host, port,
clientHost, clientPort);
}
public Socket createSocket(String host, int port)
throws IOException, UnknownHostException {
return this.sslDefaultFactory.createSocket(host, port);
}
public synchronized Socket createSocket(String host, int port,
InetAddress localAddress, int localPort, HttpConnectionParams params)
throws IOException, UnknownHostException {
// Below code is from the DefaultSSLProtocolSocketFactory#createSocket
// method only it has workarounds to deal with pre-1.4 JVMs. I've
// cut these out.
if (params == null) {
throw new IllegalArgumentException("Parameters may not be null");
}
Socket socket = null;
int timeout = params.getConnectionTimeout();
if (timeout == 0) {
socket = createSocket(host, port, localAddress, localPort);
} else {
SSLSocketFactory factory = (SSLSocketFactory)params.
getParameter(LegacyFetchHTTP.SSL_FACTORY_KEY);
SSLSocketFactory f = (factory != null)? factory: this.sslDefaultFactory;
socket = f.createSocket();
Thread current = Thread.currentThread();
InetAddress hostAddress;
if (current instanceof HostResolver) {
HostResolver resolver = (HostResolver)current;
hostAddress = resolver.resolve(host);
} else {
hostAddress = null;
}
InetSocketAddress address = (hostAddress != null)?
new InetSocketAddress(hostAddress, port):
new InetSocketAddress(host, port);
socket.bind(new InetSocketAddress(localAddress, localPort));
try {
socket.connect(address, timeout);
} catch (SocketTimeoutException e) {
// Add timeout info. to the exception.
throw new SocketTimeoutException(e.getMessage() +
": timeout set at " + Integer.toString(timeout) + "ms.");
}
assert socket.isConnected(): "Socket not connected " + host;
}
return socket;
}
public Socket createSocket(Socket socket, String host, int port,
boolean autoClose)
throws IOException, UnknownHostException {
return this.sslDefaultFactory.createSocket(socket, host,
port, autoClose);
}
public boolean equals(Object obj) {
return ((obj != null) && obj.getClass().
equals(HeritrixSSLProtocolSocketFactory.class));
}
public int hashCode() {
return HeritrixSSLProtocolSocketFactory.class.hashCode();
}
}
modules/src/main/java/org/archive/modules/fetcher/LegacyFetchHTTP.java
-1640
View File
File diff suppressed because it is too large Load Diff
modules/src/main/java/org/archive/modules/fetcher/SimpleCookieStorage.java
-48
View File
@@ -1,48 +0,0 @@
/*
* This file is part of the Heritrix web crawler (crawler.archive.org).
*
* Licensed to the Internet Archive (IA) by one or more individual
* contributors.
*
* The IA licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.archive.modules.fetcher;
import java.util.Map;
import java.util.SortedMap;
import java.util.TreeMap;
import org.apache.commons.httpclient.Cookie;
public class SimpleCookieStorage extends AbstractCookieStorage {
@SuppressWarnings("unused")
private static final long serialVersionUID = 1L;
final private SortedMap<String,Cookie> map = new TreeMap<String,Cookie>();
protected SortedMap<String,Cookie> prepareMap() {
return map;
}
public SortedMap<String,Cookie> getCookiesMap() {
return map;
}
public void innerSaveCookiesMap(Map<String,Cookie> map) {
// no-op
}
}