From 33df8faaf0accf1d8ae2b5581aebe92ae8768c67 Mon Sep 17 00:00:00 2001 From: Noah Levitt Date: Sat, 8 Sep 2012 00:49:30 -0700 Subject: [PATCH] remove LegacyFetchHTTP and its classes --- .../httpclient/HttpRecorderGetMethod.java | 120 -- .../httpclient/HttpRecorderMethod.java | 107 -- .../httpclient/HttpRecorderPostMethod.java | 82 - .../SingleHttpConnectionManager.java | 70 - .../ThreadLocalHttpConnectionManager.java | 291 --- .../modules/fetcher/FetchHTTPTest.java | 54 - .../credential/CommonsHttpCredentialUtil.java | 147 -- .../fetcher/AbstractCookieStorage.java | 236 --- .../modules/fetcher/AbstractCookieStore.java | 2 +- .../modules/fetcher/BdbCookieStorage.java | 116 -- .../modules/fetcher/CookieStorage.java | 33 - .../HeritrixHttpMethodRetryHandler.java | 84 - .../HeritrixProtocolSocketFactory.java | 196 -- .../HeritrixSSLProtocolSocketFactory.java | 151 -- .../modules/fetcher/LegacyFetchHTTP.java | 1640 ----------------- .../modules/fetcher/SimpleCookieStorage.java | 48 - 16 files changed, 1 insertion(+), 3376 deletions(-) delete mode 100644 commons/src/main/java/org/archive/httpclient/HttpRecorderGetMethod.java delete mode 100644 commons/src/main/java/org/archive/httpclient/HttpRecorderMethod.java delete mode 100644 commons/src/main/java/org/archive/httpclient/HttpRecorderPostMethod.java delete mode 100644 commons/src/main/java/org/archive/httpclient/SingleHttpConnectionManager.java delete mode 100644 commons/src/main/java/org/archive/httpclient/ThreadLocalHttpConnectionManager.java delete mode 100644 engine/src/test/java/org/archive/modules/fetcher/FetchHTTPTest.java delete mode 100644 modules/src/main/java/org/archive/modules/credential/CommonsHttpCredentialUtil.java delete mode 100644 modules/src/main/java/org/archive/modules/fetcher/AbstractCookieStorage.java delete mode 100644 modules/src/main/java/org/archive/modules/fetcher/BdbCookieStorage.java delete mode 100644 modules/src/main/java/org/archive/modules/fetcher/CookieStorage.java delete mode 100644 modules/src/main/java/org/archive/modules/fetcher/HeritrixHttpMethodRetryHandler.java delete mode 100644 modules/src/main/java/org/archive/modules/fetcher/HeritrixProtocolSocketFactory.java delete mode 100644 modules/src/main/java/org/archive/modules/fetcher/HeritrixSSLProtocolSocketFactory.java delete mode 100644 modules/src/main/java/org/archive/modules/fetcher/LegacyFetchHTTP.java delete mode 100644 modules/src/main/java/org/archive/modules/fetcher/SimpleCookieStorage.java diff --git a/commons/src/main/java/org/archive/httpclient/HttpRecorderGetMethod.java b/commons/src/main/java/org/archive/httpclient/HttpRecorderGetMethod.java deleted file mode 100644 index 105c4f7e..00000000 --- a/commons/src/main/java/org/archive/httpclient/HttpRecorderGetMethod.java +++ /dev/null @@ -1,120 +0,0 @@ -/* - * This file is part of the Heritrix web crawler (crawler.archive.org). - * - * Licensed to the Internet Archive (IA) by one or more individual - * contributors. - * - * The IA licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.archive.httpclient; - -import java.io.IOException; -import java.util.logging.Logger; - -import org.apache.commons.httpclient.HttpConnection; -import org.apache.commons.httpclient.HttpException; -import org.apache.commons.httpclient.HttpState; -import org.apache.commons.httpclient.methods.GetMethod; -import org.archive.util.Recorder; - - -/** - * Override of GetMethod that marks the passed HttpRecorder w/ the transition - * from HTTP head to body and that forces a close on the http connection. - * - * The actions done in this subclass used to be done by copying - * org.apache.commons.HttpMethodBase, overlaying our version in place of the - * one that came w/ httpclient. Here is the patch of the difference between - * shipped httpclient code and our mods: - * 
- *    -- -1338,6 +1346,12 --
- *
- *        public void releaseConnection() {
- *
- *   +        // HERITRIX always ants the streams closed.
- *   +        if (responseConnection != null)
- *   +        {
- *   +            responseConnection.close();
- *   +        }
- *   +
- *            if (responseStream != null) {
- *                try {
- *                    // FYI - this may indirectly invoke responseBodyConsumed.
- *   -- -1959,6 +1973,11 --
- *                        this.statusLine = null;
- *                    }
- *                }
- *   +            // HERITRIX mark transition from header to content.
- *   +            if (this.httpRecorder != null)
- *   +            {
- *   +                this.httpRecorder.markContentBegin();
- *   +            }
- *                readResponseBody(state, conn);
- *                processResponseBody(state, conn);
- *            } catch (IOException e) {
- *
- * - *

We're not supposed to have access to the underlying connection object; - * am only violating contract because see cases where httpclient is skipping - * out w/o cleaning up after itself. - * - * @author stack - * @version $Revision$, $Date$ - */ -public class HttpRecorderGetMethod extends GetMethod { - - protected static Logger logger = - Logger.getLogger(HttpRecorderGetMethod.class.getName()); - - /** - * Instance of http recorder method. - */ - protected HttpRecorderMethod httpRecorderMethod = null; - - - public HttpRecorderGetMethod(String uri, Recorder recorder) { - super(uri); - this.httpRecorderMethod = new HttpRecorderMethod(recorder); - } - - protected void readResponseBody(HttpState state, HttpConnection connection) - throws IOException, HttpException { - // We're about to read the body. Mark transition in http recorder. - this.httpRecorderMethod.markContentBegin(connection); - super.readResponseBody(state, connection); - } - - protected boolean shouldCloseConnection(HttpConnection conn) { - // Always close connection after each request. As best I can tell, this - // is superfluous -- we've set our client to be HTTP/1.0. Doing this - // out of paranoia. - return true; - } - - public int execute(HttpState state, HttpConnection conn) - throws HttpException, IOException { - // Save off the connection so we can close it on our way out in case - // httpclient fails to (We're not supposed to have access to the - // underlying connection object; am only violating contract because - // see cases where httpclient is skipping out w/o cleaning up - // after itself). - this.httpRecorderMethod.setConnection(conn); - return super.execute(state, conn); - } - - protected void addProxyConnectionHeader(HttpState state, HttpConnection conn) - throws IOException, HttpException { - super.addProxyConnectionHeader(state, conn); - this.httpRecorderMethod.handleAddProxyConnectionHeader(this); - } -} diff --git a/commons/src/main/java/org/archive/httpclient/HttpRecorderMethod.java b/commons/src/main/java/org/archive/httpclient/HttpRecorderMethod.java deleted file mode 100644 index 932e7e98..00000000 --- a/commons/src/main/java/org/archive/httpclient/HttpRecorderMethod.java +++ /dev/null @@ -1,107 +0,0 @@ -/* - * This file is part of the Heritrix web crawler (crawler.archive.org). - * - * Licensed to the Internet Archive (IA) by one or more individual - * contributors. - * - * The IA licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.archive.httpclient; - -import java.util.logging.Logger; - -import org.apache.commons.httpclient.Header; -import org.apache.commons.httpclient.HttpConnection; -import org.apache.commons.httpclient.HttpMethod; -import org.archive.util.Recorder; - - -/** - * This class encapsulates the specializations supplied by the - * overrides {@link HttpRecorderGetMethod} and {@link HttpRecorderPostMethod}. - * - * It keeps instance of HttpRecorder and HttpConnection. - * - * @author stack - * @version $Revision$, $Date$ - */ -public class HttpRecorderMethod { - protected static Logger logger = - Logger.getLogger(HttpRecorderMethod.class.getName()); - - /** - * Instance of http recorder we're using recording this http get. - */ - private Recorder httpRecorder = null; - - /** - * Save around so can force close. - * - * See [ 922080 ] IllegalArgumentException (size is wrong). - * https://sourceforge.net/tracker/?func=detail&aid=922080&group_id=73833&atid=539099 - */ - private HttpConnection connection = null; - - - public HttpRecorderMethod(Recorder recorder) { - this.httpRecorder = recorder; - } - - public void markContentBegin(HttpConnection c) { - if (c != this.connection) { - // We're checking that we're not being asked to work on - // a connection that is other than the one we started - // this method#execute with. - throw new IllegalArgumentException("Connections differ: " + - this.connection + " " + c + " " + - Thread.currentThread().getName()); - } - this.httpRecorder.markContentBegin(); - } - - /** - * @return Returns the connection. - */ - public HttpConnection getConnection() { - return this.connection; - } - - /** - * @param connection The connection to set. - */ - public void setConnection(HttpConnection connection) { - this.connection = connection; - } - /** - * @return Returns the httpRecorder. - */ - public Recorder getHttpRecorder() { - return httpRecorder; - } - - /** - * If a 'Proxy-Connection' header has been added to the request, - * it'll be of a 'keep-alive' type. Until we support 'keep-alives', - * override the Proxy-Connection setting and instead pass a 'close' - * (Otherwise every request has to timeout before we notice - * end-of-document). - * @param method Method to find proxy-connection header in. - */ - public void handleAddProxyConnectionHeader(HttpMethod method) { - Header h = method.getRequestHeader("Proxy-Connection"); - if (h != null) { - h.setValue("close"); - method.setRequestHeader(h); - } - } -} diff --git a/commons/src/main/java/org/archive/httpclient/HttpRecorderPostMethod.java b/commons/src/main/java/org/archive/httpclient/HttpRecorderPostMethod.java deleted file mode 100644 index 20f1bfd1..00000000 --- a/commons/src/main/java/org/archive/httpclient/HttpRecorderPostMethod.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * This file is part of the Heritrix web crawler (crawler.archive.org). - * - * Licensed to the Internet Archive (IA) by one or more individual - * contributors. - * - * The IA licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.archive.httpclient; - -import java.io.IOException; - -import org.apache.commons.httpclient.HttpConnection; -import org.apache.commons.httpclient.HttpException; -import org.apache.commons.httpclient.HttpState; -import org.apache.commons.httpclient.methods.PostMethod; -import org.archive.util.Recorder; - - -/** - * Override of PostMethod that marks the passed HttpRecorder w/ the transition - * from HTTP head to body and that forces a close on the responseConnection. - * - * This is a copy of {@link HttpRecorderGetMethod}. Only difference is the - * parent subclass. - * - * @author stack - * @version $Date$ $Revision$ - */ -public class HttpRecorderPostMethod extends PostMethod { - /** - * Instance of http recorder method. - */ - protected HttpRecorderMethod httpRecorderMethod = null; - - - public HttpRecorderPostMethod(String uri, Recorder recorder) { - super(uri); - this.httpRecorderMethod = new HttpRecorderMethod(recorder); - } - - protected void readResponseBody(HttpState state, HttpConnection connection) - throws IOException, HttpException { - // We're about to read the body. Mark transition in http recorder. - this.httpRecorderMethod.markContentBegin(connection); - super.readResponseBody(state, connection); - } - - protected boolean shouldCloseConnection(HttpConnection conn) { - // Always close connection after each request. As best I can tell, this - // is superfluous -- we've set our client to be HTTP/1.0. Doing this - // out of paranoia. - return true; - } - - public int execute(HttpState state, HttpConnection conn) - throws HttpException, IOException { - // Save off the connection so we can close it on our way out in case - // httpclient fails to (We're not supposed to have access to the - // underlying connection object; am only violating contract because - // see cases where httpclient is skipping out w/o cleaning up - // after itself). - this.httpRecorderMethod.setConnection(conn); - return super.execute(state, conn); - } - - protected void addProxyConnectionHeader(HttpState state, HttpConnection conn) - throws IOException, HttpException { - super.addProxyConnectionHeader(state, conn); - this.httpRecorderMethod.handleAddProxyConnectionHeader(this); - } -} diff --git a/commons/src/main/java/org/archive/httpclient/SingleHttpConnectionManager.java b/commons/src/main/java/org/archive/httpclient/SingleHttpConnectionManager.java deleted file mode 100644 index 4ba6a837..00000000 --- a/commons/src/main/java/org/archive/httpclient/SingleHttpConnectionManager.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * This file is part of the Heritrix web crawler (crawler.archive.org). - * - * Licensed to the Internet Archive (IA) by one or more individual - * contributors. - * - * The IA licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.archive.httpclient; - -import java.io.IOException; -import java.io.InputStream; - -import org.apache.commons.httpclient.HostConfiguration; -import org.apache.commons.httpclient.HttpConnection; -import org.apache.commons.httpclient.SimpleHttpConnectionManager; - -/** - * An HttpClient-compatible HttpConnection "manager" that actually - * just gives out a new connection each time -- skipping the overhead - * of connection management, since we already throttle our crawler - * with external mechanisms. - * - * @author gojomo - */ -public class SingleHttpConnectionManager extends SimpleHttpConnectionManager { - - public SingleHttpConnectionManager() { - super(); - } - - public HttpConnection getConnectionWithTimeout( - HostConfiguration hostConfiguration, long timeout) { - - HttpConnection conn = new HttpConnection(hostConfiguration); - conn.setHttpConnectionManager(this); - conn.getParams().setDefaults(this.getParams()); - return conn; - } - - public void releaseConnection(HttpConnection conn) { - // ensure connection is closed - conn.close(); - finishLast(conn); - } - - protected static void finishLast(HttpConnection conn) { - // copied from superclass because it wasn't made available to subclasses - InputStream lastResponse = conn.getLastResponseInputStream(); - if (lastResponse != null) { - conn.setLastResponseInputStream(null); - try { - lastResponse.close(); - } catch (IOException ioe) { - //FIXME: badness - close to force reconnect. - conn.close(); - } - } - } -} diff --git a/commons/src/main/java/org/archive/httpclient/ThreadLocalHttpConnectionManager.java b/commons/src/main/java/org/archive/httpclient/ThreadLocalHttpConnectionManager.java deleted file mode 100644 index 91e850ea..00000000 --- a/commons/src/main/java/org/archive/httpclient/ThreadLocalHttpConnectionManager.java +++ /dev/null @@ -1,291 +0,0 @@ -/** - * ==================================================================== - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ==================================================================== - * - */ -package org.archive.httpclient; - -import java.io.IOException; -import java.io.InputStream; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; -import java.util.logging.Level; -import java.util.logging.Logger; - -import org.apache.commons.httpclient.HostConfiguration; -import org.apache.commons.httpclient.HttpConnection; -import org.apache.commons.httpclient.HttpConnectionManager; -import org.apache.commons.httpclient.params.HttpConnectionManagerParams; - -/** - * A simple, but thread-safe HttpClient {@link HttpConnectionManager}. - * Based on {@link org.apache.commons.httpclient.SimpleHttpConnectionManager}. - * - * Java >= 1.4 is recommended. - * - * @author Christian Kohlschuetter - */ -public final class ThreadLocalHttpConnectionManager implements - HttpConnectionManager { - - private static final CloserThread closer = new CloserThread(); - private static final Logger logger = Logger - .getLogger(ThreadLocalHttpConnectionManager.class.getName()); - - private final ThreadLocal tl = new ThreadLocal() { - protected synchronized ConnectionInfo initialValue() { - return new ConnectionInfo(); - } - }; - - private ConnectionInfo getConnectionInfo() { - return (ConnectionInfo) tl.get(); - } - - private static final class ConnectionInfo { - /** The http connection */ - private HttpConnection conn = null; - - /** - * The time the connection was made idle. - */ - private long idleStartTime = Long.MAX_VALUE; - } - - public ThreadLocalHttpConnectionManager() { - } - - /** - * Since the same connection is about to be reused, make sure the - * previous request was completely processed, and if not - * consume it now. - * @param conn The connection - * @return true, if the connection is reusable - */ - private static boolean finishLastResponse(final HttpConnection conn) { - InputStream lastResponse = conn.getLastResponseInputStream(); - if(lastResponse != null) { - conn.setLastResponseInputStream(null); - try { - lastResponse.close(); - return true; - } catch (IOException ioe) { - // force reconnect. - return false; - } - } else { - return false; - } - } - - /** - * Collection of parameters associated with this connection manager. - */ - private HttpConnectionManagerParams params = new HttpConnectionManagerParams(); - - /** - * @see HttpConnectionManager#getConnection(HostConfiguration) - */ - public HttpConnection getConnection( - final HostConfiguration hostConfiguration) { - return getConnection(hostConfiguration, 0); - } - - /** - * Gets the staleCheckingEnabled value to be set on HttpConnections that are created. - * - * @return true if stale checking will be enabled on HttpConections - * - * @see HttpConnection#isStaleCheckingEnabled() - * - * @deprecated Use {@link HttpConnectionManagerParams#isStaleCheckingEnabled()}, - * {@link HttpConnectionManager#getParams()}. - */ - public boolean isConnectionStaleCheckingEnabled() { - return this.params.isStaleCheckingEnabled(); - } - - /** - * Sets the staleCheckingEnabled value to be set on HttpConnections that are created. - * - * @param connectionStaleCheckingEnabled true if stale checking will be enabled - * on HttpConections - * - * @see HttpConnection#setStaleCheckingEnabled(boolean) - * - * @deprecated Use {@link HttpConnectionManagerParams#setStaleCheckingEnabled(boolean)}, - * {@link HttpConnectionManager#getParams()}. - */ - public void setConnectionStaleCheckingEnabled( - final boolean connectionStaleCheckingEnabled) { - this.params.setStaleCheckingEnabled(connectionStaleCheckingEnabled); - } - - /** - * @see HttpConnectionManager#getConnectionWithTimeout(HostConfiguration, long) - * - * @since 3.0 - */ - public HttpConnection getConnectionWithTimeout( - final HostConfiguration hostConfiguration, final long timeout) { - - final ConnectionInfo ci = getConnectionInfo(); - HttpConnection httpConnection = ci.conn; - - // make sure the host and proxy are correct for this connection - // close it and set the values if they are not - if(httpConnection == null || !finishLastResponse(httpConnection) - || !hostConfiguration.hostEquals(httpConnection) - || !hostConfiguration.proxyEquals(httpConnection)) { - - if(httpConnection != null && httpConnection.isOpen()) { - closer.closeConnection(httpConnection); - } - - httpConnection = new HttpConnection(hostConfiguration); - httpConnection.setHttpConnectionManager(this); - httpConnection.getParams().setDefaults(this.params); - ci.conn = httpConnection; - - httpConnection.setHost(hostConfiguration.getHost()); - httpConnection.setPort(hostConfiguration.getPort()); - httpConnection.setProtocol(hostConfiguration.getProtocol()); - httpConnection.setLocalAddress(hostConfiguration.getLocalAddress()); - - httpConnection.setProxyHost(hostConfiguration.getProxyHost()); - httpConnection.setProxyPort(hostConfiguration.getProxyPort()); - } - - // remove the connection from the timeout handler - ci.idleStartTime = Long.MAX_VALUE; - - return httpConnection; - } - - /** - * @see HttpConnectionManager#getConnection(HostConfiguration, long) - * - * @deprecated Use #getConnectionWithTimeout(HostConfiguration, long) - */ - public HttpConnection getConnection( - final HostConfiguration hostConfiguration, final long timeout) { - return getConnectionWithTimeout(hostConfiguration, timeout); - } - - /** - * @see HttpConnectionManager#releaseConnection(org.apache.commons.httpclient.HttpConnection) - */ - public void releaseConnection(final HttpConnection conn) { - final ConnectionInfo ci = getConnectionInfo(); - HttpConnection httpConnection = ci.conn; - - if(conn != httpConnection) { - throw new IllegalStateException( - "Unexpected release of an unknown connection."); - } - - finishLastResponse(httpConnection); - - // track the time the connection was made idle - ci.idleStartTime = System.currentTimeMillis(); - } - - /** - * Returns {@link HttpConnectionManagerParams parameters} associated - * with this connection manager. - * - * @since 2.1 - * - * @see HttpConnectionManagerParams - */ - public HttpConnectionManagerParams getParams() { - return this.params; - } - - /** - * Assigns {@link HttpConnectionManagerParams parameters} for this - * connection manager. - * - * @since 2.1 - * - * @see HttpConnectionManagerParams - */ - public void setParams(final HttpConnectionManagerParams p) { - if(p == null) { - throw new IllegalArgumentException("Parameters may not be null"); - } - this.params = p; - } - - /** - * @since 3.0 - */ - public void closeIdleConnections(final long idleTimeout) { - long maxIdleTime = System.currentTimeMillis() - idleTimeout; - - final ConnectionInfo ci = getConnectionInfo(); - - if(ci.idleStartTime <= maxIdleTime) { - ci.conn.close(); - } - } - - private static final class CloserThread extends Thread { - private List connections - = new ArrayList(); - - private static final int SLEEP_INTERVAL = 5000; - - public CloserThread() { - super("HttpConnection closer"); - // Make this a daemon thread so it can't be responsible for the JVM - // not shutting down. - setDaemon(true); - start(); - } - - public void closeConnection(final HttpConnection conn) { - synchronized (connections) { - connections.add(conn); - } - } - - public void run() { - try { - while (!Thread.interrupted()) { - Thread.sleep(SLEEP_INTERVAL); - - List s; - synchronized (connections) { - s = connections; - connections = new ArrayList(); - } - logger.log(Level.INFO, "Closing " + s.size() - + " HttpConnections"); - for(final Iterator it = s.iterator(); - it.hasNext();) { - HttpConnection conn = it.next(); - conn.close(); - conn.setHttpConnectionManager(null); - it.remove(); - } - } - } catch (InterruptedException e) { - return; - } - } - } -} diff --git a/engine/src/test/java/org/archive/modules/fetcher/FetchHTTPTest.java b/engine/src/test/java/org/archive/modules/fetcher/FetchHTTPTest.java deleted file mode 100644 index e61b2b82..00000000 --- a/engine/src/test/java/org/archive/modules/fetcher/FetchHTTPTest.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * This file is part of the Heritrix web crawler (crawler.archive.org). - * - * Licensed to the Internet Archive (IA) by one or more individual - * contributors. - * - * The IA licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.archive.modules.fetcher; - - -import java.io.IOException; - -import org.archive.modules.CrawlMetadata; - - -/** - * - */ -public class FetchHTTPTest extends FetchHTTPTestBase { - - @Override - protected AbstractFetchHTTP makeModule() throws IOException { - LegacyFetchHTTP fetchHttp = new LegacyFetchHTTP(); - fetchHttp.setCookieStorage(new SimpleCookieStorage()); - fetchHttp.setServerCache(new DefaultServerCache()); - CrawlMetadata uap = new CrawlMetadata(); - uap.setUserAgentTemplate(getUserAgentString()); - fetchHttp.setUserAgentProvider(uap); - - fetchHttp.start(); - return fetchHttp; - } - - @Override - public void testHttpProxyAuth() throws Exception { - // XXX skip cuz it's slow in FetchHTTP for some reason - } - - @Override - public void testConnectionTimeout() throws Exception { - // XXX skip cuz it's slow cuz you can't change the connection timeout after FetchHTTP.start() has run - } -} diff --git a/modules/src/main/java/org/archive/modules/credential/CommonsHttpCredentialUtil.java b/modules/src/main/java/org/archive/modules/credential/CommonsHttpCredentialUtil.java deleted file mode 100644 index 58a41bec..00000000 --- a/modules/src/main/java/org/archive/modules/credential/CommonsHttpCredentialUtil.java +++ /dev/null @@ -1,147 +0,0 @@ -/* - * This file is part of the Heritrix web crawler (crawler.archive.org). - * - * Licensed to the Internet Archive (IA) by one or more individual - * contributors. - * - * The IA licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.archive.modules.credential; - -import java.util.Iterator; -import java.util.Map; -import java.util.logging.Logger; - -import org.apache.commons.httpclient.HttpClient; -import org.apache.commons.httpclient.HttpMethod; -import org.apache.commons.httpclient.HttpMethodBase; -import org.apache.commons.httpclient.NameValuePair; -import org.apache.commons.httpclient.URIException; -import org.apache.commons.httpclient.UsernamePasswordCredentials; -import org.apache.commons.httpclient.auth.AuthChallengeProcessor; -import org.apache.commons.httpclient.auth.AuthScheme; -import org.apache.commons.httpclient.auth.AuthScope; -import org.apache.commons.httpclient.auth.AuthenticationException; -import org.apache.commons.httpclient.auth.MalformedChallengeException; -import org.apache.commons.httpclient.methods.GetMethod; -import org.apache.commons.httpclient.methods.PostMethod; -import org.apache.commons.lang.StringUtils; -import org.archive.modules.CrawlURI; - -public class CommonsHttpCredentialUtil { - - private static Logger logger = Logger.getLogger(CommonsHttpCredentialUtil.class.getName()); - - public static boolean populate(CrawlURI curi, HttpClient http, - HttpMethod method, Credential cred, Map httpAuthChallenges) { - if (cred instanceof HttpAuthenticationCredential) { - return populate(curi, http, method, (HttpAuthenticationCredential) cred, httpAuthChallenges); - } else if (cred instanceof HtmlFormCredential) { - return populate(curi, http, method, (HtmlFormCredential) cred); - } else { - throw new RuntimeException("not implemented for Credential subtype " + cred.getClass()); - } - } - - public static boolean populate(CrawlURI curi, HttpClient http, - HttpMethod method, HtmlFormCredential cred) { - // http is not used - boolean result = false; - Map formItems = cred.getFormItems(); - if (formItems == null || formItems.size() <= 0) { - try { - logger.severe("No form items for " + method.getURI()); - } - catch (URIException e) { - logger.severe("No form items and exception getting uri: " + - e.getMessage()); - } - return result; - } - - NameValuePair[] data = new NameValuePair[formItems.size()]; - int index = 0; - String key = null; - for (Iterator i = formItems.keySet().iterator(); i.hasNext();) { - key = i.next(); - data[index++] = new NameValuePair(key, (String)formItems.get(key)); - } - if (method instanceof PostMethod) { - ((PostMethod)method).setRequestBody(data); - result = true; - } else if (method instanceof GetMethod) { - // Append these values to the query string. - // Get current query string, then add data, then get it again - // only this time its our data only... then append. - HttpMethodBase hmb = (HttpMethodBase)method; - String currentQuery = hmb.getQueryString(); - hmb.setQueryString(data); - String newQuery = hmb.getQueryString(); - hmb.setQueryString( - ((StringUtils.isNotEmpty(currentQuery)) - ? currentQuery + "&" - : "") - + newQuery); - result = true; - } else { - logger.severe("Unknown method type: " + method); - } - return result; - } - - public static boolean populate(CrawlURI curi, HttpClient http, - HttpMethod method, HttpAuthenticationCredential cred, Map httpAuthChallenges) { - boolean result = false; - - AuthChallengeProcessor authChallengeProcessor = new AuthChallengeProcessor(http.getParams()); - try { - AuthScheme authScheme = authChallengeProcessor.processChallenge(method.getHostAuthState(), httpAuthChallenges); - method.getHostAuthState().setAuthScheme(authScheme); - } catch (MalformedChallengeException e) { - return result; - } catch (AuthenticationException e) { - return result; - } - - // Always add the credential to HttpState. Doing this because no way of - // removing the credential once added AND there is a bug in the - // credentials management system in that it always sets URI root to - // null: it means the key used to find a credential is NOT realm + root - // URI but just the realm. Unless I set it everytime, there is - // possibility that as this thread progresses, it might come across a - // realm already loaded but the login and password are from another - // server. We'll get a failed authentication that'd be difficult to - // explain. - // - // Have to make a UsernamePasswordCredentials. The httpclient auth code - // does an instanceof down in its guts. - UsernamePasswordCredentials upc = null; - try { - upc = new UsernamePasswordCredentials(cred.getLogin(), - cred.getPassword()); - http.getState().setCredentials(new AuthScope(curi.getUURI().getHost(), - curi.getUURI().getPort(), cred.getRealm()), upc); - logger.fine("Credentials for realm " + cred.getRealm() + - " for CrawlURI " + curi.toString() + " added to request"); - - result = true; - } catch (URIException e) { - logger.severe("Failed to parse host from " + curi + ": " + - e.getMessage()); - } - - return result; - } - -} diff --git a/modules/src/main/java/org/archive/modules/fetcher/AbstractCookieStorage.java b/modules/src/main/java/org/archive/modules/fetcher/AbstractCookieStorage.java deleted file mode 100644 index ace25cb3..00000000 --- a/modules/src/main/java/org/archive/modules/fetcher/AbstractCookieStorage.java +++ /dev/null @@ -1,236 +0,0 @@ -/* - * This file is part of the Heritrix web crawler (crawler.archive.org). - * - * Licensed to the Internet Archive (IA) by one or more individual - * contributors. - * - * The IA licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.archive.modules.fetcher; - -import it.unimi.dsi.mg4j.util.MutableString; - -import java.io.BufferedReader; -import java.io.Closeable; -import java.io.File; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; -import java.io.FileReader; -import java.io.IOException; -import java.io.Reader; -import java.util.Date; -import java.util.Map; -import java.util.SortedMap; -import java.util.logging.Level; -import java.util.logging.Logger; - -import org.apache.commons.httpclient.Cookie; -import org.apache.commons.io.IOUtils; -import org.archive.spring.ConfigFile; -import org.archive.spring.ConfigPath; -import org.springframework.context.Lifecycle; - -/** - * @author pjack - * - */ -public abstract class AbstractCookieStorage - implements CookieStorage, - Lifecycle, // InitializingBean, - Closeable { - - final private static Logger LOGGER = - Logger.getLogger(AbstractCookieStorage.class.getName()); - - protected ConfigFile cookiesLoadFile = null; - public ConfigFile getCookiesLoadFile() { - return cookiesLoadFile; - } - public void setCookiesLoadFile(ConfigFile cookiesLoadFile) { - this.cookiesLoadFile = cookiesLoadFile; - } - - - protected ConfigPath cookiesSaveFile = null; - public ConfigPath getCookiesSaveFile() { - return cookiesSaveFile; - } - public void setCookiesSaveFile(ConfigPath cookiesSaveFile) { - this.cookiesSaveFile = cookiesSaveFile; - } - - protected boolean isRunning = false; - public void start() { - if(isRunning()) { - return; - } - SortedMap cookies = prepareMap(); - if (getCookiesLoadFile()!=null) { - loadCookies(getCookiesLoadFile(), cookies); - } - isRunning = true; - } - - public boolean isRunning() { - return isRunning; - } - - public void stop() { - isRunning = false; - } - - protected abstract SortedMap prepareMap(); - - /** - * Load cookies. The input is text in the Netscape's 'cookies.txt' file - * format. Example entry of cookies.txt file: - *

- * www.archive.org FALSE / FALSE 1311699995 details-visit texts-cralond - *

- *

- * Each line has 7 tab-separated fields: - *

- *
    - *
  1. DOMAIN: The domain that created and have access to the cookie value.
    2. - *
  2. FLAG: A TRUE or FALSE value indicating if hosts within the given - * domain can access the cookie value.
    3. - *
  3. PATH: The path within the domain that the cookie value is valid for.
    4. - *
  4. SECURE: A TRUE or FALSE value indicating if to use a secure - * connection to access the cookie value.
    5. - *
  5. EXPIRATION: The expiration time of the cookie value, or -1 for no - * expiration
    6. - *
  6. NAME: The name of the cookie value
    7. - *
  7. VALUE: The cookie value
    8. - *
- * - * @param reader - * input - * @param cookiesFile - * file in the Netscape's 'cookies.txt' format. - */ - public static void loadCookies(Reader reader, - SortedMap cookies) { - BufferedReader br = new BufferedReader(reader); - try { - String line; - int lineNo = 1; - while ((line = br.readLine()) != null) { - if (!line.matches("\\s*(?:#.*)?")) { // skip blank links and comments - String[] tokens = line.split("\\t"); - if (tokens.length == 7) { - long epochSeconds = Long.parseLong(tokens[4]); - Date expirationDate = (epochSeconds >= 0 ? new Date(epochSeconds * 1000) : null); - Cookie cookie = new Cookie(tokens[0], tokens[5], - tokens[6], tokens[2], expirationDate, - Boolean.valueOf(tokens[3]).booleanValue()); - cookie.setDomainAttributeSpecified(Boolean.valueOf(tokens[1]).booleanValue()); - - LOGGER.fine("Adding cookie: domain " + cookie.getDomain() + " cookie " + cookie.toExternalForm()); - cookies.put(cookie.getSortKey(), cookie); - } else { - LOGGER.warning("cookies input line " + lineNo + " invalid, expected 7 tab-delimited tokens"); - } - } - - lineNo++; - } - } catch (IOException e) { - LOGGER.log(Level.WARNING,e.getMessage(), e); - } - } - - protected static void loadCookies(ConfigFile file, - SortedMap cookies) { - - Reader reader = null; - try { - reader = file.obtainReader(); - loadCookies(reader, cookies); - } finally { - IOUtils.closeQuietly(reader); - } - } - - public static void loadCookies(String cookiesFile, - SortedMap result) { - - // Do nothing if cookiesFile is not specified. - if (cookiesFile == null || cookiesFile.length() <= 0) { - return; - } - - FileReader reader = null; - try { - reader = new FileReader(cookiesFile); - loadCookies(reader, result); - } catch (FileNotFoundException e) { - LOGGER.log(Level.WARNING,"Could not find file: " + cookiesFile, e); - } finally { - IOUtils.closeQuietly(reader); - } - } - - public static void saveCookies(String saveCookiesFile, Map cookies) { - // Do nothing if cookiesFile is not specified. - if (saveCookiesFile == null || saveCookiesFile.length() <= 0) { - return; - } - - FileOutputStream out = null; - try { - out = new FileOutputStream(new File(saveCookiesFile)); - String tab ="\t"; - out.write("# Heritrix Cookie File\n".getBytes()); - out.write("# This file is the Netscape cookies.txt format\n\n".getBytes()); - for (Cookie cookie: cookies.values()) { - // Guess an initial size - MutableString line = new MutableString(1024 * 2); - line.append(cookie.getDomain()); - line.append(tab); - line.append(cookie.isDomainAttributeSpecified() ? "TRUE" : "FALSE"); - line.append(tab); - line.append(cookie.getPath()); - line.append(tab); - line.append(cookie.getSecure() ? "TRUE" : "FALSE"); - line.append(tab); - line.append(cookie.getExpiryDate() != null ? cookie.getExpiryDate().getTime() / 1000 : -1); - line.append(tab); - line.append(cookie.getName()); - line.append(tab); - line.append(cookie.getValue() != null ? cookie.getValue() : ""); - line.append("\n"); - out.write(line.toString().getBytes()); - } - } catch (IOException e) { - LOGGER.log(Level.SEVERE, "Unable to write " + saveCookiesFile, e); - } finally { - IOUtils.closeQuietly(out); - } - } - - public abstract SortedMap getCookiesMap(); - - public void saveCookiesMap(Map map) { - innerSaveCookiesMap(map); - if (getCookiesSaveFile()!=null) { - saveCookies(getCookiesSaveFile().getFile().getAbsolutePath(), map); - } - } - - protected abstract void innerSaveCookiesMap(Map map); - - public void close() throws IOException { - } - -} diff --git a/modules/src/main/java/org/archive/modules/fetcher/AbstractCookieStore.java b/modules/src/main/java/org/archive/modules/fetcher/AbstractCookieStore.java index f1b4f0d2..10354aa9 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/AbstractCookieStore.java +++ b/modules/src/main/java/org/archive/modules/fetcher/AbstractCookieStore.java @@ -44,7 +44,7 @@ public abstract class AbstractCookieStore implements CookieStore, Lifecycle, Clo Checkpointable { private static final Logger logger = - Logger.getLogger(AbstractCookieStorage.class.getName()); + Logger.getLogger(AbstractCookieStore.class.getName()); protected ConfigFile cookiesLoadFile = null; public ConfigFile getCookiesLoadFile() { diff --git a/modules/src/main/java/org/archive/modules/fetcher/BdbCookieStorage.java b/modules/src/main/java/org/archive/modules/fetcher/BdbCookieStorage.java deleted file mode 100644 index da078b2c..00000000 --- a/modules/src/main/java/org/archive/modules/fetcher/BdbCookieStorage.java +++ /dev/null @@ -1,116 +0,0 @@ -/* - * This file is part of the Heritrix web crawler (crawler.archive.org). - * - * Licensed to the Internet Archive (IA) by one or more individual - * contributors. - * - * The IA licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.archive.modules.fetcher; - -import java.io.IOException; -import java.util.Map; -import java.util.SortedMap; - -import org.apache.commons.httpclient.Cookie; -import org.archive.bdb.BdbModule; -import org.archive.checkpointing.Checkpoint; -import org.archive.checkpointing.Checkpointable; -import org.springframework.beans.factory.annotation.Autowired; - -import com.sleepycat.bind.serial.SerialBinding; -import com.sleepycat.bind.serial.StoredClassCatalog; -import com.sleepycat.bind.tuple.StringBinding; -import com.sleepycat.collections.StoredSortedMap; -import com.sleepycat.je.Database; -import com.sleepycat.je.DatabaseException; - -/** - * CookieStorage using BDB, so that cookies accumulated in large crawls - * do not outgrow RAM. - * - * @author pjack - */ -public class BdbCookieStorage extends AbstractCookieStorage implements Checkpointable { - @SuppressWarnings("unused") - private static final long serialVersionUID = 1L; - - protected BdbModule bdb; - @Autowired - public void setBdbModule(BdbModule bdb) { - this.bdb = bdb; - } - - /** are we a checkpoint recovery? (in which case, reuse stored cookie data?) */ - protected boolean isCheckpointRecovery = false; - - public static String COOKIEDB_NAME = "http_cookies"; - - private transient Database cookieDb; - private transient StoredSortedMap cookies; - - public BdbCookieStorage() { - } - - protected SortedMap prepareMap() { - try { - StoredClassCatalog classCatalog = bdb.getClassCatalog(); - BdbModule.BdbConfig dbConfig = new BdbModule.BdbConfig(); - dbConfig.setTransactional(false); - dbConfig.setAllowCreate(true); - cookieDb = bdb.openDatabase(COOKIEDB_NAME, dbConfig, isCheckpointRecovery); - cookies = - new StoredSortedMap( - cookieDb, - new StringBinding(), - new SerialBinding(classCatalog,Cookie.class), - true); - return cookies; - } catch (DatabaseException e) { - throw new RuntimeException(e); - } - } - - public SortedMap getCookiesMap() { -// assert cookies != null : "cookie map not set up"; - return cookies; - } - - protected void innerSaveCookiesMap(Map map) { - } - - - - @Override - public void startCheckpoint(Checkpoint checkpointInProgress) { - // do nothing; handled by map checkpoint via BdbModule - } - @Override - public void doCheckpoint(Checkpoint checkpointInProgress) - throws IOException { - // do nothing; handled by map checkpoint via BdbModule - } - @Override - public void finishCheckpoint(Checkpoint checkpointInProgress) { - // do nothing; handled by map checkpoint via BdbModule - } - - @Override - public void setRecoveryCheckpoint(Checkpoint recoveryCheckpoint) { - // just remember that we are doing checkpoint-recovery; - // actual state recovery happens via BdbModule - isCheckpointRecovery = true; - } - -} diff --git a/modules/src/main/java/org/archive/modules/fetcher/CookieStorage.java b/modules/src/main/java/org/archive/modules/fetcher/CookieStorage.java deleted file mode 100644 index 3442eb51..00000000 --- a/modules/src/main/java/org/archive/modules/fetcher/CookieStorage.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * This file is part of the Heritrix web crawler (crawler.archive.org). - * - * Licensed to the Internet Archive (IA) by one or more individual - * contributors. - * - * The IA licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.archive.modules.fetcher; - -import java.util.Map; -import java.util.SortedMap; - -import org.apache.commons.httpclient.Cookie; -import org.springframework.context.Lifecycle; - -public interface CookieStorage extends Lifecycle { - - SortedMap getCookiesMap(); - - void saveCookiesMap(Map map); - -} diff --git a/modules/src/main/java/org/archive/modules/fetcher/HeritrixHttpMethodRetryHandler.java b/modules/src/main/java/org/archive/modules/fetcher/HeritrixHttpMethodRetryHandler.java deleted file mode 100644 index 2be4af14..00000000 --- a/modules/src/main/java/org/archive/modules/fetcher/HeritrixHttpMethodRetryHandler.java +++ /dev/null @@ -1,84 +0,0 @@ -/* - * This file is part of the Heritrix web crawler (crawler.archive.org). - * - * Licensed to the Internet Archive (IA) by one or more individual - * contributors. - * - * The IA licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.archive.modules.fetcher; - -import java.io.IOException; -import java.net.SocketTimeoutException; - -import org.apache.commons.httpclient.HttpMethod; -import org.apache.commons.httpclient.HttpMethodRetryHandler; -import org.apache.commons.httpclient.NoHttpResponseException; -import org.apache.commons.httpclient.methods.PostMethod; - -/** - * Retry handler that tries ten times to establish connection and then once - * established, if a GET method, tries ten times to get response (If POST, - * it tries once only). - * - * Its unsafe retrying POSTs. See 'Rule of Thumb' under 'Method Recovery' - * here: - * HttpClient Tutorial. - * - * @author stack - * @version $Date$, $Revision$ - */ -public class HeritrixHttpMethodRetryHandler implements HttpMethodRetryHandler { - private static final int DEFAULT_RETRY_COUNT = 10; - - private final int maxRetryCount; - - /** - * Constructor. - */ - public HeritrixHttpMethodRetryHandler() { - this(DEFAULT_RETRY_COUNT); - } - - /** - * Constructor. - * @param maxRetryCount Maximum amount of times to retry. - */ - public HeritrixHttpMethodRetryHandler(int maxRetryCount) { - this.maxRetryCount = maxRetryCount; - } - - public boolean retryMethod(HttpMethod method, IOException exception, - int executionCount) { - if(exception instanceof SocketTimeoutException) { - // already waited for the configured amount of time with no reply; - // do not retry further until next go round - return false; - } - if (executionCount >= this.maxRetryCount) { - // Do not retry if over max retry count - return false; - } - if (exception instanceof NoHttpResponseException) { - // Retry if the server dropped connection on us - return true; - } - if (!method.isRequestSent() && (!(method instanceof PostMethod))) { - // Retry if the request has not been sent fully or - // if it's OK to retry methods that have been sent - return true; - } - // otherwise do not retry - return false; - } -} diff --git a/modules/src/main/java/org/archive/modules/fetcher/HeritrixProtocolSocketFactory.java b/modules/src/main/java/org/archive/modules/fetcher/HeritrixProtocolSocketFactory.java deleted file mode 100644 index ae4df087..00000000 --- a/modules/src/main/java/org/archive/modules/fetcher/HeritrixProtocolSocketFactory.java +++ /dev/null @@ -1,196 +0,0 @@ -/* - * This file is part of the Heritrix web crawler (crawler.archive.org). - * - * Licensed to the Internet Archive (IA) by one or more individual - * contributors. - * - * The IA licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.archive.modules.fetcher; - -import java.io.IOException; -import java.net.InetAddress; -import java.net.InetSocketAddress; -import java.net.Socket; -import java.net.SocketTimeoutException; -import java.net.UnknownHostException; - -import org.apache.commons.httpclient.ConnectTimeoutException; -import org.apache.commons.httpclient.params.HttpConnectionParams; -import org.apache.commons.httpclient.protocol.ProtocolSocketFactory; - - -/** - * Version of protocol socket factory that tries to get IP from heritrix IP - * cache -- if its been set into the HttpConnectionParameters. - * - * Copied the guts of DefaultProtocolSocketFactory. This factory gets - * setup by {@link LegacyFetchHTTP}. - * - * @author stack - * @version $Date$, $Revision$ - */ -public class HeritrixProtocolSocketFactory -implements ProtocolSocketFactory { - /** - * Constructor. - */ - public HeritrixProtocolSocketFactory() { - super(); - } - - /** - * @see #createSocket(java.lang.String,int,java.net.InetAddress,int) - */ - public Socket createSocket( - String host, - int port, - InetAddress localAddress, - int localPort - ) throws IOException, UnknownHostException { - return new Socket(host, port, localAddress, localPort); - } - - /** - * Attempts to get a new socket connection to the given host within the - * given time limit. - *

- * This method employs several techniques to circumvent the limitations - * of older JREs that do not support connect timeout. When running in - * JRE 1.4 or above reflection is used to call - * Socket#connect(SocketAddress endpoint, int timeout) method. When - * executing in older JREs a controller thread is executed. The - * controller thread attempts to create a new socket within the given - * limit of time. If socket constructor does not return until the - * timeout expires, the controller terminates and throws an - * {@link ConnectTimeoutException} - *

- * - * @param host the host name/IP - * @param port the port on the host - * @param localAddress the local host name/IP to bind the socket to - * @param localPort the port on the local machine - * @param params {@link HttpConnectionParams Http connection parameters} - * - * @return Socket a new socket - * - * @throws IOException if an I/O error occurs while creating the socket - * @throws UnknownHostException if the IP address of the host cannot be - * @throws IOException if an I/O error occurs while creating the socket - * @throws UnknownHostException if the IP address of the host cannot be - * determined - * @throws ConnectTimeoutException if socket cannot be connected within the - * given time limit - * - * @since 3.0 - */ - public Socket createSocket( - final String host, - final int port, - final InetAddress localAddress, - final int localPort, - final HttpConnectionParams params) - throws IOException, UnknownHostException, ConnectTimeoutException { - // Below code is from the DefaultSSLProtocolSocketFactory#createSocket - // method only it has workarounds to deal with pre-1.4 JVMs. I've - // cut these out. - if (params == null) { - throw new IllegalArgumentException("Parameters may not be null"); - } - Socket socket = null; - int timeout = params.getConnectionTimeout(); - if (timeout == 0) { - socket = createSocket(host, port, localAddress, localPort); - } else { - socket = new Socket(); - - InetAddress hostAddress; - Thread current = Thread.currentThread(); - if (current instanceof HostResolver) { - HostResolver resolver = (HostResolver)current; - hostAddress = resolver.resolve(host); - } else { - hostAddress = null; - } - InetSocketAddress address = (hostAddress != null)? - new InetSocketAddress(hostAddress, port): - new InetSocketAddress(host, port); - socket.bind(new InetSocketAddress(localAddress, localPort)); - try { - socket.connect(address, timeout); - } catch (SocketTimeoutException e) { - // Add timeout info. to the exception. - throw new SocketTimeoutException(e.getMessage() + - ": timeout set at " + Integer.toString(timeout) + "ms."); - } - assert socket.isConnected(): "Socket not connected " + host; - } - return socket; - } - - /** - * Get host address using first the heritrix cache of addresses, then, - * failing that, go to the dnsjava cache. - * - * Default access and static so can be used by other classes in this - * package. - * - * @param host Host whose address we're to fetch. - * @return an IP address for this host or null if one can't be found - * in caches. - * @exception IOException If we fail to get host IP from ServerCache. - */ - /* - static InetAddress getHostAddress(final ServerCache cache, - final String host) throws IOException { - InetAddress result = null; - if (cache != null) { - CrawlHost ch = cache.getHostFor(host); - if (ch != null) { - result = ch.getIP(); - } - } - if (result == null) { - throw new IOException("Failed to get host " + host + - " address from ServerCache"); - } - return result; - } - */ - - /** - * @see ProtocolSocketFactory#createSocket(java.lang.String,int) - */ - public Socket createSocket(String host, int port) - throws IOException, UnknownHostException { - return new Socket(host, port); - } - - /** - * All instances of DefaultProtocolSocketFactory are the same. - * @param obj Object to compare. - * @return True if equal - */ - public boolean equals(Object obj) { - return ((obj != null) && - obj.getClass().equals(HeritrixProtocolSocketFactory.class)); - } - - /** - * All instances of DefaultProtocolSocketFactory have the same hash code. - * @return Hash code for this object. - */ - public int hashCode() { - return HeritrixProtocolSocketFactory.class.hashCode(); - } -} diff --git a/modules/src/main/java/org/archive/modules/fetcher/HeritrixSSLProtocolSocketFactory.java b/modules/src/main/java/org/archive/modules/fetcher/HeritrixSSLProtocolSocketFactory.java deleted file mode 100644 index 6e74c4cb..00000000 --- a/modules/src/main/java/org/archive/modules/fetcher/HeritrixSSLProtocolSocketFactory.java +++ /dev/null @@ -1,151 +0,0 @@ -/* - * This file is part of the Heritrix web crawler (crawler.archive.org). - * - * Licensed to the Internet Archive (IA) by one or more individual - * contributors. - * - * The IA licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.archive.modules.fetcher; - -import java.io.IOException; -import java.net.InetAddress; -import java.net.InetSocketAddress; -import java.net.Socket; -import java.net.SocketTimeoutException; -import java.net.UnknownHostException; -import java.security.KeyManagementException; -import java.security.KeyStoreException; -import java.security.NoSuchAlgorithmException; - -import javax.net.ssl.SSLContext; -import javax.net.ssl.SSLSocketFactory; -import javax.net.ssl.TrustManager; - -import org.apache.commons.httpclient.params.HttpConnectionParams; -import org.apache.commons.httpclient.protocol.SecureProtocolSocketFactory; -import org.archive.httpclient.ConfigurableX509TrustManager; - - -/** - * Implementation of the commons-httpclient SSLProtocolSocketFactory so we - * can return SSLSockets whose trust manager is - * {@link org.archive.httpclient.ConfigurableX509TrustManager}. - * - * We also go to the heritrix cache to get IPs to use making connection. - * To this, we have dependency on {@link HeritrixProtocolSocketFactory}; - * its assumed this class and it are used together. - * See {@link HeritrixProtocolSocketFactory#getHostAddress(ServerCache,String)}. - * - * @author stack - * @version $Id$ - * @see org.archive.httpclient.ConfigurableX509TrustManager - */ -public class HeritrixSSLProtocolSocketFactory -implements SecureProtocolSocketFactory { - /*** - * Socket factory with default trust manager installed. - */ - private SSLSocketFactory sslDefaultFactory = null; - - /** - * Shutdown constructor. - * @throws KeyManagementException - * @throws KeyStoreException - * @throws NoSuchAlgorithmException - */ - public HeritrixSSLProtocolSocketFactory() - throws KeyManagementException, KeyStoreException, NoSuchAlgorithmException{ - // Get an SSL context and initialize it. - SSLContext context = SSLContext.getInstance("SSL"); - - // I tried to get the default KeyManagers but doesn't work unless you - // point at a physical keystore. Passing null seems to do the right - // thing so we'll go w/ that. - context.init(null, new TrustManager[] { - new ConfigurableX509TrustManager( - ConfigurableX509TrustManager.DEFAULT)}, null); - this.sslDefaultFactory = context.getSocketFactory(); - } - - public Socket createSocket(String host, int port, InetAddress clientHost, - int clientPort) - throws IOException, UnknownHostException { - return this.sslDefaultFactory.createSocket(host, port, - clientHost, clientPort); - } - - public Socket createSocket(String host, int port) - throws IOException, UnknownHostException { - return this.sslDefaultFactory.createSocket(host, port); - } - - public synchronized Socket createSocket(String host, int port, - InetAddress localAddress, int localPort, HttpConnectionParams params) - throws IOException, UnknownHostException { - // Below code is from the DefaultSSLProtocolSocketFactory#createSocket - // method only it has workarounds to deal with pre-1.4 JVMs. I've - // cut these out. - if (params == null) { - throw new IllegalArgumentException("Parameters may not be null"); - } - Socket socket = null; - int timeout = params.getConnectionTimeout(); - if (timeout == 0) { - socket = createSocket(host, port, localAddress, localPort); - } else { - SSLSocketFactory factory = (SSLSocketFactory)params. - getParameter(LegacyFetchHTTP.SSL_FACTORY_KEY); - SSLSocketFactory f = (factory != null)? factory: this.sslDefaultFactory; - socket = f.createSocket(); - - Thread current = Thread.currentThread(); - InetAddress hostAddress; - if (current instanceof HostResolver) { - HostResolver resolver = (HostResolver)current; - hostAddress = resolver.resolve(host); - } else { - hostAddress = null; - } - InetSocketAddress address = (hostAddress != null)? - new InetSocketAddress(hostAddress, port): - new InetSocketAddress(host, port); - socket.bind(new InetSocketAddress(localAddress, localPort)); - try { - socket.connect(address, timeout); - } catch (SocketTimeoutException e) { - // Add timeout info. to the exception. - throw new SocketTimeoutException(e.getMessage() + - ": timeout set at " + Integer.toString(timeout) + "ms."); - } - assert socket.isConnected(): "Socket not connected " + host; - } - return socket; - } - - public Socket createSocket(Socket socket, String host, int port, - boolean autoClose) - throws IOException, UnknownHostException { - return this.sslDefaultFactory.createSocket(socket, host, - port, autoClose); - } - - public boolean equals(Object obj) { - return ((obj != null) && obj.getClass(). - equals(HeritrixSSLProtocolSocketFactory.class)); - } - - public int hashCode() { - return HeritrixSSLProtocolSocketFactory.class.hashCode(); - } -} \ No newline at end of file diff --git a/modules/src/main/java/org/archive/modules/fetcher/LegacyFetchHTTP.java b/modules/src/main/java/org/archive/modules/fetcher/LegacyFetchHTTP.java deleted file mode 100644 index b1da2887..00000000 --- a/modules/src/main/java/org/archive/modules/fetcher/LegacyFetchHTTP.java +++ /dev/null @@ -1,1640 +0,0 @@ -/* - * This file is part of the Heritrix web crawler (crawler.archive.org). - * - * Licensed to the Internet Archive (IA) by one or more individual - * contributors. - * - * The IA licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.archive.modules.fetcher; - -import static org.archive.modules.CrawlURI.FetchType.HTTP_POST; -import static org.archive.modules.fetcher.FetchErrors.HEADER_TRUNC; -import static org.archive.modules.fetcher.FetchErrors.LENGTH_TRUNC; -import static org.archive.modules.fetcher.FetchErrors.TIMER_TRUNC; -import static org.archive.modules.fetcher.FetchStatusCodes.S_CONNECT_FAILED; -import static org.archive.modules.fetcher.FetchStatusCodes.S_CONNECT_LOST; -import static org.archive.modules.fetcher.FetchStatusCodes.S_DOMAIN_PREREQUISITE_FAILURE; -import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_ETAG_HEADER; -import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_FETCH_HISTORY; -import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_LAST_MODIFIED_HEADER; -import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_REFERENCE_LENGTH; -import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_STATUS; - -import java.io.IOException; -import java.io.ObjectInputStream; -import java.io.ObjectOutputStream; -import java.net.InetAddress; -import java.net.UnknownHostException; -import java.nio.charset.Charset; -import java.security.KeyManagementException; -import java.security.KeyStoreException; -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; -import java.util.Arrays; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.logging.Level; -import java.util.logging.Logger; - -import javax.net.ssl.SSLContext; -import javax.net.ssl.SSLSocketFactory; -import javax.net.ssl.TrustManager; - -import org.apache.commons.httpclient.Cookie; -import org.apache.commons.httpclient.Credentials; -import org.apache.commons.httpclient.Header; -import org.apache.commons.httpclient.HostConfiguration; -import org.apache.commons.httpclient.HttpClient; -import org.apache.commons.httpclient.HttpConnection; -import org.apache.commons.httpclient.HttpConnectionManager; -import org.apache.commons.httpclient.HttpException; -import org.apache.commons.httpclient.HttpMethod; -import org.apache.commons.httpclient.HttpMethodBase; -import org.apache.commons.httpclient.HttpState; -import org.apache.commons.httpclient.HttpStatus; -import org.apache.commons.httpclient.HttpVersion; -import org.apache.commons.httpclient.NTCredentials; -import org.apache.commons.httpclient.URIException; -import org.apache.commons.httpclient.auth.AuthChallengeParser; -import org.apache.commons.httpclient.auth.AuthPolicy; -import org.apache.commons.httpclient.auth.AuthScheme; -import org.apache.commons.httpclient.auth.AuthScope; -import org.apache.commons.httpclient.auth.MalformedChallengeException; -import org.apache.commons.httpclient.cookie.CookiePolicy; -import org.apache.commons.httpclient.params.HttpClientParams; -import org.apache.commons.httpclient.params.HttpConnectionManagerParams; -import org.apache.commons.httpclient.params.HttpMethodParams; -import org.apache.commons.httpclient.protocol.Protocol; -import org.apache.commons.httpclient.protocol.ProtocolSocketFactory; -import org.apache.commons.lang.StringUtils; -import org.archive.httpclient.ConfigurableX509TrustManager; -import org.archive.httpclient.ConfigurableX509TrustManager.TrustLevel; -import org.archive.httpclient.HttpRecorderGetMethod; -import org.archive.httpclient.HttpRecorderMethod; -import org.archive.httpclient.HttpRecorderPostMethod; -import org.archive.httpclient.SingleHttpConnectionManager; -import org.archive.io.RecorderLengthExceededException; -import org.archive.io.RecorderTimeoutException; -import org.archive.io.RecorderTooMuchHeaderException; -import org.archive.modules.CrawlURI; -import org.archive.modules.CrawlURI.FetchType; -import org.archive.modules.ProcessResult; -import org.archive.modules.credential.CommonsHttpCredentialUtil; -import org.archive.modules.credential.Credential; -import org.archive.modules.credential.CredentialStore; -import org.archive.modules.credential.HttpAuthenticationCredential; -import org.archive.modules.deciderules.AcceptDecideRule; -import org.archive.modules.deciderules.DecideResult; -import org.archive.modules.deciderules.DecideRule; -import org.archive.modules.extractor.LinkContext; -import org.archive.modules.net.CrawlHost; -import org.archive.modules.net.CrawlServer; -import org.archive.modules.net.ServerCache; -import org.archive.util.Recorder; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.Lifecycle; - -/** - * HTTP fetcher that uses Apache Jakarta Commons - * HttpClient library. - * - * @contributor gojomo - * @contributor Igor Ranitovic - * @contributor stack - * @contributor others - * @version $Id$ - */ -public class LegacyFetchHTTP extends AbstractFetchHTTP implements Lifecycle { - @SuppressWarnings("unused") - private static final long serialVersionUID = 1L; - private static Logger logger = Logger.getLogger(LegacyFetchHTTP.class.getName()); - - /** - * Proxy host IP (set only if needed). - */ - { - setHttpProxyHost(""); - } - public String getHttpProxyHost() { - return (String) kp.get("httpProxyHost"); - } - public void setHttpProxyHost(String host) { - kp.put("httpProxyHost",host); - } - - /** - * Proxy port (set only if needed). - */ - { - setHttpProxyPort(0); - } - public int getHttpProxyPort() { - return (Integer) kp.get("httpProxyPort"); - } - public void setHttpProxyPort(int port) { - kp.put("httpProxyPort",port); - } - - /** - * Proxy user (set only if needed). - */ - { - setHttpProxyUser(""); - } - public String getHttpProxyUser() { - return (String) kp.get("httpProxyUser"); - } - public void setHttpProxyUser(String user) { - kp.put("httpProxyUser",user); - } - - /** - * Proxy password (set only if needed). - */ - { - setHttpProxyPassword(""); - } - public String getHttpProxyPassword() { - return (String) kp.get("httpProxyPassword"); - } - public void setHttpProxyPassword(String password) { - kp.put("httpProxyPassword",password); - } - - /** - * If the fetch is not completed in this number of seconds, give up (and - * retry later). - */ - { - setTimeoutSeconds(20*60); // 20 minutes - } - public int getTimeoutSeconds() { - return (Integer) kp.get("timeoutSeconds"); - } - public void setTimeoutSeconds(int timeout) { - kp.put("timeoutSeconds",timeout); - } - - /** - * If the socket is unresponsive for this number of milliseconds, give up. - * Set to zero for no timeout (Not. recommended. Could hang a thread on an - * unresponsive server). This timeout is used timing out socket opens and - * for timing out each socket read. Make sure this value is < - * {@link #TIMEOUT_SECONDS} for optimal configuration: ensures at least one - * retry read. - */ - { - setSoTimeoutMs(20*1000); // 20 seconds - } - public int getSoTimeoutMs() { - return (Integer) kp.get("soTimeoutMs"); - } - public void setSoTimeoutMs(int timeout) { - kp.put("soTimeoutMs",timeout); - } - - /** - * Maximum length in bytes to fetch. Fetch is truncated at this length. A - * value of 0 means no limit. - */ - { - setMaxLengthBytes(0L); // no limit - } - public long getMaxLengthBytes() { - return (Long) kp.get("maxLengthBytes"); - } - public void setMaxLengthBytes(long maxLengthBytes) { - kp.put("maxLengthBytes",maxLengthBytes); - } - - /** - * Accept Headers to include in each request. Each must be the complete - * header, e.g., 'Accept-Language: en'. (Thus, this can also be used to - * other headers not beginning 'Accept-' as well.) By default heritrix sends - * an Accept header similar to what a typical browser would send (the value - * comes from Firefox 4.0). - */ - { - setAcceptHeaders(Arrays.asList("Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")); - } - @SuppressWarnings("unchecked") - public List getAcceptHeaders() { - return (List) kp.get("acceptHeaders"); - } - public void setAcceptHeaders(List headers) { - kp.put("acceptHeaders",headers); - } - - /** - * The character encoding to use for files that do not have one specified in - * the HTTP response headers. Default: ISO-8859-1. - */ - { - setDefaultEncoding("ISO-8859-1"); - } - public String getDefaultEncoding() { - return getDefaultCharset().name(); - } - public void setDefaultEncoding(String encoding) { - kp.put("defaultEncoding",Charset.forName(encoding)); - } - public Charset getDefaultCharset() { - return (Charset)kp.get("defaultEncoding"); - } - - /** - * Whether or not to perform an on-the-fly digest hash of retrieved - * content-bodies. - */ - { - setDigestContent(true); - } - public boolean getDigestContent() { - return (Boolean) kp.get("digestContent"); - } - public void setDigestContent(boolean digest) { - kp.put("digestContent",digest); - } - - /** - * Which algorithm (for example MD5 or SHA-1) to use to perform an - * on-the-fly digest hash of retrieved content-bodies. - */ - protected String digestAlgorithm = "sha1"; - public String getDigestAlgorithm() { - return digestAlgorithm; - } - public void setDigestAlgorithm(String digestAlgorithm) { - this.digestAlgorithm = digestAlgorithm; - } - - /** - * The maximum KB/sec to use when fetching data from a server. The default - * of 0 means no maximum. - */ - { - setMaxFetchKBSec(0); // no limit - } - public int getMaxFetchKBSec() { - return (Integer) kp.get("maxFetchKBSec"); - } - public void setMaxFetchKBSec(int rate) { - kp.put("maxFetchKBSec",rate); - } - - public UserAgentProvider getUserAgentProvider() { - return (UserAgentProvider) kp.get("userAgentProvider"); - } - @Autowired - public void setUserAgentProvider(UserAgentProvider provider) { - kp.put("userAgentProvider",provider); - } - - /** - * SSL certificate trust level. Range is from the default 'open' (trust all - * certs including expired, selfsigned, and those for which we do not have a - * CA) through 'loose' (trust all valid certificates including selfsigned), - * 'normal' (all valid certificates not including selfsigned) to 'strict' - * (Cert is valid and DN must match servername). - */ - { - setSslTrustLevel(TrustLevel.OPEN); - } - public TrustLevel getSslTrustLevel() { - return (TrustLevel) kp.get("trustLevel"); - } - public void setSslTrustLevel(TrustLevel trustLevel) { - kp.put("sslTrustLevel",trustLevel); - } - - private transient HttpClient http = null; - - /** - * How many 'instant retries' of HttpRecoverableExceptions have occurred - * - * Would like it to be 'long', but longs aren't atomic - */ - private int recoveryRetries = 0; - - - /** - * DecideRules applied after receipt of HTTP response headers but before we - * start to download the body. If any filter returns FALSE, the fetch is - * aborted. Prerequisites such as robots.txt by-pass filtering (i.e. they - * cannot be midfetch aborted. - */ - { - setShouldFetchBodyRule(new AcceptDecideRule()); - } - public DecideRule getShouldFetchBodyRule() { - return (DecideRule) kp.get("shouldFetchBodyRule"); - } - public void setShouldFetchBodyRule(DecideRule rule) { - kp.put("shouldFetchBodyRule", rule); - } - - // see [ 1379040 ] regex for midfetch filter not being stored in crawl order - // http://sourceforge.net/support/tracker.php?aid=1379040 - // this.midfetchfilters.setExpertSetting(true); - - /** - * What to log if midfetch abort. - */ - private static final String MIDFETCH_ABORT_LOG = "midFetchAbort"; - - /** - * Use HTTP/1.1. Note: even when offering an HTTP/1.1 request, - * Heritrix may not properly handle persistent/keep-alive connections, - * so the sendConnectionClose parameter should remain 'true'. - */ - { - setUseHTTP11(false); - } - public boolean getUseHTTP11() { - return (Boolean) kp.get("useHTTP11"); - } - public void setUseHTTP11(boolean useHTTP11) { - kp.put("useHTTP11",useHTTP11); - } - - /** - * Set headers to accept compressed responses. - */ - { - setAcceptCompression(false); - } - public boolean getAcceptCompression() { - return (Boolean) kp.get("acceptCompression"); - } - public void setAcceptCompression(boolean acceptCompression) { - kp.put("acceptCompression",acceptCompression); - } - - /** - * Send 'Connection: close' header with every request. - */ - { - setSendConnectionClose(true); - } - public boolean getSendConnectionClose() { - return (Boolean) kp.get("sendConnectionClose"); - } - public void setSendConnectionClose(boolean sendClose) { - kp.put("sendConnectionClose",sendClose); - } - - private static final Header HEADER_SEND_CONNECTION_CLOSE = new Header( - "Connection", "close"); - - /** - * Send 'Referer' header with every request. - *

- * The 'Referer' header contans the location the crawler came from, the page - * the current URI was discovered in. The 'Referer' usually is logged on the - * remote server and can be of assistance to webmasters trying to figure how - * a crawler got to a particular area on a site. - */ - { - setSendReferer(true); - } - public boolean getSendReferer() { - return (Boolean) kp.get("sendReferer"); - } - public void setSendReferer(boolean sendClose) { - kp.put("sendReferer",sendClose); - } - - /** - * Send 'Range' header when a limit ({@link #MAX_LENGTH_BYTES}) on - * document size. - *

- * Be polite to the HTTP servers and send the 'Range' header, stating that - * you are only interested in the first n bytes. Only pertinent if - * {@link #MAX_LENGTH_BYTES} > 0. Sending the 'Range' header results in a - * '206 Partial Content' status response, which is better than just cutting - * the response mid-download. On rare occasion, sending 'Range' will - * generate '416 Request Range Not Satisfiable' response. - */ - { - setSendRange(false); - } - public boolean getSendRange() { - return (Boolean) kp.get("sendRange"); - } - public void setSendRange(boolean sendRange) { - kp.put("sendRange",sendRange); - } - - /** - * Send 'If-Modified-Since' header, if previous 'Last-Modified' fetch - * history information is available in URI history. - */ - { - setSendIfModifiedSince(true); - } - public boolean getSendIfModifiedSince() { - return (Boolean) kp.get("sendIfModifiedSince"); - } - public void setSendIfModifiedSince(boolean sendIfModifiedSince) { - kp.put("sendIfModifiedSince",sendIfModifiedSince); - } - - /** - * Send 'If-None-Match' header, if previous 'Etag' fetch history information - * is available in URI history. - */ - { - setSendIfNoneMatch(true); - } - public boolean getSendIfNoneMatch() { - return (Boolean) kp.get("sendIfNoneMatch"); - } - public void setSendIfNoneMatch(boolean sendIfNoneMatch) { - kp.put("sendIfNoneMatch",sendIfNoneMatch); - } - - public static final String REFERER = "Referer"; - - public static final String RANGE = "Range"; - - public static final String RANGE_PREFIX = "bytes=0-"; - - public static final String HTTP_SCHEME = "http"; - - public static final String HTTPS_SCHEME = "https"; - - - protected CookieStorage cookieStorage = new BdbCookieStorage(); - @Autowired(required=false) - public void setCookieStorage(CookieStorage storage) { - this.cookieStorage = storage; - } - public CookieStorage getCookieStorage() { - return this.cookieStorage; - } - - /** - * Disable cookie handling. - */ - { - setIgnoreCookies(false); - } - public boolean getIgnoreCookies() { - return (Boolean) kp.get("ignoreCookies"); - } - public void setIgnoreCookies(boolean ignoreCookies) { - kp.put("ignoreCookies",ignoreCookies); - } - - /** - * Local IP address or hostname to use when making connections (binding - * sockets). When not specified, uses default local address(es). - */ - public String getHttpBindAddress(){ - return (String) kp.get(HTTP_BIND_ADDRESS); - } - public void setHttpBindAddress(String address) { - kp.put(HTTP_BIND_ADDRESS, address); - } - public static final String HTTP_BIND_ADDRESS = "httpBindAddress"; - - /** - * Used to store credentials. - */ - { - // initialize with empty store so declaration not required - setCredentialStore(new CredentialStore()); - } - public CredentialStore getCredentialStore() { - return (CredentialStore) kp.get("credentialStore"); - } - @Autowired(required=false) - public void setCredentialStore(CredentialStore credentials) { - kp.put("credentialStore",credentials); - } - - /** - * Used to do DNS lookups. - */ - protected ServerCache serverCache; - public ServerCache getServerCache() { - return this.serverCache; - } - @Autowired - public void setServerCache(ServerCache serverCache) { - this.serverCache = serverCache; - } - - static { - Protocol.registerProtocol("http", new Protocol("http", - new HeritrixProtocolSocketFactory(), 80)); - try { - ProtocolSocketFactory psf = new HeritrixSSLProtocolSocketFactory(); - Protocol p = new Protocol("https", psf, 443); - Protocol.registerProtocol("https", p); - } catch (KeyManagementException e) { - e.printStackTrace(); - } catch (KeyStoreException e) { - e.printStackTrace(); - } catch (NoSuchAlgorithmException e) { - e.printStackTrace(); - } - } - - - // static final String SERVER_CACHE_KEY = "heritrix.server.cache"; - static final String SSL_FACTORY_KEY = "heritrix.ssl.factory"; - - /*************************************************************************** - * Socket factory that has the configurable trust manager installed. - */ - private transient SSLSocketFactory sslfactory = null; - - /** - * Constructor. - */ - public LegacyFetchHTTP() { - } - - protected void innerProcess(final CrawlURI curi) - throws InterruptedException { - // Note begin time - curi.setFetchBeginTime(System.currentTimeMillis()); - - // Get a reference to the HttpRecorder that is set into this ToeThread. - Recorder rec = curi.getRecorder(); - - // Shall we get a digest on the content downloaded? - boolean digestContent = getDigestContent(); - String algorithm = null; - if (digestContent) { - algorithm = getDigestAlgorithm(); - rec.getRecordedInput().setDigest(algorithm); - } else { - // clear - rec.getRecordedInput().setDigest((MessageDigest)null); - } - - // Below we do two inner classes that add check of midfetch - // filters just as we're about to receive the response body. - String curiString = curi.getUURI().toString(); - HttpMethodBase method = null; - if (curi.getFetchType() == HTTP_POST) { - method = new HttpRecorderPostMethod(curiString, rec) { - protected void readResponseBody(HttpState state, - HttpConnection conn) throws IOException, HttpException { - addResponseContent(this, curi); - if (checkMidfetchAbort(curi, this.httpRecorderMethod, conn)) { - doAbort(curi, this, MIDFETCH_ABORT_LOG); - } else { - super.readResponseBody(state, conn); - } - } - }; - curi.setFetchType(FetchType.HTTP_POST); - } else { - method = new HttpRecorderGetMethod(curiString, rec) { - protected void readResponseBody(HttpState state, - HttpConnection conn) throws IOException, HttpException { - addResponseContent(this, curi); - if (checkMidfetchAbort(curi, this.httpRecorderMethod, conn)) { - doAbort(curi, this, MIDFETCH_ABORT_LOG); - } else { - super.readResponseBody(state, conn); - } - } - }; - curi.setFetchType(FetchType.HTTP_GET); - } - - - HostConfiguration customConfigOrNull = configureMethod(curi, method); - - // Populate credentials. Set config so auth. is not automatic. - boolean addedCredentials = populateCredentials(curi, method); - if (http.getState().getProxyCredentials(new AuthScope(getProxyHost(), getProxyPort())) != null) { - addedCredentials = true; - } - - // set hardMax on bytes (if set by operator) - long hardMax = getMaxLengthBytes(); - // set overall timeout (if set by operator) - long timeoutMs = 1000 * getTimeoutSeconds(); - // Get max fetch rate (bytes/ms). It comes in in KB/sec - long maxRateKBps = getMaxFetchKBSec(); - rec.getRecordedInput().setLimits(hardMax, timeoutMs, maxRateKBps); - - try { - this.http.executeMethod(customConfigOrNull, method); - } catch (RecorderTooMuchHeaderException ex) { - // when too much header material, abort like other truncations - doAbort(curi, method, HEADER_TRUNC); - } catch (IOException e) { - failedExecuteCleanup(method, curi, e); - return; - } catch (ArrayIndexOutOfBoundsException e) { - // For weird windows-only ArrayIndex exceptions in native - // code... see - // http://forum.java.sun.com/thread.jsp?forum=11&thread=378356 - // treating as if it were an IOException - failedExecuteCleanup(method, curi, e); - return; - } - - // set softMax on bytes to get (if implied by content-length) - long softMax = method.getResponseContentLength(); - - try { - if (!method.isAborted()) { - // Force read-to-end, so that any socket hangs occur here, - // not in later modules. - rec.getRecordedInput().readFullyOrUntil(softMax); - } - } catch (RecorderTimeoutException ex) { - doAbort(curi, method, TIMER_TRUNC); - } catch (RecorderLengthExceededException ex) { - doAbort(curi, method, LENGTH_TRUNC); - } catch (IOException e) { - cleanup(curi, e, "readFully", S_CONNECT_LOST); - return; - } catch (ArrayIndexOutOfBoundsException e) { - // For weird windows-only ArrayIndex exceptions from native code - // see http://forum.java.sun.com/thread.jsp?forum=11&thread=378356 - // treating as if it were an IOException - cleanup(curi, e, "readFully", S_CONNECT_LOST); - return; - } finally { - // ensure recording has stopped - rec.closeRecorders(); - if (!method.isAborted()) { - method.releaseConnection(); - } - // Note completion time - curi.setFetchCompletedTime(System.currentTimeMillis()); - // Set the response charset into the HttpRecord if available. - setCharacterEncoding(curi, rec, method); - setSizes(curi, rec); - setOtherCodings(curi, rec, method); - } - - if (digestContent) { - curi.setContentDigest(algorithm, - rec.getRecordedInput().getDigestValue()); - } - if (logger.isLoggable(Level.FINE)) { - logger.fine(((curi.getFetchType() == HTTP_POST) ? "POST" : "GET") - + " " + curi.getUURI().toString() + " " - + method.getStatusCode() + " " - + rec.getRecordedInput().getSize() + " " - + curi.getContentType()); - } - - if (isSuccess(curi) && addedCredentials) { - // Promote the credentials from the CrawlURI to the CrawlServer - // so they are available for all subsequent CrawlURIs on this - // server. - promoteCredentials(curi); - if (logger.isLoggable(Level.FINE)) { - // Print out the cookie. Might help with the debugging. - Header setCookie = method.getResponseHeader("set-cookie"); - if (setCookie != null) { - logger.fine(setCookie.toString().trim()); - } - } - } else if (method.getStatusCode() == HttpStatus.SC_UNAUTHORIZED) { - // 401 is not 'success'. - handle401(method, curi); - } - - if (rec.getRecordedInput().isOpen()) { - logger.severe(curi.toString() + " RIS still open. Should have" - + " been closed by method release: " - + Thread.currentThread().getName()); - try { - rec.getRecordedInput().close(); - } catch (IOException e) { - logger.log(Level.SEVERE, "second-chance RIS close failed", e); - } - } - } - - /** - * Update CrawlURI internal sizes based on current transaction (and - * in the case of 304s, history) - * - * @param curi CrawlURI - * @param rec HttpRecorder - */ - protected void setSizes(CrawlURI curi, Recorder rec) { - // set reporting size - curi.setContentSize(rec.getRecordedInput().getSize()); - // special handling for 304-not modified - if (curi.getFetchStatus() == HttpStatus.SC_NOT_MODIFIED - && curi.containsDataKey(A_FETCH_HISTORY)) { - Map[] history = curi.getFetchHistory(); - if (history[0] != null - && history[0].containsKey(A_REFERENCE_LENGTH)) { - long referenceLength = (Long) history[0].get(A_REFERENCE_LENGTH); - // carry-forward previous 'reference-length' for future - curi.getData().put(A_REFERENCE_LENGTH, referenceLength); - // increase content-size to virtual-size for reporting - curi.setContentSize(rec.getRecordedInput().getSize() - + referenceLength); - } - } - } - - protected void doAbort(CrawlURI curi, HttpMethod method, - String annotation) { - curi.getAnnotations().add(annotation); - curi.getRecorder().close(); - method.abort(); - } - - protected boolean checkMidfetchAbort(CrawlURI curi, - HttpRecorderMethod method, HttpConnection conn) { - if (curi.isPrerequisite()) { - return false; - } - DecideResult r = getShouldFetchBodyRule().decisionFor(curi); - if (r != DecideResult.REJECT) { - return false; - } - method.markContentBegin(conn); - return true; - } - - /** - * This method populates curi with response status and - * content type. - * - * @param curi - * CrawlURI to populate. - * @param method - * Method to get response status and headers from. - */ - protected void addResponseContent(HttpMethod method, CrawlURI curi) { - curi.setFetchStatus(method.getStatusCode()); - Header ct = method.getResponseHeader("content-type"); - curi.setContentType((ct == null) ? null : ct.getValue()); - - for (Header h: method.getResponseHeaders()) { - curi.putHttpResponseHeader(h.getName(), h.getValue()); - } - } - - /** - * Set the character encoding based on the result headers or default. - * - * The HttpClient returns its own default encoding ("ISO-8859-1") if one - * isn't specified in the Content-Type response header. We give the user the - * option of overriding this, so we need to detect the case where the - * default is returned. - * - * Now, it may well be the case that the default returned by HttpClient and - * the default defined by the user are the same. - * - * TODO:FIXME?: This method does not do the "detect the case where the - * [HttpClient] default is returned" mentioned above! Why not? - * - * @param rec - * Recorder for this request. - * @param method - * Method used for the request. - */ - private void setCharacterEncoding(CrawlURI curi, final Recorder rec, - final HttpMethod method) { - String encoding = ((HttpMethodBase) method).getResponseCharSet(); - try { - rec.setCharset(Charset.forName(encoding)); - } catch (IllegalArgumentException e) { - curi.getAnnotations().add("unsatisfiableCharsetInHeader:"+StringUtils.stripToEmpty(encoding)); - rec.setCharset(getDefaultCharset()); - } - } - - /** - * Set the transfer, content encodings based on headers (if necessary). - * - * @param rec - * Recorder for this request. - * @param method - * Method used for the request. - */ - private void setOtherCodings(CrawlURI uri, final Recorder rec, - final HttpMethod method) { - Header transferCodingHeader = ((HttpMethodBase) method).getResponseHeader("Transfer-Encoding"); - if (transferCodingHeader !=null) { - String te = transferCodingHeader.getValue().trim(); - if(te.equalsIgnoreCase("chunked")) { - rec.setInputIsChunked(true); - } else { - logger.log(Level.WARNING,"Unknown transfer-encoding '"+te+"' for "+uri.getURI()); - } - } - Header contentEncodingHeader = ((HttpMethodBase) method).getResponseHeader("Content-Encoding"); - if (contentEncodingHeader!=null) { - String ce = contentEncodingHeader.getValue().trim(); - try { - rec.setContentEncoding(ce); - } catch (IllegalArgumentException e) { - uri.getAnnotations().add("unsatisfiableContentEncoding:"+StringUtils.stripToEmpty(ce)); - } - } - } - - /** - * Cleanup after a failed method execute. - * - * @param curi - * CrawlURI we failed on. - * @param method - * Method we failed on. - * @param exception - * Exception we failed with. - */ - private void failedExecuteCleanup(final HttpMethod method, - final CrawlURI curi, final Exception exception) { - cleanup(curi, exception, "executeMethod", (method.isRequestSent() ? S_CONNECT_LOST : S_CONNECT_FAILED)); - method.releaseConnection(); - } - - /** - * Cleanup after a failed method execute. - * - * @param curi - * CrawlURI we failed on. - * @param exception - * Exception we failed with. - * @param message - * Message to log with failure. FIXME: Seems ignored - * @param status - * Status to set on the fetch. - */ - private void cleanup(final CrawlURI curi, final Exception exception, - final String message, final int status) { - // message ignored! - curi.getNonFatalFailures().add(exception); - curi.setFetchStatus(status); - curi.getRecorder().close(); - } - - @Override - public ProcessResult process(CrawlURI uri) throws InterruptedException { - if (uri.getFetchStatus() < 0) { - // already marked as errored, this pass through - // skip to end - return ProcessResult.FINISH; - } else { - return super.process(uri); - } - } - - /** - * Can this processor fetch the given CrawlURI. May set a fetch status - * if this processor would usually handle the CrawlURI, but cannot in - * this instance. - * - * @param curi - * @return True if processor can fetch. - */ - @Override - protected boolean shouldProcess(CrawlURI curi) { - String scheme = curi.getUURI().getScheme(); - if (!(scheme.equals("http") || scheme.equals("https"))) { - // handles only plain http and https - return false; - } - - CrawlHost host = serverCache.getHostFor(curi.getUURI()); - if (host.getIP() == null && host.hasBeenLookedUp()) { - curi.setFetchStatus(S_DOMAIN_PREREQUISITE_FAILURE); - return false; - } - - return true; - } - - /** - * Configure the HttpMethod setting options and headers. - * - * @param curi - * CrawlURI from which we pull configuration. - * @param method - * The Method to configure. - */ - protected HostConfiguration configureMethod(CrawlURI curi, - HttpMethod method) { - // Don't try to handle 401s internally. - method.setDoAuthentication(false); - - // Don't auto-follow redirects - method.setFollowRedirects(false); - - // // set soTimeout - // method.getParams().setSoTimeout( - // ((Integer) getUncheckedAttribute(curi, ATTR_SOTIMEOUT_MS)) - // .intValue()); - - // Set cookie policy. - boolean ignoreCookies = getIgnoreCookies(); - method.getParams().setCookiePolicy( - ignoreCookies ? CookiePolicy.IGNORE_COOKIES - : CookiePolicy.BROWSER_COMPATIBILITY); - - method.getParams().setVersion(getUseHTTP11() - ? HttpVersion.HTTP_1_1 - : HttpVersion.HTTP_1_0); - - UserAgentProvider uap = getUserAgentProvider(); - String from = uap.getFrom(); - String userAgent = curi.getUserAgent(); - if (userAgent == null) { - userAgent = uap.getUserAgent(); - } - - method.setRequestHeader("User-Agent", userAgent); - if(StringUtils.isNotBlank(from)) { - method.setRequestHeader("From", from); - } - - // Set retry handler. - method.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, - new HeritrixHttpMethodRetryHandler()); - - final long maxLength = getMaxLengthBytes(); - if (maxLength > 0 && getSendRange()) { - method.addRequestHeader(RANGE, RANGE_PREFIX.concat(Long - .toString(maxLength - 1))); - } - - if (getSendConnectionClose()) { - method.addRequestHeader(HEADER_SEND_CONNECTION_CLOSE); - } - - if (getSendReferer() && !LinkContext.PREREQ_MISC.equals(curi.getViaContext())) { - // RFC2616 says no referer header if referer is https and the url - // is not - String via = flattenVia(curi); - if (via != null - && via.length() > 0 - && !(via.startsWith(HTTPS_SCHEME) && curi.getUURI() - .getScheme().equals(HTTP_SCHEME))) { - method.setRequestHeader(REFERER, via); - } - } - - if (!curi.isPrerequisite()) { - setConditionalGetHeader(curi, method, getSendIfModifiedSince(), - A_LAST_MODIFIED_HEADER, "If-Modified-Since"); - setConditionalGetHeader(curi, method, getSendIfNoneMatch(), - A_ETAG_HEADER, "If-None-Match"); - } - - // TODO: What happens if below method adds a header already - // added above: e.g. Connection, Range, or Referer? - setAcceptHeaders(curi, method); - - HostConfiguration config = - new HostConfiguration(http.getHostConfiguration()); - configureProxy(curi, config); - configureBindAddress(curi, config); - return config; - } - - /** - * Set the given conditional-GET header, if the setting is enabled and - * a suitable value is available in the URI history. - * @param curi source CrawlURI - * @param method HTTP operation pending - * @param setting true/false enablement setting name to consult - * @param sourceHeader header to consult in URI history - * @param targetHeader header to set if possible - */ - protected void setConditionalGetHeader(CrawlURI curi, HttpMethod method, - boolean conditional, String sourceHeader, String targetHeader) { - if (conditional) { - try { - HashMap[] history = curi.getFetchHistory(); - int previousStatus = (Integer) history[0].get(A_STATUS); - if (previousStatus <= 0) { - // do not reuse headers from any broken fetch - return; - } - String previousValue = (String) history[0].get(sourceHeader); - if (previousValue != null) { - method.setRequestHeader(targetHeader, previousValue); - } - } catch (RuntimeException e) { - // for absent key, bad index, etc. just do nothing - } - } - } - - /** - * Setup proxy, based on attributes in CrawlURI and settings, - * in given HostConfiguration - */ - private void configureProxy(CrawlURI curi, HostConfiguration config) { - String proxy = (String) getAttributeEither(curi, "httpProxyHost"); - int port = (Integer) getAttributeEither(curi, "httpProxyPort"); - String user = (String) getAttributeEither(curi, "httpProxyUser"); - String password = (String) getAttributeEither(curi, "httpProxyPassword"); - configureProxy(proxy, port, user, password, config); - } - - private void configureProxy(String proxy, int port, String user, String password, - HostConfiguration config) { - if(StringUtils.isNotEmpty(proxy)) { - config.setProxy(proxy, port); - if (StringUtils.isNotEmpty(user)) { - Credentials credentials = new NTCredentials(user, password, "", ""); - AuthScope authScope = new AuthScope(proxy, port); - this.http.getState().setProxyCredentials(authScope, credentials); - } - } - } - - /** - * Setup local bind address, based on attributes in CrawlURI and - * settings, in given HostConfiguration - */ - private void configureBindAddress(CrawlURI curi, HostConfiguration config) { - String addressString = (String) getAttributeEither(curi, HTTP_BIND_ADDRESS); - configureBindAddress(addressString,config); - } - - private void configureBindAddress(String address, HostConfiguration config) { - if (StringUtils.isNotEmpty(address)) { - try { - InetAddress localAddress = InetAddress.getByName(address); - config.setLocalAddress(localAddress); - } catch (UnknownHostException e) { - // Convert all to RuntimeException so get an exception out - // if initialization fails. - throw new RuntimeException("Unknown host " + address - + " in local-address"); - } - } - } - - /** - * Get a value either from inside the CrawlURI instance, or from - * settings (module attributes). - * - * @param curi - * CrawlURI to consult - * @param key - * key to lookup - * @return value from either CrawlURI (preferred) or settings - */ - protected Object getAttributeEither(CrawlURI curi, String key) { - - Object r = curi.getData().get(key); - if (r != null) { - return r; - } - return kp.get(key); - } - - /** - * Add credentials if any to passed method. - * - * Do credential handling. Credentials are in two places. 1. Credentials - * that succeeded are added to the CrawlServer (Or rather, avatars for - * credentials are whats added because its not safe to keep around - * references to credentials). 2. Credentials to be tried are in the curi. - * Returns true if found credentials to be tried. - * - * @param curi - * Current CrawlURI. - * @param method - * The method to add to. - * @return True if prepopulated method with credentials AND - * the credentials came from the curi, not from the - * CrawlServer. The former is special in that if the - * curi credentials - * succeed, then the caller needs to promote them from the CrawlURI to the - * CrawlServer so they are available for all subsequent CrawlURIs on this - * server. - */ - private boolean populateCredentials(CrawlURI curi, HttpMethod method) { - // First look at the server avatars. Add any that are to be volunteered - // on every request (e.g. RFC2617 credentials). Every time creds will - // return true when we call 'isEveryTime(). - String serverKey; - try { - serverKey = CrawlServer.getServerKey(curi.getUURI()); - } catch (URIException e) { - return false; - } - CrawlServer server = serverCache.getServerFor(serverKey); - if (server.hasCredentials()) { - for (Credential cred : server.getCredentials()) { - if (cred.isEveryTime()) { - CommonsHttpCredentialUtil.populate(curi, this.http, method, cred, server.getHttpAuthChallenges()); - } - } - } - - boolean result = false; - - // Now look in the curi. The Curi will have credentials loaded either - // by the handle401 method if its a rfc2617 or it'll have been set into - // the curi by the preconditionenforcer as this login uri came through. - for (Credential c: curi.getCredentials()) { - if (CommonsHttpCredentialUtil.populate(curi, this.http, method, c, curi.getHttpAuthChallenges())) { - result = true; - } - } - - return result; - } - - /** - * Promote successful credential to the server. - * - * @param curi - * CrawlURI whose credentials we are to promote. - */ - private void promoteCredentials(final CrawlURI curi) { - Set credentials = curi.getCredentials(); - for (Iterator i = credentials.iterator(); i.hasNext();) { - Credential c = i.next(); - i.remove(); - // The server to attach too may not be the server that hosts - // this passed curi. It might be of another subdomain. - // The avatar needs to be added to the server that is dependent - // on this precondition. Find it by name. Get the name from - // the credential this avatar represents. - String cd = c.getDomain(); - if (cd != null) { - CrawlServer cs = serverCache.getServerFor(cd); - if (cs != null) { - cs.addCredential(c); - cs.setHttpAuthChallenges(curi.getHttpAuthChallenges()); - } - } - } - } - - /** - * Server is looking for basic/digest auth credentials (RFC2617). If we have - * any, put them into the CrawlURI and have it come around again. - * Presence of the credential serves as flag to frontier to requeue - * promptly. If we already tried this domain and still got a 401, then our - * credentials are bad. Remove them and let this curi die. - * - * @param method - * Method that got a 401. - * @param curi - * CrawlURI that got a 401. - */ - protected void handle401(final HttpMethod method, final CrawlURI curi) { - AuthScheme authscheme = getAuthScheme(method, curi); - if (authscheme == null) { - return; - } - String realm = authscheme.getRealm(); - - // Look to see if this curi had rfc2617 avatars loaded. If so, are - // any of them for this realm? If so, then the credential failed - // if we got a 401 and it should be let die a natural 401 death. - Set curiRfc2617Credentials = getCredentials(curi, - HttpAuthenticationCredential.class); - HttpAuthenticationCredential extant = HttpAuthenticationCredential.getByRealm( - curiRfc2617Credentials, realm, curi); - if (extant != null) { - // Then, already tried this credential. Remove ANY rfc2617 - // credential since presence of a rfc2617 credential serves - // as flag to frontier to requeue this curi and let the curi - // die a natural death. - extant.detachAll(curi); - logger.warning("Auth failed (401) though supplied realm " + realm - + " to " + curi.toString()); - } else { - // Look see if we have a credential that corresponds to this - // realm in credential store. Filter by type and credential - // domain. If not, let this curi die. Else, add it to the - // curi and let it come around again. Add in the AuthScheme - // we got too. Its needed when we go to run the Auth on - // second time around. - String serverKey = getServerKey(curi); - CrawlServer server = serverCache.getServerFor(serverKey); - Set storeRfc2617Credentials = getCredentialStore().subset(curi, - HttpAuthenticationCredential.class, server.getName()); - if (storeRfc2617Credentials == null - || storeRfc2617Credentials.size() <= 0) { - logger.fine("No rfc2617 credentials for " + curi); - } else { - HttpAuthenticationCredential found = HttpAuthenticationCredential.getByRealm( - storeRfc2617Credentials, realm, curi); - if (found == null) { - logger.fine("No rfc2617 credentials for realm " + realm - + " in " + curi); - } else { - found.attach(curi); - logger.fine("Found credential for realm " + realm - + " in store for " + curi.toString()); - } - } - } - } - - /** - * @param method - * Method that got a 401. - * @param curi - * CrawlURI that got a 401. - * @return Returns first wholesome authscheme found else null. - */ - protected AuthScheme getAuthScheme(final HttpMethod method, - final CrawlURI curi) { - Header[] headers = method.getResponseHeaders("WWW-Authenticate"); - if (headers == null || headers.length <= 0) { - logger.fine("We got a 401 but no WWW-Authenticate challenge: " - + curi.toString()); - return null; - } - - Map authChallenges = null; - try { - @SuppressWarnings("unchecked") - Map parsedChallenges = AuthChallengeParser.parseChallenges(headers); - authChallenges = parsedChallenges; - - // remember WWW-Authenticate headers for later use - curi.setHttpAuthChallenges(authChallenges); - } catch (MalformedChallengeException e) { - logger.fine("Failed challenge parse: " + e.getMessage()); - } - if (authChallenges == null || authChallenges.size() <= 0) { - logger.fine("We got a 401 and WWW-Authenticate challenge" - + " but failed parse of the header " + curi.toString()); - return null; - } - - // XXX there's a lot of overlap below with AuthChallengeProcessor.processChallenge() - - AuthScheme result = null; - // Use the first auth found. - for (Iterator i = authChallenges.keySet().iterator(); result == null - && i.hasNext();) { - String key = (String) i.next(); - String challenge = (String) authChallenges.get(key); - if (key == null || key.length() <= 0 || challenge == null - || challenge.length() <= 0) { - logger.warning("Empty scheme: " + curi.toString() + ": " - + Arrays.toString(headers)); - continue; - } - AuthScheme authscheme; - try { - authscheme = AuthPolicy.getAuthScheme(key); - } catch (IllegalStateException e) { - logger.info("Unsupported auth scheme '" + key + "' at " + curi + " - " + e); - continue; - } - - try { - authscheme.processChallenge(challenge); - } catch (MalformedChallengeException e) { - logger.fine(e.getMessage() + " " + curi + " " + Arrays.toString(headers)); - continue; - } - if (authscheme.isConnectionBased()) { - logger.fine("Connection based " + authscheme); - continue; - } - - if (authscheme.getRealm() == null - || authscheme.getRealm().length() <= 0) { - logger.fine("Empty realm " + authscheme + " for " + curi); - continue; - } - result = authscheme; - } - - return result; - } - - /** - * @param curi - * CrawlURI that got a 401. - * @param type - * Class of credential to get from curi. - * @return Set of credentials attached to this curi. - */ - private Set getCredentials(CrawlURI curi, Class type) { - Set result = null; - - if (curi.hasCredentials()) { - for (Credential c : curi.getCredentials()) { - if (type.isInstance(c)) { - if (result == null) { - result = new HashSet(); - } - result.add(c); - } - } - } - return result; - } - - public void start() { - if(isRunning()) { - return; - } - super.start(); - - configureHttp(); - - if (cookieStorage != null) { - cookieStorage.start(); - http.getState().setCookiesMap(cookieStorage.getCookiesMap()); - } - - setSSLFactory(); - } - - public boolean isRunning() { - return this.http != null; - } - - public void stop() { - if(!isRunning()) { - return; - } - super.stop(); - // At the end save cookies to the file specified in the order file. - if (cookieStorage != null) { - Map map = http.getState().getCookiesMap(); - cookieStorage.saveCookiesMap(map); - cookieStorage.stop(); - } - cleanupHttp(); // XXX happens at finish; move to teardown? - } - - /** - * Perform any final cleanup related to the HttpClient instance. - */ - protected void cleanupHttp() { - this.http = null; - } - - private void setSSLFactory() { - // I tried to get the default KeyManagers but doesn't work unless you - // point at a physical keystore. Passing null seems to do the right - // thing so we'll go w/ that. - try { - SSLContext context = SSLContext.getInstance("SSL"); - context.init(null, - new TrustManager[] { new ConfigurableX509TrustManager( - getSslTrustLevel()) }, null); - this.sslfactory = context.getSocketFactory(); - } catch (Exception e) { - logger.log(Level.WARNING, "Failed configure of ssl context " - + e.getMessage(), e); - } - - } - - protected void configureHttp() { - int soTimeout = getSoTimeoutMs(); - String addressStr = getHttpBindAddress(); - String proxy = getHttpProxyHost(); - int port = -1; - String user = ""; - String password = ""; - if (proxy.length() == 0) { - proxy = null; - } else { - port = getHttpProxyPort(); - user = getHttpProxyUser(); - password = getHttpProxyPassword(); - } - configureHttp(soTimeout, addressStr, proxy, port, user, password); - } - - protected void configureHttp(int soTimeout, String addressStr, - String proxy, int port, String user, String password) { - // Get timeout. Use it for socket and for connection timeout. - int timeout = (soTimeout > 0) ? soTimeout : 0; - - // HttpConnectionManager cm = new ThreadLocalHttpConnectionManager(); - HttpConnectionManager cm = new SingleHttpConnectionManager(); - - // TODO: The following settings should be made in the corresponding - // HttpConnectionManager, not here. - HttpConnectionManagerParams hcmp = cm.getParams(); - hcmp.setConnectionTimeout(timeout); - hcmp.setStaleCheckingEnabled(true); - // Minimizes bandwidth usage. Setting to true disables Nagle's - // algorithm. IBM JVMs < 142 give an NPE setting this boolean - // on ssl sockets. - hcmp.setTcpNoDelay(false); - - this.http = new HttpClient(cm); - HttpClientParams hcp = this.http.getParams(); - // Set default socket timeout. - hcp.setSoTimeout(timeout); - // Set client to be version 1.0. - hcp.setVersion(HttpVersion.HTTP_1_0); - - // configureHttpCookies(defaults); - - // Configure how we want the method to act. - this.http.getParams().setParameter( - HttpMethodParams.SINGLE_COOKIE_HEADER, new Boolean(true)); - this.http.getParams().setParameter( - HttpMethodParams.UNAMBIGUOUS_STATUS_LINE, new Boolean(false)); - this.http.getParams().setParameter( - HttpMethodParams.STRICT_TRANSFER_ENCODING, new Boolean(false)); - this.http.getParams().setIntParameter( - HttpMethodParams.STATUS_LINE_GARBAGE_LIMIT, 10); - - if ((proxy != null) && (proxy.length() == 0)) { - proxy = null; - } - HostConfiguration config = http.getHostConfiguration(); - configureProxy(proxy, port, user, password, config); - configureBindAddress(addressStr,config); - - hcmp.setParameter(SSL_FACTORY_KEY, this.sslfactory); - } - - /* - * (non-Javadoc) - * - * @see org.archive.crawler.framework.Processor#report() - */ - public String report() { - StringBuffer ret = new StringBuffer(); - ret.append(super.report()); - ret.append(" Function: Fetch HTTP URIs\n"); - ret.append(" CrawlURIs handled: " + this.getURICount() + "\n"); - ret.append(" Recovery retries: " + this.recoveryRetries + "\n"); - - return ret.toString(); - } - - - private void setAcceptHeaders(CrawlURI curi, HttpMethod get) { - if(getAcceptCompression()) { - // we match the Firefox header exactly (ordering and whitespace) - // as a favor to caches - get.setRequestHeader("Accept-Encoding","gzip,deflate"); - } - List acceptHeaders = getAcceptHeaders(); - if (acceptHeaders.isEmpty()) { - return; - } - for (String hdr : acceptHeaders) { - String[] nvp = hdr.split(": +"); - if (nvp.length == 2) { - get.setRequestHeader(nvp[0], nvp[1]); - } else { - logger.warning("Invalid accept header: " + hdr); - } - } - } - - // custom serialization - - private String getLocalAddress() { - HostConfiguration hc = http.getHostConfiguration(); - if (hc == null) { - return ""; - } - - InetAddress addr = hc.getLocalAddress(); - if (addr == null) { - return ""; - } - - String r = addr.getCanonicalHostName(); - if (r == null) { - return ""; - } - - return r; - } - - - private String getProxyHost() { - HostConfiguration hc = http.getHostConfiguration(); - if (hc == null) { - return ""; - } - - String r = hc.getProxyHost(); - if (r == null) { - return ""; - } - - return r; - } - - - private int getProxyPort() { - HostConfiguration hc = http.getHostConfiguration(); - if (hc == null) { - return -1; - } - - return hc.getProxyPort(); - } - - - private String getProxyUser() { - NTCredentials credentials = (NTCredentials)http.getState().getProxyCredentials(new AuthScope(getProxyHost(), getProxyPort())); - if (credentials == null) { - return ""; - } - - String r = credentials.getUserName(); - if (r == null) { - return ""; - } - - return r; - } - - private String getProxyPassword() { - NTCredentials credentials = (NTCredentials)http.getState().getProxyCredentials(new AuthScope(getProxyHost(), getProxyPort())); - if (credentials == null) { - return ""; - } - - String r = credentials.getPassword(); - if (r == null) { - return ""; - } - - return r; - } - - - private void writeObject(ObjectOutputStream stream) throws IOException { - stream.defaultWriteObject(); - - // Special handling for http since it isn't Serializable itself - stream.writeInt(http.getParams().getSoTimeout()); - stream.writeUTF(getLocalAddress()); - stream.writeUTF(getProxyHost()); - stream.writeInt(getProxyPort()); - stream.writeUTF(getProxyUser()); - stream.writeUTF(getProxyPassword()); - } - - - private void readObject(ObjectInputStream stream) - throws IOException, ClassNotFoundException { - stream.defaultReadObject(); - - int soTimeout = stream.readInt(); - String localAddress = stream.readUTF(); - String proxy = stream.readUTF(); - int port = stream.readInt(); - String user = stream.readUTF(); - String password = stream.readUTF(); - - configureHttp(soTimeout, localAddress, proxy, port, user, password); - setSSLFactory(); - } - - - /** - * @return Returns the http instance. - */ - protected HttpClient getHttp() { - return this.http; - } - - private static String getServerKey(CrawlURI uri) { - try { - return CrawlServer.getServerKey(uri.getUURI()); - } catch (URIException e) { - logger.severe(e.getMessage() + ": " + uri); - e.printStackTrace(); - return null; - } - } -} diff --git a/modules/src/main/java/org/archive/modules/fetcher/SimpleCookieStorage.java b/modules/src/main/java/org/archive/modules/fetcher/SimpleCookieStorage.java deleted file mode 100644 index d815a375..00000000 --- a/modules/src/main/java/org/archive/modules/fetcher/SimpleCookieStorage.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * This file is part of the Heritrix web crawler (crawler.archive.org). - * - * Licensed to the Internet Archive (IA) by one or more individual - * contributors. - * - * The IA licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.archive.modules.fetcher; - -import java.util.Map; -import java.util.SortedMap; -import java.util.TreeMap; - -import org.apache.commons.httpclient.Cookie; - -public class SimpleCookieStorage extends AbstractCookieStorage { - - @SuppressWarnings("unused") - private static final long serialVersionUID = 1L; - - final private SortedMap map = new TreeMap(); - - - protected SortedMap prepareMap() { - return map; - } - - - public SortedMap getCookiesMap() { - return map; - } - - - public void innerSaveCookiesMap(Map map) { - // no-op - } -}