diff --git a/commons/src/main/java/org/archive/httpclient/HttpRecorderGetMethod.java b/commons/src/main/java/org/archive/httpclient/HttpRecorderGetMethod.java
deleted file mode 100644
index 105c4f7e..00000000
--- a/commons/src/main/java/org/archive/httpclient/HttpRecorderGetMethod.java
+++ /dev/null
@@ -1,120 +0,0 @@
-/*
- * This file is part of the Heritrix web crawler (crawler.archive.org).
- *
- * Licensed to the Internet Archive (IA) by one or more individual
- * contributors.
- *
- * The IA licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.archive.httpclient;
-
-import java.io.IOException;
-import java.util.logging.Logger;
-
-import org.apache.commons.httpclient.HttpConnection;
-import org.apache.commons.httpclient.HttpException;
-import org.apache.commons.httpclient.HttpState;
-import org.apache.commons.httpclient.methods.GetMethod;
-import org.archive.util.Recorder;
-
-
-/**
- * Override of GetMethod that marks the passed HttpRecorder w/ the transition
- * from HTTP head to body and that forces a close on the http connection.
- *
- * The actions done in this subclass used to be done by copying
- * org.apache.commons.HttpMethodBase, overlaying our version in place of the
- * one that came w/ httpclient. Here is the patch of the difference between
- * shipped httpclient code and our mods:
- *
- * -- -1338,6 +1346,12 --
- *
- * public void releaseConnection() {
- *
- * + // HERITRIX always ants the streams closed.
- * + if (responseConnection != null)
- * + {
- * + responseConnection.close();
- * + }
- * +
- * if (responseStream != null) {
- * try {
- * // FYI - this may indirectly invoke responseBodyConsumed.
- * -- -1959,6 +1973,11 --
- * this.statusLine = null;
- * }
- * }
- * + // HERITRIX mark transition from header to content.
- * + if (this.httpRecorder != null)
- * + {
- * + this.httpRecorder.markContentBegin();
- * + }
- * readResponseBody(state, conn);
- * processResponseBody(state, conn);
- * } catch (IOException e) {
- *
- *
- * We're not supposed to have access to the underlying connection object;
- * am only violating contract because see cases where httpclient is skipping
- * out w/o cleaning up after itself.
- *
- * @author stack
- * @version $Revision$, $Date$
- */
-public class HttpRecorderGetMethod extends GetMethod {
-
- protected static Logger logger =
- Logger.getLogger(HttpRecorderGetMethod.class.getName());
-
- /**
- * Instance of http recorder method.
- */
- protected HttpRecorderMethod httpRecorderMethod = null;
-
-
- public HttpRecorderGetMethod(String uri, Recorder recorder) {
- super(uri);
- this.httpRecorderMethod = new HttpRecorderMethod(recorder);
- }
-
- protected void readResponseBody(HttpState state, HttpConnection connection)
- throws IOException, HttpException {
- // We're about to read the body. Mark transition in http recorder.
- this.httpRecorderMethod.markContentBegin(connection);
- super.readResponseBody(state, connection);
- }
-
- protected boolean shouldCloseConnection(HttpConnection conn) {
- // Always close connection after each request. As best I can tell, this
- // is superfluous -- we've set our client to be HTTP/1.0. Doing this
- // out of paranoia.
- return true;
- }
-
- public int execute(HttpState state, HttpConnection conn)
- throws HttpException, IOException {
- // Save off the connection so we can close it on our way out in case
- // httpclient fails to (We're not supposed to have access to the
- // underlying connection object; am only violating contract because
- // see cases where httpclient is skipping out w/o cleaning up
- // after itself).
- this.httpRecorderMethod.setConnection(conn);
- return super.execute(state, conn);
- }
-
- protected void addProxyConnectionHeader(HttpState state, HttpConnection conn)
- throws IOException, HttpException {
- super.addProxyConnectionHeader(state, conn);
- this.httpRecorderMethod.handleAddProxyConnectionHeader(this);
- }
-}
diff --git a/commons/src/main/java/org/archive/httpclient/HttpRecorderMethod.java b/commons/src/main/java/org/archive/httpclient/HttpRecorderMethod.java
deleted file mode 100644
index 932e7e98..00000000
--- a/commons/src/main/java/org/archive/httpclient/HttpRecorderMethod.java
+++ /dev/null
@@ -1,107 +0,0 @@
-/*
- * This file is part of the Heritrix web crawler (crawler.archive.org).
- *
- * Licensed to the Internet Archive (IA) by one or more individual
- * contributors.
- *
- * The IA licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.archive.httpclient;
-
-import java.util.logging.Logger;
-
-import org.apache.commons.httpclient.Header;
-import org.apache.commons.httpclient.HttpConnection;
-import org.apache.commons.httpclient.HttpMethod;
-import org.archive.util.Recorder;
-
-
-/**
- * This class encapsulates the specializations supplied by the
- * overrides {@link HttpRecorderGetMethod} and {@link HttpRecorderPostMethod}.
- *
- * It keeps instance of HttpRecorder and HttpConnection.
- *
- * @author stack
- * @version $Revision$, $Date$
- */
-public class HttpRecorderMethod {
- protected static Logger logger =
- Logger.getLogger(HttpRecorderMethod.class.getName());
-
- /**
- * Instance of http recorder we're using recording this http get.
- */
- private Recorder httpRecorder = null;
-
- /**
- * Save around so can force close.
- *
- * See [ 922080 ] IllegalArgumentException (size is wrong).
- * https://sourceforge.net/tracker/?func=detail&aid=922080&group_id=73833&atid=539099
- */
- private HttpConnection connection = null;
-
-
- public HttpRecorderMethod(Recorder recorder) {
- this.httpRecorder = recorder;
- }
-
- public void markContentBegin(HttpConnection c) {
- if (c != this.connection) {
- // We're checking that we're not being asked to work on
- // a connection that is other than the one we started
- // this method#execute with.
- throw new IllegalArgumentException("Connections differ: " +
- this.connection + " " + c + " " +
- Thread.currentThread().getName());
- }
- this.httpRecorder.markContentBegin();
- }
-
- /**
- * @return Returns the connection.
- */
- public HttpConnection getConnection() {
- return this.connection;
- }
-
- /**
- * @param connection The connection to set.
- */
- public void setConnection(HttpConnection connection) {
- this.connection = connection;
- }
- /**
- * @return Returns the httpRecorder.
- */
- public Recorder getHttpRecorder() {
- return httpRecorder;
- }
-
- /**
- * If a 'Proxy-Connection' header has been added to the request,
- * it'll be of a 'keep-alive' type. Until we support 'keep-alives',
- * override the Proxy-Connection setting and instead pass a 'close'
- * (Otherwise every request has to timeout before we notice
- * end-of-document).
- * @param method Method to find proxy-connection header in.
- */
- public void handleAddProxyConnectionHeader(HttpMethod method) {
- Header h = method.getRequestHeader("Proxy-Connection");
- if (h != null) {
- h.setValue("close");
- method.setRequestHeader(h);
- }
- }
-}
diff --git a/commons/src/main/java/org/archive/httpclient/HttpRecorderPostMethod.java b/commons/src/main/java/org/archive/httpclient/HttpRecorderPostMethod.java
deleted file mode 100644
index 20f1bfd1..00000000
--- a/commons/src/main/java/org/archive/httpclient/HttpRecorderPostMethod.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * This file is part of the Heritrix web crawler (crawler.archive.org).
- *
- * Licensed to the Internet Archive (IA) by one or more individual
- * contributors.
- *
- * The IA licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.archive.httpclient;
-
-import java.io.IOException;
-
-import org.apache.commons.httpclient.HttpConnection;
-import org.apache.commons.httpclient.HttpException;
-import org.apache.commons.httpclient.HttpState;
-import org.apache.commons.httpclient.methods.PostMethod;
-import org.archive.util.Recorder;
-
-
-/**
- * Override of PostMethod that marks the passed HttpRecorder w/ the transition
- * from HTTP head to body and that forces a close on the responseConnection.
- *
- * This is a copy of {@link HttpRecorderGetMethod}. Only difference is the
- * parent subclass.
- *
- * @author stack
- * @version $Date$ $Revision$
- */
-public class HttpRecorderPostMethod extends PostMethod {
- /**
- * Instance of http recorder method.
- */
- protected HttpRecorderMethod httpRecorderMethod = null;
-
-
- public HttpRecorderPostMethod(String uri, Recorder recorder) {
- super(uri);
- this.httpRecorderMethod = new HttpRecorderMethod(recorder);
- }
-
- protected void readResponseBody(HttpState state, HttpConnection connection)
- throws IOException, HttpException {
- // We're about to read the body. Mark transition in http recorder.
- this.httpRecorderMethod.markContentBegin(connection);
- super.readResponseBody(state, connection);
- }
-
- protected boolean shouldCloseConnection(HttpConnection conn) {
- // Always close connection after each request. As best I can tell, this
- // is superfluous -- we've set our client to be HTTP/1.0. Doing this
- // out of paranoia.
- return true;
- }
-
- public int execute(HttpState state, HttpConnection conn)
- throws HttpException, IOException {
- // Save off the connection so we can close it on our way out in case
- // httpclient fails to (We're not supposed to have access to the
- // underlying connection object; am only violating contract because
- // see cases where httpclient is skipping out w/o cleaning up
- // after itself).
- this.httpRecorderMethod.setConnection(conn);
- return super.execute(state, conn);
- }
-
- protected void addProxyConnectionHeader(HttpState state, HttpConnection conn)
- throws IOException, HttpException {
- super.addProxyConnectionHeader(state, conn);
- this.httpRecorderMethod.handleAddProxyConnectionHeader(this);
- }
-}
diff --git a/commons/src/main/java/org/archive/httpclient/SingleHttpConnectionManager.java b/commons/src/main/java/org/archive/httpclient/SingleHttpConnectionManager.java
deleted file mode 100644
index 4ba6a837..00000000
--- a/commons/src/main/java/org/archive/httpclient/SingleHttpConnectionManager.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * This file is part of the Heritrix web crawler (crawler.archive.org).
- *
- * Licensed to the Internet Archive (IA) by one or more individual
- * contributors.
- *
- * The IA licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.archive.httpclient;
-
-import java.io.IOException;
-import java.io.InputStream;
-
-import org.apache.commons.httpclient.HostConfiguration;
-import org.apache.commons.httpclient.HttpConnection;
-import org.apache.commons.httpclient.SimpleHttpConnectionManager;
-
-/**
- * An HttpClient-compatible HttpConnection "manager" that actually
- * just gives out a new connection each time -- skipping the overhead
- * of connection management, since we already throttle our crawler
- * with external mechanisms.
- *
- * @author gojomo
- */
-public class SingleHttpConnectionManager extends SimpleHttpConnectionManager {
-
- public SingleHttpConnectionManager() {
- super();
- }
-
- public HttpConnection getConnectionWithTimeout(
- HostConfiguration hostConfiguration, long timeout) {
-
- HttpConnection conn = new HttpConnection(hostConfiguration);
- conn.setHttpConnectionManager(this);
- conn.getParams().setDefaults(this.getParams());
- return conn;
- }
-
- public void releaseConnection(HttpConnection conn) {
- // ensure connection is closed
- conn.close();
- finishLast(conn);
- }
-
- protected static void finishLast(HttpConnection conn) {
- // copied from superclass because it wasn't made available to subclasses
- InputStream lastResponse = conn.getLastResponseInputStream();
- if (lastResponse != null) {
- conn.setLastResponseInputStream(null);
- try {
- lastResponse.close();
- } catch (IOException ioe) {
- //FIXME: badness - close to force reconnect.
- conn.close();
- }
- }
- }
-}
diff --git a/commons/src/main/java/org/archive/httpclient/ThreadLocalHttpConnectionManager.java b/commons/src/main/java/org/archive/httpclient/ThreadLocalHttpConnectionManager.java
deleted file mode 100644
index 91e850ea..00000000
--- a/commons/src/main/java/org/archive/httpclient/ThreadLocalHttpConnectionManager.java
+++ /dev/null
@@ -1,291 +0,0 @@
-/**
- * ====================================================================
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- * ====================================================================
- *
- */
-package org.archive.httpclient;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-import org.apache.commons.httpclient.HostConfiguration;
-import org.apache.commons.httpclient.HttpConnection;
-import org.apache.commons.httpclient.HttpConnectionManager;
-import org.apache.commons.httpclient.params.HttpConnectionManagerParams;
-
-/**
- * A simple, but thread-safe HttpClient {@link HttpConnectionManager}.
- * Based on {@link org.apache.commons.httpclient.SimpleHttpConnectionManager}.
- *
- * Java >= 1.4 is recommended.
- *
- * @author Christian Kohlschuetter
- */
-public final class ThreadLocalHttpConnectionManager implements
- HttpConnectionManager {
-
- private static final CloserThread closer = new CloserThread();
- private static final Logger logger = Logger
- .getLogger(ThreadLocalHttpConnectionManager.class.getName());
-
- private final ThreadLocal tl = new ThreadLocal() {
- protected synchronized ConnectionInfo initialValue() {
- return new ConnectionInfo();
- }
- };
-
- private ConnectionInfo getConnectionInfo() {
- return (ConnectionInfo) tl.get();
- }
-
- private static final class ConnectionInfo {
- /** The http connection */
- private HttpConnection conn = null;
-
- /**
- * The time the connection was made idle.
- */
- private long idleStartTime = Long.MAX_VALUE;
- }
-
- public ThreadLocalHttpConnectionManager() {
- }
-
- /**
- * Since the same connection is about to be reused, make sure the
- * previous request was completely processed, and if not
- * consume it now.
- * @param conn The connection
- * @return true, if the connection is reusable
- */
- private static boolean finishLastResponse(final HttpConnection conn) {
- InputStream lastResponse = conn.getLastResponseInputStream();
- if(lastResponse != null) {
- conn.setLastResponseInputStream(null);
- try {
- lastResponse.close();
- return true;
- } catch (IOException ioe) {
- // force reconnect.
- return false;
- }
- } else {
- return false;
- }
- }
-
- /**
- * Collection of parameters associated with this connection manager.
- */
- private HttpConnectionManagerParams params = new HttpConnectionManagerParams();
-
- /**
- * @see HttpConnectionManager#getConnection(HostConfiguration)
- */
- public HttpConnection getConnection(
- final HostConfiguration hostConfiguration) {
- return getConnection(hostConfiguration, 0);
- }
-
- /**
- * Gets the staleCheckingEnabled value to be set on HttpConnections that are created.
- *
- * @return true if stale checking will be enabled on HttpConections
- *
- * @see HttpConnection#isStaleCheckingEnabled()
- *
- * @deprecated Use {@link HttpConnectionManagerParams#isStaleCheckingEnabled()},
- * {@link HttpConnectionManager#getParams()}.
- */
- public boolean isConnectionStaleCheckingEnabled() {
- return this.params.isStaleCheckingEnabled();
- }
-
- /**
- * Sets the staleCheckingEnabled value to be set on HttpConnections that are created.
- *
- * @param connectionStaleCheckingEnabled true if stale checking will be enabled
- * on HttpConections
- *
- * @see HttpConnection#setStaleCheckingEnabled(boolean)
- *
- * @deprecated Use {@link HttpConnectionManagerParams#setStaleCheckingEnabled(boolean)},
- * {@link HttpConnectionManager#getParams()}.
- */
- public void setConnectionStaleCheckingEnabled(
- final boolean connectionStaleCheckingEnabled) {
- this.params.setStaleCheckingEnabled(connectionStaleCheckingEnabled);
- }
-
- /**
- * @see HttpConnectionManager#getConnectionWithTimeout(HostConfiguration, long)
- *
- * @since 3.0
- */
- public HttpConnection getConnectionWithTimeout(
- final HostConfiguration hostConfiguration, final long timeout) {
-
- final ConnectionInfo ci = getConnectionInfo();
- HttpConnection httpConnection = ci.conn;
-
- // make sure the host and proxy are correct for this connection
- // close it and set the values if they are not
- if(httpConnection == null || !finishLastResponse(httpConnection)
- || !hostConfiguration.hostEquals(httpConnection)
- || !hostConfiguration.proxyEquals(httpConnection)) {
-
- if(httpConnection != null && httpConnection.isOpen()) {
- closer.closeConnection(httpConnection);
- }
-
- httpConnection = new HttpConnection(hostConfiguration);
- httpConnection.setHttpConnectionManager(this);
- httpConnection.getParams().setDefaults(this.params);
- ci.conn = httpConnection;
-
- httpConnection.setHost(hostConfiguration.getHost());
- httpConnection.setPort(hostConfiguration.getPort());
- httpConnection.setProtocol(hostConfiguration.getProtocol());
- httpConnection.setLocalAddress(hostConfiguration.getLocalAddress());
-
- httpConnection.setProxyHost(hostConfiguration.getProxyHost());
- httpConnection.setProxyPort(hostConfiguration.getProxyPort());
- }
-
- // remove the connection from the timeout handler
- ci.idleStartTime = Long.MAX_VALUE;
-
- return httpConnection;
- }
-
- /**
- * @see HttpConnectionManager#getConnection(HostConfiguration, long)
- *
- * @deprecated Use #getConnectionWithTimeout(HostConfiguration, long)
- */
- public HttpConnection getConnection(
- final HostConfiguration hostConfiguration, final long timeout) {
- return getConnectionWithTimeout(hostConfiguration, timeout);
- }
-
- /**
- * @see HttpConnectionManager#releaseConnection(org.apache.commons.httpclient.HttpConnection)
- */
- public void releaseConnection(final HttpConnection conn) {
- final ConnectionInfo ci = getConnectionInfo();
- HttpConnection httpConnection = ci.conn;
-
- if(conn != httpConnection) {
- throw new IllegalStateException(
- "Unexpected release of an unknown connection.");
- }
-
- finishLastResponse(httpConnection);
-
- // track the time the connection was made idle
- ci.idleStartTime = System.currentTimeMillis();
- }
-
- /**
- * Returns {@link HttpConnectionManagerParams parameters} associated
- * with this connection manager.
- *
- * @since 2.1
- *
- * @see HttpConnectionManagerParams
- */
- public HttpConnectionManagerParams getParams() {
- return this.params;
- }
-
- /**
- * Assigns {@link HttpConnectionManagerParams parameters} for this
- * connection manager.
- *
- * @since 2.1
- *
- * @see HttpConnectionManagerParams
- */
- public void setParams(final HttpConnectionManagerParams p) {
- if(p == null) {
- throw new IllegalArgumentException("Parameters may not be null");
- }
- this.params = p;
- }
-
- /**
- * @since 3.0
- */
- public void closeIdleConnections(final long idleTimeout) {
- long maxIdleTime = System.currentTimeMillis() - idleTimeout;
-
- final ConnectionInfo ci = getConnectionInfo();
-
- if(ci.idleStartTime <= maxIdleTime) {
- ci.conn.close();
- }
- }
-
- private static final class CloserThread extends Thread {
- private List connections
- = new ArrayList();
-
- private static final int SLEEP_INTERVAL = 5000;
-
- public CloserThread() {
- super("HttpConnection closer");
- // Make this a daemon thread so it can't be responsible for the JVM
- // not shutting down.
- setDaemon(true);
- start();
- }
-
- public void closeConnection(final HttpConnection conn) {
- synchronized (connections) {
- connections.add(conn);
- }
- }
-
- public void run() {
- try {
- while (!Thread.interrupted()) {
- Thread.sleep(SLEEP_INTERVAL);
-
- List s;
- synchronized (connections) {
- s = connections;
- connections = new ArrayList();
- }
- logger.log(Level.INFO, "Closing " + s.size()
- + " HttpConnections");
- for(final Iterator it = s.iterator();
- it.hasNext();) {
- HttpConnection conn = it.next();
- conn.close();
- conn.setHttpConnectionManager(null);
- it.remove();
- }
- }
- } catch (InterruptedException e) {
- return;
- }
- }
- }
-}
diff --git a/engine/src/test/java/org/archive/modules/fetcher/FetchHTTPTest.java b/engine/src/test/java/org/archive/modules/fetcher/FetchHTTPTest.java
deleted file mode 100644
index e61b2b82..00000000
--- a/engine/src/test/java/org/archive/modules/fetcher/FetchHTTPTest.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * This file is part of the Heritrix web crawler (crawler.archive.org).
- *
- * Licensed to the Internet Archive (IA) by one or more individual
- * contributors.
- *
- * The IA licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.archive.modules.fetcher;
-
-
-import java.io.IOException;
-
-import org.archive.modules.CrawlMetadata;
-
-
-/**
- *
- */
-public class FetchHTTPTest extends FetchHTTPTestBase {
-
- @Override
- protected AbstractFetchHTTP makeModule() throws IOException {
- LegacyFetchHTTP fetchHttp = new LegacyFetchHTTP();
- fetchHttp.setCookieStorage(new SimpleCookieStorage());
- fetchHttp.setServerCache(new DefaultServerCache());
- CrawlMetadata uap = new CrawlMetadata();
- uap.setUserAgentTemplate(getUserAgentString());
- fetchHttp.setUserAgentProvider(uap);
-
- fetchHttp.start();
- return fetchHttp;
- }
-
- @Override
- public void testHttpProxyAuth() throws Exception {
- // XXX skip cuz it's slow in FetchHTTP for some reason
- }
-
- @Override
- public void testConnectionTimeout() throws Exception {
- // XXX skip cuz it's slow cuz you can't change the connection timeout after FetchHTTP.start() has run
- }
-}
diff --git a/modules/src/main/java/org/archive/modules/credential/CommonsHttpCredentialUtil.java b/modules/src/main/java/org/archive/modules/credential/CommonsHttpCredentialUtil.java
deleted file mode 100644
index 58a41bec..00000000
--- a/modules/src/main/java/org/archive/modules/credential/CommonsHttpCredentialUtil.java
+++ /dev/null
@@ -1,147 +0,0 @@
-/*
- * This file is part of the Heritrix web crawler (crawler.archive.org).
- *
- * Licensed to the Internet Archive (IA) by one or more individual
- * contributors.
- *
- * The IA licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.archive.modules.credential;
-
-import java.util.Iterator;
-import java.util.Map;
-import java.util.logging.Logger;
-
-import org.apache.commons.httpclient.HttpClient;
-import org.apache.commons.httpclient.HttpMethod;
-import org.apache.commons.httpclient.HttpMethodBase;
-import org.apache.commons.httpclient.NameValuePair;
-import org.apache.commons.httpclient.URIException;
-import org.apache.commons.httpclient.UsernamePasswordCredentials;
-import org.apache.commons.httpclient.auth.AuthChallengeProcessor;
-import org.apache.commons.httpclient.auth.AuthScheme;
-import org.apache.commons.httpclient.auth.AuthScope;
-import org.apache.commons.httpclient.auth.AuthenticationException;
-import org.apache.commons.httpclient.auth.MalformedChallengeException;
-import org.apache.commons.httpclient.methods.GetMethod;
-import org.apache.commons.httpclient.methods.PostMethod;
-import org.apache.commons.lang.StringUtils;
-import org.archive.modules.CrawlURI;
-
-public class CommonsHttpCredentialUtil {
-
- private static Logger logger = Logger.getLogger(CommonsHttpCredentialUtil.class.getName());
-
- public static boolean populate(CrawlURI curi, HttpClient http,
- HttpMethod method, Credential cred, Map httpAuthChallenges) {
- if (cred instanceof HttpAuthenticationCredential) {
- return populate(curi, http, method, (HttpAuthenticationCredential) cred, httpAuthChallenges);
- } else if (cred instanceof HtmlFormCredential) {
- return populate(curi, http, method, (HtmlFormCredential) cred);
- } else {
- throw new RuntimeException("not implemented for Credential subtype " + cred.getClass());
- }
- }
-
- public static boolean populate(CrawlURI curi, HttpClient http,
- HttpMethod method, HtmlFormCredential cred) {
- // http is not used
- boolean result = false;
- Map formItems = cred.getFormItems();
- if (formItems == null || formItems.size() <= 0) {
- try {
- logger.severe("No form items for " + method.getURI());
- }
- catch (URIException e) {
- logger.severe("No form items and exception getting uri: " +
- e.getMessage());
- }
- return result;
- }
-
- NameValuePair[] data = new NameValuePair[formItems.size()];
- int index = 0;
- String key = null;
- for (Iterator i = formItems.keySet().iterator(); i.hasNext();) {
- key = i.next();
- data[index++] = new NameValuePair(key, (String)formItems.get(key));
- }
- if (method instanceof PostMethod) {
- ((PostMethod)method).setRequestBody(data);
- result = true;
- } else if (method instanceof GetMethod) {
- // Append these values to the query string.
- // Get current query string, then add data, then get it again
- // only this time its our data only... then append.
- HttpMethodBase hmb = (HttpMethodBase)method;
- String currentQuery = hmb.getQueryString();
- hmb.setQueryString(data);
- String newQuery = hmb.getQueryString();
- hmb.setQueryString(
- ((StringUtils.isNotEmpty(currentQuery))
- ? currentQuery + "&"
- : "")
- + newQuery);
- result = true;
- } else {
- logger.severe("Unknown method type: " + method);
- }
- return result;
- }
-
- public static boolean populate(CrawlURI curi, HttpClient http,
- HttpMethod method, HttpAuthenticationCredential cred, Map httpAuthChallenges) {
- boolean result = false;
-
- AuthChallengeProcessor authChallengeProcessor = new AuthChallengeProcessor(http.getParams());
- try {
- AuthScheme authScheme = authChallengeProcessor.processChallenge(method.getHostAuthState(), httpAuthChallenges);
- method.getHostAuthState().setAuthScheme(authScheme);
- } catch (MalformedChallengeException e) {
- return result;
- } catch (AuthenticationException e) {
- return result;
- }
-
- // Always add the credential to HttpState. Doing this because no way of
- // removing the credential once added AND there is a bug in the
- // credentials management system in that it always sets URI root to
- // null: it means the key used to find a credential is NOT realm + root
- // URI but just the realm. Unless I set it everytime, there is
- // possibility that as this thread progresses, it might come across a
- // realm already loaded but the login and password are from another
- // server. We'll get a failed authentication that'd be difficult to
- // explain.
- //
- // Have to make a UsernamePasswordCredentials. The httpclient auth code
- // does an instanceof down in its guts.
- UsernamePasswordCredentials upc = null;
- try {
- upc = new UsernamePasswordCredentials(cred.getLogin(),
- cred.getPassword());
- http.getState().setCredentials(new AuthScope(curi.getUURI().getHost(),
- curi.getUURI().getPort(), cred.getRealm()), upc);
- logger.fine("Credentials for realm " + cred.getRealm() +
- " for CrawlURI " + curi.toString() + " added to request");
-
- result = true;
- } catch (URIException e) {
- logger.severe("Failed to parse host from " + curi + ": " +
- e.getMessage());
- }
-
- return result;
- }
-
-}
diff --git a/modules/src/main/java/org/archive/modules/fetcher/AbstractCookieStorage.java b/modules/src/main/java/org/archive/modules/fetcher/AbstractCookieStorage.java
deleted file mode 100644
index ace25cb3..00000000
--- a/modules/src/main/java/org/archive/modules/fetcher/AbstractCookieStorage.java
+++ /dev/null
@@ -1,236 +0,0 @@
-/*
- * This file is part of the Heritrix web crawler (crawler.archive.org).
- *
- * Licensed to the Internet Archive (IA) by one or more individual
- * contributors.
- *
- * The IA licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.archive.modules.fetcher;
-
-import it.unimi.dsi.mg4j.util.MutableString;
-
-import java.io.BufferedReader;
-import java.io.Closeable;
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
-import java.io.FileReader;
-import java.io.IOException;
-import java.io.Reader;
-import java.util.Date;
-import java.util.Map;
-import java.util.SortedMap;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-import org.apache.commons.httpclient.Cookie;
-import org.apache.commons.io.IOUtils;
-import org.archive.spring.ConfigFile;
-import org.archive.spring.ConfigPath;
-import org.springframework.context.Lifecycle;
-
-/**
- * @author pjack
- *
- */
-public abstract class AbstractCookieStorage
- implements CookieStorage,
- Lifecycle, // InitializingBean,
- Closeable {
-
- final private static Logger LOGGER =
- Logger.getLogger(AbstractCookieStorage.class.getName());
-
- protected ConfigFile cookiesLoadFile = null;
- public ConfigFile getCookiesLoadFile() {
- return cookiesLoadFile;
- }
- public void setCookiesLoadFile(ConfigFile cookiesLoadFile) {
- this.cookiesLoadFile = cookiesLoadFile;
- }
-
-
- protected ConfigPath cookiesSaveFile = null;
- public ConfigPath getCookiesSaveFile() {
- return cookiesSaveFile;
- }
- public void setCookiesSaveFile(ConfigPath cookiesSaveFile) {
- this.cookiesSaveFile = cookiesSaveFile;
- }
-
- protected boolean isRunning = false;
- public void start() {
- if(isRunning()) {
- return;
- }
- SortedMap cookies = prepareMap();
- if (getCookiesLoadFile()!=null) {
- loadCookies(getCookiesLoadFile(), cookies);
- }
- isRunning = true;
- }
-
- public boolean isRunning() {
- return isRunning;
- }
-
- public void stop() {
- isRunning = false;
- }
-
- protected abstract SortedMap prepareMap();
-
- /**
- * Load cookies. The input is text in the Netscape's 'cookies.txt' file
- * format. Example entry of cookies.txt file:
- *
- * www.archive.org FALSE / FALSE 1311699995 details-visit texts-cralond
- *
- *
- * Each line has 7 tab-separated fields:
- *
- *
- * - DOMAIN: The domain that created and have access to the cookie value.
- * - FLAG: A TRUE or FALSE value indicating if hosts within the given
- * domain can access the cookie value.
- * - PATH: The path within the domain that the cookie value is valid for.
- * - SECURE: A TRUE or FALSE value indicating if to use a secure
- * connection to access the cookie value.
- * - EXPIRATION: The expiration time of the cookie value, or -1 for no
- * expiration
- * - NAME: The name of the cookie value
- * - VALUE: The cookie value
- *
- *
- * @param reader
- * input
- * @param cookiesFile
- * file in the Netscape's 'cookies.txt' format.
- */
- public static void loadCookies(Reader reader,
- SortedMap cookies) {
- BufferedReader br = new BufferedReader(reader);
- try {
- String line;
- int lineNo = 1;
- while ((line = br.readLine()) != null) {
- if (!line.matches("\\s*(?:#.*)?")) { // skip blank links and comments
- String[] tokens = line.split("\\t");
- if (tokens.length == 7) {
- long epochSeconds = Long.parseLong(tokens[4]);
- Date expirationDate = (epochSeconds >= 0 ? new Date(epochSeconds * 1000) : null);
- Cookie cookie = new Cookie(tokens[0], tokens[5],
- tokens[6], tokens[2], expirationDate,
- Boolean.valueOf(tokens[3]).booleanValue());
- cookie.setDomainAttributeSpecified(Boolean.valueOf(tokens[1]).booleanValue());
-
- LOGGER.fine("Adding cookie: domain " + cookie.getDomain() + " cookie " + cookie.toExternalForm());
- cookies.put(cookie.getSortKey(), cookie);
- } else {
- LOGGER.warning("cookies input line " + lineNo + " invalid, expected 7 tab-delimited tokens");
- }
- }
-
- lineNo++;
- }
- } catch (IOException e) {
- LOGGER.log(Level.WARNING,e.getMessage(), e);
- }
- }
-
- protected static void loadCookies(ConfigFile file,
- SortedMap cookies) {
-
- Reader reader = null;
- try {
- reader = file.obtainReader();
- loadCookies(reader, cookies);
- } finally {
- IOUtils.closeQuietly(reader);
- }
- }
-
- public static void loadCookies(String cookiesFile,
- SortedMap result) {
-
- // Do nothing if cookiesFile is not specified.
- if (cookiesFile == null || cookiesFile.length() <= 0) {
- return;
- }
-
- FileReader reader = null;
- try {
- reader = new FileReader(cookiesFile);
- loadCookies(reader, result);
- } catch (FileNotFoundException e) {
- LOGGER.log(Level.WARNING,"Could not find file: " + cookiesFile, e);
- } finally {
- IOUtils.closeQuietly(reader);
- }
- }
-
- public static void saveCookies(String saveCookiesFile, Map cookies) {
- // Do nothing if cookiesFile is not specified.
- if (saveCookiesFile == null || saveCookiesFile.length() <= 0) {
- return;
- }
-
- FileOutputStream out = null;
- try {
- out = new FileOutputStream(new File(saveCookiesFile));
- String tab ="\t";
- out.write("# Heritrix Cookie File\n".getBytes());
- out.write("# This file is the Netscape cookies.txt format\n\n".getBytes());
- for (Cookie cookie: cookies.values()) {
- // Guess an initial size
- MutableString line = new MutableString(1024 * 2);
- line.append(cookie.getDomain());
- line.append(tab);
- line.append(cookie.isDomainAttributeSpecified() ? "TRUE" : "FALSE");
- line.append(tab);
- line.append(cookie.getPath());
- line.append(tab);
- line.append(cookie.getSecure() ? "TRUE" : "FALSE");
- line.append(tab);
- line.append(cookie.getExpiryDate() != null ? cookie.getExpiryDate().getTime() / 1000 : -1);
- line.append(tab);
- line.append(cookie.getName());
- line.append(tab);
- line.append(cookie.getValue() != null ? cookie.getValue() : "");
- line.append("\n");
- out.write(line.toString().getBytes());
- }
- } catch (IOException e) {
- LOGGER.log(Level.SEVERE, "Unable to write " + saveCookiesFile, e);
- } finally {
- IOUtils.closeQuietly(out);
- }
- }
-
- public abstract SortedMap getCookiesMap();
-
- public void saveCookiesMap(Map map) {
- innerSaveCookiesMap(map);
- if (getCookiesSaveFile()!=null) {
- saveCookies(getCookiesSaveFile().getFile().getAbsolutePath(), map);
- }
- }
-
- protected abstract void innerSaveCookiesMap(Map map);
-
- public void close() throws IOException {
- }
-
-}
diff --git a/modules/src/main/java/org/archive/modules/fetcher/AbstractCookieStore.java b/modules/src/main/java/org/archive/modules/fetcher/AbstractCookieStore.java
index f1b4f0d2..10354aa9 100644
--- a/modules/src/main/java/org/archive/modules/fetcher/AbstractCookieStore.java
+++ b/modules/src/main/java/org/archive/modules/fetcher/AbstractCookieStore.java
@@ -44,7 +44,7 @@ public abstract class AbstractCookieStore implements CookieStore, Lifecycle, Clo
Checkpointable {
private static final Logger logger =
- Logger.getLogger(AbstractCookieStorage.class.getName());
+ Logger.getLogger(AbstractCookieStore.class.getName());
protected ConfigFile cookiesLoadFile = null;
public ConfigFile getCookiesLoadFile() {
diff --git a/modules/src/main/java/org/archive/modules/fetcher/BdbCookieStorage.java b/modules/src/main/java/org/archive/modules/fetcher/BdbCookieStorage.java
deleted file mode 100644
index da078b2c..00000000
--- a/modules/src/main/java/org/archive/modules/fetcher/BdbCookieStorage.java
+++ /dev/null
@@ -1,116 +0,0 @@
-/*
- * This file is part of the Heritrix web crawler (crawler.archive.org).
- *
- * Licensed to the Internet Archive (IA) by one or more individual
- * contributors.
- *
- * The IA licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.archive.modules.fetcher;
-
-import java.io.IOException;
-import java.util.Map;
-import java.util.SortedMap;
-
-import org.apache.commons.httpclient.Cookie;
-import org.archive.bdb.BdbModule;
-import org.archive.checkpointing.Checkpoint;
-import org.archive.checkpointing.Checkpointable;
-import org.springframework.beans.factory.annotation.Autowired;
-
-import com.sleepycat.bind.serial.SerialBinding;
-import com.sleepycat.bind.serial.StoredClassCatalog;
-import com.sleepycat.bind.tuple.StringBinding;
-import com.sleepycat.collections.StoredSortedMap;
-import com.sleepycat.je.Database;
-import com.sleepycat.je.DatabaseException;
-
-/**
- * CookieStorage using BDB, so that cookies accumulated in large crawls
- * do not outgrow RAM.
- *
- * @author pjack
- */
-public class BdbCookieStorage extends AbstractCookieStorage implements Checkpointable {
- @SuppressWarnings("unused")
- private static final long serialVersionUID = 1L;
-
- protected BdbModule bdb;
- @Autowired
- public void setBdbModule(BdbModule bdb) {
- this.bdb = bdb;
- }
-
- /** are we a checkpoint recovery? (in which case, reuse stored cookie data?) */
- protected boolean isCheckpointRecovery = false;
-
- public static String COOKIEDB_NAME = "http_cookies";
-
- private transient Database cookieDb;
- private transient StoredSortedMap cookies;
-
- public BdbCookieStorage() {
- }
-
- protected SortedMap prepareMap() {
- try {
- StoredClassCatalog classCatalog = bdb.getClassCatalog();
- BdbModule.BdbConfig dbConfig = new BdbModule.BdbConfig();
- dbConfig.setTransactional(false);
- dbConfig.setAllowCreate(true);
- cookieDb = bdb.openDatabase(COOKIEDB_NAME, dbConfig, isCheckpointRecovery);
- cookies =
- new StoredSortedMap(
- cookieDb,
- new StringBinding(),
- new SerialBinding(classCatalog,Cookie.class),
- true);
- return cookies;
- } catch (DatabaseException e) {
- throw new RuntimeException(e);
- }
- }
-
- public SortedMap getCookiesMap() {
-// assert cookies != null : "cookie map not set up";
- return cookies;
- }
-
- protected void innerSaveCookiesMap(Map map) {
- }
-
-
-
- @Override
- public void startCheckpoint(Checkpoint checkpointInProgress) {
- // do nothing; handled by map checkpoint via BdbModule
- }
- @Override
- public void doCheckpoint(Checkpoint checkpointInProgress)
- throws IOException {
- // do nothing; handled by map checkpoint via BdbModule
- }
- @Override
- public void finishCheckpoint(Checkpoint checkpointInProgress) {
- // do nothing; handled by map checkpoint via BdbModule
- }
-
- @Override
- public void setRecoveryCheckpoint(Checkpoint recoveryCheckpoint) {
- // just remember that we are doing checkpoint-recovery;
- // actual state recovery happens via BdbModule
- isCheckpointRecovery = true;
- }
-
-}
diff --git a/modules/src/main/java/org/archive/modules/fetcher/CookieStorage.java b/modules/src/main/java/org/archive/modules/fetcher/CookieStorage.java
deleted file mode 100644
index 3442eb51..00000000
--- a/modules/src/main/java/org/archive/modules/fetcher/CookieStorage.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * This file is part of the Heritrix web crawler (crawler.archive.org).
- *
- * Licensed to the Internet Archive (IA) by one or more individual
- * contributors.
- *
- * The IA licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.archive.modules.fetcher;
-
-import java.util.Map;
-import java.util.SortedMap;
-
-import org.apache.commons.httpclient.Cookie;
-import org.springframework.context.Lifecycle;
-
-public interface CookieStorage extends Lifecycle {
-
- SortedMap getCookiesMap();
-
- void saveCookiesMap(Map map);
-
-}
diff --git a/modules/src/main/java/org/archive/modules/fetcher/HeritrixHttpMethodRetryHandler.java b/modules/src/main/java/org/archive/modules/fetcher/HeritrixHttpMethodRetryHandler.java
deleted file mode 100644
index 2be4af14..00000000
--- a/modules/src/main/java/org/archive/modules/fetcher/HeritrixHttpMethodRetryHandler.java
+++ /dev/null
@@ -1,84 +0,0 @@
-/*
- * This file is part of the Heritrix web crawler (crawler.archive.org).
- *
- * Licensed to the Internet Archive (IA) by one or more individual
- * contributors.
- *
- * The IA licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.archive.modules.fetcher;
-
-import java.io.IOException;
-import java.net.SocketTimeoutException;
-
-import org.apache.commons.httpclient.HttpMethod;
-import org.apache.commons.httpclient.HttpMethodRetryHandler;
-import org.apache.commons.httpclient.NoHttpResponseException;
-import org.apache.commons.httpclient.methods.PostMethod;
-
-/**
- * Retry handler that tries ten times to establish connection and then once
- * established, if a GET method, tries ten times to get response (If POST,
- * it tries once only).
- *
- * Its unsafe retrying POSTs. See 'Rule of Thumb' under 'Method Recovery'
- * here:
- * HttpClient Tutorial.
- *
- * @author stack
- * @version $Date$, $Revision$
- */
-public class HeritrixHttpMethodRetryHandler implements HttpMethodRetryHandler {
- private static final int DEFAULT_RETRY_COUNT = 10;
-
- private final int maxRetryCount;
-
- /**
- * Constructor.
- */
- public HeritrixHttpMethodRetryHandler() {
- this(DEFAULT_RETRY_COUNT);
- }
-
- /**
- * Constructor.
- * @param maxRetryCount Maximum amount of times to retry.
- */
- public HeritrixHttpMethodRetryHandler(int maxRetryCount) {
- this.maxRetryCount = maxRetryCount;
- }
-
- public boolean retryMethod(HttpMethod method, IOException exception,
- int executionCount) {
- if(exception instanceof SocketTimeoutException) {
- // already waited for the configured amount of time with no reply;
- // do not retry further until next go round
- return false;
- }
- if (executionCount >= this.maxRetryCount) {
- // Do not retry if over max retry count
- return false;
- }
- if (exception instanceof NoHttpResponseException) {
- // Retry if the server dropped connection on us
- return true;
- }
- if (!method.isRequestSent() && (!(method instanceof PostMethod))) {
- // Retry if the request has not been sent fully or
- // if it's OK to retry methods that have been sent
- return true;
- }
- // otherwise do not retry
- return false;
- }
-}
diff --git a/modules/src/main/java/org/archive/modules/fetcher/HeritrixProtocolSocketFactory.java b/modules/src/main/java/org/archive/modules/fetcher/HeritrixProtocolSocketFactory.java
deleted file mode 100644
index ae4df087..00000000
--- a/modules/src/main/java/org/archive/modules/fetcher/HeritrixProtocolSocketFactory.java
+++ /dev/null
@@ -1,196 +0,0 @@
-/*
- * This file is part of the Heritrix web crawler (crawler.archive.org).
- *
- * Licensed to the Internet Archive (IA) by one or more individual
- * contributors.
- *
- * The IA licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.archive.modules.fetcher;
-
-import java.io.IOException;
-import java.net.InetAddress;
-import java.net.InetSocketAddress;
-import java.net.Socket;
-import java.net.SocketTimeoutException;
-import java.net.UnknownHostException;
-
-import org.apache.commons.httpclient.ConnectTimeoutException;
-import org.apache.commons.httpclient.params.HttpConnectionParams;
-import org.apache.commons.httpclient.protocol.ProtocolSocketFactory;
-
-
-/**
- * Version of protocol socket factory that tries to get IP from heritrix IP
- * cache -- if its been set into the HttpConnectionParameters.
- *
- * Copied the guts of DefaultProtocolSocketFactory. This factory gets
- * setup by {@link LegacyFetchHTTP}.
- *
- * @author stack
- * @version $Date$, $Revision$
- */
-public class HeritrixProtocolSocketFactory
-implements ProtocolSocketFactory {
- /**
- * Constructor.
- */
- public HeritrixProtocolSocketFactory() {
- super();
- }
-
- /**
- * @see #createSocket(java.lang.String,int,java.net.InetAddress,int)
- */
- public Socket createSocket(
- String host,
- int port,
- InetAddress localAddress,
- int localPort
- ) throws IOException, UnknownHostException {
- return new Socket(host, port, localAddress, localPort);
- }
-
- /**
- * Attempts to get a new socket connection to the given host within the
- * given time limit.
- *
- * This method employs several techniques to circumvent the limitations
- * of older JREs that do not support connect timeout. When running in
- * JRE 1.4 or above reflection is used to call
- * Socket#connect(SocketAddress endpoint, int timeout) method. When
- * executing in older JREs a controller thread is executed. The
- * controller thread attempts to create a new socket within the given
- * limit of time. If socket constructor does not return until the
- * timeout expires, the controller terminates and throws an
- * {@link ConnectTimeoutException}
- *
- *
- * @param host the host name/IP
- * @param port the port on the host
- * @param localAddress the local host name/IP to bind the socket to
- * @param localPort the port on the local machine
- * @param params {@link HttpConnectionParams Http connection parameters}
- *
- * @return Socket a new socket
- *
- * @throws IOException if an I/O error occurs while creating the socket
- * @throws UnknownHostException if the IP address of the host cannot be
- * @throws IOException if an I/O error occurs while creating the socket
- * @throws UnknownHostException if the IP address of the host cannot be
- * determined
- * @throws ConnectTimeoutException if socket cannot be connected within the
- * given time limit
- *
- * @since 3.0
- */
- public Socket createSocket(
- final String host,
- final int port,
- final InetAddress localAddress,
- final int localPort,
- final HttpConnectionParams params)
- throws IOException, UnknownHostException, ConnectTimeoutException {
- // Below code is from the DefaultSSLProtocolSocketFactory#createSocket
- // method only it has workarounds to deal with pre-1.4 JVMs. I've
- // cut these out.
- if (params == null) {
- throw new IllegalArgumentException("Parameters may not be null");
- }
- Socket socket = null;
- int timeout = params.getConnectionTimeout();
- if (timeout == 0) {
- socket = createSocket(host, port, localAddress, localPort);
- } else {
- socket = new Socket();
-
- InetAddress hostAddress;
- Thread current = Thread.currentThread();
- if (current instanceof HostResolver) {
- HostResolver resolver = (HostResolver)current;
- hostAddress = resolver.resolve(host);
- } else {
- hostAddress = null;
- }
- InetSocketAddress address = (hostAddress != null)?
- new InetSocketAddress(hostAddress, port):
- new InetSocketAddress(host, port);
- socket.bind(new InetSocketAddress(localAddress, localPort));
- try {
- socket.connect(address, timeout);
- } catch (SocketTimeoutException e) {
- // Add timeout info. to the exception.
- throw new SocketTimeoutException(e.getMessage() +
- ": timeout set at " + Integer.toString(timeout) + "ms.");
- }
- assert socket.isConnected(): "Socket not connected " + host;
- }
- return socket;
- }
-
- /**
- * Get host address using first the heritrix cache of addresses, then,
- * failing that, go to the dnsjava cache.
- *
- * Default access and static so can be used by other classes in this
- * package.
- *
- * @param host Host whose address we're to fetch.
- * @return an IP address for this host or null if one can't be found
- * in caches.
- * @exception IOException If we fail to get host IP from ServerCache.
- */
- /*
- static InetAddress getHostAddress(final ServerCache cache,
- final String host) throws IOException {
- InetAddress result = null;
- if (cache != null) {
- CrawlHost ch = cache.getHostFor(host);
- if (ch != null) {
- result = ch.getIP();
- }
- }
- if (result == null) {
- throw new IOException("Failed to get host " + host +
- " address from ServerCache");
- }
- return result;
- }
- */
-
- /**
- * @see ProtocolSocketFactory#createSocket(java.lang.String,int)
- */
- public Socket createSocket(String host, int port)
- throws IOException, UnknownHostException {
- return new Socket(host, port);
- }
-
- /**
- * All instances of DefaultProtocolSocketFactory are the same.
- * @param obj Object to compare.
- * @return True if equal
- */
- public boolean equals(Object obj) {
- return ((obj != null) &&
- obj.getClass().equals(HeritrixProtocolSocketFactory.class));
- }
-
- /**
- * All instances of DefaultProtocolSocketFactory have the same hash code.
- * @return Hash code for this object.
- */
- public int hashCode() {
- return HeritrixProtocolSocketFactory.class.hashCode();
- }
-}
diff --git a/modules/src/main/java/org/archive/modules/fetcher/HeritrixSSLProtocolSocketFactory.java b/modules/src/main/java/org/archive/modules/fetcher/HeritrixSSLProtocolSocketFactory.java
deleted file mode 100644
index 6e74c4cb..00000000
--- a/modules/src/main/java/org/archive/modules/fetcher/HeritrixSSLProtocolSocketFactory.java
+++ /dev/null
@@ -1,151 +0,0 @@
-/*
- * This file is part of the Heritrix web crawler (crawler.archive.org).
- *
- * Licensed to the Internet Archive (IA) by one or more individual
- * contributors.
- *
- * The IA licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.archive.modules.fetcher;
-
-import java.io.IOException;
-import java.net.InetAddress;
-import java.net.InetSocketAddress;
-import java.net.Socket;
-import java.net.SocketTimeoutException;
-import java.net.UnknownHostException;
-import java.security.KeyManagementException;
-import java.security.KeyStoreException;
-import java.security.NoSuchAlgorithmException;
-
-import javax.net.ssl.SSLContext;
-import javax.net.ssl.SSLSocketFactory;
-import javax.net.ssl.TrustManager;
-
-import org.apache.commons.httpclient.params.HttpConnectionParams;
-import org.apache.commons.httpclient.protocol.SecureProtocolSocketFactory;
-import org.archive.httpclient.ConfigurableX509TrustManager;
-
-
-/**
- * Implementation of the commons-httpclient SSLProtocolSocketFactory so we
- * can return SSLSockets whose trust manager is
- * {@link org.archive.httpclient.ConfigurableX509TrustManager}.
- *
- * We also go to the heritrix cache to get IPs to use making connection.
- * To this, we have dependency on {@link HeritrixProtocolSocketFactory};
- * its assumed this class and it are used together.
- * See {@link HeritrixProtocolSocketFactory#getHostAddress(ServerCache,String)}.
- *
- * @author stack
- * @version $Id$
- * @see org.archive.httpclient.ConfigurableX509TrustManager
- */
-public class HeritrixSSLProtocolSocketFactory
-implements SecureProtocolSocketFactory {
- /***
- * Socket factory with default trust manager installed.
- */
- private SSLSocketFactory sslDefaultFactory = null;
-
- /**
- * Shutdown constructor.
- * @throws KeyManagementException
- * @throws KeyStoreException
- * @throws NoSuchAlgorithmException
- */
- public HeritrixSSLProtocolSocketFactory()
- throws KeyManagementException, KeyStoreException, NoSuchAlgorithmException{
- // Get an SSL context and initialize it.
- SSLContext context = SSLContext.getInstance("SSL");
-
- // I tried to get the default KeyManagers but doesn't work unless you
- // point at a physical keystore. Passing null seems to do the right
- // thing so we'll go w/ that.
- context.init(null, new TrustManager[] {
- new ConfigurableX509TrustManager(
- ConfigurableX509TrustManager.DEFAULT)}, null);
- this.sslDefaultFactory = context.getSocketFactory();
- }
-
- public Socket createSocket(String host, int port, InetAddress clientHost,
- int clientPort)
- throws IOException, UnknownHostException {
- return this.sslDefaultFactory.createSocket(host, port,
- clientHost, clientPort);
- }
-
- public Socket createSocket(String host, int port)
- throws IOException, UnknownHostException {
- return this.sslDefaultFactory.createSocket(host, port);
- }
-
- public synchronized Socket createSocket(String host, int port,
- InetAddress localAddress, int localPort, HttpConnectionParams params)
- throws IOException, UnknownHostException {
- // Below code is from the DefaultSSLProtocolSocketFactory#createSocket
- // method only it has workarounds to deal with pre-1.4 JVMs. I've
- // cut these out.
- if (params == null) {
- throw new IllegalArgumentException("Parameters may not be null");
- }
- Socket socket = null;
- int timeout = params.getConnectionTimeout();
- if (timeout == 0) {
- socket = createSocket(host, port, localAddress, localPort);
- } else {
- SSLSocketFactory factory = (SSLSocketFactory)params.
- getParameter(LegacyFetchHTTP.SSL_FACTORY_KEY);
- SSLSocketFactory f = (factory != null)? factory: this.sslDefaultFactory;
- socket = f.createSocket();
-
- Thread current = Thread.currentThread();
- InetAddress hostAddress;
- if (current instanceof HostResolver) {
- HostResolver resolver = (HostResolver)current;
- hostAddress = resolver.resolve(host);
- } else {
- hostAddress = null;
- }
- InetSocketAddress address = (hostAddress != null)?
- new InetSocketAddress(hostAddress, port):
- new InetSocketAddress(host, port);
- socket.bind(new InetSocketAddress(localAddress, localPort));
- try {
- socket.connect(address, timeout);
- } catch (SocketTimeoutException e) {
- // Add timeout info. to the exception.
- throw new SocketTimeoutException(e.getMessage() +
- ": timeout set at " + Integer.toString(timeout) + "ms.");
- }
- assert socket.isConnected(): "Socket not connected " + host;
- }
- return socket;
- }
-
- public Socket createSocket(Socket socket, String host, int port,
- boolean autoClose)
- throws IOException, UnknownHostException {
- return this.sslDefaultFactory.createSocket(socket, host,
- port, autoClose);
- }
-
- public boolean equals(Object obj) {
- return ((obj != null) && obj.getClass().
- equals(HeritrixSSLProtocolSocketFactory.class));
- }
-
- public int hashCode() {
- return HeritrixSSLProtocolSocketFactory.class.hashCode();
- }
-}
\ No newline at end of file
diff --git a/modules/src/main/java/org/archive/modules/fetcher/LegacyFetchHTTP.java b/modules/src/main/java/org/archive/modules/fetcher/LegacyFetchHTTP.java
deleted file mode 100644
index b1da2887..00000000
--- a/modules/src/main/java/org/archive/modules/fetcher/LegacyFetchHTTP.java
+++ /dev/null
@@ -1,1640 +0,0 @@
-/*
- * This file is part of the Heritrix web crawler (crawler.archive.org).
- *
- * Licensed to the Internet Archive (IA) by one or more individual
- * contributors.
- *
- * The IA licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.archive.modules.fetcher;
-
-import static org.archive.modules.CrawlURI.FetchType.HTTP_POST;
-import static org.archive.modules.fetcher.FetchErrors.HEADER_TRUNC;
-import static org.archive.modules.fetcher.FetchErrors.LENGTH_TRUNC;
-import static org.archive.modules.fetcher.FetchErrors.TIMER_TRUNC;
-import static org.archive.modules.fetcher.FetchStatusCodes.S_CONNECT_FAILED;
-import static org.archive.modules.fetcher.FetchStatusCodes.S_CONNECT_LOST;
-import static org.archive.modules.fetcher.FetchStatusCodes.S_DOMAIN_PREREQUISITE_FAILURE;
-import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_ETAG_HEADER;
-import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_FETCH_HISTORY;
-import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_LAST_MODIFIED_HEADER;
-import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_REFERENCE_LENGTH;
-import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_STATUS;
-
-import java.io.IOException;
-import java.io.ObjectInputStream;
-import java.io.ObjectOutputStream;
-import java.net.InetAddress;
-import java.net.UnknownHostException;
-import java.nio.charset.Charset;
-import java.security.KeyManagementException;
-import java.security.KeyStoreException;
-import java.security.MessageDigest;
-import java.security.NoSuchAlgorithmException;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-import javax.net.ssl.SSLContext;
-import javax.net.ssl.SSLSocketFactory;
-import javax.net.ssl.TrustManager;
-
-import org.apache.commons.httpclient.Cookie;
-import org.apache.commons.httpclient.Credentials;
-import org.apache.commons.httpclient.Header;
-import org.apache.commons.httpclient.HostConfiguration;
-import org.apache.commons.httpclient.HttpClient;
-import org.apache.commons.httpclient.HttpConnection;
-import org.apache.commons.httpclient.HttpConnectionManager;
-import org.apache.commons.httpclient.HttpException;
-import org.apache.commons.httpclient.HttpMethod;
-import org.apache.commons.httpclient.HttpMethodBase;
-import org.apache.commons.httpclient.HttpState;
-import org.apache.commons.httpclient.HttpStatus;
-import org.apache.commons.httpclient.HttpVersion;
-import org.apache.commons.httpclient.NTCredentials;
-import org.apache.commons.httpclient.URIException;
-import org.apache.commons.httpclient.auth.AuthChallengeParser;
-import org.apache.commons.httpclient.auth.AuthPolicy;
-import org.apache.commons.httpclient.auth.AuthScheme;
-import org.apache.commons.httpclient.auth.AuthScope;
-import org.apache.commons.httpclient.auth.MalformedChallengeException;
-import org.apache.commons.httpclient.cookie.CookiePolicy;
-import org.apache.commons.httpclient.params.HttpClientParams;
-import org.apache.commons.httpclient.params.HttpConnectionManagerParams;
-import org.apache.commons.httpclient.params.HttpMethodParams;
-import org.apache.commons.httpclient.protocol.Protocol;
-import org.apache.commons.httpclient.protocol.ProtocolSocketFactory;
-import org.apache.commons.lang.StringUtils;
-import org.archive.httpclient.ConfigurableX509TrustManager;
-import org.archive.httpclient.ConfigurableX509TrustManager.TrustLevel;
-import org.archive.httpclient.HttpRecorderGetMethod;
-import org.archive.httpclient.HttpRecorderMethod;
-import org.archive.httpclient.HttpRecorderPostMethod;
-import org.archive.httpclient.SingleHttpConnectionManager;
-import org.archive.io.RecorderLengthExceededException;
-import org.archive.io.RecorderTimeoutException;
-import org.archive.io.RecorderTooMuchHeaderException;
-import org.archive.modules.CrawlURI;
-import org.archive.modules.CrawlURI.FetchType;
-import org.archive.modules.ProcessResult;
-import org.archive.modules.credential.CommonsHttpCredentialUtil;
-import org.archive.modules.credential.Credential;
-import org.archive.modules.credential.CredentialStore;
-import org.archive.modules.credential.HttpAuthenticationCredential;
-import org.archive.modules.deciderules.AcceptDecideRule;
-import org.archive.modules.deciderules.DecideResult;
-import org.archive.modules.deciderules.DecideRule;
-import org.archive.modules.extractor.LinkContext;
-import org.archive.modules.net.CrawlHost;
-import org.archive.modules.net.CrawlServer;
-import org.archive.modules.net.ServerCache;
-import org.archive.util.Recorder;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.context.Lifecycle;
-
-/**
- * HTTP fetcher that uses Apache Jakarta Commons
- * HttpClient library.
- *
- * @contributor gojomo
- * @contributor Igor Ranitovic
- * @contributor stack
- * @contributor others
- * @version $Id$
- */
-public class LegacyFetchHTTP extends AbstractFetchHTTP implements Lifecycle {
- @SuppressWarnings("unused")
- private static final long serialVersionUID = 1L;
- private static Logger logger = Logger.getLogger(LegacyFetchHTTP.class.getName());
-
- /**
- * Proxy host IP (set only if needed).
- */
- {
- setHttpProxyHost("");
- }
- public String getHttpProxyHost() {
- return (String) kp.get("httpProxyHost");
- }
- public void setHttpProxyHost(String host) {
- kp.put("httpProxyHost",host);
- }
-
- /**
- * Proxy port (set only if needed).
- */
- {
- setHttpProxyPort(0);
- }
- public int getHttpProxyPort() {
- return (Integer) kp.get("httpProxyPort");
- }
- public void setHttpProxyPort(int port) {
- kp.put("httpProxyPort",port);
- }
-
- /**
- * Proxy user (set only if needed).
- */
- {
- setHttpProxyUser("");
- }
- public String getHttpProxyUser() {
- return (String) kp.get("httpProxyUser");
- }
- public void setHttpProxyUser(String user) {
- kp.put("httpProxyUser",user);
- }
-
- /**
- * Proxy password (set only if needed).
- */
- {
- setHttpProxyPassword("");
- }
- public String getHttpProxyPassword() {
- return (String) kp.get("httpProxyPassword");
- }
- public void setHttpProxyPassword(String password) {
- kp.put("httpProxyPassword",password);
- }
-
- /**
- * If the fetch is not completed in this number of seconds, give up (and
- * retry later).
- */
- {
- setTimeoutSeconds(20*60); // 20 minutes
- }
- public int getTimeoutSeconds() {
- return (Integer) kp.get("timeoutSeconds");
- }
- public void setTimeoutSeconds(int timeout) {
- kp.put("timeoutSeconds",timeout);
- }
-
- /**
- * If the socket is unresponsive for this number of milliseconds, give up.
- * Set to zero for no timeout (Not. recommended. Could hang a thread on an
- * unresponsive server). This timeout is used timing out socket opens and
- * for timing out each socket read. Make sure this value is <
- * {@link #TIMEOUT_SECONDS} for optimal configuration: ensures at least one
- * retry read.
- */
- {
- setSoTimeoutMs(20*1000); // 20 seconds
- }
- public int getSoTimeoutMs() {
- return (Integer) kp.get("soTimeoutMs");
- }
- public void setSoTimeoutMs(int timeout) {
- kp.put("soTimeoutMs",timeout);
- }
-
- /**
- * Maximum length in bytes to fetch. Fetch is truncated at this length. A
- * value of 0 means no limit.
- */
- {
- setMaxLengthBytes(0L); // no limit
- }
- public long getMaxLengthBytes() {
- return (Long) kp.get("maxLengthBytes");
- }
- public void setMaxLengthBytes(long maxLengthBytes) {
- kp.put("maxLengthBytes",maxLengthBytes);
- }
-
- /**
- * Accept Headers to include in each request. Each must be the complete
- * header, e.g., 'Accept-Language: en'. (Thus, this can also be used to
- * other headers not beginning 'Accept-' as well.) By default heritrix sends
- * an Accept header similar to what a typical browser would send (the value
- * comes from Firefox 4.0).
- */
- {
- setAcceptHeaders(Arrays.asList("Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"));
- }
- @SuppressWarnings("unchecked")
- public List getAcceptHeaders() {
- return (List) kp.get("acceptHeaders");
- }
- public void setAcceptHeaders(List headers) {
- kp.put("acceptHeaders",headers);
- }
-
- /**
- * The character encoding to use for files that do not have one specified in
- * the HTTP response headers. Default: ISO-8859-1.
- */
- {
- setDefaultEncoding("ISO-8859-1");
- }
- public String getDefaultEncoding() {
- return getDefaultCharset().name();
- }
- public void setDefaultEncoding(String encoding) {
- kp.put("defaultEncoding",Charset.forName(encoding));
- }
- public Charset getDefaultCharset() {
- return (Charset)kp.get("defaultEncoding");
- }
-
- /**
- * Whether or not to perform an on-the-fly digest hash of retrieved
- * content-bodies.
- */
- {
- setDigestContent(true);
- }
- public boolean getDigestContent() {
- return (Boolean) kp.get("digestContent");
- }
- public void setDigestContent(boolean digest) {
- kp.put("digestContent",digest);
- }
-
- /**
- * Which algorithm (for example MD5 or SHA-1) to use to perform an
- * on-the-fly digest hash of retrieved content-bodies.
- */
- protected String digestAlgorithm = "sha1";
- public String getDigestAlgorithm() {
- return digestAlgorithm;
- }
- public void setDigestAlgorithm(String digestAlgorithm) {
- this.digestAlgorithm = digestAlgorithm;
- }
-
- /**
- * The maximum KB/sec to use when fetching data from a server. The default
- * of 0 means no maximum.
- */
- {
- setMaxFetchKBSec(0); // no limit
- }
- public int getMaxFetchKBSec() {
- return (Integer) kp.get("maxFetchKBSec");
- }
- public void setMaxFetchKBSec(int rate) {
- kp.put("maxFetchKBSec",rate);
- }
-
- public UserAgentProvider getUserAgentProvider() {
- return (UserAgentProvider) kp.get("userAgentProvider");
- }
- @Autowired
- public void setUserAgentProvider(UserAgentProvider provider) {
- kp.put("userAgentProvider",provider);
- }
-
- /**
- * SSL certificate trust level. Range is from the default 'open' (trust all
- * certs including expired, selfsigned, and those for which we do not have a
- * CA) through 'loose' (trust all valid certificates including selfsigned),
- * 'normal' (all valid certificates not including selfsigned) to 'strict'
- * (Cert is valid and DN must match servername).
- */
- {
- setSslTrustLevel(TrustLevel.OPEN);
- }
- public TrustLevel getSslTrustLevel() {
- return (TrustLevel) kp.get("trustLevel");
- }
- public void setSslTrustLevel(TrustLevel trustLevel) {
- kp.put("sslTrustLevel",trustLevel);
- }
-
- private transient HttpClient http = null;
-
- /**
- * How many 'instant retries' of HttpRecoverableExceptions have occurred
- *
- * Would like it to be 'long', but longs aren't atomic
- */
- private int recoveryRetries = 0;
-
-
- /**
- * DecideRules applied after receipt of HTTP response headers but before we
- * start to download the body. If any filter returns FALSE, the fetch is
- * aborted. Prerequisites such as robots.txt by-pass filtering (i.e. they
- * cannot be midfetch aborted.
- */
- {
- setShouldFetchBodyRule(new AcceptDecideRule());
- }
- public DecideRule getShouldFetchBodyRule() {
- return (DecideRule) kp.get("shouldFetchBodyRule");
- }
- public void setShouldFetchBodyRule(DecideRule rule) {
- kp.put("shouldFetchBodyRule", rule);
- }
-
- // see [ 1379040 ] regex for midfetch filter not being stored in crawl order
- // http://sourceforge.net/support/tracker.php?aid=1379040
- // this.midfetchfilters.setExpertSetting(true);
-
- /**
- * What to log if midfetch abort.
- */
- private static final String MIDFETCH_ABORT_LOG = "midFetchAbort";
-
- /**
- * Use HTTP/1.1. Note: even when offering an HTTP/1.1 request,
- * Heritrix may not properly handle persistent/keep-alive connections,
- * so the sendConnectionClose parameter should remain 'true'.
- */
- {
- setUseHTTP11(false);
- }
- public boolean getUseHTTP11() {
- return (Boolean) kp.get("useHTTP11");
- }
- public void setUseHTTP11(boolean useHTTP11) {
- kp.put("useHTTP11",useHTTP11);
- }
-
- /**
- * Set headers to accept compressed responses.
- */
- {
- setAcceptCompression(false);
- }
- public boolean getAcceptCompression() {
- return (Boolean) kp.get("acceptCompression");
- }
- public void setAcceptCompression(boolean acceptCompression) {
- kp.put("acceptCompression",acceptCompression);
- }
-
- /**
- * Send 'Connection: close' header with every request.
- */
- {
- setSendConnectionClose(true);
- }
- public boolean getSendConnectionClose() {
- return (Boolean) kp.get("sendConnectionClose");
- }
- public void setSendConnectionClose(boolean sendClose) {
- kp.put("sendConnectionClose",sendClose);
- }
-
- private static final Header HEADER_SEND_CONNECTION_CLOSE = new Header(
- "Connection", "close");
-
- /**
- * Send 'Referer' header with every request.
- *
- * The 'Referer' header contans the location the crawler came from, the page
- * the current URI was discovered in. The 'Referer' usually is logged on the
- * remote server and can be of assistance to webmasters trying to figure how
- * a crawler got to a particular area on a site.
- */
- {
- setSendReferer(true);
- }
- public boolean getSendReferer() {
- return (Boolean) kp.get("sendReferer");
- }
- public void setSendReferer(boolean sendClose) {
- kp.put("sendReferer",sendClose);
- }
-
- /**
- * Send 'Range' header when a limit ({@link #MAX_LENGTH_BYTES}) on
- * document size.
- *
- * Be polite to the HTTP servers and send the 'Range' header, stating that
- * you are only interested in the first n bytes. Only pertinent if
- * {@link #MAX_LENGTH_BYTES} > 0. Sending the 'Range' header results in a
- * '206 Partial Content' status response, which is better than just cutting
- * the response mid-download. On rare occasion, sending 'Range' will
- * generate '416 Request Range Not Satisfiable' response.
- */
- {
- setSendRange(false);
- }
- public boolean getSendRange() {
- return (Boolean) kp.get("sendRange");
- }
- public void setSendRange(boolean sendRange) {
- kp.put("sendRange",sendRange);
- }
-
- /**
- * Send 'If-Modified-Since' header, if previous 'Last-Modified' fetch
- * history information is available in URI history.
- */
- {
- setSendIfModifiedSince(true);
- }
- public boolean getSendIfModifiedSince() {
- return (Boolean) kp.get("sendIfModifiedSince");
- }
- public void setSendIfModifiedSince(boolean sendIfModifiedSince) {
- kp.put("sendIfModifiedSince",sendIfModifiedSince);
- }
-
- /**
- * Send 'If-None-Match' header, if previous 'Etag' fetch history information
- * is available in URI history.
- */
- {
- setSendIfNoneMatch(true);
- }
- public boolean getSendIfNoneMatch() {
- return (Boolean) kp.get("sendIfNoneMatch");
- }
- public void setSendIfNoneMatch(boolean sendIfNoneMatch) {
- kp.put("sendIfNoneMatch",sendIfNoneMatch);
- }
-
- public static final String REFERER = "Referer";
-
- public static final String RANGE = "Range";
-
- public static final String RANGE_PREFIX = "bytes=0-";
-
- public static final String HTTP_SCHEME = "http";
-
- public static final String HTTPS_SCHEME = "https";
-
-
- protected CookieStorage cookieStorage = new BdbCookieStorage();
- @Autowired(required=false)
- public void setCookieStorage(CookieStorage storage) {
- this.cookieStorage = storage;
- }
- public CookieStorage getCookieStorage() {
- return this.cookieStorage;
- }
-
- /**
- * Disable cookie handling.
- */
- {
- setIgnoreCookies(false);
- }
- public boolean getIgnoreCookies() {
- return (Boolean) kp.get("ignoreCookies");
- }
- public void setIgnoreCookies(boolean ignoreCookies) {
- kp.put("ignoreCookies",ignoreCookies);
- }
-
- /**
- * Local IP address or hostname to use when making connections (binding
- * sockets). When not specified, uses default local address(es).
- */
- public String getHttpBindAddress(){
- return (String) kp.get(HTTP_BIND_ADDRESS);
- }
- public void setHttpBindAddress(String address) {
- kp.put(HTTP_BIND_ADDRESS, address);
- }
- public static final String HTTP_BIND_ADDRESS = "httpBindAddress";
-
- /**
- * Used to store credentials.
- */
- {
- // initialize with empty store so declaration not required
- setCredentialStore(new CredentialStore());
- }
- public CredentialStore getCredentialStore() {
- return (CredentialStore) kp.get("credentialStore");
- }
- @Autowired(required=false)
- public void setCredentialStore(CredentialStore credentials) {
- kp.put("credentialStore",credentials);
- }
-
- /**
- * Used to do DNS lookups.
- */
- protected ServerCache serverCache;
- public ServerCache getServerCache() {
- return this.serverCache;
- }
- @Autowired
- public void setServerCache(ServerCache serverCache) {
- this.serverCache = serverCache;
- }
-
- static {
- Protocol.registerProtocol("http", new Protocol("http",
- new HeritrixProtocolSocketFactory(), 80));
- try {
- ProtocolSocketFactory psf = new HeritrixSSLProtocolSocketFactory();
- Protocol p = new Protocol("https", psf, 443);
- Protocol.registerProtocol("https", p);
- } catch (KeyManagementException e) {
- e.printStackTrace();
- } catch (KeyStoreException e) {
- e.printStackTrace();
- } catch (NoSuchAlgorithmException e) {
- e.printStackTrace();
- }
- }
-
-
- // static final String SERVER_CACHE_KEY = "heritrix.server.cache";
- static final String SSL_FACTORY_KEY = "heritrix.ssl.factory";
-
- /***************************************************************************
- * Socket factory that has the configurable trust manager installed.
- */
- private transient SSLSocketFactory sslfactory = null;
-
- /**
- * Constructor.
- */
- public LegacyFetchHTTP() {
- }
-
- protected void innerProcess(final CrawlURI curi)
- throws InterruptedException {
- // Note begin time
- curi.setFetchBeginTime(System.currentTimeMillis());
-
- // Get a reference to the HttpRecorder that is set into this ToeThread.
- Recorder rec = curi.getRecorder();
-
- // Shall we get a digest on the content downloaded?
- boolean digestContent = getDigestContent();
- String algorithm = null;
- if (digestContent) {
- algorithm = getDigestAlgorithm();
- rec.getRecordedInput().setDigest(algorithm);
- } else {
- // clear
- rec.getRecordedInput().setDigest((MessageDigest)null);
- }
-
- // Below we do two inner classes that add check of midfetch
- // filters just as we're about to receive the response body.
- String curiString = curi.getUURI().toString();
- HttpMethodBase method = null;
- if (curi.getFetchType() == HTTP_POST) {
- method = new HttpRecorderPostMethod(curiString, rec) {
- protected void readResponseBody(HttpState state,
- HttpConnection conn) throws IOException, HttpException {
- addResponseContent(this, curi);
- if (checkMidfetchAbort(curi, this.httpRecorderMethod, conn)) {
- doAbort(curi, this, MIDFETCH_ABORT_LOG);
- } else {
- super.readResponseBody(state, conn);
- }
- }
- };
- curi.setFetchType(FetchType.HTTP_POST);
- } else {
- method = new HttpRecorderGetMethod(curiString, rec) {
- protected void readResponseBody(HttpState state,
- HttpConnection conn) throws IOException, HttpException {
- addResponseContent(this, curi);
- if (checkMidfetchAbort(curi, this.httpRecorderMethod, conn)) {
- doAbort(curi, this, MIDFETCH_ABORT_LOG);
- } else {
- super.readResponseBody(state, conn);
- }
- }
- };
- curi.setFetchType(FetchType.HTTP_GET);
- }
-
-
- HostConfiguration customConfigOrNull = configureMethod(curi, method);
-
- // Populate credentials. Set config so auth. is not automatic.
- boolean addedCredentials = populateCredentials(curi, method);
- if (http.getState().getProxyCredentials(new AuthScope(getProxyHost(), getProxyPort())) != null) {
- addedCredentials = true;
- }
-
- // set hardMax on bytes (if set by operator)
- long hardMax = getMaxLengthBytes();
- // set overall timeout (if set by operator)
- long timeoutMs = 1000 * getTimeoutSeconds();
- // Get max fetch rate (bytes/ms). It comes in in KB/sec
- long maxRateKBps = getMaxFetchKBSec();
- rec.getRecordedInput().setLimits(hardMax, timeoutMs, maxRateKBps);
-
- try {
- this.http.executeMethod(customConfigOrNull, method);
- } catch (RecorderTooMuchHeaderException ex) {
- // when too much header material, abort like other truncations
- doAbort(curi, method, HEADER_TRUNC);
- } catch (IOException e) {
- failedExecuteCleanup(method, curi, e);
- return;
- } catch (ArrayIndexOutOfBoundsException e) {
- // For weird windows-only ArrayIndex exceptions in native
- // code... see
- // http://forum.java.sun.com/thread.jsp?forum=11&thread=378356
- // treating as if it were an IOException
- failedExecuteCleanup(method, curi, e);
- return;
- }
-
- // set softMax on bytes to get (if implied by content-length)
- long softMax = method.getResponseContentLength();
-
- try {
- if (!method.isAborted()) {
- // Force read-to-end, so that any socket hangs occur here,
- // not in later modules.
- rec.getRecordedInput().readFullyOrUntil(softMax);
- }
- } catch (RecorderTimeoutException ex) {
- doAbort(curi, method, TIMER_TRUNC);
- } catch (RecorderLengthExceededException ex) {
- doAbort(curi, method, LENGTH_TRUNC);
- } catch (IOException e) {
- cleanup(curi, e, "readFully", S_CONNECT_LOST);
- return;
- } catch (ArrayIndexOutOfBoundsException e) {
- // For weird windows-only ArrayIndex exceptions from native code
- // see http://forum.java.sun.com/thread.jsp?forum=11&thread=378356
- // treating as if it were an IOException
- cleanup(curi, e, "readFully", S_CONNECT_LOST);
- return;
- } finally {
- // ensure recording has stopped
- rec.closeRecorders();
- if (!method.isAborted()) {
- method.releaseConnection();
- }
- // Note completion time
- curi.setFetchCompletedTime(System.currentTimeMillis());
- // Set the response charset into the HttpRecord if available.
- setCharacterEncoding(curi, rec, method);
- setSizes(curi, rec);
- setOtherCodings(curi, rec, method);
- }
-
- if (digestContent) {
- curi.setContentDigest(algorithm,
- rec.getRecordedInput().getDigestValue());
- }
- if (logger.isLoggable(Level.FINE)) {
- logger.fine(((curi.getFetchType() == HTTP_POST) ? "POST" : "GET")
- + " " + curi.getUURI().toString() + " "
- + method.getStatusCode() + " "
- + rec.getRecordedInput().getSize() + " "
- + curi.getContentType());
- }
-
- if (isSuccess(curi) && addedCredentials) {
- // Promote the credentials from the CrawlURI to the CrawlServer
- // so they are available for all subsequent CrawlURIs on this
- // server.
- promoteCredentials(curi);
- if (logger.isLoggable(Level.FINE)) {
- // Print out the cookie. Might help with the debugging.
- Header setCookie = method.getResponseHeader("set-cookie");
- if (setCookie != null) {
- logger.fine(setCookie.toString().trim());
- }
- }
- } else if (method.getStatusCode() == HttpStatus.SC_UNAUTHORIZED) {
- // 401 is not 'success'.
- handle401(method, curi);
- }
-
- if (rec.getRecordedInput().isOpen()) {
- logger.severe(curi.toString() + " RIS still open. Should have"
- + " been closed by method release: "
- + Thread.currentThread().getName());
- try {
- rec.getRecordedInput().close();
- } catch (IOException e) {
- logger.log(Level.SEVERE, "second-chance RIS close failed", e);
- }
- }
- }
-
- /**
- * Update CrawlURI internal sizes based on current transaction (and
- * in the case of 304s, history)
- *
- * @param curi CrawlURI
- * @param rec HttpRecorder
- */
- protected void setSizes(CrawlURI curi, Recorder rec) {
- // set reporting size
- curi.setContentSize(rec.getRecordedInput().getSize());
- // special handling for 304-not modified
- if (curi.getFetchStatus() == HttpStatus.SC_NOT_MODIFIED
- && curi.containsDataKey(A_FETCH_HISTORY)) {
- Map[] history = curi.getFetchHistory();
- if (history[0] != null
- && history[0].containsKey(A_REFERENCE_LENGTH)) {
- long referenceLength = (Long) history[0].get(A_REFERENCE_LENGTH);
- // carry-forward previous 'reference-length' for future
- curi.getData().put(A_REFERENCE_LENGTH, referenceLength);
- // increase content-size to virtual-size for reporting
- curi.setContentSize(rec.getRecordedInput().getSize()
- + referenceLength);
- }
- }
- }
-
- protected void doAbort(CrawlURI curi, HttpMethod method,
- String annotation) {
- curi.getAnnotations().add(annotation);
- curi.getRecorder().close();
- method.abort();
- }
-
- protected boolean checkMidfetchAbort(CrawlURI curi,
- HttpRecorderMethod method, HttpConnection conn) {
- if (curi.isPrerequisite()) {
- return false;
- }
- DecideResult r = getShouldFetchBodyRule().decisionFor(curi);
- if (r != DecideResult.REJECT) {
- return false;
- }
- method.markContentBegin(conn);
- return true;
- }
-
- /**
- * This method populates curi with response status and
- * content type.
- *
- * @param curi
- * CrawlURI to populate.
- * @param method
- * Method to get response status and headers from.
- */
- protected void addResponseContent(HttpMethod method, CrawlURI curi) {
- curi.setFetchStatus(method.getStatusCode());
- Header ct = method.getResponseHeader("content-type");
- curi.setContentType((ct == null) ? null : ct.getValue());
-
- for (Header h: method.getResponseHeaders()) {
- curi.putHttpResponseHeader(h.getName(), h.getValue());
- }
- }
-
- /**
- * Set the character encoding based on the result headers or default.
- *
- * The HttpClient returns its own default encoding ("ISO-8859-1") if one
- * isn't specified in the Content-Type response header. We give the user the
- * option of overriding this, so we need to detect the case where the
- * default is returned.
- *
- * Now, it may well be the case that the default returned by HttpClient and
- * the default defined by the user are the same.
- *
- * TODO:FIXME?: This method does not do the "detect the case where the
- * [HttpClient] default is returned" mentioned above! Why not?
- *
- * @param rec
- * Recorder for this request.
- * @param method
- * Method used for the request.
- */
- private void setCharacterEncoding(CrawlURI curi, final Recorder rec,
- final HttpMethod method) {
- String encoding = ((HttpMethodBase) method).getResponseCharSet();
- try {
- rec.setCharset(Charset.forName(encoding));
- } catch (IllegalArgumentException e) {
- curi.getAnnotations().add("unsatisfiableCharsetInHeader:"+StringUtils.stripToEmpty(encoding));
- rec.setCharset(getDefaultCharset());
- }
- }
-
- /**
- * Set the transfer, content encodings based on headers (if necessary).
- *
- * @param rec
- * Recorder for this request.
- * @param method
- * Method used for the request.
- */
- private void setOtherCodings(CrawlURI uri, final Recorder rec,
- final HttpMethod method) {
- Header transferCodingHeader = ((HttpMethodBase) method).getResponseHeader("Transfer-Encoding");
- if (transferCodingHeader !=null) {
- String te = transferCodingHeader.getValue().trim();
- if(te.equalsIgnoreCase("chunked")) {
- rec.setInputIsChunked(true);
- } else {
- logger.log(Level.WARNING,"Unknown transfer-encoding '"+te+"' for "+uri.getURI());
- }
- }
- Header contentEncodingHeader = ((HttpMethodBase) method).getResponseHeader("Content-Encoding");
- if (contentEncodingHeader!=null) {
- String ce = contentEncodingHeader.getValue().trim();
- try {
- rec.setContentEncoding(ce);
- } catch (IllegalArgumentException e) {
- uri.getAnnotations().add("unsatisfiableContentEncoding:"+StringUtils.stripToEmpty(ce));
- }
- }
- }
-
- /**
- * Cleanup after a failed method execute.
- *
- * @param curi
- * CrawlURI we failed on.
- * @param method
- * Method we failed on.
- * @param exception
- * Exception we failed with.
- */
- private void failedExecuteCleanup(final HttpMethod method,
- final CrawlURI curi, final Exception exception) {
- cleanup(curi, exception, "executeMethod", (method.isRequestSent() ? S_CONNECT_LOST : S_CONNECT_FAILED));
- method.releaseConnection();
- }
-
- /**
- * Cleanup after a failed method execute.
- *
- * @param curi
- * CrawlURI we failed on.
- * @param exception
- * Exception we failed with.
- * @param message
- * Message to log with failure. FIXME: Seems ignored
- * @param status
- * Status to set on the fetch.
- */
- private void cleanup(final CrawlURI curi, final Exception exception,
- final String message, final int status) {
- // message ignored!
- curi.getNonFatalFailures().add(exception);
- curi.setFetchStatus(status);
- curi.getRecorder().close();
- }
-
- @Override
- public ProcessResult process(CrawlURI uri) throws InterruptedException {
- if (uri.getFetchStatus() < 0) {
- // already marked as errored, this pass through
- // skip to end
- return ProcessResult.FINISH;
- } else {
- return super.process(uri);
- }
- }
-
- /**
- * Can this processor fetch the given CrawlURI. May set a fetch status
- * if this processor would usually handle the CrawlURI, but cannot in
- * this instance.
- *
- * @param curi
- * @return True if processor can fetch.
- */
- @Override
- protected boolean shouldProcess(CrawlURI curi) {
- String scheme = curi.getUURI().getScheme();
- if (!(scheme.equals("http") || scheme.equals("https"))) {
- // handles only plain http and https
- return false;
- }
-
- CrawlHost host = serverCache.getHostFor(curi.getUURI());
- if (host.getIP() == null && host.hasBeenLookedUp()) {
- curi.setFetchStatus(S_DOMAIN_PREREQUISITE_FAILURE);
- return false;
- }
-
- return true;
- }
-
- /**
- * Configure the HttpMethod setting options and headers.
- *
- * @param curi
- * CrawlURI from which we pull configuration.
- * @param method
- * The Method to configure.
- */
- protected HostConfiguration configureMethod(CrawlURI curi,
- HttpMethod method) {
- // Don't try to handle 401s internally.
- method.setDoAuthentication(false);
-
- // Don't auto-follow redirects
- method.setFollowRedirects(false);
-
- // // set soTimeout
- // method.getParams().setSoTimeout(
- // ((Integer) getUncheckedAttribute(curi, ATTR_SOTIMEOUT_MS))
- // .intValue());
-
- // Set cookie policy.
- boolean ignoreCookies = getIgnoreCookies();
- method.getParams().setCookiePolicy(
- ignoreCookies ? CookiePolicy.IGNORE_COOKIES
- : CookiePolicy.BROWSER_COMPATIBILITY);
-
- method.getParams().setVersion(getUseHTTP11()
- ? HttpVersion.HTTP_1_1
- : HttpVersion.HTTP_1_0);
-
- UserAgentProvider uap = getUserAgentProvider();
- String from = uap.getFrom();
- String userAgent = curi.getUserAgent();
- if (userAgent == null) {
- userAgent = uap.getUserAgent();
- }
-
- method.setRequestHeader("User-Agent", userAgent);
- if(StringUtils.isNotBlank(from)) {
- method.setRequestHeader("From", from);
- }
-
- // Set retry handler.
- method.getParams().setParameter(HttpMethodParams.RETRY_HANDLER,
- new HeritrixHttpMethodRetryHandler());
-
- final long maxLength = getMaxLengthBytes();
- if (maxLength > 0 && getSendRange()) {
- method.addRequestHeader(RANGE, RANGE_PREFIX.concat(Long
- .toString(maxLength - 1)));
- }
-
- if (getSendConnectionClose()) {
- method.addRequestHeader(HEADER_SEND_CONNECTION_CLOSE);
- }
-
- if (getSendReferer() && !LinkContext.PREREQ_MISC.equals(curi.getViaContext())) {
- // RFC2616 says no referer header if referer is https and the url
- // is not
- String via = flattenVia(curi);
- if (via != null
- && via.length() > 0
- && !(via.startsWith(HTTPS_SCHEME) && curi.getUURI()
- .getScheme().equals(HTTP_SCHEME))) {
- method.setRequestHeader(REFERER, via);
- }
- }
-
- if (!curi.isPrerequisite()) {
- setConditionalGetHeader(curi, method, getSendIfModifiedSince(),
- A_LAST_MODIFIED_HEADER, "If-Modified-Since");
- setConditionalGetHeader(curi, method, getSendIfNoneMatch(),
- A_ETAG_HEADER, "If-None-Match");
- }
-
- // TODO: What happens if below method adds a header already
- // added above: e.g. Connection, Range, or Referer?
- setAcceptHeaders(curi, method);
-
- HostConfiguration config =
- new HostConfiguration(http.getHostConfiguration());
- configureProxy(curi, config);
- configureBindAddress(curi, config);
- return config;
- }
-
- /**
- * Set the given conditional-GET header, if the setting is enabled and
- * a suitable value is available in the URI history.
- * @param curi source CrawlURI
- * @param method HTTP operation pending
- * @param setting true/false enablement setting name to consult
- * @param sourceHeader header to consult in URI history
- * @param targetHeader header to set if possible
- */
- protected void setConditionalGetHeader(CrawlURI curi, HttpMethod method,
- boolean conditional, String sourceHeader, String targetHeader) {
- if (conditional) {
- try {
- HashMap[] history = curi.getFetchHistory();
- int previousStatus = (Integer) history[0].get(A_STATUS);
- if (previousStatus <= 0) {
- // do not reuse headers from any broken fetch
- return;
- }
- String previousValue = (String) history[0].get(sourceHeader);
- if (previousValue != null) {
- method.setRequestHeader(targetHeader, previousValue);
- }
- } catch (RuntimeException e) {
- // for absent key, bad index, etc. just do nothing
- }
- }
- }
-
- /**
- * Setup proxy, based on attributes in CrawlURI and settings,
- * in given HostConfiguration
- */
- private void configureProxy(CrawlURI curi, HostConfiguration config) {
- String proxy = (String) getAttributeEither(curi, "httpProxyHost");
- int port = (Integer) getAttributeEither(curi, "httpProxyPort");
- String user = (String) getAttributeEither(curi, "httpProxyUser");
- String password = (String) getAttributeEither(curi, "httpProxyPassword");
- configureProxy(proxy, port, user, password, config);
- }
-
- private void configureProxy(String proxy, int port, String user, String password,
- HostConfiguration config) {
- if(StringUtils.isNotEmpty(proxy)) {
- config.setProxy(proxy, port);
- if (StringUtils.isNotEmpty(user)) {
- Credentials credentials = new NTCredentials(user, password, "", "");
- AuthScope authScope = new AuthScope(proxy, port);
- this.http.getState().setProxyCredentials(authScope, credentials);
- }
- }
- }
-
- /**
- * Setup local bind address, based on attributes in CrawlURI and
- * settings, in given HostConfiguration
- */
- private void configureBindAddress(CrawlURI curi, HostConfiguration config) {
- String addressString = (String) getAttributeEither(curi, HTTP_BIND_ADDRESS);
- configureBindAddress(addressString,config);
- }
-
- private void configureBindAddress(String address, HostConfiguration config) {
- if (StringUtils.isNotEmpty(address)) {
- try {
- InetAddress localAddress = InetAddress.getByName(address);
- config.setLocalAddress(localAddress);
- } catch (UnknownHostException e) {
- // Convert all to RuntimeException so get an exception out
- // if initialization fails.
- throw new RuntimeException("Unknown host " + address
- + " in local-address");
- }
- }
- }
-
- /**
- * Get a value either from inside the CrawlURI instance, or from
- * settings (module attributes).
- *
- * @param curi
- * CrawlURI to consult
- * @param key
- * key to lookup
- * @return value from either CrawlURI (preferred) or settings
- */
- protected Object getAttributeEither(CrawlURI curi, String key) {
-
- Object r = curi.getData().get(key);
- if (r != null) {
- return r;
- }
- return kp.get(key);
- }
-
- /**
- * Add credentials if any to passed method.
- *
- * Do credential handling. Credentials are in two places. 1. Credentials
- * that succeeded are added to the CrawlServer (Or rather, avatars for
- * credentials are whats added because its not safe to keep around
- * references to credentials). 2. Credentials to be tried are in the curi.
- * Returns true if found credentials to be tried.
- *
- * @param curi
- * Current CrawlURI.
- * @param method
- * The method to add to.
- * @return True if prepopulated method with credentials AND
- * the credentials came from the curi, not from the
- * CrawlServer. The former is special in that if the
- * curi credentials
- * succeed, then the caller needs to promote them from the CrawlURI to the
- * CrawlServer so they are available for all subsequent CrawlURIs on this
- * server.
- */
- private boolean populateCredentials(CrawlURI curi, HttpMethod method) {
- // First look at the server avatars. Add any that are to be volunteered
- // on every request (e.g. RFC2617 credentials). Every time creds will
- // return true when we call 'isEveryTime().
- String serverKey;
- try {
- serverKey = CrawlServer.getServerKey(curi.getUURI());
- } catch (URIException e) {
- return false;
- }
- CrawlServer server = serverCache.getServerFor(serverKey);
- if (server.hasCredentials()) {
- for (Credential cred : server.getCredentials()) {
- if (cred.isEveryTime()) {
- CommonsHttpCredentialUtil.populate(curi, this.http, method, cred, server.getHttpAuthChallenges());
- }
- }
- }
-
- boolean result = false;
-
- // Now look in the curi. The Curi will have credentials loaded either
- // by the handle401 method if its a rfc2617 or it'll have been set into
- // the curi by the preconditionenforcer as this login uri came through.
- for (Credential c: curi.getCredentials()) {
- if (CommonsHttpCredentialUtil.populate(curi, this.http, method, c, curi.getHttpAuthChallenges())) {
- result = true;
- }
- }
-
- return result;
- }
-
- /**
- * Promote successful credential to the server.
- *
- * @param curi
- * CrawlURI whose credentials we are to promote.
- */
- private void promoteCredentials(final CrawlURI curi) {
- Set credentials = curi.getCredentials();
- for (Iterator i = credentials.iterator(); i.hasNext();) {
- Credential c = i.next();
- i.remove();
- // The server to attach too may not be the server that hosts
- // this passed curi. It might be of another subdomain.
- // The avatar needs to be added to the server that is dependent
- // on this precondition. Find it by name. Get the name from
- // the credential this avatar represents.
- String cd = c.getDomain();
- if (cd != null) {
- CrawlServer cs = serverCache.getServerFor(cd);
- if (cs != null) {
- cs.addCredential(c);
- cs.setHttpAuthChallenges(curi.getHttpAuthChallenges());
- }
- }
- }
- }
-
- /**
- * Server is looking for basic/digest auth credentials (RFC2617). If we have
- * any, put them into the CrawlURI and have it come around again.
- * Presence of the credential serves as flag to frontier to requeue
- * promptly. If we already tried this domain and still got a 401, then our
- * credentials are bad. Remove them and let this curi die.
- *
- * @param method
- * Method that got a 401.
- * @param curi
- * CrawlURI that got a 401.
- */
- protected void handle401(final HttpMethod method, final CrawlURI curi) {
- AuthScheme authscheme = getAuthScheme(method, curi);
- if (authscheme == null) {
- return;
- }
- String realm = authscheme.getRealm();
-
- // Look to see if this curi had rfc2617 avatars loaded. If so, are
- // any of them for this realm? If so, then the credential failed
- // if we got a 401 and it should be let die a natural 401 death.
- Set curiRfc2617Credentials = getCredentials(curi,
- HttpAuthenticationCredential.class);
- HttpAuthenticationCredential extant = HttpAuthenticationCredential.getByRealm(
- curiRfc2617Credentials, realm, curi);
- if (extant != null) {
- // Then, already tried this credential. Remove ANY rfc2617
- // credential since presence of a rfc2617 credential serves
- // as flag to frontier to requeue this curi and let the curi
- // die a natural death.
- extant.detachAll(curi);
- logger.warning("Auth failed (401) though supplied realm " + realm
- + " to " + curi.toString());
- } else {
- // Look see if we have a credential that corresponds to this
- // realm in credential store. Filter by type and credential
- // domain. If not, let this curi die. Else, add it to the
- // curi and let it come around again. Add in the AuthScheme
- // we got too. Its needed when we go to run the Auth on
- // second time around.
- String serverKey = getServerKey(curi);
- CrawlServer server = serverCache.getServerFor(serverKey);
- Set storeRfc2617Credentials = getCredentialStore().subset(curi,
- HttpAuthenticationCredential.class, server.getName());
- if (storeRfc2617Credentials == null
- || storeRfc2617Credentials.size() <= 0) {
- logger.fine("No rfc2617 credentials for " + curi);
- } else {
- HttpAuthenticationCredential found = HttpAuthenticationCredential.getByRealm(
- storeRfc2617Credentials, realm, curi);
- if (found == null) {
- logger.fine("No rfc2617 credentials for realm " + realm
- + " in " + curi);
- } else {
- found.attach(curi);
- logger.fine("Found credential for realm " + realm
- + " in store for " + curi.toString());
- }
- }
- }
- }
-
- /**
- * @param method
- * Method that got a 401.
- * @param curi
- * CrawlURI that got a 401.
- * @return Returns first wholesome authscheme found else null.
- */
- protected AuthScheme getAuthScheme(final HttpMethod method,
- final CrawlURI curi) {
- Header[] headers = method.getResponseHeaders("WWW-Authenticate");
- if (headers == null || headers.length <= 0) {
- logger.fine("We got a 401 but no WWW-Authenticate challenge: "
- + curi.toString());
- return null;
- }
-
- Map authChallenges = null;
- try {
- @SuppressWarnings("unchecked")
- Map parsedChallenges = AuthChallengeParser.parseChallenges(headers);
- authChallenges = parsedChallenges;
-
- // remember WWW-Authenticate headers for later use
- curi.setHttpAuthChallenges(authChallenges);
- } catch (MalformedChallengeException e) {
- logger.fine("Failed challenge parse: " + e.getMessage());
- }
- if (authChallenges == null || authChallenges.size() <= 0) {
- logger.fine("We got a 401 and WWW-Authenticate challenge"
- + " but failed parse of the header " + curi.toString());
- return null;
- }
-
- // XXX there's a lot of overlap below with AuthChallengeProcessor.processChallenge()
-
- AuthScheme result = null;
- // Use the first auth found.
- for (Iterator i = authChallenges.keySet().iterator(); result == null
- && i.hasNext();) {
- String key = (String) i.next();
- String challenge = (String) authChallenges.get(key);
- if (key == null || key.length() <= 0 || challenge == null
- || challenge.length() <= 0) {
- logger.warning("Empty scheme: " + curi.toString() + ": "
- + Arrays.toString(headers));
- continue;
- }
- AuthScheme authscheme;
- try {
- authscheme = AuthPolicy.getAuthScheme(key);
- } catch (IllegalStateException e) {
- logger.info("Unsupported auth scheme '" + key + "' at " + curi + " - " + e);
- continue;
- }
-
- try {
- authscheme.processChallenge(challenge);
- } catch (MalformedChallengeException e) {
- logger.fine(e.getMessage() + " " + curi + " " + Arrays.toString(headers));
- continue;
- }
- if (authscheme.isConnectionBased()) {
- logger.fine("Connection based " + authscheme);
- continue;
- }
-
- if (authscheme.getRealm() == null
- || authscheme.getRealm().length() <= 0) {
- logger.fine("Empty realm " + authscheme + " for " + curi);
- continue;
- }
- result = authscheme;
- }
-
- return result;
- }
-
- /**
- * @param curi
- * CrawlURI that got a 401.
- * @param type
- * Class of credential to get from curi.
- * @return Set of credentials attached to this curi.
- */
- private Set getCredentials(CrawlURI curi, Class type) {
- Set result = null;
-
- if (curi.hasCredentials()) {
- for (Credential c : curi.getCredentials()) {
- if (type.isInstance(c)) {
- if (result == null) {
- result = new HashSet();
- }
- result.add(c);
- }
- }
- }
- return result;
- }
-
- public void start() {
- if(isRunning()) {
- return;
- }
- super.start();
-
- configureHttp();
-
- if (cookieStorage != null) {
- cookieStorage.start();
- http.getState().setCookiesMap(cookieStorage.getCookiesMap());
- }
-
- setSSLFactory();
- }
-
- public boolean isRunning() {
- return this.http != null;
- }
-
- public void stop() {
- if(!isRunning()) {
- return;
- }
- super.stop();
- // At the end save cookies to the file specified in the order file.
- if (cookieStorage != null) {
- Map map = http.getState().getCookiesMap();
- cookieStorage.saveCookiesMap(map);
- cookieStorage.stop();
- }
- cleanupHttp(); // XXX happens at finish; move to teardown?
- }
-
- /**
- * Perform any final cleanup related to the HttpClient instance.
- */
- protected void cleanupHttp() {
- this.http = null;
- }
-
- private void setSSLFactory() {
- // I tried to get the default KeyManagers but doesn't work unless you
- // point at a physical keystore. Passing null seems to do the right
- // thing so we'll go w/ that.
- try {
- SSLContext context = SSLContext.getInstance("SSL");
- context.init(null,
- new TrustManager[] { new ConfigurableX509TrustManager(
- getSslTrustLevel()) }, null);
- this.sslfactory = context.getSocketFactory();
- } catch (Exception e) {
- logger.log(Level.WARNING, "Failed configure of ssl context "
- + e.getMessage(), e);
- }
-
- }
-
- protected void configureHttp() {
- int soTimeout = getSoTimeoutMs();
- String addressStr = getHttpBindAddress();
- String proxy = getHttpProxyHost();
- int port = -1;
- String user = "";
- String password = "";
- if (proxy.length() == 0) {
- proxy = null;
- } else {
- port = getHttpProxyPort();
- user = getHttpProxyUser();
- password = getHttpProxyPassword();
- }
- configureHttp(soTimeout, addressStr, proxy, port, user, password);
- }
-
- protected void configureHttp(int soTimeout, String addressStr,
- String proxy, int port, String user, String password) {
- // Get timeout. Use it for socket and for connection timeout.
- int timeout = (soTimeout > 0) ? soTimeout : 0;
-
- // HttpConnectionManager cm = new ThreadLocalHttpConnectionManager();
- HttpConnectionManager cm = new SingleHttpConnectionManager();
-
- // TODO: The following settings should be made in the corresponding
- // HttpConnectionManager, not here.
- HttpConnectionManagerParams hcmp = cm.getParams();
- hcmp.setConnectionTimeout(timeout);
- hcmp.setStaleCheckingEnabled(true);
- // Minimizes bandwidth usage. Setting to true disables Nagle's
- // algorithm. IBM JVMs < 142 give an NPE setting this boolean
- // on ssl sockets.
- hcmp.setTcpNoDelay(false);
-
- this.http = new HttpClient(cm);
- HttpClientParams hcp = this.http.getParams();
- // Set default socket timeout.
- hcp.setSoTimeout(timeout);
- // Set client to be version 1.0.
- hcp.setVersion(HttpVersion.HTTP_1_0);
-
- // configureHttpCookies(defaults);
-
- // Configure how we want the method to act.
- this.http.getParams().setParameter(
- HttpMethodParams.SINGLE_COOKIE_HEADER, new Boolean(true));
- this.http.getParams().setParameter(
- HttpMethodParams.UNAMBIGUOUS_STATUS_LINE, new Boolean(false));
- this.http.getParams().setParameter(
- HttpMethodParams.STRICT_TRANSFER_ENCODING, new Boolean(false));
- this.http.getParams().setIntParameter(
- HttpMethodParams.STATUS_LINE_GARBAGE_LIMIT, 10);
-
- if ((proxy != null) && (proxy.length() == 0)) {
- proxy = null;
- }
- HostConfiguration config = http.getHostConfiguration();
- configureProxy(proxy, port, user, password, config);
- configureBindAddress(addressStr,config);
-
- hcmp.setParameter(SSL_FACTORY_KEY, this.sslfactory);
- }
-
- /*
- * (non-Javadoc)
- *
- * @see org.archive.crawler.framework.Processor#report()
- */
- public String report() {
- StringBuffer ret = new StringBuffer();
- ret.append(super.report());
- ret.append(" Function: Fetch HTTP URIs\n");
- ret.append(" CrawlURIs handled: " + this.getURICount() + "\n");
- ret.append(" Recovery retries: " + this.recoveryRetries + "\n");
-
- return ret.toString();
- }
-
-
- private void setAcceptHeaders(CrawlURI curi, HttpMethod get) {
- if(getAcceptCompression()) {
- // we match the Firefox header exactly (ordering and whitespace)
- // as a favor to caches
- get.setRequestHeader("Accept-Encoding","gzip,deflate");
- }
- List acceptHeaders = getAcceptHeaders();
- if (acceptHeaders.isEmpty()) {
- return;
- }
- for (String hdr : acceptHeaders) {
- String[] nvp = hdr.split(": +");
- if (nvp.length == 2) {
- get.setRequestHeader(nvp[0], nvp[1]);
- } else {
- logger.warning("Invalid accept header: " + hdr);
- }
- }
- }
-
- // custom serialization
-
- private String getLocalAddress() {
- HostConfiguration hc = http.getHostConfiguration();
- if (hc == null) {
- return "";
- }
-
- InetAddress addr = hc.getLocalAddress();
- if (addr == null) {
- return "";
- }
-
- String r = addr.getCanonicalHostName();
- if (r == null) {
- return "";
- }
-
- return r;
- }
-
-
- private String getProxyHost() {
- HostConfiguration hc = http.getHostConfiguration();
- if (hc == null) {
- return "";
- }
-
- String r = hc.getProxyHost();
- if (r == null) {
- return "";
- }
-
- return r;
- }
-
-
- private int getProxyPort() {
- HostConfiguration hc = http.getHostConfiguration();
- if (hc == null) {
- return -1;
- }
-
- return hc.getProxyPort();
- }
-
-
- private String getProxyUser() {
- NTCredentials credentials = (NTCredentials)http.getState().getProxyCredentials(new AuthScope(getProxyHost(), getProxyPort()));
- if (credentials == null) {
- return "";
- }
-
- String r = credentials.getUserName();
- if (r == null) {
- return "";
- }
-
- return r;
- }
-
- private String getProxyPassword() {
- NTCredentials credentials = (NTCredentials)http.getState().getProxyCredentials(new AuthScope(getProxyHost(), getProxyPort()));
- if (credentials == null) {
- return "";
- }
-
- String r = credentials.getPassword();
- if (r == null) {
- return "";
- }
-
- return r;
- }
-
-
- private void writeObject(ObjectOutputStream stream) throws IOException {
- stream.defaultWriteObject();
-
- // Special handling for http since it isn't Serializable itself
- stream.writeInt(http.getParams().getSoTimeout());
- stream.writeUTF(getLocalAddress());
- stream.writeUTF(getProxyHost());
- stream.writeInt(getProxyPort());
- stream.writeUTF(getProxyUser());
- stream.writeUTF(getProxyPassword());
- }
-
-
- private void readObject(ObjectInputStream stream)
- throws IOException, ClassNotFoundException {
- stream.defaultReadObject();
-
- int soTimeout = stream.readInt();
- String localAddress = stream.readUTF();
- String proxy = stream.readUTF();
- int port = stream.readInt();
- String user = stream.readUTF();
- String password = stream.readUTF();
-
- configureHttp(soTimeout, localAddress, proxy, port, user, password);
- setSSLFactory();
- }
-
-
- /**
- * @return Returns the http instance.
- */
- protected HttpClient getHttp() {
- return this.http;
- }
-
- private static String getServerKey(CrawlURI uri) {
- try {
- return CrawlServer.getServerKey(uri.getUURI());
- } catch (URIException e) {
- logger.severe(e.getMessage() + ": " + uri);
- e.printStackTrace();
- return null;
- }
- }
-}
diff --git a/modules/src/main/java/org/archive/modules/fetcher/SimpleCookieStorage.java b/modules/src/main/java/org/archive/modules/fetcher/SimpleCookieStorage.java
deleted file mode 100644
index d815a375..00000000
--- a/modules/src/main/java/org/archive/modules/fetcher/SimpleCookieStorage.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * This file is part of the Heritrix web crawler (crawler.archive.org).
- *
- * Licensed to the Internet Archive (IA) by one or more individual
- * contributors.
- *
- * The IA licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.archive.modules.fetcher;
-
-import java.util.Map;
-import java.util.SortedMap;
-import java.util.TreeMap;
-
-import org.apache.commons.httpclient.Cookie;
-
-public class SimpleCookieStorage extends AbstractCookieStorage {
-
- @SuppressWarnings("unused")
- private static final long serialVersionUID = 1L;
-
- final private SortedMap map = new TreeMap();
-
-
- protected SortedMap prepareMap() {
- return map;
- }
-
-
- public SortedMap getCookiesMap() {
- return map;
- }
-
-
- public void innerSaveCookiesMap(Map map) {
- // no-op
- }
-}