diff --git a/commons/src/main/java/org/archive/net/http/RecordingHttpClient.java b/commons/src/main/java/org/archive/net/http/RecordingHttpClient.java
index e0855f96..e1eb163e 100644
--- a/commons/src/main/java/org/archive/net/http/RecordingHttpClient.java
+++ b/commons/src/main/java/org/archive/net/http/RecordingHttpClient.java
@@ -96,7 +96,8 @@ public class RecordingHttpClient extends DefaultHttpClient {
return new BasicClientConnectionManager(SchemeRegistryFactory.createDefault()) {
@Override
protected ClientConnectionOperator createConnectionOperator(SchemeRegistry schreg) {
- return new RecordingClientConnectionOperator(schreg, new ServerCacheResolver(getServerCache()));
+ return new RecordingClientConnectionOperator(schreg,
+ new ServerCacheResolver(RecordingHttpClient.this.getServerCache()));
}
};
}
diff --git a/dist/src/main/conf/logging.properties b/dist/src/main/conf/logging.properties
index 0238eeed..5a8e7af6 100644
--- a/dist/src/main/conf/logging.properties
+++ b/dist/src/main/conf/logging.properties
@@ -11,7 +11,3 @@ org.archive.level = INFO
handlers = java.util.logging.ConsoleHandler
java.util.logging.ConsoleHandler.level = ALL
java.util.logging.ConsoleHandler.formatter= org.archive.util.OneLineSimpleLogger
-
-org.apache.http.level = ALL
-org.archive.modules.fetcher.level = ALL
-# org.apache.http.impl.conn.level = ALL
diff --git a/modules/src/main/java/org/archive/modules/fetcher/AbstractCookieStore.java b/modules/src/main/java/org/archive/modules/fetcher/AbstractCookieStore.java
new file mode 100644
index 00000000..d964e9b3
--- /dev/null
+++ b/modules/src/main/java/org/archive/modules/fetcher/AbstractCookieStore.java
@@ -0,0 +1,193 @@
+package org.archive.modules.fetcher;
+
+import it.unimi.dsi.mg4j.util.MutableString;
+
+import java.io.BufferedReader;
+import java.io.Closeable;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.Reader;
+import java.util.Date;
+import java.util.Set;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.http.client.CookieStore;
+import org.apache.http.cookie.Cookie;
+import org.apache.http.impl.cookie.BasicClientCookie;
+import org.archive.checkpointing.Checkpointable;
+import org.archive.spring.ConfigFile;
+import org.archive.spring.ConfigPath;
+import org.springframework.context.Lifecycle;
+
+public abstract class AbstractCookieStore implements CookieStore, Lifecycle, Closeable,
+ Checkpointable {
+
+ private static final Logger logger =
+ Logger.getLogger(AbstractCookieStorage.class.getName());
+
+ protected ConfigFile cookiesLoadFile = null;
+ public ConfigFile getCookiesLoadFile() {
+ return cookiesLoadFile;
+ }
+ public void setCookiesLoadFile(ConfigFile cookiesLoadFile) {
+ this.cookiesLoadFile = cookiesLoadFile;
+ }
+
+ protected ConfigPath cookiesSaveFile = null;
+ public ConfigPath getCookiesSaveFile() {
+ return cookiesSaveFile;
+ }
+ public void setCookiesSaveFile(ConfigPath cookiesSaveFile) {
+ this.cookiesSaveFile = cookiesSaveFile;
+ }
+
+ @Override
+ public void close() throws IOException {
+ // XXX only here because old BdbCookie also implements Closeable and does nothing... why?
+ }
+
+ protected boolean isRunning = false;
+
+ @Override
+ public void start() {
+ if (isRunning()) {
+ return;
+ }
+ prepare();
+ if (getCookiesLoadFile()!=null) {
+ loadCookies(getCookiesLoadFile());
+ }
+ isRunning = true;
+ }
+
+
+ @Override
+ public void stop() {
+ isRunning = false;
+ }
+
+ @Override
+ public boolean isRunning() {
+ return isRunning;
+ }
+
+ public void saveCookies() {
+ if (getCookiesSaveFile()!=null) {
+ saveCookies(getCookiesSaveFile().getFile().getAbsolutePath());
+ }
+ }
+
+ protected void loadCookies(ConfigFile file) {
+ Reader reader = null;
+ try {
+ reader = file.obtainReader();
+ loadCookies(reader);
+ } finally {
+ IOUtils.closeQuietly(reader);
+ }
+
+ }
+
+ /**
+ * Load cookies. The input is text in the Netscape's 'cookies.txt' file
+ * format. Example entry of cookies.txt file:
+ *
+ * www.archive.org FALSE / FALSE 1311699995 details-visit texts-cralond
+ *
+ *
+ * Each line has 7 tab-separated fields:
+ *
+ *
+ * - DOMAIN: The domain that created and have access to the cookie value.
+ * - FLAG: A TRUE or FALSE value indicating if hosts within the given
+ * domain can access the cookie value.
+ * - PATH: The path within the domain that the cookie value is valid for.
+ * - SECURE: A TRUE or FALSE value indicating if to use a secure
+ * connection to access the cookie value.
+ * - EXPIRATION: The expiration time of the cookie value, or -1 for no
+ * expiration
+ * - NAME: The name of the cookie value
+ * - VALUE: The cookie value
+ *
+ *
+ * @param reader
+ * input in the Netscape's 'cookies.txt' format.
+ */
+ public static void loadCookies(Reader reader, Set cookies) {
+ BufferedReader br = new BufferedReader(reader);
+ try {
+ String line;
+ int lineNo = 1;
+ while ((line = br.readLine()) != null) {
+ if (!line.matches("\\s*(?:#.*)?")) { // skip blank links and comments
+ String[] tokens = line.split("\\t");
+ if (tokens.length == 7) {
+ long epochSeconds = Long.parseLong(tokens[4]);
+ Date expirationDate = (epochSeconds >= 0 ? new Date(epochSeconds * 1000) : null);
+ BasicClientCookie cookie = new BasicClientCookie(tokens[5], tokens[6]);
+ cookie.setDomain(tokens[0]);
+ cookie.setExpiryDate(expirationDate);
+ cookie.setSecure(Boolean.valueOf(tokens[3]).booleanValue());
+ cookie.setPath(tokens[2]);
+// XXX httpclient cookie doesn't have this thing?
+// cookie.setDomainAttributeSpecified(Boolean.valueOf(tokens[1]).booleanValue());
+ logger.fine("Adding cookie: domain " + cookie.getDomain() + " cookie " + cookie);
+ cookies.add(cookie);
+ } else {
+ logger.warning("cookies input line " + lineNo + " invalid, expected 7 tab-delimited tokens");
+ }
+ }
+
+ lineNo++;
+ }
+ } catch (IOException e) {
+ logger.log(Level.WARNING,e.getMessage(), e);
+ }
+ }
+
+ public static void saveCookies(String saveCookiesFile, Set cookies) {
+ // Do nothing if cookiesFile is not specified.
+ if (saveCookiesFile == null || saveCookiesFile.length() <= 0) {
+ return;
+ }
+
+ FileOutputStream out = null;
+ try {
+ out = new FileOutputStream(new File(saveCookiesFile));
+ String tab ="\t";
+ out.write("# Heritrix Cookie File\n".getBytes());
+ out.write("# This file is the Netscape cookies.txt format\n\n".getBytes());
+ for (Cookie cookie: cookies) {
+ // Guess an initial size
+ MutableString line = new MutableString(1024 * 2);
+ line.append(cookie.getDomain());
+ line.append(tab);
+ // XXX line.append(cookie.isDomainAttributeSpecified() ? "TRUE" : "FALSE");
+ line.append("TRUE");
+ line.append(tab);
+ line.append(cookie.getPath());
+ line.append(tab);
+ line.append(cookie.isSecure() ? "TRUE" : "FALSE");
+ line.append(tab);
+ line.append(cookie.getExpiryDate() != null ? cookie.getExpiryDate().getTime() / 1000 : -1);
+ line.append(tab);
+ line.append(cookie.getName());
+ line.append(tab);
+ line.append(cookie.getValue() != null ? cookie.getValue() : "");
+ line.append("\n");
+ out.write(line.toString().getBytes());
+ }
+ } catch (IOException e) {
+ logger.log(Level.SEVERE, "Unable to write " + saveCookiesFile, e);
+ } finally {
+ IOUtils.closeQuietly(out);
+ }
+ }
+
+ abstract protected void prepare();
+ abstract protected void loadCookies(Reader reader);
+ abstract protected void saveCookies(String absolutePath);
+}
diff --git a/modules/src/main/java/org/archive/modules/fetcher/BdbCookieStore.java b/modules/src/main/java/org/archive/modules/fetcher/BdbCookieStore.java
new file mode 100644
index 00000000..dec2b124
--- /dev/null
+++ b/modules/src/main/java/org/archive/modules/fetcher/BdbCookieStore.java
@@ -0,0 +1,150 @@
+package org.archive.modules.fetcher;
+
+import java.io.IOException;
+import java.io.Reader;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.http.cookie.Cookie;
+import org.archive.bdb.BdbModule;
+import org.archive.checkpointing.Checkpoint;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import com.sleepycat.bind.serial.SerialBinding;
+import com.sleepycat.bind.serial.StoredClassCatalog;
+import com.sleepycat.collections.StoredSortedKeySet;
+import com.sleepycat.je.Database;
+import com.sleepycat.je.DatabaseException;
+
+public class BdbCookieStore extends AbstractCookieStore {
+
+ protected BdbModule bdb;
+ @Autowired
+ public void setBdbModule(BdbModule bdb) {
+ this.bdb = bdb;
+ }
+
+ /** are we a checkpoint recovery? (in which case, reuse stored cookie data?) */
+ protected boolean isCheckpointRecovery = false;
+
+ public static String COOKIEDB_NAME = "hc_httpclient_cookies";
+
+ private transient Database cookieDb;
+ private transient StoredSortedKeySet cookies;
+
+ public void prepare() {
+ try {
+ StoredClassCatalog classCatalog = bdb.getClassCatalog();
+ BdbModule.BdbConfig dbConfig = new BdbModule.BdbConfig();
+ dbConfig.setTransactional(false);
+ dbConfig.setAllowCreate(true);
+ cookieDb = bdb.openDatabase(COOKIEDB_NAME, dbConfig,
+ isCheckpointRecovery);
+ cookies = new StoredSortedKeySet(cookieDb,
+ new SerialBinding(classCatalog, Cookie.class), true);
+ } catch (DatabaseException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ * Adds an {@link Cookie HTTP cookie}, replacing any existing equivalent cookies.
+ * If the given cookie has already expired it will not be added, but existing
+ * values will still be removed.
+ *
+ * @param cookie the {@link Cookie cookie} to be added
+ */
+ @Override
+ public synchronized void addCookie(Cookie cookie) {
+ if (cookie != null) {
+ // first remove any old cookie that is equivalent
+ cookies.remove(cookie);
+ if (!cookie.isExpired(new Date())) {
+ cookies.add(cookie);
+ }
+ }
+ }
+
+ /**
+ * Returns an immutable array of {@link Cookie cookies} that this HTTP
+ * state currently contains.
+ *
+ * @return an array of {@link Cookie cookies}.
+ */
+ @Override
+ public synchronized List getCookies() {
+ if (cookies != null) {
+ //create defensive copy so it won't be concurrently modified
+ return new ArrayList(cookies);
+ } else {
+ return null;
+ }
+ }
+
+ /**
+ * Removes all of {@link Cookie cookies} in this HTTP state
+ * that have expired by the specified {@link java.util.Date date}.
+ *
+ * @return true if any cookies were purged.
+ *
+ * @see Cookie#isExpired(Date)
+ */
+ @Override
+ public synchronized boolean clearExpired(final Date date) {
+ if (date == null) {
+ return false;
+ }
+ boolean removed = false;
+ for (Iterator it = cookies.iterator(); it.hasNext();) {
+ if (it.next().isExpired(date)) {
+ it.remove();
+ removed = true;
+ }
+ }
+ return removed;
+ }
+
+ /**
+ * Clears all cookies.
+ */
+ @Override
+ public synchronized void clear() {
+ cookies.clear();
+ }
+
+ @Override
+ public void startCheckpoint(Checkpoint checkpointInProgress) {
+ // do nothing; handled by map checkpoint via BdbModule
+ }
+
+ @Override
+ public void doCheckpoint(Checkpoint checkpointInProgress)
+ throws IOException {
+ // do nothing; handled by map checkpoint via BdbModule
+ }
+
+ @Override
+ public void finishCheckpoint(Checkpoint checkpointInProgress) {
+ // do nothing; handled by map checkpoint via BdbModule
+ }
+
+ @Override
+ public void setRecoveryCheckpoint(Checkpoint recoveryCheckpoint) {
+ // just remember that we are doing checkpoint-recovery;
+ // actual state recovery happens via BdbModule
+ isCheckpointRecovery = true;
+ }
+
+
+ @Override
+ protected void loadCookies(Reader reader) {
+ loadCookies(reader, cookies);
+ }
+
+ @Override
+ protected void saveCookies(String absolutePath) {
+ saveCookies(absolutePath, cookies);
+ }
+}
diff --git a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP2.java b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP2.java
index f7f0d7a0..3bf48ddc 100644
--- a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP2.java
+++ b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP2.java
@@ -41,13 +41,11 @@ import org.apache.http.HttpResponse;
import org.apache.http.HttpStatus;
import org.apache.http.HttpVersion;
import org.apache.http.client.ClientProtocolException;
-import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.methods.HttpRequestBase;
import org.apache.http.client.params.HttpClientParams;
import org.apache.http.entity.ContentType;
-import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.message.BasicHeader;
import org.apache.http.params.HttpProtocolParams;
import org.apache.http.protocol.HTTP;
@@ -68,7 +66,7 @@ public class FetchHTTP2 extends Processor implements Lifecycle {
private static Logger logger = Logger.getLogger(FetchHTTP2.class.getName());
- protected DefaultHttpClient httpClient;
+ protected RecordingHttpClient httpClient;
public static final String REFERER = "Referer";
public static final String RANGE = "Range";
@@ -229,6 +227,15 @@ public class FetchHTTP2 extends Processor implements Lifecycle {
kp.put("acceptHeaders",headers);
}
+ protected AbstractCookieStore cookieStore;
+ @Autowired(required=false)
+ public void setCookieStore(AbstractCookieStore store) {
+ this.cookieStore = store;
+ }
+ public AbstractCookieStore getCookieStore() {
+ return cookieStore;
+ }
+
protected static final Header HEADER_SEND_CONNECTION_CLOSE = new BasicHeader(
HTTP.CONN_DIRECTIVE, HTTP.CONN_CLOSE);
@@ -475,7 +482,7 @@ public class FetchHTTP2 extends Processor implements Lifecycle {
}
}
- protected HttpClient getHttpClient() {
+ protected RecordingHttpClient getHttpClient() {
if (httpClient == null) {
httpClient = new RecordingHttpClient(getServerCache());
}
@@ -573,4 +580,38 @@ public class FetchHTTP2 extends Processor implements Lifecycle {
curi.setFetchStatus(status);
curi.getRecorder().close();
}
+
+ public void start() {
+ if(isRunning()) {
+ return;
+ }
+ super.start();
+
+ // configureHttp();
+
+ if (getCookieStore() != null) {
+ getCookieStore().start();
+ getHttpClient().setCookieStore(getCookieStore());
+ }
+
+ // setSSLFactory();
+ }
+
+ public boolean isRunning() {
+ return this.httpClient != null;
+ }
+
+ public void stop() {
+ if (!isRunning()) {
+ return;
+ }
+ super.stop();
+ // At the end save cookies to the file specified in the order file.
+ if (cookieStore != null) {
+ cookieStore.saveCookies();
+ cookieStore.stop();
+ }
+ // cleanupHttp(); // XXX happens at finish; move to teardown?
+ }
+
}