mirror of
https://github.com/internetarchive/heritrix3.git
synced 2026-09-24 06:36:12 +00:00
bdb cookie store for FetchHTTP2
This commit is contained in:
Noah Levitt
@@ -96,7 +96,8 @@ public class RecordingHttpClient extends DefaultHttpClient {
|
||||
return new BasicClientConnectionManager(SchemeRegistryFactory.createDefault()) {
|
||||
@Override
|
||||
protected ClientConnectionOperator createConnectionOperator(SchemeRegistry schreg) {
|
||||
return new RecordingClientConnectionOperator(schreg, new ServerCacheResolver(getServerCache()));
|
||||
return new RecordingClientConnectionOperator(schreg,
|
||||
new ServerCacheResolver(RecordingHttpClient.this.getServerCache()));
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
-4
@@ -11,7 +11,3 @@ org.archive.level = INFO
|
||||
handlers = java.util.logging.ConsoleHandler
|
||||
java.util.logging.ConsoleHandler.level = ALL
|
||||
java.util.logging.ConsoleHandler.formatter= org.archive.util.OneLineSimpleLogger
|
||||
|
||||
org.apache.http.level = ALL
|
||||
org.archive.modules.fetcher.level = ALL
|
||||
# org.apache.http.impl.conn.level = ALL
|
||||
|
||||
@@ -0,0 +1,193 @@
|
||||
package org.archive.modules.fetcher;
|
||||
|
||||
import it.unimi.dsi.mg4j.util.MutableString;
|
||||
|
||||
import java.io.BufferedReader;
|
||||
import java.io.Closeable;
|
||||
import java.io.File;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.Reader;
|
||||
import java.util.Date;
|
||||
import java.util.Set;
|
||||
import java.util.logging.Level;
|
||||
import java.util.logging.Logger;
|
||||
|
||||
import org.apache.commons.io.IOUtils;
|
||||
import org.apache.http.client.CookieStore;
|
||||
import org.apache.http.cookie.Cookie;
|
||||
import org.apache.http.impl.cookie.BasicClientCookie;
|
||||
import org.archive.checkpointing.Checkpointable;
|
||||
import org.archive.spring.ConfigFile;
|
||||
import org.archive.spring.ConfigPath;
|
||||
import org.springframework.context.Lifecycle;
|
||||
|
||||
public abstract class AbstractCookieStore implements CookieStore, Lifecycle, Closeable,
|
||||
Checkpointable {
|
||||
|
||||
private static final Logger logger =
|
||||
Logger.getLogger(AbstractCookieStorage.class.getName());
|
||||
|
||||
protected ConfigFile cookiesLoadFile = null;
|
||||
public ConfigFile getCookiesLoadFile() {
|
||||
return cookiesLoadFile;
|
||||
}
|
||||
public void setCookiesLoadFile(ConfigFile cookiesLoadFile) {
|
||||
this.cookiesLoadFile = cookiesLoadFile;
|
||||
}
|
||||
|
||||
protected ConfigPath cookiesSaveFile = null;
|
||||
public ConfigPath getCookiesSaveFile() {
|
||||
return cookiesSaveFile;
|
||||
}
|
||||
public void setCookiesSaveFile(ConfigPath cookiesSaveFile) {
|
||||
this.cookiesSaveFile = cookiesSaveFile;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() throws IOException {
|
||||
// XXX only here because old BdbCookie also implements Closeable and does nothing... why?
|
||||
}
|
||||
|
||||
protected boolean isRunning = false;
|
||||
|
||||
@Override
|
||||
public void start() {
|
||||
if (isRunning()) {
|
||||
return;
|
||||
}
|
||||
prepare();
|
||||
if (getCookiesLoadFile()!=null) {
|
||||
loadCookies(getCookiesLoadFile());
|
||||
}
|
||||
isRunning = true;
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public void stop() {
|
||||
isRunning = false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isRunning() {
|
||||
return isRunning;
|
||||
}
|
||||
|
||||
public void saveCookies() {
|
||||
if (getCookiesSaveFile()!=null) {
|
||||
saveCookies(getCookiesSaveFile().getFile().getAbsolutePath());
|
||||
}
|
||||
}
|
||||
|
||||
protected void loadCookies(ConfigFile file) {
|
||||
Reader reader = null;
|
||||
try {
|
||||
reader = file.obtainReader();
|
||||
loadCookies(reader);
|
||||
} finally {
|
||||
IOUtils.closeQuietly(reader);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* Load cookies. The input is text in the Netscape's 'cookies.txt' file
|
||||
* format. Example entry of cookies.txt file:
|
||||
* <p>
|
||||
* www.archive.org FALSE / FALSE 1311699995 details-visit texts-cralond
|
||||
* </p>
|
||||
* <p>
|
||||
* Each line has 7 tab-separated fields:
|
||||
* </p>
|
||||
* <ol>
|
||||
* <li>DOMAIN: The domain that created and have access to the cookie value.</li>
|
||||
* <li>FLAG: A TRUE or FALSE value indicating if hosts within the given
|
||||
* domain can access the cookie value.</li>
|
||||
* <li>PATH: The path within the domain that the cookie value is valid for.</li>
|
||||
* <li>SECURE: A TRUE or FALSE value indicating if to use a secure
|
||||
* connection to access the cookie value.</li>
|
||||
* <li>EXPIRATION: The expiration time of the cookie value, or -1 for no
|
||||
* expiration</li>
|
||||
* <li>NAME: The name of the cookie value</li>
|
||||
* <li>VALUE: The cookie value</li>
|
||||
* </ol>
|
||||
*
|
||||
* @param reader
|
||||
* input in the Netscape's 'cookies.txt' format.
|
||||
*/
|
||||
public static void loadCookies(Reader reader, Set<Cookie> cookies) {
|
||||
BufferedReader br = new BufferedReader(reader);
|
||||
try {
|
||||
String line;
|
||||
int lineNo = 1;
|
||||
while ((line = br.readLine()) != null) {
|
||||
if (!line.matches("\\s*(?:#.*)?")) { // skip blank links and comments
|
||||
String[] tokens = line.split("\\t");
|
||||
if (tokens.length == 7) {
|
||||
long epochSeconds = Long.parseLong(tokens[4]);
|
||||
Date expirationDate = (epochSeconds >= 0 ? new Date(epochSeconds * 1000) : null);
|
||||
BasicClientCookie cookie = new BasicClientCookie(tokens[5], tokens[6]);
|
||||
cookie.setDomain(tokens[0]);
|
||||
cookie.setExpiryDate(expirationDate);
|
||||
cookie.setSecure(Boolean.valueOf(tokens[3]).booleanValue());
|
||||
cookie.setPath(tokens[2]);
|
||||
// XXX httpclient cookie doesn't have this thing?
|
||||
// cookie.setDomainAttributeSpecified(Boolean.valueOf(tokens[1]).booleanValue());
|
||||
logger.fine("Adding cookie: domain " + cookie.getDomain() + " cookie " + cookie);
|
||||
cookies.add(cookie);
|
||||
} else {
|
||||
logger.warning("cookies input line " + lineNo + " invalid, expected 7 tab-delimited tokens");
|
||||
}
|
||||
}
|
||||
|
||||
lineNo++;
|
||||
}
|
||||
} catch (IOException e) {
|
||||
logger.log(Level.WARNING,e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
|
||||
public static void saveCookies(String saveCookiesFile, Set<Cookie> cookies) {
|
||||
// Do nothing if cookiesFile is not specified.
|
||||
if (saveCookiesFile == null || saveCookiesFile.length() <= 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
FileOutputStream out = null;
|
||||
try {
|
||||
out = new FileOutputStream(new File(saveCookiesFile));
|
||||
String tab ="\t";
|
||||
out.write("# Heritrix Cookie File\n".getBytes());
|
||||
out.write("# This file is the Netscape cookies.txt format\n\n".getBytes());
|
||||
for (Cookie cookie: cookies) {
|
||||
// Guess an initial size
|
||||
MutableString line = new MutableString(1024 * 2);
|
||||
line.append(cookie.getDomain());
|
||||
line.append(tab);
|
||||
// XXX line.append(cookie.isDomainAttributeSpecified() ? "TRUE" : "FALSE");
|
||||
line.append("TRUE");
|
||||
line.append(tab);
|
||||
line.append(cookie.getPath());
|
||||
line.append(tab);
|
||||
line.append(cookie.isSecure() ? "TRUE" : "FALSE");
|
||||
line.append(tab);
|
||||
line.append(cookie.getExpiryDate() != null ? cookie.getExpiryDate().getTime() / 1000 : -1);
|
||||
line.append(tab);
|
||||
line.append(cookie.getName());
|
||||
line.append(tab);
|
||||
line.append(cookie.getValue() != null ? cookie.getValue() : "");
|
||||
line.append("\n");
|
||||
out.write(line.toString().getBytes());
|
||||
}
|
||||
} catch (IOException e) {
|
||||
logger.log(Level.SEVERE, "Unable to write " + saveCookiesFile, e);
|
||||
} finally {
|
||||
IOUtils.closeQuietly(out);
|
||||
}
|
||||
}
|
||||
|
||||
abstract protected void prepare();
|
||||
abstract protected void loadCookies(Reader reader);
|
||||
abstract protected void saveCookies(String absolutePath);
|
||||
}
|
||||
@@ -0,0 +1,150 @@
|
||||
package org.archive.modules.fetcher;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.Reader;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Date;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.http.cookie.Cookie;
|
||||
import org.archive.bdb.BdbModule;
|
||||
import org.archive.checkpointing.Checkpoint;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
|
||||
import com.sleepycat.bind.serial.SerialBinding;
|
||||
import com.sleepycat.bind.serial.StoredClassCatalog;
|
||||
import com.sleepycat.collections.StoredSortedKeySet;
|
||||
import com.sleepycat.je.Database;
|
||||
import com.sleepycat.je.DatabaseException;
|
||||
|
||||
public class BdbCookieStore extends AbstractCookieStore {
|
||||
|
||||
protected BdbModule bdb;
|
||||
@Autowired
|
||||
public void setBdbModule(BdbModule bdb) {
|
||||
this.bdb = bdb;
|
||||
}
|
||||
|
||||
/** are we a checkpoint recovery? (in which case, reuse stored cookie data?) */
|
||||
protected boolean isCheckpointRecovery = false;
|
||||
|
||||
public static String COOKIEDB_NAME = "hc_httpclient_cookies";
|
||||
|
||||
private transient Database cookieDb;
|
||||
private transient StoredSortedKeySet<Cookie> cookies;
|
||||
|
||||
public void prepare() {
|
||||
try {
|
||||
StoredClassCatalog classCatalog = bdb.getClassCatalog();
|
||||
BdbModule.BdbConfig dbConfig = new BdbModule.BdbConfig();
|
||||
dbConfig.setTransactional(false);
|
||||
dbConfig.setAllowCreate(true);
|
||||
cookieDb = bdb.openDatabase(COOKIEDB_NAME, dbConfig,
|
||||
isCheckpointRecovery);
|
||||
cookies = new StoredSortedKeySet<Cookie>(cookieDb,
|
||||
new SerialBinding<Cookie>(classCatalog, Cookie.class), true);
|
||||
} catch (DatabaseException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Adds an {@link Cookie HTTP cookie}, replacing any existing equivalent cookies.
|
||||
* If the given cookie has already expired it will not be added, but existing
|
||||
* values will still be removed.
|
||||
*
|
||||
* @param cookie the {@link Cookie cookie} to be added
|
||||
*/
|
||||
@Override
|
||||
public synchronized void addCookie(Cookie cookie) {
|
||||
if (cookie != null) {
|
||||
// first remove any old cookie that is equivalent
|
||||
cookies.remove(cookie);
|
||||
if (!cookie.isExpired(new Date())) {
|
||||
cookies.add(cookie);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns an immutable array of {@link Cookie cookies} that this HTTP
|
||||
* state currently contains.
|
||||
*
|
||||
* @return an array of {@link Cookie cookies}.
|
||||
*/
|
||||
@Override
|
||||
public synchronized List<Cookie> getCookies() {
|
||||
if (cookies != null) {
|
||||
//create defensive copy so it won't be concurrently modified
|
||||
return new ArrayList<Cookie>(cookies);
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Removes all of {@link Cookie cookies} in this HTTP state
|
||||
* that have expired by the specified {@link java.util.Date date}.
|
||||
*
|
||||
* @return true if any cookies were purged.
|
||||
*
|
||||
* @see Cookie#isExpired(Date)
|
||||
*/
|
||||
@Override
|
||||
public synchronized boolean clearExpired(final Date date) {
|
||||
if (date == null) {
|
||||
return false;
|
||||
}
|
||||
boolean removed = false;
|
||||
for (Iterator<Cookie> it = cookies.iterator(); it.hasNext();) {
|
||||
if (it.next().isExpired(date)) {
|
||||
it.remove();
|
||||
removed = true;
|
||||
}
|
||||
}
|
||||
return removed;
|
||||
}
|
||||
|
||||
/**
|
||||
* Clears all cookies.
|
||||
*/
|
||||
@Override
|
||||
public synchronized void clear() {
|
||||
cookies.clear();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void startCheckpoint(Checkpoint checkpointInProgress) {
|
||||
// do nothing; handled by map checkpoint via BdbModule
|
||||
}
|
||||
|
||||
@Override
|
||||
public void doCheckpoint(Checkpoint checkpointInProgress)
|
||||
throws IOException {
|
||||
// do nothing; handled by map checkpoint via BdbModule
|
||||
}
|
||||
|
||||
@Override
|
||||
public void finishCheckpoint(Checkpoint checkpointInProgress) {
|
||||
// do nothing; handled by map checkpoint via BdbModule
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setRecoveryCheckpoint(Checkpoint recoveryCheckpoint) {
|
||||
// just remember that we are doing checkpoint-recovery;
|
||||
// actual state recovery happens via BdbModule
|
||||
isCheckpointRecovery = true;
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
protected void loadCookies(Reader reader) {
|
||||
loadCookies(reader, cookies);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void saveCookies(String absolutePath) {
|
||||
saveCookies(absolutePath, cookies);
|
||||
}
|
||||
}
|
||||
@@ -41,13 +41,11 @@ import org.apache.http.HttpResponse;
|
||||
import org.apache.http.HttpStatus;
|
||||
import org.apache.http.HttpVersion;
|
||||
import org.apache.http.client.ClientProtocolException;
|
||||
import org.apache.http.client.HttpClient;
|
||||
import org.apache.http.client.methods.HttpGet;
|
||||
import org.apache.http.client.methods.HttpPost;
|
||||
import org.apache.http.client.methods.HttpRequestBase;
|
||||
import org.apache.http.client.params.HttpClientParams;
|
||||
import org.apache.http.entity.ContentType;
|
||||
import org.apache.http.impl.client.DefaultHttpClient;
|
||||
import org.apache.http.message.BasicHeader;
|
||||
import org.apache.http.params.HttpProtocolParams;
|
||||
import org.apache.http.protocol.HTTP;
|
||||
@@ -68,7 +66,7 @@ public class FetchHTTP2 extends Processor implements Lifecycle {
|
||||
|
||||
private static Logger logger = Logger.getLogger(FetchHTTP2.class.getName());
|
||||
|
||||
protected DefaultHttpClient httpClient;
|
||||
protected RecordingHttpClient httpClient;
|
||||
|
||||
public static final String REFERER = "Referer";
|
||||
public static final String RANGE = "Range";
|
||||
@@ -229,6 +227,15 @@ public class FetchHTTP2 extends Processor implements Lifecycle {
|
||||
kp.put("acceptHeaders",headers);
|
||||
}
|
||||
|
||||
protected AbstractCookieStore cookieStore;
|
||||
@Autowired(required=false)
|
||||
public void setCookieStore(AbstractCookieStore store) {
|
||||
this.cookieStore = store;
|
||||
}
|
||||
public AbstractCookieStore getCookieStore() {
|
||||
return cookieStore;
|
||||
}
|
||||
|
||||
protected static final Header HEADER_SEND_CONNECTION_CLOSE = new BasicHeader(
|
||||
HTTP.CONN_DIRECTIVE, HTTP.CONN_CLOSE);
|
||||
|
||||
@@ -475,7 +482,7 @@ public class FetchHTTP2 extends Processor implements Lifecycle {
|
||||
}
|
||||
}
|
||||
|
||||
protected HttpClient getHttpClient() {
|
||||
protected RecordingHttpClient getHttpClient() {
|
||||
if (httpClient == null) {
|
||||
httpClient = new RecordingHttpClient(getServerCache());
|
||||
}
|
||||
@@ -573,4 +580,38 @@ public class FetchHTTP2 extends Processor implements Lifecycle {
|
||||
curi.setFetchStatus(status);
|
||||
curi.getRecorder().close();
|
||||
}
|
||||
|
||||
public void start() {
|
||||
if(isRunning()) {
|
||||
return;
|
||||
}
|
||||
super.start();
|
||||
|
||||
// configureHttp();
|
||||
|
||||
if (getCookieStore() != null) {
|
||||
getCookieStore().start();
|
||||
getHttpClient().setCookieStore(getCookieStore());
|
||||
}
|
||||
|
||||
// setSSLFactory();
|
||||
}
|
||||
|
||||
public boolean isRunning() {
|
||||
return this.httpClient != null;
|
||||
}
|
||||
|
||||
public void stop() {
|
||||
if (!isRunning()) {
|
||||
return;
|
||||
}
|
||||
super.stop();
|
||||
// At the end save cookies to the file specified in the order file.
|
||||
if (cookieStore != null) {
|
||||
cookieStore.saveCookies();
|
||||
cookieStore.stop();
|
||||
}
|
||||
// cleanupHttp(); // XXX happens at finish; move to teardown?
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user