Merge pull request #30 from nlevitt/fix-bdb-cookie-store

Fix bdb cookie store (cookie is piling up due to lack of proper equality method in Cookie)
This commit is contained in:
Kenji Nagahashi
2014-01-21 12:01:10 -08:00
4 changed files with 356 additions and 20 deletions
@@ -49,7 +49,9 @@ public class ConfigFile extends ConfigPath implements ReadSource, WriteTarget {
getFile().createNewFile();
}
configurer.snapshotToLaunchDir(getFile());
if (configurer != null) {
configurer.snapshotToLaunchDir(getFile());
}
return new InputStreamReader(
new FileInputStream(getFile()),
@@ -26,18 +26,22 @@ import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.Reader;
import java.util.Collection;
import java.util.Date;
import java.util.Set;
import java.util.LinkedList;
import java.util.Locale;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.apache.commons.io.IOUtils;
import org.apache.http.client.CookieStore;
import org.apache.http.cookie.Cookie;
import org.apache.http.cookie.CookieIdentityComparator;
import org.apache.http.impl.cookie.BasicClientCookie;
import org.archive.checkpointing.Checkpointable;
import org.archive.spring.ConfigFile;
import org.archive.spring.ConfigPath;
import org.json.JSONArray;
import org.springframework.context.Lifecycle;
public abstract class AbstractCookieStore implements CookieStore, Lifecycle, Closeable,
@@ -134,7 +138,8 @@ public abstract class AbstractCookieStore implements CookieStore, Lifecycle, Clo
* @param reader
* input in the Netscape's 'cookies.txt' format.
*/
public static void loadCookies(Reader reader, Set<Cookie> cookies) {
public static Collection<Cookie> readCookies(Reader reader) {
LinkedList<Cookie> cookies = new LinkedList<Cookie>();
BufferedReader br = new BufferedReader(reader);
try {
String line;
@@ -164,9 +169,10 @@ public abstract class AbstractCookieStore implements CookieStore, Lifecycle, Clo
} catch (IOException e) {
logger.log(Level.WARNING,e.getMessage(), e);
}
return cookies;
}
public static void saveCookies(String saveCookiesFile, Set<Cookie> cookies) {
public static void saveCookies(String saveCookiesFile, Collection<Cookie> collection) {
// Do nothing if cookiesFile is not specified.
if (saveCookiesFile == null || saveCookiesFile.length() <= 0) {
return;
@@ -178,7 +184,7 @@ public abstract class AbstractCookieStore implements CookieStore, Lifecycle, Clo
String tab ="\t";
out.write("# Heritrix Cookie File\n".getBytes());
out.write("# This file is the Netscape cookies.txt format\n\n".getBytes());
for (Cookie cookie: cookies) {
for (Cookie cookie: collection) {
// Guess an initial size
MutableString line = new MutableString(1024 * 2);
line.append(cookie.getDomain());
@@ -186,7 +192,7 @@ public abstract class AbstractCookieStore implements CookieStore, Lifecycle, Clo
// XXX line.append(cookie.isDomainAttributeSpecified() ? "TRUE" : "FALSE");
line.append("TRUE");
line.append(tab);
line.append(cookie.getPath());
line.append(cookie.getPath() != null ? cookie.getPath() : "/");
line.append(tab);
line.append(cookie.isSecure() ? "TRUE" : "FALSE");
line.append(tab);
@@ -204,6 +210,31 @@ public abstract class AbstractCookieStore implements CookieStore, Lifecycle, Clo
IOUtils.closeQuietly(out);
}
}
/**
* @see {@link CookieIdentityComparator#compare(Cookie, Cookie)}
*/
protected String makeKey(Cookie cookie) {
JSONArray a = new JSONArray();
a.put(cookie.getName());
String d = cookie.getDomain();
if (d == null) {
d = "";
} else if (d.indexOf('.') == -1) {
d = d + ".local";
}
d = d.toLowerCase(Locale.ENGLISH);
a.put(d);
String p = cookie.getPath();
if (p == null) {
p = "/";
}
a.put(p);
return a.toString();
}
abstract protected void prepare();
abstract protected void loadCookies(Reader reader);
@@ -21,8 +21,8 @@ package org.archive.modules.fetcher;
import java.io.IOException;
import java.io.Reader;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Date;
import java.util.Iterator;
import java.util.List;
import org.apache.http.cookie.Cookie;
@@ -32,7 +32,8 @@ import org.springframework.beans.factory.annotation.Autowired;
import com.sleepycat.bind.serial.SerialBinding;
import com.sleepycat.bind.serial.StoredClassCatalog;
import com.sleepycat.collections.StoredSortedKeySet;
import com.sleepycat.bind.tuple.StringBinding;
import com.sleepycat.collections.StoredSortedMap;
import com.sleepycat.je.Database;
import com.sleepycat.je.DatabaseException;
@@ -50,7 +51,7 @@ public class BdbCookieStore extends AbstractCookieStore {
public static String COOKIEDB_NAME = "hc_httpclient_cookies";
private transient Database cookieDb;
private transient StoredSortedKeySet<Cookie> cookies;
private transient StoredSortedMap<String,Cookie> cookies;
public void prepare() {
try {
@@ -60,8 +61,9 @@ public class BdbCookieStore extends AbstractCookieStore {
dbConfig.setAllowCreate(true);
cookieDb = bdb.openDatabase(COOKIEDB_NAME, dbConfig,
isCheckpointRecovery);
cookies = new StoredSortedKeySet<Cookie>(cookieDb,
new SerialBinding<Cookie>(classCatalog, Cookie.class), true);
cookies = new StoredSortedMap<String, Cookie>(cookieDb,
new StringBinding(), new SerialBinding<Cookie>(
classCatalog, Cookie.class), true);
} catch (DatabaseException e) {
throw new RuntimeException(e);
}
@@ -77,10 +79,11 @@ public class BdbCookieStore extends AbstractCookieStore {
@Override
public synchronized void addCookie(Cookie cookie) {
if (cookie != null) {
String key = makeKey(cookie);
// first remove any old cookie that is equivalent
cookies.remove(cookie);
cookies.remove(key);
if (!cookie.isExpired(new Date())) {
cookies.add(cookie);
cookies.put(key, cookie);
}
}
}
@@ -94,8 +97,8 @@ public class BdbCookieStore extends AbstractCookieStore {
@Override
public synchronized List<Cookie> getCookies() {
if (cookies != null) {
//create defensive copy so it won't be concurrently modified
return new ArrayList<Cookie>(cookies);
// create defensive copy so it won't be concurrently modified
return new ArrayList<Cookie>(cookies.values());
} else {
return null;
}
@@ -115,9 +118,9 @@ public class BdbCookieStore extends AbstractCookieStore {
return false;
}
boolean removed = false;
for (Iterator<Cookie> it = cookies.iterator(); it.hasNext();) {
if (it.next().isExpired(date)) {
it.remove();
for (String key: cookies.keySet()) {
if (cookies.get(key).isExpired(date)) {
cookies.remove(key);
removed = true;
}
}
@@ -158,11 +161,14 @@ public class BdbCookieStore extends AbstractCookieStore {
@Override
protected void loadCookies(Reader reader) {
loadCookies(reader, cookies);
Collection<Cookie> loadedCookies = readCookies(reader);
for (Cookie cookie: loadedCookies) {
addCookie(cookie);
}
}
@Override
protected void saveCookies(String absolutePath) {
saveCookies(absolutePath, cookies);
saveCookies(absolutePath, cookies.values());
}
}
@@ -0,0 +1,297 @@
/*
* This file is part of the Heritrix web crawler (crawler.archive.org).
*
* Licensed to the Internet Archive (IA) by one or more individual
* contributors.
*
* The IA licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.archive.modules.fetcher;
import java.io.IOException;
import java.util.Arrays;
import java.util.Date;
import java.util.List;
import java.util.logging.Logger;
import org.apache.commons.io.FileUtils;
import org.apache.http.cookie.Cookie;
import org.apache.http.impl.client.BasicCookieStore;
import org.apache.http.impl.cookie.BasicClientCookie;
import org.archive.bdb.BdbModule;
import org.archive.spring.ConfigFile;
import org.archive.spring.ConfigPath;
import org.archive.util.TmpDirTestCase;
/**
* Tests that BdbCookieStore matches behavior of SimpleCookieStore, which uses
* the reference implementation {@link BasicCookieStore} under the hood.
*
* @contributor nlevitt
*/
public class BdbCookieStoreTest extends TmpDirTestCase {
private static Logger logger = Logger.getLogger(BdbCookieStoreTest.class.getName());
protected BdbModule bdb;
protected BdbCookieStore bdbCookieStore;
protected SimpleCookieStore simpleCookieStore;
protected BdbModule bdb() throws IOException {
if (bdb == null) {
ConfigPath basePath = new ConfigPath("testBase",
getTmpDir().getAbsolutePath());
ConfigPath bdbDir = new ConfigPath("bdb", "bdb");
bdbDir.setBase(basePath);
FileUtils.deleteDirectory(bdbDir.getFile());
bdb = new BdbModule();
bdb.setDir(bdbDir);
bdb.start();
logger.info("created " + bdb);
}
return bdb;
}
protected BdbCookieStore bdbCookieStore() throws IOException {
if (bdbCookieStore == null) {
bdbCookieStore = new BdbCookieStore();
ConfigPath basePath = new ConfigPath("testBase",
getTmpDir().getAbsolutePath());
ConfigFile cookiesSaveFile = new ConfigFile("cookiesSaveFile", "cookies.txt");
cookiesSaveFile.setBase(basePath);
bdbCookieStore.setCookiesSaveFile(cookiesSaveFile);
bdbCookieStore.setBdbModule(bdb());
bdbCookieStore.start();
}
return bdbCookieStore;
}
protected SimpleCookieStore simpleCookieStore() {
if (simpleCookieStore == null) {
simpleCookieStore = new SimpleCookieStore();
simpleCookieStore.start();
}
return simpleCookieStore;
}
@Override
protected void tearDown() throws Exception {
super.tearDown();
bdb.close();
}
public void testBasics() throws IOException {
bdbCookieStore().clear();
simpleCookieStore().clear();
assertCookieListsIdentical(simpleCookieStore().getCookies(), bdbCookieStore().getCookies());
BasicClientCookie cookie = new BasicClientCookie("name1", "value1");
bdbCookieStore().addCookie(cookie);
simpleCookieStore().addCookie(cookie);
assertCookieListsIdentical(simpleCookieStore().getCookies(), bdbCookieStore().getCookies());
}
public void testSimpleReplace() throws IOException {
bdbCookieStore().clear();
simpleCookieStore().clear();
assertCookieListsIdentical(simpleCookieStore().getCookies(), bdbCookieStore().getCookies());
BasicClientCookie cookie = new BasicClientCookie("name1", "value1");
bdbCookieStore().addCookie(cookie);
simpleCookieStore().addCookie(cookie);
assertCookieListsIdentical(simpleCookieStore().getCookies(), bdbCookieStore().getCookies());
// should replace existing cookie
cookie = new BasicClientCookie("name1", "value2");
bdbCookieStore().addCookie(cookie);
simpleCookieStore().addCookie(cookie);
assertCookieListsIdentical(simpleCookieStore().getCookies(), bdbCookieStore().getCookies());
}
public void testDomains() throws IOException {
bdbCookieStore().clear();
simpleCookieStore().clear();
assertCookieListsIdentical(simpleCookieStore().getCookies(), bdbCookieStore().getCookies());
BasicClientCookie cookie = new BasicClientCookie("name1", "value1");
cookie.setDomain("example.org");
bdbCookieStore().addCookie(cookie);
simpleCookieStore().addCookie(cookie);
assertCookieListsIdentical(simpleCookieStore().getCookies(), bdbCookieStore().getCookies());
// add a 2nd cookie, same name, different domain
cookie = new BasicClientCookie("name1", "value2");
cookie.setDomain("example.com");
bdbCookieStore().addCookie(cookie);
simpleCookieStore().addCookie(cookie);
assertCookieListsIdentical(simpleCookieStore().getCookies(), bdbCookieStore().getCookies());
// add a 3rd cookie, same name, different domain
cookie = new BasicClientCookie("name1", "value3");
cookie.setDomain("foo.example.com");
bdbCookieStore().addCookie(cookie);
simpleCookieStore().addCookie(cookie);
assertCookieListsIdentical(simpleCookieStore().getCookies(), bdbCookieStore().getCookies());
// replace 1st cookie
cookie = new BasicClientCookie("name1", "value4");
cookie.setDomain("example.org");
bdbCookieStore().addCookie(cookie);
simpleCookieStore().addCookie(cookie);
assertCookieListsIdentical(simpleCookieStore().getCookies(), bdbCookieStore().getCookies());
// replace 2nd cookie, case-insensitive domain
cookie = new BasicClientCookie("name1", "value5");
cookie.setDomain("eXaMpLe.CoM");
bdbCookieStore().addCookie(cookie);
simpleCookieStore().addCookie(cookie);
assertCookieListsIdentical(simpleCookieStore().getCookies(), bdbCookieStore().getCookies());
}
public void testPaths() throws IOException {
bdbCookieStore().clear();
simpleCookieStore().clear();
assertCookieListsIdentical(simpleCookieStore().getCookies(), bdbCookieStore().getCookies());
BasicClientCookie cookie = new BasicClientCookie("name1", "value1");
bdbCookieStore().addCookie(cookie);
simpleCookieStore().addCookie(cookie);
assertCookieListsIdentical(simpleCookieStore().getCookies(), bdbCookieStore().getCookies());
// replace 1st cookie, with explicit path "/", which is the implied path if not specified
cookie = new BasicClientCookie("name1", "value2");
cookie.setPath("/");
bdbCookieStore().addCookie(cookie);
simpleCookieStore().addCookie(cookie);
assertEquals(1, simpleCookieStore().getCookies().size());
assertEquals(cookie, simpleCookieStore().getCookies().get(0));
assertCookieListsIdentical(simpleCookieStore().getCookies(), bdbCookieStore().getCookies());
// add a 2nd cookie at a subpath
cookie = new BasicClientCookie("name1", "value3");
cookie.setPath("/path1");
bdbCookieStore().addCookie(cookie);
simpleCookieStore().addCookie(cookie);
assertEquals(2, simpleCookieStore().getCookies().size());
assertCookieListsIdentical(simpleCookieStore().getCookies(), bdbCookieStore().getCookies());
// add a 3rd cookie at a subpath
cookie = new BasicClientCookie("name1", "value4");
cookie.setPath("/path2");
bdbCookieStore().addCookie(cookie);
simpleCookieStore().addCookie(cookie);
assertEquals(3, simpleCookieStore().getCookies().size());
assertCookieListsIdentical(simpleCookieStore().getCookies(), bdbCookieStore().getCookies());
// replace 2nd cookie
cookie = new BasicClientCookie("name1", "value5");
cookie.setPath("/path1");
bdbCookieStore().addCookie(cookie);
simpleCookieStore().addCookie(cookie);
assertEquals(3, simpleCookieStore().getCookies().size());
assertCookieListsIdentical(simpleCookieStore().getCookies(), bdbCookieStore().getCookies());
// add 4th cookie at previously used path
cookie = new BasicClientCookie("name2", "value6");
cookie.setPath("/path1");
bdbCookieStore().addCookie(cookie);
simpleCookieStore().addCookie(cookie);
assertEquals(4, simpleCookieStore().getCookies().size());
assertCookieListsIdentical(simpleCookieStore().getCookies(), bdbCookieStore().getCookies());
// add 5th cookie at different path (case sensitive)
cookie = new BasicClientCookie("name1", "value7");
cookie.setPath("/pAtH1");
bdbCookieStore().addCookie(cookie);
simpleCookieStore().addCookie(cookie);
assertEquals(5, simpleCookieStore().getCookies().size());
assertCookieListsIdentical(simpleCookieStore().getCookies(), bdbCookieStore().getCookies());
}
/*
* saveCookies() expects non-null domain, and real world cookies always have
* a domain. And only test attributes that are saved in cookies.txt.
*/
public void testSaveLoadCookies() throws IOException {
bdbCookieStore().clear();
BasicClientCookie cookie = new BasicClientCookie("name1", "value1");
cookie.setDomain("example.com");
bdbCookieStore().addCookie(cookie);
cookie = new BasicClientCookie("name2", "value2");
cookie.setDomain("example.com");
bdbCookieStore().addCookie(cookie);
cookie = new BasicClientCookie("name3", "value3");
cookie.setDomain("example.com");
cookie.setSecure(true);
bdbCookieStore().addCookie(cookie);
cookie = new BasicClientCookie("name4", "value4");
cookie.setDomain("example.org");
bdbCookieStore().addCookie(cookie);
cookie = new BasicClientCookie("name5", "value5");
cookie.setDomain("example.com");
// make sure date is in the future so cookie doesn't expire, and has no
// millisecond value, so save/load loses no info
long someFutureDateMs = ((System.currentTimeMillis() + 9999999999l)/1000l)*1000l;
cookie.setExpiryDate(new Date(someFutureDateMs));
bdbCookieStore().addCookie(cookie);
cookie = new BasicClientCookie("name6", "value6");
cookie.setDomain("example.com");
cookie.setPath("/path1");
bdbCookieStore().addCookie(cookie);
assertEquals(6, bdbCookieStore().getCookies().size());
List<Cookie> cookiesBefore = bdbCookieStore().getCookies();
bdbCookieStore().saveCookies();
bdbCookieStore().clear();
assertEquals(0, bdbCookieStore().getCookies().size());
bdbCookieStore().loadCookies((ConfigFile) bdbCookieStore().getCookiesSaveFile());
assertEquals(6, bdbCookieStore().getCookies().size());
logger.info("before: " + cookiesBefore);
logger.info(" after: " + bdbCookieStore().getCookies());
assertCookieListsIdentical(cookiesBefore, bdbCookieStore().getCookies());
}
protected void assertCookieListsIdentical(List<Cookie> list1,
List<Cookie> list2) {
assertEquals(list1.size(), list2.size());
for (int i = 0; i < list1.size(); i++) {
Cookie c1 = list1.get(i);
Cookie c2 = list2.get(i);
assertCookiesIdentical(c1, c2);
}
}
protected void assertCookiesIdentical(Cookie c1, Cookie c2) {
assertEquals(c1.getComment(), c2.getComment());
assertEquals(c1.getCommentURL(), c2.getCommentURL());
assertEquals(c1.getDomain(), c2.getDomain());
assertEquals(c1.getName(), c2.getName());
String p1 = c1.getPath() != null ? c1.getPath() : "/";
String p2 = c2.getPath() != null ? c2.getPath() : "/";
assertEquals(p1, p2);
assertEquals(c1.getValue(), c2.getValue());
assertEquals(c1.getVersion(), c2.getVersion());
assertEquals(c1.getExpiryDate(), c2.getExpiryDate());
assertTrue(Arrays.equals(c1.getPorts(), c2.getPorts()));
}
}