diff --git a/commons/src/main/java/org/archive/spring/ConfigFile.java b/commons/src/main/java/org/archive/spring/ConfigFile.java index e0293dbb..92d0c221 100644 --- a/commons/src/main/java/org/archive/spring/ConfigFile.java +++ b/commons/src/main/java/org/archive/spring/ConfigFile.java @@ -49,7 +49,9 @@ public class ConfigFile extends ConfigPath implements ReadSource, WriteTarget { getFile().createNewFile(); } - configurer.snapshotToLaunchDir(getFile()); + if (configurer != null) { + configurer.snapshotToLaunchDir(getFile()); + } return new InputStreamReader( new FileInputStream(getFile()), diff --git a/modules/src/main/java/org/archive/modules/fetcher/AbstractCookieStore.java b/modules/src/main/java/org/archive/modules/fetcher/AbstractCookieStore.java index 10354aa9..6239f3f3 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/AbstractCookieStore.java +++ b/modules/src/main/java/org/archive/modules/fetcher/AbstractCookieStore.java @@ -26,18 +26,22 @@ import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.io.Reader; +import java.util.Collection; import java.util.Date; -import java.util.Set; +import java.util.LinkedList; +import java.util.Locale; import java.util.logging.Level; import java.util.logging.Logger; import org.apache.commons.io.IOUtils; import org.apache.http.client.CookieStore; import org.apache.http.cookie.Cookie; +import org.apache.http.cookie.CookieIdentityComparator; import org.apache.http.impl.cookie.BasicClientCookie; import org.archive.checkpointing.Checkpointable; import org.archive.spring.ConfigFile; import org.archive.spring.ConfigPath; +import org.json.JSONArray; import org.springframework.context.Lifecycle; public abstract class AbstractCookieStore implements CookieStore, Lifecycle, Closeable, @@ -134,7 +138,8 @@ public abstract class AbstractCookieStore implements CookieStore, Lifecycle, Clo * @param reader * input in the Netscape's 'cookies.txt' format. */ - public static void loadCookies(Reader reader, Set cookies) { + public static Collection readCookies(Reader reader) { + LinkedList cookies = new LinkedList(); BufferedReader br = new BufferedReader(reader); try { String line; @@ -164,9 +169,10 @@ public abstract class AbstractCookieStore implements CookieStore, Lifecycle, Clo } catch (IOException e) { logger.log(Level.WARNING,e.getMessage(), e); } + return cookies; } - public static void saveCookies(String saveCookiesFile, Set cookies) { + public static void saveCookies(String saveCookiesFile, Collection collection) { // Do nothing if cookiesFile is not specified. if (saveCookiesFile == null || saveCookiesFile.length() <= 0) { return; @@ -178,7 +184,7 @@ public abstract class AbstractCookieStore implements CookieStore, Lifecycle, Clo String tab ="\t"; out.write("# Heritrix Cookie File\n".getBytes()); out.write("# This file is the Netscape cookies.txt format\n\n".getBytes()); - for (Cookie cookie: cookies) { + for (Cookie cookie: collection) { // Guess an initial size MutableString line = new MutableString(1024 * 2); line.append(cookie.getDomain()); @@ -186,7 +192,7 @@ public abstract class AbstractCookieStore implements CookieStore, Lifecycle, Clo // XXX line.append(cookie.isDomainAttributeSpecified() ? "TRUE" : "FALSE"); line.append("TRUE"); line.append(tab); - line.append(cookie.getPath()); + line.append(cookie.getPath() != null ? cookie.getPath() : "/"); line.append(tab); line.append(cookie.isSecure() ? "TRUE" : "FALSE"); line.append(tab); @@ -204,6 +210,31 @@ public abstract class AbstractCookieStore implements CookieStore, Lifecycle, Clo IOUtils.closeQuietly(out); } } + + /** + * @see {@link CookieIdentityComparator#compare(Cookie, Cookie)} + */ + protected String makeKey(Cookie cookie) { + JSONArray a = new JSONArray(); + a.put(cookie.getName()); + + String d = cookie.getDomain(); + if (d == null) { + d = ""; + } else if (d.indexOf('.') == -1) { + d = d + ".local"; + } + d = d.toLowerCase(Locale.ENGLISH); + a.put(d); + + String p = cookie.getPath(); + if (p == null) { + p = "/"; + } + a.put(p); + + return a.toString(); + } abstract protected void prepare(); abstract protected void loadCookies(Reader reader); diff --git a/modules/src/main/java/org/archive/modules/fetcher/BdbCookieStore.java b/modules/src/main/java/org/archive/modules/fetcher/BdbCookieStore.java index 1ea094e8..091fe78c 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/BdbCookieStore.java +++ b/modules/src/main/java/org/archive/modules/fetcher/BdbCookieStore.java @@ -21,8 +21,8 @@ package org.archive.modules.fetcher; import java.io.IOException; import java.io.Reader; import java.util.ArrayList; +import java.util.Collection; import java.util.Date; -import java.util.Iterator; import java.util.List; import org.apache.http.cookie.Cookie; @@ -32,7 +32,8 @@ import org.springframework.beans.factory.annotation.Autowired; import com.sleepycat.bind.serial.SerialBinding; import com.sleepycat.bind.serial.StoredClassCatalog; -import com.sleepycat.collections.StoredSortedKeySet; +import com.sleepycat.bind.tuple.StringBinding; +import com.sleepycat.collections.StoredSortedMap; import com.sleepycat.je.Database; import com.sleepycat.je.DatabaseException; @@ -50,7 +51,7 @@ public class BdbCookieStore extends AbstractCookieStore { public static String COOKIEDB_NAME = "hc_httpclient_cookies"; private transient Database cookieDb; - private transient StoredSortedKeySet cookies; + private transient StoredSortedMap cookies; public void prepare() { try { @@ -60,8 +61,9 @@ public class BdbCookieStore extends AbstractCookieStore { dbConfig.setAllowCreate(true); cookieDb = bdb.openDatabase(COOKIEDB_NAME, dbConfig, isCheckpointRecovery); - cookies = new StoredSortedKeySet(cookieDb, - new SerialBinding(classCatalog, Cookie.class), true); + cookies = new StoredSortedMap(cookieDb, + new StringBinding(), new SerialBinding( + classCatalog, Cookie.class), true); } catch (DatabaseException e) { throw new RuntimeException(e); } @@ -77,10 +79,11 @@ public class BdbCookieStore extends AbstractCookieStore { @Override public synchronized void addCookie(Cookie cookie) { if (cookie != null) { + String key = makeKey(cookie); // first remove any old cookie that is equivalent - cookies.remove(cookie); + cookies.remove(key); if (!cookie.isExpired(new Date())) { - cookies.add(cookie); + cookies.put(key, cookie); } } } @@ -94,8 +97,8 @@ public class BdbCookieStore extends AbstractCookieStore { @Override public synchronized List getCookies() { if (cookies != null) { - //create defensive copy so it won't be concurrently modified - return new ArrayList(cookies); + // create defensive copy so it won't be concurrently modified + return new ArrayList(cookies.values()); } else { return null; } @@ -115,9 +118,9 @@ public class BdbCookieStore extends AbstractCookieStore { return false; } boolean removed = false; - for (Iterator it = cookies.iterator(); it.hasNext();) { - if (it.next().isExpired(date)) { - it.remove(); + for (String key: cookies.keySet()) { + if (cookies.get(key).isExpired(date)) { + cookies.remove(key); removed = true; } } @@ -158,11 +161,14 @@ public class BdbCookieStore extends AbstractCookieStore { @Override protected void loadCookies(Reader reader) { - loadCookies(reader, cookies); + Collection loadedCookies = readCookies(reader); + for (Cookie cookie: loadedCookies) { + addCookie(cookie); + } } @Override protected void saveCookies(String absolutePath) { - saveCookies(absolutePath, cookies); + saveCookies(absolutePath, cookies.values()); } } diff --git a/modules/src/test/java/org/archive/modules/fetcher/BdbCookieStoreTest.java b/modules/src/test/java/org/archive/modules/fetcher/BdbCookieStoreTest.java new file mode 100644 index 00000000..00aedbc0 --- /dev/null +++ b/modules/src/test/java/org/archive/modules/fetcher/BdbCookieStoreTest.java @@ -0,0 +1,297 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.archive.modules.fetcher; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Date; +import java.util.List; +import java.util.logging.Logger; + +import org.apache.commons.io.FileUtils; +import org.apache.http.cookie.Cookie; +import org.apache.http.impl.client.BasicCookieStore; +import org.apache.http.impl.cookie.BasicClientCookie; +import org.archive.bdb.BdbModule; +import org.archive.spring.ConfigFile; +import org.archive.spring.ConfigPath; +import org.archive.util.TmpDirTestCase; + +/** + * Tests that BdbCookieStore matches behavior of SimpleCookieStore, which uses + * the reference implementation {@link BasicCookieStore} under the hood. + * + * @contributor nlevitt + */ +public class BdbCookieStoreTest extends TmpDirTestCase { + + private static Logger logger = Logger.getLogger(BdbCookieStoreTest.class.getName()); + + protected BdbModule bdb; + protected BdbCookieStore bdbCookieStore; + protected SimpleCookieStore simpleCookieStore; + + protected BdbModule bdb() throws IOException { + if (bdb == null) { + ConfigPath basePath = new ConfigPath("testBase", + getTmpDir().getAbsolutePath()); + ConfigPath bdbDir = new ConfigPath("bdb", "bdb"); + bdbDir.setBase(basePath); + FileUtils.deleteDirectory(bdbDir.getFile()); + + bdb = new BdbModule(); + bdb.setDir(bdbDir); + bdb.start(); + logger.info("created " + bdb); + } + return bdb; + } + + protected BdbCookieStore bdbCookieStore() throws IOException { + if (bdbCookieStore == null) { + bdbCookieStore = new BdbCookieStore(); + ConfigPath basePath = new ConfigPath("testBase", + getTmpDir().getAbsolutePath()); + ConfigFile cookiesSaveFile = new ConfigFile("cookiesSaveFile", "cookies.txt"); + cookiesSaveFile.setBase(basePath); + bdbCookieStore.setCookiesSaveFile(cookiesSaveFile); + bdbCookieStore.setBdbModule(bdb()); + bdbCookieStore.start(); + } + return bdbCookieStore; + } + + protected SimpleCookieStore simpleCookieStore() { + if (simpleCookieStore == null) { + simpleCookieStore = new SimpleCookieStore(); + simpleCookieStore.start(); + } + return simpleCookieStore; + } + + @Override + protected void tearDown() throws Exception { + super.tearDown(); + bdb.close(); + } + + public void testBasics() throws IOException { + bdbCookieStore().clear(); + simpleCookieStore().clear(); + assertCookieListsIdentical(simpleCookieStore().getCookies(), bdbCookieStore().getCookies()); + + BasicClientCookie cookie = new BasicClientCookie("name1", "value1"); + bdbCookieStore().addCookie(cookie); + simpleCookieStore().addCookie(cookie); + assertCookieListsIdentical(simpleCookieStore().getCookies(), bdbCookieStore().getCookies()); + } + + public void testSimpleReplace() throws IOException { + bdbCookieStore().clear(); + simpleCookieStore().clear(); + assertCookieListsIdentical(simpleCookieStore().getCookies(), bdbCookieStore().getCookies()); + + BasicClientCookie cookie = new BasicClientCookie("name1", "value1"); + bdbCookieStore().addCookie(cookie); + simpleCookieStore().addCookie(cookie); + assertCookieListsIdentical(simpleCookieStore().getCookies(), bdbCookieStore().getCookies()); + + // should replace existing cookie + cookie = new BasicClientCookie("name1", "value2"); + bdbCookieStore().addCookie(cookie); + simpleCookieStore().addCookie(cookie); + assertCookieListsIdentical(simpleCookieStore().getCookies(), bdbCookieStore().getCookies()); + } + + public void testDomains() throws IOException { + bdbCookieStore().clear(); + simpleCookieStore().clear(); + assertCookieListsIdentical(simpleCookieStore().getCookies(), bdbCookieStore().getCookies()); + + BasicClientCookie cookie = new BasicClientCookie("name1", "value1"); + cookie.setDomain("example.org"); + bdbCookieStore().addCookie(cookie); + simpleCookieStore().addCookie(cookie); + assertCookieListsIdentical(simpleCookieStore().getCookies(), bdbCookieStore().getCookies()); + + // add a 2nd cookie, same name, different domain + cookie = new BasicClientCookie("name1", "value2"); + cookie.setDomain("example.com"); + bdbCookieStore().addCookie(cookie); + simpleCookieStore().addCookie(cookie); + assertCookieListsIdentical(simpleCookieStore().getCookies(), bdbCookieStore().getCookies()); + + // add a 3rd cookie, same name, different domain + cookie = new BasicClientCookie("name1", "value3"); + cookie.setDomain("foo.example.com"); + bdbCookieStore().addCookie(cookie); + simpleCookieStore().addCookie(cookie); + assertCookieListsIdentical(simpleCookieStore().getCookies(), bdbCookieStore().getCookies()); + + // replace 1st cookie + cookie = new BasicClientCookie("name1", "value4"); + cookie.setDomain("example.org"); + bdbCookieStore().addCookie(cookie); + simpleCookieStore().addCookie(cookie); + assertCookieListsIdentical(simpleCookieStore().getCookies(), bdbCookieStore().getCookies()); + + // replace 2nd cookie, case-insensitive domain + cookie = new BasicClientCookie("name1", "value5"); + cookie.setDomain("eXaMpLe.CoM"); + bdbCookieStore().addCookie(cookie); + simpleCookieStore().addCookie(cookie); + assertCookieListsIdentical(simpleCookieStore().getCookies(), bdbCookieStore().getCookies()); + } + + public void testPaths() throws IOException { + bdbCookieStore().clear(); + simpleCookieStore().clear(); + assertCookieListsIdentical(simpleCookieStore().getCookies(), bdbCookieStore().getCookies()); + + BasicClientCookie cookie = new BasicClientCookie("name1", "value1"); + bdbCookieStore().addCookie(cookie); + simpleCookieStore().addCookie(cookie); + assertCookieListsIdentical(simpleCookieStore().getCookies(), bdbCookieStore().getCookies()); + + // replace 1st cookie, with explicit path "/", which is the implied path if not specified + cookie = new BasicClientCookie("name1", "value2"); + cookie.setPath("/"); + bdbCookieStore().addCookie(cookie); + simpleCookieStore().addCookie(cookie); + assertEquals(1, simpleCookieStore().getCookies().size()); + assertEquals(cookie, simpleCookieStore().getCookies().get(0)); + assertCookieListsIdentical(simpleCookieStore().getCookies(), bdbCookieStore().getCookies()); + + // add a 2nd cookie at a subpath + cookie = new BasicClientCookie("name1", "value3"); + cookie.setPath("/path1"); + bdbCookieStore().addCookie(cookie); + simpleCookieStore().addCookie(cookie); + assertEquals(2, simpleCookieStore().getCookies().size()); + assertCookieListsIdentical(simpleCookieStore().getCookies(), bdbCookieStore().getCookies()); + + // add a 3rd cookie at a subpath + cookie = new BasicClientCookie("name1", "value4"); + cookie.setPath("/path2"); + bdbCookieStore().addCookie(cookie); + simpleCookieStore().addCookie(cookie); + assertEquals(3, simpleCookieStore().getCookies().size()); + assertCookieListsIdentical(simpleCookieStore().getCookies(), bdbCookieStore().getCookies()); + + // replace 2nd cookie + cookie = new BasicClientCookie("name1", "value5"); + cookie.setPath("/path1"); + bdbCookieStore().addCookie(cookie); + simpleCookieStore().addCookie(cookie); + assertEquals(3, simpleCookieStore().getCookies().size()); + assertCookieListsIdentical(simpleCookieStore().getCookies(), bdbCookieStore().getCookies()); + + // add 4th cookie at previously used path + cookie = new BasicClientCookie("name2", "value6"); + cookie.setPath("/path1"); + bdbCookieStore().addCookie(cookie); + simpleCookieStore().addCookie(cookie); + assertEquals(4, simpleCookieStore().getCookies().size()); + assertCookieListsIdentical(simpleCookieStore().getCookies(), bdbCookieStore().getCookies()); + + // add 5th cookie at different path (case sensitive) + cookie = new BasicClientCookie("name1", "value7"); + cookie.setPath("/pAtH1"); + bdbCookieStore().addCookie(cookie); + simpleCookieStore().addCookie(cookie); + assertEquals(5, simpleCookieStore().getCookies().size()); + assertCookieListsIdentical(simpleCookieStore().getCookies(), bdbCookieStore().getCookies()); + } + + /* + * saveCookies() expects non-null domain, and real world cookies always have + * a domain. And only test attributes that are saved in cookies.txt. + */ + public void testSaveLoadCookies() throws IOException { + bdbCookieStore().clear(); + + BasicClientCookie cookie = new BasicClientCookie("name1", "value1"); + cookie.setDomain("example.com"); + bdbCookieStore().addCookie(cookie); + + cookie = new BasicClientCookie("name2", "value2"); + cookie.setDomain("example.com"); + bdbCookieStore().addCookie(cookie); + + cookie = new BasicClientCookie("name3", "value3"); + cookie.setDomain("example.com"); + cookie.setSecure(true); + bdbCookieStore().addCookie(cookie); + + cookie = new BasicClientCookie("name4", "value4"); + cookie.setDomain("example.org"); + bdbCookieStore().addCookie(cookie); + + cookie = new BasicClientCookie("name5", "value5"); + cookie.setDomain("example.com"); + // make sure date is in the future so cookie doesn't expire, and has no + // millisecond value, so save/load loses no info + long someFutureDateMs = ((System.currentTimeMillis() + 9999999999l)/1000l)*1000l; + cookie.setExpiryDate(new Date(someFutureDateMs)); + bdbCookieStore().addCookie(cookie); + + cookie = new BasicClientCookie("name6", "value6"); + cookie.setDomain("example.com"); + cookie.setPath("/path1"); + bdbCookieStore().addCookie(cookie); + + assertEquals(6, bdbCookieStore().getCookies().size()); + List cookiesBefore = bdbCookieStore().getCookies(); + bdbCookieStore().saveCookies(); + + bdbCookieStore().clear(); + assertEquals(0, bdbCookieStore().getCookies().size()); + + bdbCookieStore().loadCookies((ConfigFile) bdbCookieStore().getCookiesSaveFile()); + assertEquals(6, bdbCookieStore().getCookies().size()); + logger.info("before: " + cookiesBefore); + logger.info(" after: " + bdbCookieStore().getCookies()); + assertCookieListsIdentical(cookiesBefore, bdbCookieStore().getCookies()); + } + + protected void assertCookieListsIdentical(List list1, + List list2) { + assertEquals(list1.size(), list2.size()); + for (int i = 0; i < list1.size(); i++) { + Cookie c1 = list1.get(i); + Cookie c2 = list2.get(i); + assertCookiesIdentical(c1, c2); + } + } + + protected void assertCookiesIdentical(Cookie c1, Cookie c2) { + assertEquals(c1.getComment(), c2.getComment()); + assertEquals(c1.getCommentURL(), c2.getCommentURL()); + assertEquals(c1.getDomain(), c2.getDomain()); + assertEquals(c1.getName(), c2.getName()); + String p1 = c1.getPath() != null ? c1.getPath() : "/"; + String p2 = c2.getPath() != null ? c2.getPath() : "/"; + assertEquals(p1, p2); + assertEquals(c1.getValue(), c2.getValue()); + assertEquals(c1.getVersion(), c2.getVersion()); + assertEquals(c1.getExpiryDate(), c2.getExpiryDate()); + assertTrue(Arrays.equals(c1.getPorts(), c2.getPorts())); + } + +}