diff --git a/modules/src/main/java/org/archive/modules/fetcher/AbstractCookieStore.java b/modules/src/main/java/org/archive/modules/fetcher/AbstractCookieStore.java index 6a2ae6c5..9e7b3f56 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/AbstractCookieStore.java +++ b/modules/src/main/java/org/archive/modules/fetcher/AbstractCookieStore.java @@ -18,7 +18,11 @@ */ package org.archive.modules.fetcher; +import it.unimi.dsi.mg4j.util.MutableString; + import java.io.BufferedReader; +import java.io.File; +import java.io.FileOutputStream; import java.io.IOException; import java.io.Reader; import java.util.Collection; @@ -44,7 +48,8 @@ import org.springframework.context.Lifecycle; import com.google.common.net.InternetDomainName; -abstract public class AbstractCookieStore implements Lifecycle, Checkpointable, CookieStore { +abstract public class AbstractCookieStore implements Lifecycle, Checkpointable, + CookieStore, FetchHTTPCookieStore { protected final Logger logger = Logger.getLogger(AbstractCookieStore.class.getName()); @@ -97,7 +102,6 @@ abstract public class AbstractCookieStore implements Lifecycle, Checkpointable, } } - abstract public void saveCookies(String saveCookiesFile); protected void loadCookies(ConfigFile file) { Reader reader = null; try { @@ -115,6 +119,45 @@ abstract public class AbstractCookieStore implements Lifecycle, Checkpointable, } } + public void saveCookies(String saveCookiesFile) { + // Do nothing if cookiesFile is not specified. + if (saveCookiesFile == null || saveCookiesFile.length() <= 0) { + return; + } + + FileOutputStream out = null; + try { + out = new FileOutputStream(new File(saveCookiesFile)); + String tab ="\t"; + out.write("# Heritrix Cookie File\n".getBytes()); + out.write("# This file is the Netscape cookies.txt format\n\n".getBytes()); + for (Cookie cookie: getCookies()) { + // Guess an initial size + MutableString line = new MutableString(1024 * 2); + line.append(cookie.getDomain()); + line.append(tab); + // XXX line.append(cookie.isDomainAttributeSpecified() ? "TRUE" : "FALSE"); + line.append("TRUE"); + line.append(tab); + line.append(cookie.getPath() != null ? cookie.getPath() : "/"); + line.append(tab); + line.append(cookie.isSecure() ? "TRUE" : "FALSE"); + line.append(tab); + line.append(cookie.getExpiryDate() != null ? cookie.getExpiryDate().getTime() / 1000 : -1); + line.append(tab); + line.append(cookie.getName()); + line.append(tab); + line.append(cookie.getValue() != null ? cookie.getValue() : ""); + line.append("\n"); + out.write(line.toString().getBytes()); + } + } catch (IOException e) { + logger.log(Level.SEVERE, "Unable to write " + saveCookiesFile, e); + } finally { + IOUtils.closeQuietly(out); + } + } + /** * Load cookies. The input is text in the Netscape's 'cookies.txt' file * format. Example entry of cookies.txt file: @@ -203,32 +246,42 @@ abstract public class AbstractCookieStore implements Lifecycle, Checkpointable, } /** - * Adapted from {@link CookieIdentityComparator#compare(Cookie, Cookie)} - * XXX explain about sorting - * @param cookie - * @return + * Returns a string that uniquely identifies the cookie, and is prepended + * with the top private domain (one level below the TLD) associated with the + * cookie. This way such cookies can be grouped together in a sorted list, + * for example. The format The format of the key is + * {@code "topPrivateDomain;normalizedDomain;name;path"}. Adapted from + * {@link CookieIdentityComparator#compare(Cookie, Cookie)}. */ - protected String makeSortKey(Cookie cookie) { + protected String sortableKey(Cookie cookie) { String normalizedDomain = normalizeDomain(cookie.getDomain()); String topPrivateDomain = topPrivateDomain(normalizedDomain); + // use ";" as delimiter since it is the delimiter in the cookie header, + // so presumably can't appear in any of these values StringBuilder buf = new StringBuilder(topPrivateDomain); - buf.append("-").append(normalizedDomain); - buf.append("-").append(cookie.getName()); - buf.append("-").append(cookie.getPath() != null ? cookie.getPath() : "/"); + buf.append(";").append(normalizedDomain); + buf.append(";").append(cookie.getName()); + buf.append(";").append(cookie.getPath() != null ? cookie.getPath() : "/"); return buf.toString(); } - protected String topPrivateDomain(String domain) { - String topPrivateDomain = ""; - if (InternetDomainName.isValid(domain)) { - InternetDomainName d = InternetDomainName.from(domain); + /** + * Returns the top private domain, i.e. the topmost assigned domain, one + * level below the TLD, for the supplied {@code host}. Returns + * {@code host} unaltered if a top private domain can't be identified (for + * example, if {@code host} is an IP address). + */ + protected String topPrivateDomain(String host) { + if (InternetDomainName.isValid(host)) { + InternetDomainName d = InternetDomainName.from(host); if (d.hasPublicSuffix()) { - topPrivateDomain = d.topPrivateDomain().toString(); + return d.topPrivateDomain().toString(); } } - return topPrivateDomain; + + return host; } protected String normalizeDomain(String domain) { diff --git a/modules/src/main/java/org/archive/modules/fetcher/BdbCookieStore.java b/modules/src/main/java/org/archive/modules/fetcher/BdbCookieStore.java index 83699a23..f81375da 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/BdbCookieStore.java +++ b/modules/src/main/java/org/archive/modules/fetcher/BdbCookieStore.java @@ -18,10 +18,6 @@ */ package org.archive.modules.fetcher; -import it.unimi.dsi.mg4j.util.MutableString; - -import java.io.File; -import java.io.FileOutputStream; import java.io.IOException; import java.io.UnsupportedEncodingException; import java.util.Collection; @@ -29,11 +25,8 @@ import java.util.Date; import java.util.Iterator; import java.util.List; import java.util.ListIterator; -import java.util.Map.Entry; import java.util.SortedMap; -import java.util.logging.Level; -import org.apache.commons.io.IOUtils; import org.apache.http.client.CookieStore; import org.apache.http.cookie.Cookie; import org.archive.bdb.BdbModule; @@ -43,23 +36,47 @@ import org.springframework.beans.factory.annotation.Autowired; import com.sleepycat.bind.ByteArrayBinding; import com.sleepycat.bind.serial.SerialBinding; import com.sleepycat.bind.serial.StoredClassCatalog; +import com.sleepycat.collections.StoredCollection; import com.sleepycat.collections.StoredSortedMap; import com.sleepycat.je.Database; import com.sleepycat.je.DatabaseException; -public class BdbCookieStore extends AbstractCookieStore implements CookieStore { +/** + * Cookie store using bdb for storage. Cookies are stored in a SortedMap keyed + * by {@link #sortableKey(Cookie)}, so they are grouped together by top private + * domain. {@link #cookieStoreFor(String)} returns a facade whose + * {@link CookieStore#getCookies()} returns a list of cookies limited to + * subdomains of the supplied top private domain. + * + * @see https://webarchive.jira.com/browse/HER-2070 + * @see https://github.com/internetarchive/heritrix3/pull/96 + * @see https://groups.yahoo.com/neo/groups/archive-crawler/conversations/messages/8620 + * + * @contributor nlevitt + */ +public class BdbCookieStore extends AbstractCookieStore implements +FetchHTTPCookieStore, CookieStore { /** - * Needed because httpclient requires List even though it only uses - * methods available on Collection. (+1 for python, -1 for java on this one) + * A {@link List} implementation that wraps a {@link Collection}. Needed + * because httpclient requires {@code List}. + * + *

+ * This class is "restricted" in the sense that it is immutable, and also + * because some methods throw {@link RuntimeException} for other reasons. + * For example, {@link #iterator()} is not implemented, because we use this + * class to wrap a bdb {@link StoredCollection}, and iterators from that + * class need to be explicitly closed. Since this class hides the fact that + * a StoredCollection underlies it, we simply prevent {@link #iterator()} + * from being used. */ - public static class CollectionListFacade implements List { + public static class RestrictedCollectionWrappedList implements List { private Collection wrapped; - public CollectionListFacade(Collection wrapped) { this.wrapped = wrapped; } + public RestrictedCollectionWrappedList(Collection wrapped) { this.wrapped = wrapped; } @Override public int size() { return wrapped.size(); } - @Override public boolean isEmpty() { return wrapped.isEmpty(); } - @Override public boolean contains(Object o) { return wrapped.contains(o); } - @Override public Iterator iterator() { return wrapped.iterator(); } + @Override public boolean isEmpty() { throw new RuntimeException("not implemented"); } + @Override public boolean contains(Object o) { throw new RuntimeException("not implemented"); } + @Override public Iterator iterator() { throw new RuntimeException("not implemented"); } @Override public Object[] toArray() { return wrapped.toArray(); } @SuppressWarnings("hiding") @Override public T[] toArray(T[] a) { return wrapped.toArray(a); } @Override public boolean add(T e) { throw new RuntimeException("immutable list"); } @@ -109,11 +126,11 @@ public class BdbCookieStore extends AbstractCookieStore implements CookieStore { throw new RuntimeException(e); } } - + public void addCookie(Cookie cookie) { byte[] key; try { - key = makeSortKey(cookie).getBytes("UTF-8"); + key = sortableKey(cookie).getBytes("UTF-8"); } catch (UnsupportedEncodingException e) { throw new RuntimeException(e); // impossible } @@ -124,21 +141,26 @@ public class BdbCookieStore extends AbstractCookieStore implements CookieStore { cookies.remove(key); } } - - public CookieStore cookieStoreFor(String topPrivateDomain) { - // XXX "the natural ordering of a stored collection is data byte order" -- explain more about subMap + + /** + * Returns a {@link LimitedCookieStoreFacade} whose + * {@link LimitedCookieStoreFacade#getCookies()} method returns only the + * cookies from the domain {@code topPrivateDomainOrIP} and subdomains. + */ + public CookieStore cookieStoreFor(String topPrivateDomainOrIP) { SortedMap domainCookiesSubMap; try { - byte[] startKey = topPrivateDomain.getBytes("UTF-8"); - byte[] endKey = (topPrivateDomain + ".").getBytes("UTF-8"); + byte[] startKey = topPrivateDomainOrIP.getBytes("UTF-8"); + char chAfterDelim = (char)(((int)';')+1); + byte[] endKey = (topPrivateDomainOrIP + chAfterDelim).getBytes("UTF-8"); domainCookiesSubMap = cookies.subMap(startKey, endKey); } catch (UnsupportedEncodingException e) { throw new RuntimeException(e); // impossible } - + Collection domainCookiesCollection = domainCookiesSubMap.values(); - List domainCookiesList = new CollectionListFacade(domainCookiesCollection); - + List domainCookiesList = new RestrictedCollectionWrappedList(domainCookiesCollection); + return new LimitedCookieStoreFacade(domainCookiesList); } @@ -155,7 +177,7 @@ public class BdbCookieStore extends AbstractCookieStore implements CookieStore { public void finishCheckpoint(Checkpoint checkpointInProgress) { // do nothing; handled by map checkpoint via BdbModule } - + /** are we a checkpoint recovery? (in which case, reuse stored cookie data?) */ protected boolean isCheckpointRecovery = false; @Override @@ -164,45 +186,6 @@ public class BdbCookieStore extends AbstractCookieStore implements CookieStore { // actual state recovery happens via BdbModule isCheckpointRecovery = true; } - - public void saveCookies(String saveCookiesFile) { - // Do nothing if cookiesFile is not specified. - if (saveCookiesFile == null || saveCookiesFile.length() <= 0) { - return; - } - - FileOutputStream out = null; - try { - out = new FileOutputStream(new File(saveCookiesFile)); - String tab ="\t"; - out.write("# Heritrix Cookie File\n".getBytes()); - out.write("# This file is the Netscape cookies.txt format\n\n".getBytes()); - for (Cookie cookie: cookies.values()) { - // Guess an initial size - MutableString line = new MutableString(1024 * 2); - line.append(cookie.getDomain()); - line.append(tab); - // XXX line.append(cookie.isDomainAttributeSpecified() ? "TRUE" : "FALSE"); - line.append("TRUE"); - line.append(tab); - line.append(cookie.getPath() != null ? cookie.getPath() : "/"); - line.append(tab); - line.append(cookie.isSecure() ? "TRUE" : "FALSE"); - line.append(tab); - line.append(cookie.getExpiryDate() != null ? cookie.getExpiryDate().getTime() / 1000 : -1); - line.append(tab); - line.append(cookie.getName()); - line.append(tab); - line.append(cookie.getValue() != null ? cookie.getValue() : ""); - line.append("\n"); - out.write(line.toString().getBytes()); - } - } catch (IOException e) { - logger.log(Level.SEVERE, "Unable to write " + saveCookiesFile, e); - } finally { - IOUtils.closeQuietly(out); - } - } @Override public void clear() { @@ -215,7 +198,7 @@ public class BdbCookieStore extends AbstractCookieStore implements CookieStore { @Override public List getCookies() { if (cookies != null) { - return new CollectionListFacade(cookies.values()); + return new RestrictedCollectionWrappedList(cookies.values()); } else { return null; } @@ -223,33 +206,6 @@ public class BdbCookieStore extends AbstractCookieStore implements CookieStore { @Override public boolean clearExpired(Date date) { - throw new RuntimeException("note implemented"); - } - - @Override - public String toString() { - - Iterator> i = cookies.entrySet().iterator(); - if (! i.hasNext()) - return "{}"; - - StringBuilder sb = new StringBuilder(); - sb.append('{'); - for (;;) { - Entry e = i.next(); - String key; - try { - key = new String(e.getKey(), "UTF-8"); - } catch (UnsupportedEncodingException e1) { - throw new RuntimeException(); - } - Cookie value = e.getValue(); - sb.append(key); - sb.append('='); - sb.append(value == this ? "(this Map)" : value); - if (! i.hasNext()) - return sb.append('}').toString(); - sb.append(',').append(' '); - } + throw new RuntimeException("not implemented"); } } diff --git a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTPCookieStore.java b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTPCookieStore.java new file mode 100644 index 00000000..47aa53e9 --- /dev/null +++ b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTPCookieStore.java @@ -0,0 +1,37 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.archive.modules.fetcher; + +import org.apache.commons.httpclient.URIException; +import org.apache.http.client.CookieStore; +import org.archive.modules.CrawlURI; + +public interface FetchHTTPCookieStore extends CookieStore { + /** + * Returns a {@link CookieStore} whose {@link CookieStore#getCookies()} + * returns all the cookies from {@code topPrivateDomain} and its subdomains. + */ + public CookieStore cookieStoreFor(String topPrivateDomain); + + /** + * Returns a {@link CookieStore} whose {@link CookieStore#getCookies()} + * returns all the cookies that could possibly apply {@code curi}. + */ + public CookieStore cookieStoreFor(CrawlURI curi) throws URIException; +} diff --git a/modules/src/main/java/org/archive/modules/fetcher/SimpleCookieStore.java b/modules/src/main/java/org/archive/modules/fetcher/SimpleCookieStore.java index 3ea1cf4e..960ade54 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/SimpleCookieStore.java +++ b/modules/src/main/java/org/archive/modules/fetcher/SimpleCookieStore.java @@ -28,6 +28,7 @@ import org.apache.http.impl.client.BasicCookieStore; import org.archive.checkpointing.Checkpoint; import org.springframework.beans.factory.annotation.Autowired; +/** In-memory cookie store, mostly for testing. */ public class SimpleCookieStore extends AbstractCookieStore implements CookieStore { protected BasicCookieStore cookies; @@ -66,11 +67,6 @@ public class SimpleCookieStore extends AbstractCookieStore implements CookieStor return cookies.clearExpired(date); } - @Override - public void saveCookies(String saveCookiesFile) { - throw new RuntimeException("not implemented"); - } - @Override public CookieStore cookieStoreFor(String topPrivateDomain) { return this; diff --git a/modules/src/test/java/org/archive/modules/fetcher/CookieFetchHTTPIntegrationTest.java b/modules/src/test/java/org/archive/modules/fetcher/CookieFetchHTTPIntegrationTest.java index e5d0be17..5e9674fb 100644 --- a/modules/src/test/java/org/archive/modules/fetcher/CookieFetchHTTPIntegrationTest.java +++ b/modules/src/test/java/org/archive/modules/fetcher/CookieFetchHTTPIntegrationTest.java @@ -59,15 +59,6 @@ import com.google.common.io.Files; @SuppressWarnings("restriction") public class CookieFetchHTTPIntegrationTest extends ProcessorTestBase { - // private static Logger logger = Logger.getLogger(FetchHTTPTest.class.getName()); - // static { - // Logger.getLogger("").setLevel(Level.FINE); - // for (java.util.logging.Handler h: Logger.getLogger("").getHandlers()) { - // h.setLevel(Level.ALL); - // h.setFormatter(new OneLineSimpleLogger()); - // } - // } - protected static class TestHandler extends SessionHandler { public TestHandler() { super(); @@ -219,7 +210,7 @@ public class CookieFetchHTTPIntegrationTest extends ProcessorTestBase { return bdb; } - protected BdbCookieStore bdbCookieStore() throws IOException { + protected AbstractCookieStore bdbCookieStore() throws IOException { if (bdbCookieStore == null) { bdbCookieStore = new BdbCookieStore(); ConfigPath basePath = new ConfigPath("testBase", diff --git a/modules/src/test/java/org/archive/modules/fetcher/CookieStoreTest.java b/modules/src/test/java/org/archive/modules/fetcher/CookieStoreTest.java index a68f582a..e0b311cc 100644 --- a/modules/src/test/java/org/archive/modules/fetcher/CookieStoreTest.java +++ b/modules/src/test/java/org/archive/modules/fetcher/CookieStoreTest.java @@ -69,7 +69,7 @@ public class CookieStoreTest extends TmpDirTestCase { return bdb; } - protected BdbCookieStore bdbCookieStore() throws IOException { + protected AbstractCookieStore bdbCookieStore() throws IOException { if (bdbCookieStore == null) { bdbCookieStore = new BdbCookieStore(); ConfigPath basePath = new ConfigPath("testBase", @@ -337,14 +337,14 @@ public class CookieStoreTest extends TmpDirTestCase { Iterator iter1 = sorted1.iterator(); Iterator iter2 = sorted2.iterator(); - for (int i = 0; i < list1.size(); i++) { + for (int i = 0; i < sorted1.size(); i++) { Cookie c1 = iter1.next(); Cookie c2 = iter2.next(); assertCookiesIdentical(c1, c2); } } - protected void assertCookieStoresEquivalent(BasicCookieStore simple, BdbCookieStore bdb) { + protected void assertCookieStoresEquivalent(BasicCookieStore simple, AbstractCookieStore bdb) { assertCookieListsEquivalent(simple.getCookies(), bdb.getCookies()); }