lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-09-22 21:55:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

some javadocs and other comments, some minor refactoring, change delimiter to ';' for the sortable key

Browse Source
This commit is contained in:
Noah Levitt
2014-09-29 17:51:24 -07:00
parent 87b408cc23
commit 5fd120dffd
6 changed files with 161 additions and 128 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/src/main/java/org/archive/modules/fetcher/AbstractCookieStore.java
+69 -16
View File
@@ -18,7 +18,11 @@
*/
package org.archive.modules.fetcher;
import it.unimi.dsi.mg4j.util.MutableString;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.Reader;
import java.util.Collection;
@@ -44,7 +48,8 @@ import org.springframework.context.Lifecycle;
import com.google.common.net.InternetDomainName;
abstract public class AbstractCookieStore implements Lifecycle, Checkpointable, CookieStore {
abstract public class AbstractCookieStore implements Lifecycle, Checkpointable,
CookieStore, FetchHTTPCookieStore {
protected final Logger logger =
Logger.getLogger(AbstractCookieStore.class.getName());
@@ -97,7 +102,6 @@ abstract public class AbstractCookieStore implements Lifecycle, Checkpointable,
}
}
abstract public void saveCookies(String saveCookiesFile);
protected void loadCookies(ConfigFile file) {
Reader reader = null;
try {
@@ -115,6 +119,45 @@ abstract public class AbstractCookieStore implements Lifecycle, Checkpointable,
}
}
public void saveCookies(String saveCookiesFile) {
// Do nothing if cookiesFile is not specified.
if (saveCookiesFile == null || saveCookiesFile.length() <= 0) {
return;
}
FileOutputStream out = null;
try {
out = new FileOutputStream(new File(saveCookiesFile));
String tab ="\t";
out.write("# Heritrix Cookie File\n".getBytes());
out.write("# This file is the Netscape cookies.txt format\n\n".getBytes());
for (Cookie cookie: getCookies()) {
// Guess an initial size
MutableString line = new MutableString(1024 * 2);
line.append(cookie.getDomain());
line.append(tab);
// XXX line.append(cookie.isDomainAttributeSpecified() ? "TRUE" : "FALSE");
line.append("TRUE");
line.append(tab);
line.append(cookie.getPath() != null ? cookie.getPath() : "/");
line.append(tab);
line.append(cookie.isSecure() ? "TRUE" : "FALSE");
line.append(tab);
line.append(cookie.getExpiryDate() != null ? cookie.getExpiryDate().getTime() / 1000 : -1);
line.append(tab);
line.append(cookie.getName());
line.append(tab);
line.append(cookie.getValue() != null ? cookie.getValue() : "");
line.append("\n");
out.write(line.toString().getBytes());
}
} catch (IOException e) {
logger.log(Level.SEVERE, "Unable to write " + saveCookiesFile, e);
} finally {
IOUtils.closeQuietly(out);
}
}
/**
* Load cookies. The input is text in the Netscape's 'cookies.txt' file
* format. Example entry of cookies.txt file:
@@ -203,32 +246,42 @@ abstract public class AbstractCookieStore implements Lifecycle, Checkpointable,
}
/**
* Adapted from {@link CookieIdentityComparator#compare(Cookie, Cookie)}
* XXX explain about sorting
* @param cookie
* @return
* Returns a string that uniquely identifies the cookie, and is prepended
* with the top private domain (one level below the TLD) associated with the
* cookie. This way such cookies can be grouped together in a sorted list,
* for example. The format The format of the key is
* {@code "topPrivateDomain;normalizedDomain;name;path"}. Adapted from
* {@link CookieIdentityComparator#compare(Cookie, Cookie)}.
*/
protected String makeSortKey(Cookie cookie) {
protected String sortableKey(Cookie cookie) {
String normalizedDomain = normalizeDomain(cookie.getDomain());
String topPrivateDomain = topPrivateDomain(normalizedDomain);
// use ";" as delimiter since it is the delimiter in the cookie header,
// so presumably can't appear in any of these values
StringBuilder buf = new StringBuilder(topPrivateDomain);
buf.append("-").append(normalizedDomain);
buf.append("-").append(cookie.getName());
buf.append("-").append(cookie.getPath() != null ? cookie.getPath() : "/");
buf.append(";").append(normalizedDomain);
buf.append(";").append(cookie.getName());
buf.append(";").append(cookie.getPath() != null ? cookie.getPath() : "/");
return buf.toString();
}
protected String topPrivateDomain(String domain) {
String topPrivateDomain = "";
if (InternetDomainName.isValid(domain)) {
InternetDomainName d = InternetDomainName.from(domain);
/**
* Returns the top private domain, i.e. the topmost assigned domain, one
* level below the TLD, for the supplied {@code host}. Returns
* {@code host} unaltered if a top private domain can't be identified (for
* example, if {@code host} is an IP address).
*/
protected String topPrivateDomain(String host) {
if (InternetDomainName.isValid(host)) {
InternetDomainName d = InternetDomainName.from(host);
if (d.hasPublicSuffix()) {
topPrivateDomain = d.topPrivateDomain().toString();
return d.topPrivateDomain().toString();
}
}
return topPrivateDomain;
return host;
}
protected String normalizeDomain(String domain) {
modules/src/main/java/org/archive/modules/fetcher/BdbCookieStore.java
+50 -94
View File
@@ -18,10 +18,6 @@
*/
package org.archive.modules.fetcher;
import it.unimi.dsi.mg4j.util.MutableString;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.util.Collection;
@@ -29,11 +25,8 @@ import java.util.Date;
import java.util.Iterator;
import java.util.List;
import java.util.ListIterator;
import java.util.Map.Entry;
import java.util.SortedMap;
import java.util.logging.Level;
import org.apache.commons.io.IOUtils;
import org.apache.http.client.CookieStore;
import org.apache.http.cookie.Cookie;
import org.archive.bdb.BdbModule;
@@ -43,23 +36,47 @@ import org.springframework.beans.factory.annotation.Autowired;
import com.sleepycat.bind.ByteArrayBinding;
import com.sleepycat.bind.serial.SerialBinding;
import com.sleepycat.bind.serial.StoredClassCatalog;
import com.sleepycat.collections.StoredCollection;
import com.sleepycat.collections.StoredSortedMap;
import com.sleepycat.je.Database;
import com.sleepycat.je.DatabaseException;
public class BdbCookieStore extends AbstractCookieStore implements CookieStore {
/**
* Cookie store using bdb for storage. Cookies are stored in a SortedMap keyed
* by {@link #sortableKey(Cookie)}, so they are grouped together by top private
* domain. {@link #cookieStoreFor(String)} returns a facade whose
* {@link CookieStore#getCookies()} returns a list of cookies limited to
* subdomains of the supplied top private domain.
*
* @see https://webarchive.jira.com/browse/HER-2070
* @see https://github.com/internetarchive/heritrix3/pull/96
* @see https://groups.yahoo.com/neo/groups/archive-crawler/conversations/messages/8620
*
* @contributor nlevitt
*/
public class BdbCookieStore extends AbstractCookieStore implements
FetchHTTPCookieStore, CookieStore {
/**
* Needed because httpclient requires List<Cookie> even though it only uses
* methods available on Collection. (+1 for python, -1 for java on this one)
* A {@link List} implementation that wraps a {@link Collection}. Needed
* because httpclient requires {@code List<Cookie>}.
*
* <p>
* This class is "restricted" in the sense that it is immutable, and also
* because some methods throw {@link RuntimeException} for other reasons.
* For example, {@link #iterator()} is not implemented, because we use this
* class to wrap a bdb {@link StoredCollection}, and iterators from that
* class need to be explicitly closed. Since this class hides the fact that
* a StoredCollection underlies it, we simply prevent {@link #iterator()}
* from being used.
*/
public static class CollectionListFacade<T> implements List<T> {
public static class RestrictedCollectionWrappedList<T> implements List<T> {
private Collection<T> wrapped;
public CollectionListFacade(Collection<T> wrapped) { this.wrapped = wrapped; }
public RestrictedCollectionWrappedList(Collection<T> wrapped) { this.wrapped = wrapped; }
@Override public int size() { return wrapped.size(); }
@Override public boolean isEmpty() { return wrapped.isEmpty(); }
@Override public boolean contains(Object o) { return wrapped.contains(o); }
@Override public Iterator<T> iterator() { return wrapped.iterator(); }
@Override public boolean isEmpty() { throw new RuntimeException("not implemented"); }
@Override public boolean contains(Object o) { throw new RuntimeException("not implemented"); }
@Override public Iterator<T> iterator() { throw new RuntimeException("not implemented"); }
@Override public Object[] toArray() { return wrapped.toArray(); }
@SuppressWarnings("hiding") @Override public <T> T[] toArray(T[] a) { return wrapped.toArray(a); }
@Override public boolean add(T e) { throw new RuntimeException("immutable list"); }
@@ -109,11 +126,11 @@ public class BdbCookieStore extends AbstractCookieStore implements CookieStore {
throw new RuntimeException(e);
}
}
public void addCookie(Cookie cookie) {
byte[] key;
try {
key = makeSortKey(cookie).getBytes("UTF-8");
key = sortableKey(cookie).getBytes("UTF-8");
} catch (UnsupportedEncodingException e) {
throw new RuntimeException(e); // impossible
}
@@ -124,21 +141,26 @@ public class BdbCookieStore extends AbstractCookieStore implements CookieStore {
cookies.remove(key);
}
}
public CookieStore cookieStoreFor(String topPrivateDomain) {
// XXX "the natural ordering of a stored collection is data byte order" -- explain more about subMap
/**
* Returns a {@link LimitedCookieStoreFacade} whose
* {@link LimitedCookieStoreFacade#getCookies()} method returns only the
* cookies from the domain {@code topPrivateDomainOrIP} and subdomains.
*/
public CookieStore cookieStoreFor(String topPrivateDomainOrIP) {
SortedMap<byte[], Cookie> domainCookiesSubMap;
try {
byte[] startKey = topPrivateDomain.getBytes("UTF-8");
byte[] endKey = (topPrivateDomain + ".").getBytes("UTF-8");
byte[] startKey = topPrivateDomainOrIP.getBytes("UTF-8");
char chAfterDelim = (char)(((int)';')+1);
byte[] endKey = (topPrivateDomainOrIP + chAfterDelim).getBytes("UTF-8");
domainCookiesSubMap = cookies.subMap(startKey, endKey);
} catch (UnsupportedEncodingException e) {
throw new RuntimeException(e); // impossible
}
Collection<Cookie> domainCookiesCollection = domainCookiesSubMap.values();
List<Cookie> domainCookiesList = new CollectionListFacade<Cookie>(domainCookiesCollection);
List<Cookie> domainCookiesList = new RestrictedCollectionWrappedList<Cookie>(domainCookiesCollection);
return new LimitedCookieStoreFacade(domainCookiesList);
}
@@ -155,7 +177,7 @@ public class BdbCookieStore extends AbstractCookieStore implements CookieStore {
public void finishCheckpoint(Checkpoint checkpointInProgress) {
// do nothing; handled by map checkpoint via BdbModule
}
/** are we a checkpoint recovery? (in which case, reuse stored cookie data?) */
protected boolean isCheckpointRecovery = false;
@Override
@@ -164,45 +186,6 @@ public class BdbCookieStore extends AbstractCookieStore implements CookieStore {
// actual state recovery happens via BdbModule
isCheckpointRecovery = true;
}
public void saveCookies(String saveCookiesFile) {
// Do nothing if cookiesFile is not specified.
if (saveCookiesFile == null || saveCookiesFile.length() <= 0) {
return;
}
FileOutputStream out = null;
try {
out = new FileOutputStream(new File(saveCookiesFile));
String tab ="\t";
out.write("# Heritrix Cookie File\n".getBytes());
out.write("# This file is the Netscape cookies.txt format\n\n".getBytes());
for (Cookie cookie: cookies.values()) {
// Guess an initial size
MutableString line = new MutableString(1024 * 2);
line.append(cookie.getDomain());
line.append(tab);
// XXX line.append(cookie.isDomainAttributeSpecified() ? "TRUE" : "FALSE");
line.append("TRUE");
line.append(tab);
line.append(cookie.getPath() != null ? cookie.getPath() : "/");
line.append(tab);
line.append(cookie.isSecure() ? "TRUE" : "FALSE");
line.append(tab);
line.append(cookie.getExpiryDate() != null ? cookie.getExpiryDate().getTime() / 1000 : -1);
line.append(tab);
line.append(cookie.getName());
line.append(tab);
line.append(cookie.getValue() != null ? cookie.getValue() : "");
line.append("\n");
out.write(line.toString().getBytes());
}
} catch (IOException e) {
logger.log(Level.SEVERE, "Unable to write " + saveCookiesFile, e);
} finally {
IOUtils.closeQuietly(out);
}
}
@Override
public void clear() {
@@ -215,7 +198,7 @@ public class BdbCookieStore extends AbstractCookieStore implements CookieStore {
@Override
public List<Cookie> getCookies() {
if (cookies != null) {
return new CollectionListFacade<Cookie>(cookies.values());
return new RestrictedCollectionWrappedList<Cookie>(cookies.values());
} else {
return null;
}
@@ -223,33 +206,6 @@ public class BdbCookieStore extends AbstractCookieStore implements CookieStore {
@Override
public boolean clearExpired(Date date) {
throw new RuntimeException("note implemented");
}
@Override
public String toString() {
Iterator<Entry<byte[], Cookie>> i = cookies.entrySet().iterator();
if (! i.hasNext())
return "{}";
StringBuilder sb = new StringBuilder();
sb.append('{');
for (;;) {
Entry<byte[], Cookie> e = i.next();
String key;
try {
key = new String(e.getKey(), "UTF-8");
} catch (UnsupportedEncodingException e1) {
throw new RuntimeException();
}
Cookie value = e.getValue();
sb.append(key);
sb.append('=');
sb.append(value == this ? "(this Map)" : value);
if (! i.hasNext())
return sb.append('}').toString();
sb.append(',').append(' ');
}
throw new RuntimeException("not implemented");
}
}
modules/src/main/java/org/archive/modules/fetcher/FetchHTTPCookieStore.java
+37
View File
@@ -0,0 +1,37 @@
/*
* This file is part of the Heritrix web crawler (crawler.archive.org).
*
* Licensed to the Internet Archive (IA) by one or more individual
* contributors.
*
* The IA licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.archive.modules.fetcher;
import org.apache.commons.httpclient.URIException;
import org.apache.http.client.CookieStore;
import org.archive.modules.CrawlURI;
public interface FetchHTTPCookieStore extends CookieStore {
/**
* Returns a {@link CookieStore} whose {@link CookieStore#getCookies()}
* returns all the cookies from {@code topPrivateDomain} and its subdomains.
*/
public CookieStore cookieStoreFor(String topPrivateDomain);
/**
* Returns a {@link CookieStore} whose {@link CookieStore#getCookies()}
* returns all the cookies that could possibly apply {@code curi}.
*/
public CookieStore cookieStoreFor(CrawlURI curi) throws URIException;
}
modules/src/main/java/org/archive/modules/fetcher/SimpleCookieStore.java
+1 -5
View File
@@ -28,6 +28,7 @@ import org.apache.http.impl.client.BasicCookieStore;
import org.archive.checkpointing.Checkpoint;
import org.springframework.beans.factory.annotation.Autowired;
/** In-memory cookie store, mostly for testing. */
public class SimpleCookieStore extends AbstractCookieStore implements CookieStore {
protected BasicCookieStore cookies;
@@ -66,11 +67,6 @@ public class SimpleCookieStore extends AbstractCookieStore implements CookieStor
return cookies.clearExpired(date);
}
@Override
public void saveCookies(String saveCookiesFile) {
throw new RuntimeException("not implemented");
}
@Override
public CookieStore cookieStoreFor(String topPrivateDomain) {
return this;
modules/src/test/java/org/archive/modules/fetcher/CookieFetchHTTPIntegrationTest.java
+1 -10
View File
@@ -59,15 +59,6 @@ import com.google.common.io.Files;
@SuppressWarnings("restriction")
public class CookieFetchHTTPIntegrationTest extends ProcessorTestBase {
// private static Logger logger = Logger.getLogger(FetchHTTPTest.class.getName());
// static {
// Logger.getLogger("").setLevel(Level.FINE);
// for (java.util.logging.Handler h: Logger.getLogger("").getHandlers()) {
// h.setLevel(Level.ALL);
// h.setFormatter(new OneLineSimpleLogger());
// }
// }
protected static class TestHandler extends SessionHandler {
public TestHandler() {
super();
@@ -219,7 +210,7 @@ public class CookieFetchHTTPIntegrationTest extends ProcessorTestBase {
return bdb;
}
protected BdbCookieStore bdbCookieStore() throws IOException {
protected AbstractCookieStore bdbCookieStore() throws IOException {
if (bdbCookieStore == null) {
bdbCookieStore = new BdbCookieStore();
ConfigPath basePath = new ConfigPath("testBase",
modules/src/test/java/org/archive/modules/fetcher/CookieStoreTest.java
+3 -3
View File
@@ -69,7 +69,7 @@ public class CookieStoreTest extends TmpDirTestCase {
return bdb;
}
protected BdbCookieStore bdbCookieStore() throws IOException {
protected AbstractCookieStore bdbCookieStore() throws IOException {
if (bdbCookieStore == null) {
bdbCookieStore = new BdbCookieStore();
ConfigPath basePath = new ConfigPath("testBase",
@@ -337,14 +337,14 @@ public class CookieStoreTest extends TmpDirTestCase {
Iterator<Cookie> iter1 = sorted1.iterator();
Iterator<Cookie> iter2 = sorted2.iterator();
for (int i = 0; i < list1.size(); i++) {
for (int i = 0; i < sorted1.size(); i++) {
Cookie c1 = iter1.next();
Cookie c2 = iter2.next();
assertCookiesIdentical(c1, c2);
}
}
protected void assertCookieStoresEquivalent(BasicCookieStore simple, BdbCookieStore bdb) {
protected void assertCookieStoresEquivalent(BasicCookieStore simple, AbstractCookieStore bdb) {
assertCookieListsEquivalent(simple.getCookies(), bdb.getCookies());
}