lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-09-13 01:05:47 +00:00
Code Issues Packages Projects Releases Wiki Activity

[HER-1864] cookies are being set and sent back on TLDs/public-suffixes (like '.com') (sometimes called 'supercookies')

Browse Source 
* CookieSpecBase
    use Guava InternetDomainName public-suffixes to both (1) prevent storing cookie on public-suffix; (2) prevent looking-up cookies for public-suffixes
* BdbCookieStorage
    correct checkpoint/relaunch behavior: don't reuse persisted cookies on normal relaunches; do reuse on resume-from-checkpoint
This commit is contained in:
gojomo
2011-02-16 01:18:57 +00:00
parent 5de965590c
commit 548114dbde
2 changed files with 70 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
commons/src/main/java/org/apache/commons/httpclient/cookie/CookieSpecBase.java
+38 -10
View File
@@ -45,6 +45,7 @@ import org.apache.commons.httpclient.util.DateUtil;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import com.google.common.net.InternetDomainName; // <- IA/HERITRIX CHANGE
import com.sleepycat.collections.StoredIterator; // <- IA/HERITRIX CHANGE
/**
@@ -420,7 +421,26 @@ public class CookieSpecBase implements CookieSpec {
"Illegal domain attribute \"" + cookie.getDomain()
+ "\". Domain of origin: \"" + host + "\"");
}
}
// BEGIN IA/HERITRIX ADDITION
else {
// requested domain is suffix of origin host; now make sure
// it's not a public-suffix
String requestedDomain = cookie.getDomain();
if(requestedDomain.startsWith(".")) {
requestedDomain = requestedDomain.substring(1);
}
try {
if((InternetDomainName.fromLenient(requestedDomain)).isPublicSuffix()) {
throw new MalformedCookieException(
"Illegal public-suffix domain attribute \"" + cookie.getDomain()
+ "\". Domain of origin: \"" + host + "\"");
}
} catch (IllegalArgumentException e) {
// TODO: consider if this means cookie should be invalid
}
}
// END IA/HERITRIX ADDITION
} else {
if (!host.equals(cookie.getDomain())) {
throw new MalformedCookieException(
@@ -593,17 +613,21 @@ public class CookieSpecBase implements CookieSpec {
LOG.trace("enter CookieSpecBase.match("
+ "String, int, String, boolean, SortedMap)");
// TODO: skip meaningless 'narrowing' when host is a numeric IP
// (harmless in the meantime)
if (cookies == null) {
return null;
}
List matching = new LinkedList();
String narrowHost = host;
do {
Iterator iter = cookies.subMap(narrowHost,
narrowHost + Cookie.DOMAIN_OVERBOUNDS).values().iterator();
InternetDomainName domain;
try {
domain = InternetDomainName.fromLenient(host);
} catch(IllegalArgumentException e) {
domain = null;
}
String candidate = (domain!=null) ? domain.toString() : host;
while(candidate!=null) {
Iterator iter = cookies.subMap(candidate,
candidate + Cookie.DOMAIN_OVERBOUNDS).values().iterator();
while (iter.hasNext()) {
Cookie cookie = (Cookie) (iter.next());
if (match(host, port, path, secure, cookie)) {
@@ -611,9 +635,13 @@ public class CookieSpecBase implements CookieSpec {
}
}
StoredIterator.close(iter);
int trimTo = narrowHost.indexOf('.', 1);
narrowHost = (trimTo < 0) ? null : narrowHost.substring(trimTo+1);
} while (narrowHost != null);
if(domain!=null && domain.isUnderPublicSuffix()) {
domain = domain.parent();
candidate = domain.toString();
} else {
candidate = null;
}
}
return (Cookie[]) matching.toArray(new Cookie[matching.size()]);
}
modules/src/main/java/org/archive/modules/fetcher/BdbCookieStorage.java
+32 -4
View File
@@ -19,11 +19,14 @@
package org.archive.modules.fetcher;
import java.io.IOException;
import java.util.Map;
import java.util.SortedMap;
import org.apache.commons.httpclient.Cookie;
import org.archive.bdb.BdbModule;
import org.archive.checkpointing.Checkpoint;
import org.archive.checkpointing.Checkpointable;
import org.springframework.beans.factory.annotation.Autowired;
import com.sleepycat.bind.serial.SerialBinding;
@@ -39,7 +42,7 @@ import com.sleepycat.je.DatabaseException;
*
* @author pjack
*/
public class BdbCookieStorage extends AbstractCookieStorage {
public class BdbCookieStorage extends AbstractCookieStorage implements Checkpointable {
private static final long serialVersionUID = 1L;
protected BdbModule bdb;
@@ -48,6 +51,9 @@ public class BdbCookieStorage extends AbstractCookieStorage {
this.bdb = bdb;
}
/** are we a checkpoint recovery? (in which case, reuse stored cookie data?) */
boolean isCheckpointRecovery = false;
public static String COOKIEDB_NAME = "http_cookies";
private transient Database cookieDb;
@@ -62,14 +68,13 @@ public class BdbCookieStorage extends AbstractCookieStorage {
BdbModule.BdbConfig dbConfig = new BdbModule.BdbConfig();
dbConfig.setTransactional(false);
dbConfig.setAllowCreate(true);
cookieDb = bdb.openDatabase(COOKIEDB_NAME, dbConfig, true);
cookieDb = bdb.openDatabase(COOKIEDB_NAME, dbConfig, isCheckpointRecovery);
cookies =
new StoredSortedMap<String,Cookie>(
cookieDb,
new StringBinding(),
new SerialBinding<Cookie>(classCatalog,Cookie.class),
true);
@SuppressWarnings("unchecked")
SortedMap<String,Cookie> result = cookies;
return result;
} catch (DatabaseException e) {
@@ -77,7 +82,6 @@ public class BdbCookieStorage extends AbstractCookieStorage {
}
}
@SuppressWarnings("unchecked")
public SortedMap<String, Cookie> getCookiesMap() {
// assert cookies != null : "cookie map not set up";
return cookies;
@@ -85,4 +89,28 @@ public class BdbCookieStorage extends AbstractCookieStorage {
protected void innerSaveCookiesMap(Map<String, Cookie> map) {
}
@Override
public void startCheckpoint(Checkpoint checkpointInProgress) {
// do nothing; handled by map checkpoint via BdbModule
}
@Override
public void doCheckpoint(Checkpoint checkpointInProgress)
throws IOException {
// do nothing; handled by map checkpoint via BdbModule
}
@Override
public void finishCheckpoint(Checkpoint checkpointInProgress) {
// do nothing; handled by map checkpoint via BdbModule
}
@Override
public void setRecoveryCheckpoint(Checkpoint recoveryCheckpoint) {
// just remember that we are doing checkpoint-recovery;
// actual state recovery happens via BdbModule
isCheckpointRecovery = true;
}
}