From 548114dbdeb1915e1f02ba5a4cafe4747bc503ea Mon Sep 17 00:00:00 2001 From: gojomo Date: Wed, 16 Feb 2011 01:18:57 +0000 Subject: [PATCH] [HER-1864] cookies are being set and sent back on TLDs/public-suffixes (like '.com') (sometimes called 'supercookies') * CookieSpecBase use Guava InternetDomainName public-suffixes to both (1) prevent storing cookie on public-suffix; (2) prevent looking-up cookies for public-suffixes * BdbCookieStorage correct checkpoint/relaunch behavior: don't reuse persisted cookies on normal relaunches; do reuse on resume-from-checkpoint --- .../httpclient/cookie/CookieSpecBase.java | 48 +++++++++++++++---- .../modules/fetcher/BdbCookieStorage.java | 36 ++++++++++++-- 2 files changed, 70 insertions(+), 14 deletions(-) diff --git a/commons/src/main/java/org/apache/commons/httpclient/cookie/CookieSpecBase.java b/commons/src/main/java/org/apache/commons/httpclient/cookie/CookieSpecBase.java index d15121d6..60b3d65c 100644 --- a/commons/src/main/java/org/apache/commons/httpclient/cookie/CookieSpecBase.java +++ b/commons/src/main/java/org/apache/commons/httpclient/cookie/CookieSpecBase.java @@ -45,6 +45,7 @@ import org.apache.commons.httpclient.util.DateUtil; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import com.google.common.net.InternetDomainName; // <- IA/HERITRIX CHANGE import com.sleepycat.collections.StoredIterator; // <- IA/HERITRIX CHANGE /** @@ -420,7 +421,26 @@ public class CookieSpecBase implements CookieSpec { "Illegal domain attribute \"" + cookie.getDomain() + "\". Domain of origin: \"" + host + "\""); } + } + // BEGIN IA/HERITRIX ADDITION + else { + // requested domain is suffix of origin host; now make sure + // it's not a public-suffix + String requestedDomain = cookie.getDomain(); + if(requestedDomain.startsWith(".")) { + requestedDomain = requestedDomain.substring(1); + } + try { + if((InternetDomainName.fromLenient(requestedDomain)).isPublicSuffix()) { + throw new MalformedCookieException( + "Illegal public-suffix domain attribute \"" + cookie.getDomain() + + "\". Domain of origin: \"" + host + "\""); + } + } catch (IllegalArgumentException e) { + // TODO: consider if this means cookie should be invalid + } } + // END IA/HERITRIX ADDITION } else { if (!host.equals(cookie.getDomain())) { throw new MalformedCookieException( @@ -593,17 +613,21 @@ public class CookieSpecBase implements CookieSpec { LOG.trace("enter CookieSpecBase.match(" + "String, int, String, boolean, SortedMap)"); - // TODO: skip meaningless 'narrowing' when host is a numeric IP - // (harmless in the meantime) - if (cookies == null) { return null; } List matching = new LinkedList(); - String narrowHost = host; - do { - Iterator iter = cookies.subMap(narrowHost, - narrowHost + Cookie.DOMAIN_OVERBOUNDS).values().iterator(); + InternetDomainName domain; + try { + domain = InternetDomainName.fromLenient(host); + } catch(IllegalArgumentException e) { + domain = null; + } + + String candidate = (domain!=null) ? domain.toString() : host; + while(candidate!=null) { + Iterator iter = cookies.subMap(candidate, + candidate + Cookie.DOMAIN_OVERBOUNDS).values().iterator(); while (iter.hasNext()) { Cookie cookie = (Cookie) (iter.next()); if (match(host, port, path, secure, cookie)) { @@ -611,9 +635,13 @@ public class CookieSpecBase implements CookieSpec { } } StoredIterator.close(iter); - int trimTo = narrowHost.indexOf('.', 1); - narrowHost = (trimTo < 0) ? null : narrowHost.substring(trimTo+1); - } while (narrowHost != null); + if(domain!=null && domain.isUnderPublicSuffix()) { + domain = domain.parent(); + candidate = domain.toString(); + } else { + candidate = null; + } + } return (Cookie[]) matching.toArray(new Cookie[matching.size()]); } diff --git a/modules/src/main/java/org/archive/modules/fetcher/BdbCookieStorage.java b/modules/src/main/java/org/archive/modules/fetcher/BdbCookieStorage.java index c310257c..2026c159 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/BdbCookieStorage.java +++ b/modules/src/main/java/org/archive/modules/fetcher/BdbCookieStorage.java @@ -19,11 +19,14 @@ package org.archive.modules.fetcher; +import java.io.IOException; import java.util.Map; import java.util.SortedMap; import org.apache.commons.httpclient.Cookie; import org.archive.bdb.BdbModule; +import org.archive.checkpointing.Checkpoint; +import org.archive.checkpointing.Checkpointable; import org.springframework.beans.factory.annotation.Autowired; import com.sleepycat.bind.serial.SerialBinding; @@ -39,7 +42,7 @@ import com.sleepycat.je.DatabaseException; * * @author pjack */ -public class BdbCookieStorage extends AbstractCookieStorage { +public class BdbCookieStorage extends AbstractCookieStorage implements Checkpointable { private static final long serialVersionUID = 1L; protected BdbModule bdb; @@ -48,6 +51,9 @@ public class BdbCookieStorage extends AbstractCookieStorage { this.bdb = bdb; } + /** are we a checkpoint recovery? (in which case, reuse stored cookie data?) */ + boolean isCheckpointRecovery = false; + public static String COOKIEDB_NAME = "http_cookies"; private transient Database cookieDb; @@ -62,14 +68,13 @@ public class BdbCookieStorage extends AbstractCookieStorage { BdbModule.BdbConfig dbConfig = new BdbModule.BdbConfig(); dbConfig.setTransactional(false); dbConfig.setAllowCreate(true); - cookieDb = bdb.openDatabase(COOKIEDB_NAME, dbConfig, true); + cookieDb = bdb.openDatabase(COOKIEDB_NAME, dbConfig, isCheckpointRecovery); cookies = new StoredSortedMap( cookieDb, new StringBinding(), new SerialBinding(classCatalog,Cookie.class), true); - @SuppressWarnings("unchecked") SortedMap result = cookies; return result; } catch (DatabaseException e) { @@ -77,7 +82,6 @@ public class BdbCookieStorage extends AbstractCookieStorage { } } - @SuppressWarnings("unchecked") public SortedMap getCookiesMap() { // assert cookies != null : "cookie map not set up"; return cookies; @@ -85,4 +89,28 @@ public class BdbCookieStorage extends AbstractCookieStorage { protected void innerSaveCookiesMap(Map map) { } + + + + @Override + public void startCheckpoint(Checkpoint checkpointInProgress) { + // do nothing; handled by map checkpoint via BdbModule + } + @Override + public void doCheckpoint(Checkpoint checkpointInProgress) + throws IOException { + // do nothing; handled by map checkpoint via BdbModule + } + @Override + public void finishCheckpoint(Checkpoint checkpointInProgress) { + // do nothing; handled by map checkpoint via BdbModule + } + + @Override + public void setRecoveryCheckpoint(Checkpoint recoveryCheckpoint) { + // just remember that we are doing checkpoint-recovery; + // actual state recovery happens via BdbModule + isCheckpointRecovery = true; + } + }