From 47b7caad290cc5dde37cbdca06cd4fa3ab6b0d2b Mon Sep 17 00:00:00 2001
From: Noah Levitt <nlevitt@archive.org>
Date: Fri, 26 Sep 2014 20:56:45 -0700
Subject: [PATCH] initial bdb implementation; facade by topmost assigned
 domain, since cookies can apply to subdomains

---
 .../restlet/profile-crawler-beans.cxml        |   2 +-
 .../archive/modules/fetcher/FormAuthTest.java |   2 +-
 .../archive/modules/fetcher/FetchHTTP.java    |   6 +-
 .../modules/fetcher/FetchHTTPRequest.java     |  22 ++-
 .../fetcher/SimpleHostCookieSetStore.java     | 174 ------------------
 .../modules/fetcher/FetchHTTPTests.java       |   3 +-
 6 files changed, 26 insertions(+), 183 deletions(-)
 delete mode 100644 modules/src/main/java/org/archive/modules/fetcher/SimpleHostCookieSetStore.java

diff --git a/engine/src/main/resources/org/archive/crawler/restlet/profile-crawler-beans.cxml b/engine/src/main/resources/org/archive/crawler/restlet/profile-crawler-beans.cxml
index 02dba028..f83ad838 100644
--- a/engine/src/main/resources/org/archive/crawler/restlet/profile-crawler-beans.cxml
+++ b/engine/src/main/resources/org/archive/crawler/restlet/profile-crawler-beans.cxml
@@ -679,7 +679,7 @@ http://example.example/example
  
  <!-- BDBCOOKIESTORE: disk-based cookie storage for FetchHTTP -->
  <bean id="cookieStore" 
-   class="org.archive.modules.fetcher.BdbCookieStore">
+  class="org.archive.modules.fetcher.BdbDomainCookieSetStore">
   <!-- <property name="cookiesLoadFile"><null/></property> -->
   <!-- <property name="cookiesSaveFile"><null/></property> -->
   <!-- <property name="bdbModule">
diff --git a/engine/src/test/java/org/archive/modules/fetcher/FormAuthTest.java b/engine/src/test/java/org/archive/modules/fetcher/FormAuthTest.java
index f6b77b1a..a616a809 100644
--- a/engine/src/test/java/org/archive/modules/fetcher/FormAuthTest.java
+++ b/engine/src/test/java/org/archive/modules/fetcher/FormAuthTest.java
@@ -78,7 +78,7 @@ public class FormAuthTest extends TestCase {
     protected FetchHTTP getFetcher() throws IOException {
         if (fetchHttp == null) { 
             fetchHttp = new FetchHTTP();
-            fetchHttp.setCookieStore(new SimpleHostCookieSetStore());
+            fetchHttp.setCookieStore(new SimpleDomainCookieSetStore());
             fetchHttp.setServerCache(new DefaultServerCache());
             CrawlMetadata uap = new CrawlMetadata();
             uap.setUserAgentTemplate(getClass().getName());
diff --git a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java
index a7265196..6ccd7e47 100644
--- a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java
+++ b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java
@@ -266,12 +266,12 @@ public class FetchHTTP extends Processor implements Lifecycle {
         kp.put("acceptHeaders",headers);
     }
 
-    protected SimpleHostCookieSetStore cookieStore;
+    protected AbstractDomainCookieSetStore cookieStore;
     @Autowired(required=false)
-    public void setCookieStore(SimpleHostCookieSetStore cookieStore) {
+    public void setCookieStore(AbstractDomainCookieSetStore cookieStore) {
         this.cookieStore = cookieStore;
     }
-    public SimpleHostCookieSetStore getCookieStore() {
+    public AbstractDomainCookieSetStore getCookieStore() {
         return cookieStore;
     }
     
diff --git a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTPRequest.java b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTPRequest.java
index 8627b8e2..79c1219d 100644
--- a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTPRequest.java
+++ b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTPRequest.java
@@ -35,6 +35,7 @@ import java.nio.charset.CodingErrorAction;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.concurrent.atomic.AtomicLong;
@@ -110,6 +111,8 @@ import org.archive.modules.net.CrawlServer;
 import org.archive.modules.net.ServerCache;
 import org.archive.util.Recorder;
 
+import com.google.common.net.InternetDomainName;
+
 /**
  * @contributor nlevitt
  */
@@ -452,14 +455,29 @@ class FetchHTTPRequest {
             userAgent = fetcher.getUserAgentProvider().getUserAgent();
         }
         httpClientBuilder.setUserAgent(userAgent);
-        
-        CookieStore hostCookieStore = fetcher.getCookieStore().cookieStoreFor(targetHost.getHostName());
+
+        CookieStore hostCookieStore = fetcher.getCookieStore().cookieStoreFor(topmostAssigned());
         httpClientBuilder.setDefaultCookieStore(hostCookieStore);
         
         connMan = buildConnectionManager();
         httpClientBuilder.setConnectionManager(connMan);
     }
 
+    protected String topmostAssigned() {
+        String domain = targetHost.getHostName();
+        if (domain == null) {
+            domain = "";
+        }
+        if (domain.startsWith(".")) {
+            domain = domain.substring(1);
+        }
+        domain = domain.toLowerCase(Locale.ENGLISH);
+        if (InternetDomainName.isValid(domain)) {
+            domain = InternetDomainName.from(domain).topPrivateDomain().toString();
+        }
+        return domain;
+    }
+
     protected HttpClientConnectionManager buildConnectionManager() {
         Registry<ConnectionSocketFactory> socketFactoryRegistry = RegistryBuilder.<ConnectionSocketFactory>create()
                 .register("http", PlainConnectionSocketFactory.INSTANCE)
diff --git a/modules/src/main/java/org/archive/modules/fetcher/SimpleHostCookieSetStore.java b/modules/src/main/java/org/archive/modules/fetcher/SimpleHostCookieSetStore.java
deleted file mode 100644
index ccc3e260..00000000
--- a/modules/src/main/java/org/archive/modules/fetcher/SimpleHostCookieSetStore.java
+++ /dev/null
@@ -1,174 +0,0 @@
-/*
- *  This file is part of the Heritrix web crawler (crawler.archive.org).
- *
- *  Licensed to the Internet Archive (IA) by one or more individual 
- *  contributors. 
- *
- *  The IA licenses this file to You under the Apache License, Version 2.0
- *  (the "License"); you may not use this file except in compliance with
- *  the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-package org.archive.modules.fetcher;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.Date;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.TreeSet;
-import java.util.logging.Logger;
-
-import org.apache.http.client.CookieStore;
-import org.apache.http.cookie.Cookie;
-import org.apache.http.cookie.CookieIdentityComparator;
-import org.archive.checkpointing.Checkpoint;
-import org.archive.checkpointing.Checkpointable;
-import org.archive.spring.ConfigFile;
-import org.archive.spring.ConfigPath;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.context.Lifecycle;
-
-public class SimpleHostCookieSetStore implements Lifecycle, Checkpointable {
-
-    private static final Logger logger = 
-            Logger.getLogger(SimpleHostCookieSetStore.class.getName());
-
-    protected ConfigFile cookiesLoadFile = null;
-    public ConfigFile getCookiesLoadFile() {
-        return cookiesLoadFile;
-    }
-    public void setCookiesLoadFile(ConfigFile cookiesLoadFile) {
-        this.cookiesLoadFile = cookiesLoadFile;
-    }
-
-    protected ConfigPath cookiesSaveFile = null;
-    public ConfigPath getCookiesSaveFile() {
-        return cookiesSaveFile;
-    }
-    public void setCookiesSaveFile(ConfigPath cookiesSaveFile) {
-        this.cookiesSaveFile = cookiesSaveFile;
-    }
-
-    protected static final Comparator<Cookie> cookieComparator = new CookieIdentityComparator();
-    
-    protected Map<String,TreeSet<Cookie>> cookiesByHost;
-    
-    protected void prepare() {
-        cookiesByHost = new LinkedHashMap<String, TreeSet<Cookie>>();
-    }
-
-    public CookieStore cookieStoreFor(String host) {
-        TreeSet<Cookie> hostCookieSet;
-        synchronized (this) {
-            hostCookieSet = cookiesByHost.get(host);
-        }
-        final List<Cookie> hostCookies;
-        if (hostCookieSet != null) {
-            hostCookies = new ArrayList<Cookie>(hostCookieSet);
-        } else {
-            hostCookies = Collections.emptyList();
-        }
-        
-        return new CookieStore() {
-            @Override
-            public List<Cookie> getCookies() {
-                return hostCookies;
-            }
-            
-            @Override
-            public boolean clearExpired(Date date) {
-                throw new RuntimeException("not implemented");
-            }
-            
-            @Override
-            public void clear() {
-                throw new RuntimeException("not implemented");
-            }
-            
-            @Override
-            public void addCookie(Cookie cookie) {
-                SimpleHostCookieSetStore.this.addCookie(cookie);
-            }
-        };
-    }
-
-    synchronized protected void addCookie(Cookie cookie) {
-        String host = cookie.getDomain();
-        if (host == null) {
-            host = "";
-        }
-        host = host.toLowerCase(Locale.ENGLISH);
-        
-        TreeSet<Cookie> hostCookieSet = cookiesByHost.get(host);
-        if (hostCookieSet == null) {
-            hostCookieSet = new TreeSet<Cookie>(cookieComparator);
-        }
-        hostCookieSet.add(cookie);
-
-        cookiesByHost.put(host, hostCookieSet);
-    }
-
-    @Override
-    public void startCheckpoint(Checkpoint checkpointInProgress) {
-        throw new RuntimeException("not implemented");
-    }
-
-    @Override
-    public void doCheckpoint(Checkpoint checkpointInProgress)
-            throws IOException {
-        throw new RuntimeException("not implemented");
-    }
-
-    @Override
-    public void finishCheckpoint(Checkpoint checkpointInProgress) {
-        throw new RuntimeException("not implemented");
-    }
-
-    @Override
-    @Autowired(required=false)
-    public void setRecoveryCheckpoint(Checkpoint recoveryCheckpoint) {
-        throw new RuntimeException("not implemented");
-    }
-
-    protected boolean isRunning = false;
-    
-    @Override
-    public void start() {
-        if (isRunning()) {
-            return;
-        }
-        prepare();
-//        if (getCookiesLoadFile()!=null) {
-//            loadCookies(getCookiesLoadFile());
-//        }
-        isRunning = true;
-    }
-
-    @Override
-    public void stop() {
-        isRunning = false;
-    }
-
-    @Override
-    public boolean isRunning() {
-        return isRunning;
-    }
-
-    public void saveCookies() {
-        if (getCookiesSaveFile() != null) {
-            // saveCookies(getCookiesSaveFile().getFile().getAbsolutePath());
-        }
-    }
-}
diff --git a/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTests.java b/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTests.java
index e5298b9c..80e8c25f 100644
--- a/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTests.java
+++ b/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTests.java
@@ -68,7 +68,6 @@ import org.archive.modules.deciderules.RejectDecideRule;
 import org.archive.modules.recrawl.FetchHistoryProcessor;
 import org.archive.modules.revisit.ServerNotModifiedRevisit;
 import org.archive.net.UURI;
-import org.archive.util.OneLineSimpleLogger;
 import org.jboss.netty.handler.codec.http.HttpRequest;
 import org.littleshoot.proxy.DefaultHttpProxyServer;
 import org.littleshoot.proxy.HttpFilter;
@@ -883,7 +882,7 @@ public class FetchHTTPTests extends ProcessorTestBase {
     
     public static FetchHTTP newTestFetchHttp(String userAgentString) {
         FetchHTTP fetchHttp = new FetchHTTP();
-        fetchHttp.setCookieStore(new SimpleHostCookieSetStore());
+        fetchHttp.setCookieStore(new SimpleDomainCookieSetStore());
         fetchHttp.setServerCache(new DefaultServerCache());
         CrawlMetadata uap = new CrawlMetadata();
         uap.setUserAgentTemplate(userAgentString);