lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-07-18 05:27:11 +00:00
Code Issues Packages Projects Releases Wiki Activity

initial bdb implementation; facade by topmost assigned domain, since cookies can apply to subdomains

Browse Source
This commit is contained in:
Noah Levitt
2014-09-26 20:56:45 -07:00
parent 886a0f2136
commit 47b7caad29
6 changed files with 26 additions and 183 deletions
Download Patch File Download Diff File Expand all files Collapse all files
engine/src/main/resources/org/archive/crawler/restlet/profile-crawler-beans.cxml
+1 -1
View File
@@ -679,7 +679,7 @@ http://example.example/example
<!-- BDBCOOKIESTORE: disk-based cookie storage for FetchHTTP -->
<bean id="cookieStore"
class="org.archive.modules.fetcher.BdbCookieStore">
class="org.archive.modules.fetcher.BdbDomainCookieSetStore">
<!-- <property name="cookiesLoadFile"><null/></property> -->
<!-- <property name="cookiesSaveFile"><null/></property> -->
<!-- <property name="bdbModule">
engine/src/test/java/org/archive/modules/fetcher/FormAuthTest.java
+1 -1
View File
@@ -78,7 +78,7 @@ public class FormAuthTest extends TestCase {
protected FetchHTTP getFetcher() throws IOException {
if (fetchHttp == null) {
fetchHttp = new FetchHTTP();
fetchHttp.setCookieStore(new SimpleHostCookieSetStore());
fetchHttp.setCookieStore(new SimpleDomainCookieSetStore());
fetchHttp.setServerCache(new DefaultServerCache());
CrawlMetadata uap = new CrawlMetadata();
uap.setUserAgentTemplate(getClass().getName());
modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java
+3 -3
View File
@@ -266,12 +266,12 @@ public class FetchHTTP extends Processor implements Lifecycle {
kp.put("acceptHeaders",headers);
}
protected SimpleHostCookieSetStore cookieStore;
protected AbstractDomainCookieSetStore cookieStore;
@Autowired(required=false)
public void setCookieStore(SimpleHostCookieSetStore cookieStore) {
public void setCookieStore(AbstractDomainCookieSetStore cookieStore) {
this.cookieStore = cookieStore;
}
public SimpleHostCookieSetStore getCookieStore() {
public AbstractDomainCookieSetStore getCookieStore() {
return cookieStore;
}
modules/src/main/java/org/archive/modules/fetcher/FetchHTTPRequest.java
+20 -2
View File
@@ -35,6 +35,7 @@ import java.nio.charset.CodingErrorAction;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Map.Entry;
import java.util.concurrent.atomic.AtomicLong;
@@ -110,6 +111,8 @@ import org.archive.modules.net.CrawlServer;
import org.archive.modules.net.ServerCache;
import org.archive.util.Recorder;
import com.google.common.net.InternetDomainName;
/**
* @contributor nlevitt
*/
@@ -452,14 +455,29 @@ class FetchHTTPRequest {
userAgent = fetcher.getUserAgentProvider().getUserAgent();
}
httpClientBuilder.setUserAgent(userAgent);
CookieStore hostCookieStore = fetcher.getCookieStore().cookieStoreFor(targetHost.getHostName());
CookieStore hostCookieStore = fetcher.getCookieStore().cookieStoreFor(topmostAssigned());
httpClientBuilder.setDefaultCookieStore(hostCookieStore);
connMan = buildConnectionManager();
httpClientBuilder.setConnectionManager(connMan);
}
protected String topmostAssigned() {
String domain = targetHost.getHostName();
if (domain == null) {
domain = "";
}
if (domain.startsWith(".")) {
domain = domain.substring(1);
}
domain = domain.toLowerCase(Locale.ENGLISH);
if (InternetDomainName.isValid(domain)) {
domain = InternetDomainName.from(domain).topPrivateDomain().toString();
}
return domain;
}
protected HttpClientConnectionManager buildConnectionManager() {
Registry<ConnectionSocketFactory> socketFactoryRegistry = RegistryBuilder.<ConnectionSocketFactory>create()
.register("http", PlainConnectionSocketFactory.INSTANCE)
modules/src/main/java/org/archive/modules/fetcher/SimpleHostCookieSetStore.java
-174
View File
@@ -1,174 +0,0 @@
/*
* This file is part of the Heritrix web crawler (crawler.archive.org).
*
* Licensed to the Internet Archive (IA) by one or more individual
* contributors.
*
* The IA licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.archive.modules.fetcher;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.Date;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.TreeSet;
import java.util.logging.Logger;
import org.apache.http.client.CookieStore;
import org.apache.http.cookie.Cookie;
import org.apache.http.cookie.CookieIdentityComparator;
import org.archive.checkpointing.Checkpoint;
import org.archive.checkpointing.Checkpointable;
import org.archive.spring.ConfigFile;
import org.archive.spring.ConfigPath;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.Lifecycle;
public class SimpleHostCookieSetStore implements Lifecycle, Checkpointable {
private static final Logger logger =
Logger.getLogger(SimpleHostCookieSetStore.class.getName());
protected ConfigFile cookiesLoadFile = null;
public ConfigFile getCookiesLoadFile() {
return cookiesLoadFile;
}
public void setCookiesLoadFile(ConfigFile cookiesLoadFile) {
this.cookiesLoadFile = cookiesLoadFile;
}
protected ConfigPath cookiesSaveFile = null;
public ConfigPath getCookiesSaveFile() {
return cookiesSaveFile;
}
public void setCookiesSaveFile(ConfigPath cookiesSaveFile) {
this.cookiesSaveFile = cookiesSaveFile;
}
protected static final Comparator<Cookie> cookieComparator = new CookieIdentityComparator();
protected Map<String,TreeSet<Cookie>> cookiesByHost;
protected void prepare() {
cookiesByHost = new LinkedHashMap<String, TreeSet<Cookie>>();
}
public CookieStore cookieStoreFor(String host) {
TreeSet<Cookie> hostCookieSet;
synchronized (this) {
hostCookieSet = cookiesByHost.get(host);
}
final List<Cookie> hostCookies;
if (hostCookieSet != null) {
hostCookies = new ArrayList<Cookie>(hostCookieSet);
} else {
hostCookies = Collections.emptyList();
}
return new CookieStore() {
@Override
public List<Cookie> getCookies() {
return hostCookies;
}
@Override
public boolean clearExpired(Date date) {
throw new RuntimeException("not implemented");
}
@Override
public void clear() {
throw new RuntimeException("not implemented");
}
@Override
public void addCookie(Cookie cookie) {
SimpleHostCookieSetStore.this.addCookie(cookie);
}
};
}
synchronized protected void addCookie(Cookie cookie) {
String host = cookie.getDomain();
if (host == null) {
host = "";
}
host = host.toLowerCase(Locale.ENGLISH);
TreeSet<Cookie> hostCookieSet = cookiesByHost.get(host);
if (hostCookieSet == null) {
hostCookieSet = new TreeSet<Cookie>(cookieComparator);
}
hostCookieSet.add(cookie);
cookiesByHost.put(host, hostCookieSet);
}
@Override
public void startCheckpoint(Checkpoint checkpointInProgress) {
throw new RuntimeException("not implemented");
}
@Override
public void doCheckpoint(Checkpoint checkpointInProgress)
throws IOException {
throw new RuntimeException("not implemented");
}
@Override
public void finishCheckpoint(Checkpoint checkpointInProgress) {
throw new RuntimeException("not implemented");
}
@Override
@Autowired(required=false)
public void setRecoveryCheckpoint(Checkpoint recoveryCheckpoint) {
throw new RuntimeException("not implemented");
}
protected boolean isRunning = false;
@Override
public void start() {
if (isRunning()) {
return;
}
prepare();
// if (getCookiesLoadFile()!=null) {
// loadCookies(getCookiesLoadFile());
// }
isRunning = true;
}
@Override
public void stop() {
isRunning = false;
}
@Override
public boolean isRunning() {
return isRunning;
}
public void saveCookies() {
if (getCookiesSaveFile() != null) {
// saveCookies(getCookiesSaveFile().getFile().getAbsolutePath());
}
}
}
modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTests.java
+1 -2
View File
@@ -68,7 +68,6 @@ import org.archive.modules.deciderules.RejectDecideRule;
import org.archive.modules.recrawl.FetchHistoryProcessor;
import org.archive.modules.revisit.ServerNotModifiedRevisit;
import org.archive.net.UURI;
import org.archive.util.OneLineSimpleLogger;
import org.jboss.netty.handler.codec.http.HttpRequest;
import org.littleshoot.proxy.DefaultHttpProxyServer;
import org.littleshoot.proxy.HttpFilter;
@@ -883,7 +882,7 @@ public class FetchHTTPTests extends ProcessorTestBase {
public static FetchHTTP newTestFetchHttp(String userAgentString) {
FetchHTTP fetchHttp = new FetchHTTP();
fetchHttp.setCookieStore(new SimpleHostCookieSetStore());
fetchHttp.setCookieStore(new SimpleDomainCookieSetStore());
fetchHttp.setServerCache(new DefaultServerCache());
CrawlMetadata uap = new CrawlMetadata();
uap.setUserAgentTemplate(userAgentString);