Discard 'CredentialAvatar' -- no longer needed with Spring settings -- use Credential instances directly

This commit is contained in:
gojomo
2010-11-06 03:45:00 +00:00
parent e6c0f71175
commit f12e940edd
12 changed files with 76 additions and 381 deletions
@@ -1155,7 +1155,7 @@ public abstract class AbstractFrontier
// rfc2617 credential present and if there, assume it got
// loaded in FetchHTTP on expectation that we're to go around
// again. If no rfc2617 loaded, we should not be here.
boolean loaded = curi.hasRfc2617CredentialAvatar();
boolean loaded = curi.hasRfc2617Credential();
if (!loaded && logger.isLoggable(Level.FINE)) {
logger.fine("Have 401 but no creds loaded " + curi);
}
@@ -35,7 +35,6 @@ import org.archive.modules.CrawlURI;
import org.archive.modules.ProcessResult;
import org.archive.modules.Processor;
import org.archive.modules.credential.Credential;
import org.archive.modules.credential.CredentialAvatar;
import org.archive.modules.credential.CredentialStore;
import org.archive.modules.extractor.Hop;
import org.archive.modules.extractor.Link;
@@ -447,22 +446,19 @@ public class PreconditionEnforcer extends Processor {
* @param curi CrawlURI.
* @return True if already run.
*/
private boolean authenticated(final Credential credential,
final CrawlURI curi) {
boolean result = false;
private boolean authenticated(final Credential credential, final CrawlURI curi) {
CrawlServer server = serverCache.getServerFor(curi.getUURI());
if (!server.hasCredentialAvatars()) {
return result;
if (!server.hasCredentials()) {
return false;
}
Set<CredentialAvatar> avatars = server.getCredentialAvatars();
for (CredentialAvatar ca: avatars) {
String key = null;
key = credential.getKey();
if (ca.match(credential.getClass(), key)) {
result = true;
Set<Credential> credentials = server.getCredentials();
for (Credential cred: credentials) {
if (cred.getKey().equals(credential.getKey())
&& cred.getClass().isInstance(credential)) {
return true;
}
}
return result;
return false;
}
@@ -72,8 +72,8 @@ public interface CoreAttributeConstants {
/**
* Key to get credential avatars from A_LIST.
*/
public static final String A_CREDENTIAL_AVATARS_KEY =
"credential-avatars";
public static final String A_CREDENTIALS_KEY =
"credentials";
/** a 'source' (usu. URI) that's inherited by discovered URIs */
public static String A_SOURCE_TAG = ModuleAttributeConstants.A_SOURCE_TAG;
@@ -20,7 +20,7 @@
package org.archive.modules;
import static org.archive.modules.CoreAttributeConstants.A_ANNOTATIONS;
import static org.archive.modules.CoreAttributeConstants.A_CREDENTIAL_AVATARS_KEY;
import static org.archive.modules.CoreAttributeConstants.A_CREDENTIALS_KEY;
import static org.archive.modules.CoreAttributeConstants.A_DNS_SERVER_IP_LABEL;
import static org.archive.modules.CoreAttributeConstants.A_FETCH_COMPLETED_TIME;
import static org.archive.modules.CoreAttributeConstants.A_FORCE_RETIRE;
@@ -64,7 +64,6 @@ import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@@ -78,7 +77,7 @@ import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.httpclient.methods.PostMethod;
import org.apache.commons.lang.StringUtils;
import org.archive.bdb.AutoKryo;
import org.archive.modules.credential.CredentialAvatar;
import org.archive.modules.credential.Credential;
import org.archive.modules.credential.HttpAuthenticationCredential;
import org.archive.modules.extractor.HTMLLinkContext;
import org.archive.modules.extractor.Hop;
@@ -257,7 +256,7 @@ implements MultiReporter, Serializable, OverlayContext {
*/
private static final Collection<String> persistentKeys
= new CopyOnWriteArrayList<String>(
new String [] {A_CREDENTIAL_AVATARS_KEY});
new String [] {A_CREDENTIALS_KEY});
/**
* A digest (hash, usually SHA1) of retrieved content-body.
@@ -903,12 +902,12 @@ implements MultiReporter, Serializable, OverlayContext {
/**
* @return Credential avatars. Null if none set.
*/
public Set<CredentialAvatar> getCredentialAvatars() {
public Set<Credential> getCredentials() {
@SuppressWarnings("unchecked")
Set<CredentialAvatar> r = (Set)getData().get(A_CREDENTIAL_AVATARS_KEY);
Set<Credential> r = (Set)getData().get(A_CREDENTIALS_KEY);
if (r == null) {
r = new HashSet<CredentialAvatar>();
getData().put(A_CREDENTIAL_AVATARS_KEY, r);
r = new HashSet<Credential>();
getData().put(A_CREDENTIALS_KEY, r);
}
return r;
}
@@ -916,8 +915,8 @@ implements MultiReporter, Serializable, OverlayContext {
/**
* @return True if there are avatars attached to this instance.
*/
public boolean hasCredentialAvatars() {
return containsDataKey(A_CREDENTIAL_AVATARS_KEY);
public boolean hasCredentials() {
return containsDataKey(A_CREDENTIALS_KEY);
}
@@ -942,7 +941,7 @@ implements MultiReporter, Serializable, OverlayContext {
boolean result = false;
int statusCode = this.fetchStatus;
if (statusCode == HttpStatus.SC_UNAUTHORIZED &&
hasRfc2617CredentialAvatar()) {
hasRfc2617Credential()) {
result = false;
} else {
result = (statusCode > 0);
@@ -961,22 +960,18 @@ implements MultiReporter, Serializable, OverlayContext {
/**
* @return True if we have an rfc2617 payload.
*/
public boolean hasRfc2617CredentialAvatar() {
boolean result = false;
Set<CredentialAvatar> avatars = getCredentialAvatars();
if (avatars != null && avatars.size() > 0) {
for (Iterator<CredentialAvatar> i = avatars.iterator(); i.hasNext();) {
if (((CredentialAvatar)i.next()).
match(HttpAuthenticationCredential.class)) {
result = true;
break;
public boolean hasRfc2617Credential() {
Set<Credential> credentials = getCredentials();
if (credentials != null && credentials.size() > 0) {
for (Credential credential : credentials) {
if(credential instanceof HttpAuthenticationCredential) {
return true;
}
}
}
return result;
return false;
}
/**
* Set the retained content-digest value (usu. SHA1).
*
@@ -1824,6 +1819,8 @@ implements MultiReporter, Serializable, OverlayContext {
kryo.autoregister(org.archive.modules.extractor.HTMLLinkContext.class);
kryo.autoregister(org.archive.modules.extractor.LinkContext.SimpleLinkContext.class);
kryo.autoregister(java.util.HashMap[].class);
kryo.autoregister(org.archive.modules.credential.HttpAuthenticationCredential.class);
kryo.autoregister(org.archive.modules.credential.HtmlFormCredential.class);
kryo.setRegistrationOptional(true);
}
}
@@ -26,7 +26,7 @@ import java.util.concurrent.atomic.AtomicLong;
import org.apache.commons.httpclient.HttpStatus;
import org.archive.checkpointing.Checkpoint;
import org.archive.checkpointing.Checkpointable;
import org.archive.modules.credential.CredentialAvatar;
import org.archive.modules.credential.Credential;
import org.archive.modules.credential.HttpAuthenticationCredential;
import org.archive.modules.deciderules.AcceptDecideRule;
import org.archive.modules.deciderules.DecideResult;
@@ -211,7 +211,7 @@ implements HasKeyedProperties,
boolean result = false;
int statusCode = puri.getFetchStatus();
if (statusCode == HttpStatus.SC_UNAUTHORIZED &&
hasHttpAuthenticationCredentialAvatar(puri)) {
hasHttpAuthenticationCredential(puri)) {
result = false;
} else {
result = (statusCode > 0);
@@ -232,10 +232,10 @@ implements HasKeyedProperties,
/**
* @return True if we have an HttpAuthentication (rfc2617) payload.
*/
public static boolean hasHttpAuthenticationCredentialAvatar(CrawlURI puri) {
Set<CredentialAvatar> avatars = puri.getCredentialAvatars();
for (CredentialAvatar ca: avatars) {
if (ca.match(HttpAuthenticationCredential.class)) {
public static boolean hasHttpAuthenticationCredential(CrawlURI puri) {
Set<Credential> credentials = puri.getCredentials();
for (Credential ca: credentials) {
if (ca instanceof HttpAuthenticationCredential) {
return true;
}
}
@@ -20,7 +20,6 @@ package org.archive.modules.credential;
import java.io.Serializable;
import java.util.Iterator;
import java.util.Set;
import java.util.logging.Logger;
import javax.management.AttributeNotFoundException;
@@ -96,20 +95,7 @@ public abstract class Credential implements Serializable {
* @param curi CrawlURI to load with credentials.
*/
public void attach(CrawlURI curi) {
attach(curi, null);
}
/**
* Attach this credentials avatar to the passed <code>curi</code> .
*
* @param curi CrawlURI to load with credentials.
* @param payload Payload to carry in avatar. Usually credentials.
*/
public void attach(CrawlURI curi, String payload) {
CredentialAvatar ca = (payload == null )?
new CredentialAvatar(this.getClass(), getKey()):
new CredentialAvatar(this.getClass(), getKey(), payload);
curi.getCredentialAvatars().add(ca);
curi.getCredentials().add(this);
}
/**
@@ -119,22 +105,7 @@ public abstract class Credential implements Serializable {
* @return True if we detached a Credential reference.
*/
public boolean detach(CrawlURI curi) {
boolean result = false;
Set<CredentialAvatar> avatars = curi.getCredentialAvatars();
if (avatars.isEmpty()) {
logger.severe("This curi " + curi + " has no cred when it should");
}
Iterator<CredentialAvatar> iter = avatars.iterator();
while (iter.hasNext()) {
CredentialAvatar ca = iter.next();
if (ca.match(getClass(), getKey())) {
iter.remove();
result = true;
}
}
return result;
return curi.getCredentials().remove(this);
}
/**
@@ -145,15 +116,10 @@ public abstract class Credential implements Serializable {
*/
public boolean detachAll(CrawlURI curi) {
boolean result = false;
Set<CredentialAvatar> avatars = curi.getCredentialAvatars();
if (avatars.isEmpty()) {
logger.severe("This curi " + curi +" has no creds when it should.");
return false;
}
Iterator<CredentialAvatar> iter = avatars.iterator();
Iterator<Credential> iter = curi.getCredentials().iterator();
while (iter.hasNext()) {
CredentialAvatar ca = iter.next();
if (ca.match(getClass())) {
Credential cred = iter.next();
if (cred.getClass() == this.getClass()) {
iter.remove();
result = true;
}
@@ -201,11 +167,10 @@ public abstract class Credential implements Serializable {
* @param curi CrawlURI to as for context.
* @param http Instance of httpclient.
* @param method Method to populate.
* @param payload Avatar payload to use populating the method.
* @return True if added a credentials.
*/
public abstract boolean populate(CrawlURI curi, HttpClient http,
HttpMethod method, String payload);
HttpMethod method);
/**
* @param curi CrawlURI to look at.
@@ -1,220 +0,0 @@
/*
* This file is part of the Heritrix web crawler (crawler.archive.org).
*
* Licensed to the Internet Archive (IA) by one or more individual
* contributors.
*
* The IA licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.archive.modules.credential;
import java.io.Serializable;
import java.util.Collection;
import java.util.Iterator;
import java.util.List;
import java.util.logging.Logger;
import org.archive.modules.CrawlURI;
/**
* A credential representation.
*
* Added to the CrawlServer upon successful authentication. Used as a marker
* of successful authentication event and for carrying credential
* payload to be used subsequently doing preemptive authentications (e.g.
* For case of RFC2617, needs to be offered everytime we're accessing inside
* a protected area). Also carried by the ProcessorURI when cycling through
* processing chain trying a credential to see if it will authenticate.
*
* <p>This class exists because its not safe to keep references
* to the settings derived Credential classes so instead of keeping references
* to credential classes, we carry around this avatar.
*
* <p>Scope for avatars is crawlserver. Only used within a CrawlServer
* scope.
*
* <p>Immutable.
*
* @author stack
* @version $Revision$, $Date$
*/
public class CredentialAvatar
implements Serializable {
private static final long serialVersionUID = 3L;
private static final Logger logger =
Logger.getLogger(CredentialAvatar.class.getName());
/**
* Key for this credential avatar.
*/
private final String key;
/**
* Type represented by this avatar.
*/
private final Class<?> type;
/**
* Data.
*
* May be null.
*
* <p>This used to be an Object and I used to store in here
* the httpclient AuthScheme but AuthScheme is not serializable
* and so there'd be trouble getting this payload to lie down
* in a bdb database. Changed it to String. That should be
* generic enough for credential purposes.
*/
private final String payload;
/**
* Constructor.
* @param type Type for this credential avatar.
* @param key Key for this credential avatar.
*/
public CredentialAvatar(Class<?> type, String key) {
this(type, key, null);
}
/**
* Constructor.
* @param type Type for this credential avatar.
* @param key Key for this credential avatar.
* @param payload Data credential needs rerunning or preempting. May be
* null and then just the presence is used as signifier of successful
* auth.
*/
public CredentialAvatar(Class<?> type, String key, String payload) {
if (!checkType(type)) {
throw new IllegalArgumentException("Type is unrecognized: " +
type);
}
this.key = key;
this.type = type;
this.payload = payload;
}
/**
* Shutdown default constructor.
*/
@SuppressWarnings("unused")
private CredentialAvatar() {
super();
this.key = null;
this.type = null;
this.payload = null;
}
/**
* @param candidateType Type to check.
* @return True if this is a known credential type.
*/
protected boolean checkType(Class<?> candidateType) {
boolean result = false;
List<Class<?>> types = CredentialStore.getCredentialTypes();
for (Iterator<Class<?>> i = types.iterator(); i.hasNext();) {
if (i.next().equals(candidateType)) {
result = true;
break;
}
}
return result;
}
/**
* @return Returns the payload. May be null.
*/
public String getPayload() {
return this.payload;
}
/**
* @return Returns the key.
*/
public String getKey() {
return this.key;
}
/**
* @return Type represented by this avatar.
*/
public Class<?> getType() {
return this.type;
}
/**
* @param otherType Class to match.
* @return True if this credential avatar is of same type.
*/
public boolean match(Class<?> otherType) {
return this.type.equals(otherType);
}
/**
* @param otherType Credential to match.
* @param otherKey Key to test.
* @return True if this is avatar for passed credential.
*/
public boolean match(Class<?> otherType, String otherKey) {
return match(otherType) &&
(otherKey != null && this.key != null &&
this.key.equals(otherKey));
}
public String toString() {
return getType() + "." + this.getKey();
}
/**
* @param handler Settings handler.
* @param curi ProcessorURI to use for context.
* @return The credential this avatar represents.
*/
public Credential getCredential(CredentialStore cs, CrawlURI curi) {
Credential result = null;
if (cs == null) {
logger.severe("No credential store for " + curi);
return result;
}
Collection<Credential> all = cs.getAll();
if (all == null) {
logger.severe("Have CredentialAvatar " + toString() +
" but no collection: " + curi);
return result;
}
for (Credential c: all) {
if (!this.type.isInstance(c)) {
continue;
}
String credKey = c.getKey();
if (credKey != null && credKey.equals(getKey())) {
result = c;
break;
}
}
if (result == null) {
logger.severe("Have CredentialAvatar " + toString() +
" but no corresponding credential: " + curi);
}
return result;
}
}
@@ -54,6 +54,7 @@ public class CredentialStore implements Serializable, HasKeyedProperties {
private static final long serialVersionUID = 3L;
@SuppressWarnings("unused")
private static Logger logger = Logger.getLogger(
"org.archive.crawler.datamodel.CredentialStore");
@@ -179,35 +180,4 @@ public class CredentialStore implements Serializable, HasKeyedProperties {
}
return result;
}
public Credential getCredential(CrawlURI curi, CredentialAvatar ca) {
Credential result = null;
Collection<Credential> all = getAll();
if (all == null) {
logger.severe("Have CredentialAvatar " + toString() +
" but no collection: " + curi);
return result;
}
for (Credential c: all) {
if (!ca.getType().isInstance(c)) {
continue;
}
String credKey = c.getKey();
if (credKey != null && credKey.equals(ca.getKey())) {
result = c;
break;
}
}
if (result == null) {
logger.severe("Have CredentialAvatar " + toString() +
" but no corresponding credential: " + curi);
}
return result;
}
}
@@ -133,10 +133,8 @@ public class HtmlFormCredential extends Credential {
return false;
}
public boolean populate(CrawlURI curi, HttpClient http, HttpMethod method,
String payload) {
// http is not used.
// payload is not used.
public boolean populate(CrawlURI curi, HttpClient http, HttpMethod method) {
// http is not used
boolean result = false;
Map<String,String> formItems = getFormItems();
if (formItems == null || formItems.size() <= 0) {
@@ -104,14 +104,9 @@ public class HttpAuthenticationCredential extends Credential {
return true;
}
public boolean populate(CrawlURI curi, HttpClient http, HttpMethod method,
String payload) {
public boolean populate(CrawlURI curi, HttpClient http, HttpMethod method) {
boolean result = false;
String authRealm = payload;
if (authRealm == null) {
logger.severe("No authscheme though creds: " + curi);
return result;
}
// Always add the credential to HttpState. Doing this because no way of
// removing the credential once added AND there is a bug in the
@@ -130,8 +125,8 @@ public class HttpAuthenticationCredential extends Credential {
upc = new UsernamePasswordCredentials(getLogin(),
getPassword());
http.getState().setCredentials(new AuthScope(curi.getUURI().getHost(),
curi.getUURI().getPort(), authRealm), upc);
logger.fine("Credentials for realm " + authRealm +
curi.getUURI().getPort(), getRealm()), upc);
logger.fine("Credentials for realm " + getRealm() +
" for CrawlURI " + curi.toString() + " added to request: " +
result);
result = true;
@@ -93,7 +93,6 @@ import org.archive.modules.CrawlURI;
import org.archive.modules.ProcessResult;
import org.archive.modules.Processor;
import org.archive.modules.credential.Credential;
import org.archive.modules.credential.CredentialAvatar;
import org.archive.modules.credential.CredentialStore;
import org.archive.modules.credential.HttpAuthenticationCredential;
import org.archive.modules.deciderules.AcceptDecideRule;
@@ -1043,11 +1042,10 @@ public class FetchHTTP extends Processor implements Lifecycle {
return false;
}
CrawlServer server = serverCache.getServerFor(serverKey);
if (server.hasCredentialAvatars()) {
for (CredentialAvatar ca : server.getCredentialAvatars()) {
Credential c = ca.getCredential(getCredentialStore(), curi);
if (c.isEveryTime()) {
c.populate(curi, this.http, method, ca.getPayload());
if (server.hasCredentials()) {
for (Credential cred : server.getCredentials()) {
if (cred.isEveryTime()) {
cred.populate(curi, this.http, method);
}
}
}
@@ -1057,9 +1055,8 @@ public class FetchHTTP extends Processor implements Lifecycle {
// Now look in the curi. The Curi will have credentials loaded either
// by the handle401 method if its a rfc2617 or it'll have been set into
// the curi by the preconditionenforcer as this login uri came through.
for (CredentialAvatar ca : curi.getCredentialAvatars()) {
Credential c = ca.getCredential(getCredentialStore(), curi);
if (c.populate(curi, this.http, method, ca.getPayload())) {
for (Credential c: curi.getCredentials()) {
if (c.populate(curi, this.http, method)) {
result = true;
}
}
@@ -1074,21 +1071,20 @@ public class FetchHTTP extends Processor implements Lifecycle {
* CrawlURI whose credentials we are to promote.
*/
private void promoteCredentials(final CrawlURI curi) {
Set<CredentialAvatar> avatars = curi.getCredentialAvatars();
for (Iterator<CredentialAvatar> i = avatars.iterator(); i.hasNext();) {
CredentialAvatar ca = i.next();
Set<Credential> credentials = curi.getCredentials();
for (Iterator<Credential> i = credentials.iterator(); i.hasNext();) {
Credential c = i.next();
i.remove();
// The server to attach too may not be the server that hosts
// this passed curi. It might be of another subdomain.
// The avatar needs to be added to the server that is dependent
// on this precondition. Find it by name. Get the name from
// the credential this avatar represents.
Credential c = getCredentialStore().getCredential(curi, ca);
String cd = c.getDomain();
if (cd != null) {
CrawlServer cs = serverCache.getServerFor(cd);
if (cs != null) {
cs.addCredentialAvatar(ca);
cs.addCredential(c);
}
}
}
@@ -1163,7 +1159,7 @@ public class FetchHTTP extends Processor implements Lifecycle {
logger.fine("No rfc2617 credentials for realm " + realm
+ " in " + curi);
} else {
found.attach(curi, authscheme.getRealm());
found.attach(curi);
logger.fine("Found credential for realm " + realm
+ " in store for " + curi.toString());
}
@@ -1253,15 +1249,13 @@ public class FetchHTTP extends Processor implements Lifecycle {
private Set<Credential> getCredentials(CrawlURI curi, Class<?> type) {
Set<Credential> result = null;
if (curi.hasCredentialAvatars()) {
for (Iterator<CredentialAvatar> i = curi.getCredentialAvatars().iterator(); i
.hasNext();) {
CredentialAvatar ca = (CredentialAvatar) i.next();
if (ca.match(type)) {
if (curi.hasCredentials()) {
for (Credential c : curi.getCredentials()) {
if (type.isInstance(c)) {
if (result == null) {
result = new HashSet<Credential>();
}
result.add(ca.getCredential(getCredentialStore(), curi));
result.add(c);
}
}
}
@@ -40,7 +40,7 @@ import org.apache.commons.io.IOUtils;
import org.archive.bdb.AutoKryo;
import org.archive.io.ReplayInputStream;
import org.archive.modules.CrawlURI;
import org.archive.modules.credential.CredentialAvatar;
import org.archive.modules.credential.Credential;
import org.archive.modules.fetcher.FetchStats;
import org.archive.net.UURI;
import org.archive.net.UURIFactory;
@@ -77,9 +77,9 @@ public class CrawlServer implements Serializable, FetchStats.HasFetchStats {
protected int consecutiveConnectionErrors = 0;
/**
* Set of credential avatars.
* Set of credentials.
*/
private transient Set<CredentialAvatar> avatars = null;
private transient Set<Credential> credentials = null;
/**
* Creates a new CrawlServer object.
@@ -226,15 +226,15 @@ public class CrawlServer implements Serializable, FetchStats.HasFetchStats {
/**
* @return Credential avatars for this server. Returns null if none.
*/
public Set<CredentialAvatar> getCredentialAvatars() {
return this.avatars;
public Set<Credential> getCredentials() {
return this.credentials;
}
/**
* @return True if there are avatars attached to this instance.
*/
public boolean hasCredentialAvatars() {
return this.avatars != null && this.avatars.size() > 0;
public boolean hasCredentials() {
return this.credentials != null && this.credentials.size() > 0;
}
/**
@@ -242,11 +242,11 @@ public class CrawlServer implements Serializable, FetchStats.HasFetchStats {
*
* @param ca Credential avatar to add to set of avatars.
*/
public void addCredentialAvatar(CredentialAvatar ca) {
if (this.avatars == null) {
this.avatars = new HashSet<CredentialAvatar>();
public void addCredential(Credential cred) {
if (this.credentials == null) {
this.credentials = new HashSet<Credential>();
}
this.avatars.add(ca);
this.credentials.add(cred);
}
/**