mirror of
https://github.com/internetarchive/heritrix3.git
synced 2026-09-13 09:15:37 +00:00
Discard 'CredentialAvatar' -- no longer needed with Spring settings -- use Credential instances directly
This commit is contained in:
@@ -1155,7 +1155,7 @@ public abstract class AbstractFrontier
|
||||
// rfc2617 credential present and if there, assume it got
|
||||
// loaded in FetchHTTP on expectation that we're to go around
|
||||
// again. If no rfc2617 loaded, we should not be here.
|
||||
boolean loaded = curi.hasRfc2617CredentialAvatar();
|
||||
boolean loaded = curi.hasRfc2617Credential();
|
||||
if (!loaded && logger.isLoggable(Level.FINE)) {
|
||||
logger.fine("Have 401 but no creds loaded " + curi);
|
||||
}
|
||||
|
||||
@@ -35,7 +35,6 @@ import org.archive.modules.CrawlURI;
|
||||
import org.archive.modules.ProcessResult;
|
||||
import org.archive.modules.Processor;
|
||||
import org.archive.modules.credential.Credential;
|
||||
import org.archive.modules.credential.CredentialAvatar;
|
||||
import org.archive.modules.credential.CredentialStore;
|
||||
import org.archive.modules.extractor.Hop;
|
||||
import org.archive.modules.extractor.Link;
|
||||
@@ -447,22 +446,19 @@ public class PreconditionEnforcer extends Processor {
|
||||
* @param curi CrawlURI.
|
||||
* @return True if already run.
|
||||
*/
|
||||
private boolean authenticated(final Credential credential,
|
||||
final CrawlURI curi) {
|
||||
boolean result = false;
|
||||
private boolean authenticated(final Credential credential, final CrawlURI curi) {
|
||||
CrawlServer server = serverCache.getServerFor(curi.getUURI());
|
||||
if (!server.hasCredentialAvatars()) {
|
||||
return result;
|
||||
if (!server.hasCredentials()) {
|
||||
return false;
|
||||
}
|
||||
Set<CredentialAvatar> avatars = server.getCredentialAvatars();
|
||||
for (CredentialAvatar ca: avatars) {
|
||||
String key = null;
|
||||
key = credential.getKey();
|
||||
if (ca.match(credential.getClass(), key)) {
|
||||
result = true;
|
||||
Set<Credential> credentials = server.getCredentials();
|
||||
for (Credential cred: credentials) {
|
||||
if (cred.getKey().equals(credential.getKey())
|
||||
&& cred.getClass().isInstance(credential)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return result;
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -72,8 +72,8 @@ public interface CoreAttributeConstants {
|
||||
/**
|
||||
* Key to get credential avatars from A_LIST.
|
||||
*/
|
||||
public static final String A_CREDENTIAL_AVATARS_KEY =
|
||||
"credential-avatars";
|
||||
public static final String A_CREDENTIALS_KEY =
|
||||
"credentials";
|
||||
|
||||
/** a 'source' (usu. URI) that's inherited by discovered URIs */
|
||||
public static String A_SOURCE_TAG = ModuleAttributeConstants.A_SOURCE_TAG;
|
||||
|
||||
@@ -20,7 +20,7 @@
|
||||
package org.archive.modules;
|
||||
|
||||
import static org.archive.modules.CoreAttributeConstants.A_ANNOTATIONS;
|
||||
import static org.archive.modules.CoreAttributeConstants.A_CREDENTIAL_AVATARS_KEY;
|
||||
import static org.archive.modules.CoreAttributeConstants.A_CREDENTIALS_KEY;
|
||||
import static org.archive.modules.CoreAttributeConstants.A_DNS_SERVER_IP_LABEL;
|
||||
import static org.archive.modules.CoreAttributeConstants.A_FETCH_COMPLETED_TIME;
|
||||
import static org.archive.modules.CoreAttributeConstants.A_FORCE_RETIRE;
|
||||
@@ -64,7 +64,6 @@ import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.Iterator;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
@@ -78,7 +77,7 @@ import org.apache.commons.httpclient.methods.GetMethod;
|
||||
import org.apache.commons.httpclient.methods.PostMethod;
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.archive.bdb.AutoKryo;
|
||||
import org.archive.modules.credential.CredentialAvatar;
|
||||
import org.archive.modules.credential.Credential;
|
||||
import org.archive.modules.credential.HttpAuthenticationCredential;
|
||||
import org.archive.modules.extractor.HTMLLinkContext;
|
||||
import org.archive.modules.extractor.Hop;
|
||||
@@ -257,7 +256,7 @@ implements MultiReporter, Serializable, OverlayContext {
|
||||
*/
|
||||
private static final Collection<String> persistentKeys
|
||||
= new CopyOnWriteArrayList<String>(
|
||||
new String [] {A_CREDENTIAL_AVATARS_KEY});
|
||||
new String [] {A_CREDENTIALS_KEY});
|
||||
|
||||
/**
|
||||
* A digest (hash, usually SHA1) of retrieved content-body.
|
||||
@@ -903,12 +902,12 @@ implements MultiReporter, Serializable, OverlayContext {
|
||||
/**
|
||||
* @return Credential avatars. Null if none set.
|
||||
*/
|
||||
public Set<CredentialAvatar> getCredentialAvatars() {
|
||||
public Set<Credential> getCredentials() {
|
||||
@SuppressWarnings("unchecked")
|
||||
Set<CredentialAvatar> r = (Set)getData().get(A_CREDENTIAL_AVATARS_KEY);
|
||||
Set<Credential> r = (Set)getData().get(A_CREDENTIALS_KEY);
|
||||
if (r == null) {
|
||||
r = new HashSet<CredentialAvatar>();
|
||||
getData().put(A_CREDENTIAL_AVATARS_KEY, r);
|
||||
r = new HashSet<Credential>();
|
||||
getData().put(A_CREDENTIALS_KEY, r);
|
||||
}
|
||||
return r;
|
||||
}
|
||||
@@ -916,8 +915,8 @@ implements MultiReporter, Serializable, OverlayContext {
|
||||
/**
|
||||
* @return True if there are avatars attached to this instance.
|
||||
*/
|
||||
public boolean hasCredentialAvatars() {
|
||||
return containsDataKey(A_CREDENTIAL_AVATARS_KEY);
|
||||
public boolean hasCredentials() {
|
||||
return containsDataKey(A_CREDENTIALS_KEY);
|
||||
}
|
||||
|
||||
|
||||
@@ -942,7 +941,7 @@ implements MultiReporter, Serializable, OverlayContext {
|
||||
boolean result = false;
|
||||
int statusCode = this.fetchStatus;
|
||||
if (statusCode == HttpStatus.SC_UNAUTHORIZED &&
|
||||
hasRfc2617CredentialAvatar()) {
|
||||
hasRfc2617Credential()) {
|
||||
result = false;
|
||||
} else {
|
||||
result = (statusCode > 0);
|
||||
@@ -961,22 +960,18 @@ implements MultiReporter, Serializable, OverlayContext {
|
||||
/**
|
||||
* @return True if we have an rfc2617 payload.
|
||||
*/
|
||||
public boolean hasRfc2617CredentialAvatar() {
|
||||
boolean result = false;
|
||||
Set<CredentialAvatar> avatars = getCredentialAvatars();
|
||||
if (avatars != null && avatars.size() > 0) {
|
||||
for (Iterator<CredentialAvatar> i = avatars.iterator(); i.hasNext();) {
|
||||
if (((CredentialAvatar)i.next()).
|
||||
match(HttpAuthenticationCredential.class)) {
|
||||
result = true;
|
||||
break;
|
||||
public boolean hasRfc2617Credential() {
|
||||
Set<Credential> credentials = getCredentials();
|
||||
if (credentials != null && credentials.size() > 0) {
|
||||
for (Credential credential : credentials) {
|
||||
if(credential instanceof HttpAuthenticationCredential) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
return result;
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Set the retained content-digest value (usu. SHA1).
|
||||
*
|
||||
@@ -1824,6 +1819,8 @@ implements MultiReporter, Serializable, OverlayContext {
|
||||
kryo.autoregister(org.archive.modules.extractor.HTMLLinkContext.class);
|
||||
kryo.autoregister(org.archive.modules.extractor.LinkContext.SimpleLinkContext.class);
|
||||
kryo.autoregister(java.util.HashMap[].class);
|
||||
kryo.autoregister(org.archive.modules.credential.HttpAuthenticationCredential.class);
|
||||
kryo.autoregister(org.archive.modules.credential.HtmlFormCredential.class);
|
||||
kryo.setRegistrationOptional(true);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -26,7 +26,7 @@ import java.util.concurrent.atomic.AtomicLong;
|
||||
import org.apache.commons.httpclient.HttpStatus;
|
||||
import org.archive.checkpointing.Checkpoint;
|
||||
import org.archive.checkpointing.Checkpointable;
|
||||
import org.archive.modules.credential.CredentialAvatar;
|
||||
import org.archive.modules.credential.Credential;
|
||||
import org.archive.modules.credential.HttpAuthenticationCredential;
|
||||
import org.archive.modules.deciderules.AcceptDecideRule;
|
||||
import org.archive.modules.deciderules.DecideResult;
|
||||
@@ -211,7 +211,7 @@ implements HasKeyedProperties,
|
||||
boolean result = false;
|
||||
int statusCode = puri.getFetchStatus();
|
||||
if (statusCode == HttpStatus.SC_UNAUTHORIZED &&
|
||||
hasHttpAuthenticationCredentialAvatar(puri)) {
|
||||
hasHttpAuthenticationCredential(puri)) {
|
||||
result = false;
|
||||
} else {
|
||||
result = (statusCode > 0);
|
||||
@@ -232,10 +232,10 @@ implements HasKeyedProperties,
|
||||
/**
|
||||
* @return True if we have an HttpAuthentication (rfc2617) payload.
|
||||
*/
|
||||
public static boolean hasHttpAuthenticationCredentialAvatar(CrawlURI puri) {
|
||||
Set<CredentialAvatar> avatars = puri.getCredentialAvatars();
|
||||
for (CredentialAvatar ca: avatars) {
|
||||
if (ca.match(HttpAuthenticationCredential.class)) {
|
||||
public static boolean hasHttpAuthenticationCredential(CrawlURI puri) {
|
||||
Set<Credential> credentials = puri.getCredentials();
|
||||
for (Credential ca: credentials) {
|
||||
if (ca instanceof HttpAuthenticationCredential) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -20,7 +20,6 @@ package org.archive.modules.credential;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.util.Iterator;
|
||||
import java.util.Set;
|
||||
import java.util.logging.Logger;
|
||||
|
||||
import javax.management.AttributeNotFoundException;
|
||||
@@ -96,20 +95,7 @@ public abstract class Credential implements Serializable {
|
||||
* @param curi CrawlURI to load with credentials.
|
||||
*/
|
||||
public void attach(CrawlURI curi) {
|
||||
attach(curi, null);
|
||||
}
|
||||
|
||||
/**
|
||||
* Attach this credentials avatar to the passed <code>curi</code> .
|
||||
*
|
||||
* @param curi CrawlURI to load with credentials.
|
||||
* @param payload Payload to carry in avatar. Usually credentials.
|
||||
*/
|
||||
public void attach(CrawlURI curi, String payload) {
|
||||
CredentialAvatar ca = (payload == null )?
|
||||
new CredentialAvatar(this.getClass(), getKey()):
|
||||
new CredentialAvatar(this.getClass(), getKey(), payload);
|
||||
curi.getCredentialAvatars().add(ca);
|
||||
curi.getCredentials().add(this);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -119,22 +105,7 @@ public abstract class Credential implements Serializable {
|
||||
* @return True if we detached a Credential reference.
|
||||
*/
|
||||
public boolean detach(CrawlURI curi) {
|
||||
boolean result = false;
|
||||
Set<CredentialAvatar> avatars = curi.getCredentialAvatars();
|
||||
if (avatars.isEmpty()) {
|
||||
logger.severe("This curi " + curi + " has no cred when it should");
|
||||
}
|
||||
|
||||
Iterator<CredentialAvatar> iter = avatars.iterator();
|
||||
while (iter.hasNext()) {
|
||||
CredentialAvatar ca = iter.next();
|
||||
if (ca.match(getClass(), getKey())) {
|
||||
iter.remove();
|
||||
result = true;
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
return curi.getCredentials().remove(this);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -145,15 +116,10 @@ public abstract class Credential implements Serializable {
|
||||
*/
|
||||
public boolean detachAll(CrawlURI curi) {
|
||||
boolean result = false;
|
||||
Set<CredentialAvatar> avatars = curi.getCredentialAvatars();
|
||||
if (avatars.isEmpty()) {
|
||||
logger.severe("This curi " + curi +" has no creds when it should.");
|
||||
return false;
|
||||
}
|
||||
Iterator<CredentialAvatar> iter = avatars.iterator();
|
||||
Iterator<Credential> iter = curi.getCredentials().iterator();
|
||||
while (iter.hasNext()) {
|
||||
CredentialAvatar ca = iter.next();
|
||||
if (ca.match(getClass())) {
|
||||
Credential cred = iter.next();
|
||||
if (cred.getClass() == this.getClass()) {
|
||||
iter.remove();
|
||||
result = true;
|
||||
}
|
||||
@@ -201,11 +167,10 @@ public abstract class Credential implements Serializable {
|
||||
* @param curi CrawlURI to as for context.
|
||||
* @param http Instance of httpclient.
|
||||
* @param method Method to populate.
|
||||
* @param payload Avatar payload to use populating the method.
|
||||
* @return True if added a credentials.
|
||||
*/
|
||||
public abstract boolean populate(CrawlURI curi, HttpClient http,
|
||||
HttpMethod method, String payload);
|
||||
HttpMethod method);
|
||||
|
||||
/**
|
||||
* @param curi CrawlURI to look at.
|
||||
|
||||
@@ -1,220 +0,0 @@
|
||||
/*
|
||||
* This file is part of the Heritrix web crawler (crawler.archive.org).
|
||||
*
|
||||
* Licensed to the Internet Archive (IA) by one or more individual
|
||||
* contributors.
|
||||
*
|
||||
* The IA licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.archive.modules.credential;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.util.Collection;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.logging.Logger;
|
||||
|
||||
import org.archive.modules.CrawlURI;
|
||||
|
||||
/**
|
||||
* A credential representation.
|
||||
*
|
||||
* Added to the CrawlServer upon successful authentication. Used as a marker
|
||||
* of successful authentication event and for carrying credential
|
||||
* payload to be used subsequently doing preemptive authentications (e.g.
|
||||
* For case of RFC2617, needs to be offered everytime we're accessing inside
|
||||
* a protected area). Also carried by the ProcessorURI when cycling through
|
||||
* processing chain trying a credential to see if it will authenticate.
|
||||
*
|
||||
* <p>This class exists because its not safe to keep references
|
||||
* to the settings derived Credential classes so instead of keeping references
|
||||
* to credential classes, we carry around this avatar.
|
||||
*
|
||||
* <p>Scope for avatars is crawlserver. Only used within a CrawlServer
|
||||
* scope.
|
||||
*
|
||||
* <p>Immutable.
|
||||
*
|
||||
* @author stack
|
||||
* @version $Revision$, $Date$
|
||||
*/
|
||||
public class CredentialAvatar
|
||||
implements Serializable {
|
||||
|
||||
private static final long serialVersionUID = 3L;
|
||||
|
||||
private static final Logger logger =
|
||||
Logger.getLogger(CredentialAvatar.class.getName());
|
||||
|
||||
/**
|
||||
* Key for this credential avatar.
|
||||
*/
|
||||
private final String key;
|
||||
|
||||
/**
|
||||
* Type represented by this avatar.
|
||||
*/
|
||||
private final Class<?> type;
|
||||
|
||||
/**
|
||||
* Data.
|
||||
*
|
||||
* May be null.
|
||||
*
|
||||
* <p>This used to be an Object and I used to store in here
|
||||
* the httpclient AuthScheme but AuthScheme is not serializable
|
||||
* and so there'd be trouble getting this payload to lie down
|
||||
* in a bdb database. Changed it to String. That should be
|
||||
* generic enough for credential purposes.
|
||||
*/
|
||||
private final String payload;
|
||||
|
||||
|
||||
/**
|
||||
* Constructor.
|
||||
* @param type Type for this credential avatar.
|
||||
* @param key Key for this credential avatar.
|
||||
*/
|
||||
public CredentialAvatar(Class<?> type, String key) {
|
||||
this(type, key, null);
|
||||
}
|
||||
|
||||
/**
|
||||
* Constructor.
|
||||
* @param type Type for this credential avatar.
|
||||
* @param key Key for this credential avatar.
|
||||
* @param payload Data credential needs rerunning or preempting. May be
|
||||
* null and then just the presence is used as signifier of successful
|
||||
* auth.
|
||||
*/
|
||||
public CredentialAvatar(Class<?> type, String key, String payload) {
|
||||
if (!checkType(type)) {
|
||||
throw new IllegalArgumentException("Type is unrecognized: " +
|
||||
type);
|
||||
}
|
||||
this.key = key;
|
||||
this.type = type;
|
||||
this.payload = payload;
|
||||
}
|
||||
|
||||
/**
|
||||
* Shutdown default constructor.
|
||||
*/
|
||||
@SuppressWarnings("unused")
|
||||
private CredentialAvatar() {
|
||||
super();
|
||||
this.key = null;
|
||||
this.type = null;
|
||||
this.payload = null;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param candidateType Type to check.
|
||||
* @return True if this is a known credential type.
|
||||
*/
|
||||
protected boolean checkType(Class<?> candidateType) {
|
||||
boolean result = false;
|
||||
List<Class<?>> types = CredentialStore.getCredentialTypes();
|
||||
for (Iterator<Class<?>> i = types.iterator(); i.hasNext();) {
|
||||
if (i.next().equals(candidateType)) {
|
||||
result = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return Returns the payload. May be null.
|
||||
*/
|
||||
public String getPayload() {
|
||||
return this.payload;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return Returns the key.
|
||||
*/
|
||||
public String getKey() {
|
||||
return this.key;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return Type represented by this avatar.
|
||||
*/
|
||||
public Class<?> getType() {
|
||||
return this.type;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param otherType Class to match.
|
||||
* @return True if this credential avatar is of same type.
|
||||
*/
|
||||
public boolean match(Class<?> otherType) {
|
||||
return this.type.equals(otherType);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param otherType Credential to match.
|
||||
* @param otherKey Key to test.
|
||||
* @return True if this is avatar for passed credential.
|
||||
*/
|
||||
public boolean match(Class<?> otherType, String otherKey) {
|
||||
return match(otherType) &&
|
||||
(otherKey != null && this.key != null &&
|
||||
this.key.equals(otherKey));
|
||||
}
|
||||
|
||||
public String toString() {
|
||||
return getType() + "." + this.getKey();
|
||||
}
|
||||
|
||||
/**
|
||||
* @param handler Settings handler.
|
||||
* @param curi ProcessorURI to use for context.
|
||||
* @return The credential this avatar represents.
|
||||
*/
|
||||
public Credential getCredential(CredentialStore cs, CrawlURI curi) {
|
||||
Credential result = null;
|
||||
|
||||
if (cs == null) {
|
||||
logger.severe("No credential store for " + curi);
|
||||
return result;
|
||||
}
|
||||
|
||||
Collection<Credential> all = cs.getAll();
|
||||
if (all == null) {
|
||||
logger.severe("Have CredentialAvatar " + toString() +
|
||||
" but no collection: " + curi);
|
||||
return result;
|
||||
}
|
||||
|
||||
for (Credential c: all) {
|
||||
if (!this.type.isInstance(c)) {
|
||||
continue;
|
||||
}
|
||||
String credKey = c.getKey();
|
||||
if (credKey != null && credKey.equals(getKey())) {
|
||||
result = c;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (result == null) {
|
||||
logger.severe("Have CredentialAvatar " + toString() +
|
||||
" but no corresponding credential: " + curi);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
}
|
||||
@@ -54,6 +54,7 @@ public class CredentialStore implements Serializable, HasKeyedProperties {
|
||||
|
||||
private static final long serialVersionUID = 3L;
|
||||
|
||||
@SuppressWarnings("unused")
|
||||
private static Logger logger = Logger.getLogger(
|
||||
"org.archive.crawler.datamodel.CredentialStore");
|
||||
|
||||
@@ -179,35 +180,4 @@ public class CredentialStore implements Serializable, HasKeyedProperties {
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
public Credential getCredential(CrawlURI curi, CredentialAvatar ca) {
|
||||
Credential result = null;
|
||||
|
||||
Collection<Credential> all = getAll();
|
||||
if (all == null) {
|
||||
logger.severe("Have CredentialAvatar " + toString() +
|
||||
" but no collection: " + curi);
|
||||
return result;
|
||||
}
|
||||
|
||||
for (Credential c: all) {
|
||||
if (!ca.getType().isInstance(c)) {
|
||||
continue;
|
||||
}
|
||||
String credKey = c.getKey();
|
||||
if (credKey != null && credKey.equals(ca.getKey())) {
|
||||
result = c;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (result == null) {
|
||||
logger.severe("Have CredentialAvatar " + toString() +
|
||||
" but no corresponding credential: " + curi);
|
||||
}
|
||||
|
||||
return result;
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
@@ -133,10 +133,8 @@ public class HtmlFormCredential extends Credential {
|
||||
return false;
|
||||
}
|
||||
|
||||
public boolean populate(CrawlURI curi, HttpClient http, HttpMethod method,
|
||||
String payload) {
|
||||
// http is not used.
|
||||
// payload is not used.
|
||||
public boolean populate(CrawlURI curi, HttpClient http, HttpMethod method) {
|
||||
// http is not used
|
||||
boolean result = false;
|
||||
Map<String,String> formItems = getFormItems();
|
||||
if (formItems == null || formItems.size() <= 0) {
|
||||
|
||||
+4
-9
@@ -104,14 +104,9 @@ public class HttpAuthenticationCredential extends Credential {
|
||||
return true;
|
||||
}
|
||||
|
||||
public boolean populate(CrawlURI curi, HttpClient http, HttpMethod method,
|
||||
String payload) {
|
||||
public boolean populate(CrawlURI curi, HttpClient http, HttpMethod method) {
|
||||
boolean result = false;
|
||||
String authRealm = payload;
|
||||
if (authRealm == null) {
|
||||
logger.severe("No authscheme though creds: " + curi);
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
// Always add the credential to HttpState. Doing this because no way of
|
||||
// removing the credential once added AND there is a bug in the
|
||||
@@ -130,8 +125,8 @@ public class HttpAuthenticationCredential extends Credential {
|
||||
upc = new UsernamePasswordCredentials(getLogin(),
|
||||
getPassword());
|
||||
http.getState().setCredentials(new AuthScope(curi.getUURI().getHost(),
|
||||
curi.getUURI().getPort(), authRealm), upc);
|
||||
logger.fine("Credentials for realm " + authRealm +
|
||||
curi.getUURI().getPort(), getRealm()), upc);
|
||||
logger.fine("Credentials for realm " + getRealm() +
|
||||
" for CrawlURI " + curi.toString() + " added to request: " +
|
||||
result);
|
||||
result = true;
|
||||
|
||||
@@ -93,7 +93,6 @@ import org.archive.modules.CrawlURI;
|
||||
import org.archive.modules.ProcessResult;
|
||||
import org.archive.modules.Processor;
|
||||
import org.archive.modules.credential.Credential;
|
||||
import org.archive.modules.credential.CredentialAvatar;
|
||||
import org.archive.modules.credential.CredentialStore;
|
||||
import org.archive.modules.credential.HttpAuthenticationCredential;
|
||||
import org.archive.modules.deciderules.AcceptDecideRule;
|
||||
@@ -1043,11 +1042,10 @@ public class FetchHTTP extends Processor implements Lifecycle {
|
||||
return false;
|
||||
}
|
||||
CrawlServer server = serverCache.getServerFor(serverKey);
|
||||
if (server.hasCredentialAvatars()) {
|
||||
for (CredentialAvatar ca : server.getCredentialAvatars()) {
|
||||
Credential c = ca.getCredential(getCredentialStore(), curi);
|
||||
if (c.isEveryTime()) {
|
||||
c.populate(curi, this.http, method, ca.getPayload());
|
||||
if (server.hasCredentials()) {
|
||||
for (Credential cred : server.getCredentials()) {
|
||||
if (cred.isEveryTime()) {
|
||||
cred.populate(curi, this.http, method);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1057,9 +1055,8 @@ public class FetchHTTP extends Processor implements Lifecycle {
|
||||
// Now look in the curi. The Curi will have credentials loaded either
|
||||
// by the handle401 method if its a rfc2617 or it'll have been set into
|
||||
// the curi by the preconditionenforcer as this login uri came through.
|
||||
for (CredentialAvatar ca : curi.getCredentialAvatars()) {
|
||||
Credential c = ca.getCredential(getCredentialStore(), curi);
|
||||
if (c.populate(curi, this.http, method, ca.getPayload())) {
|
||||
for (Credential c: curi.getCredentials()) {
|
||||
if (c.populate(curi, this.http, method)) {
|
||||
result = true;
|
||||
}
|
||||
}
|
||||
@@ -1074,21 +1071,20 @@ public class FetchHTTP extends Processor implements Lifecycle {
|
||||
* CrawlURI whose credentials we are to promote.
|
||||
*/
|
||||
private void promoteCredentials(final CrawlURI curi) {
|
||||
Set<CredentialAvatar> avatars = curi.getCredentialAvatars();
|
||||
for (Iterator<CredentialAvatar> i = avatars.iterator(); i.hasNext();) {
|
||||
CredentialAvatar ca = i.next();
|
||||
Set<Credential> credentials = curi.getCredentials();
|
||||
for (Iterator<Credential> i = credentials.iterator(); i.hasNext();) {
|
||||
Credential c = i.next();
|
||||
i.remove();
|
||||
// The server to attach too may not be the server that hosts
|
||||
// this passed curi. It might be of another subdomain.
|
||||
// The avatar needs to be added to the server that is dependent
|
||||
// on this precondition. Find it by name. Get the name from
|
||||
// the credential this avatar represents.
|
||||
Credential c = getCredentialStore().getCredential(curi, ca);
|
||||
String cd = c.getDomain();
|
||||
if (cd != null) {
|
||||
CrawlServer cs = serverCache.getServerFor(cd);
|
||||
if (cs != null) {
|
||||
cs.addCredentialAvatar(ca);
|
||||
cs.addCredential(c);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1163,7 +1159,7 @@ public class FetchHTTP extends Processor implements Lifecycle {
|
||||
logger.fine("No rfc2617 credentials for realm " + realm
|
||||
+ " in " + curi);
|
||||
} else {
|
||||
found.attach(curi, authscheme.getRealm());
|
||||
found.attach(curi);
|
||||
logger.fine("Found credential for realm " + realm
|
||||
+ " in store for " + curi.toString());
|
||||
}
|
||||
@@ -1253,15 +1249,13 @@ public class FetchHTTP extends Processor implements Lifecycle {
|
||||
private Set<Credential> getCredentials(CrawlURI curi, Class<?> type) {
|
||||
Set<Credential> result = null;
|
||||
|
||||
if (curi.hasCredentialAvatars()) {
|
||||
for (Iterator<CredentialAvatar> i = curi.getCredentialAvatars().iterator(); i
|
||||
.hasNext();) {
|
||||
CredentialAvatar ca = (CredentialAvatar) i.next();
|
||||
if (ca.match(type)) {
|
||||
if (curi.hasCredentials()) {
|
||||
for (Credential c : curi.getCredentials()) {
|
||||
if (type.isInstance(c)) {
|
||||
if (result == null) {
|
||||
result = new HashSet<Credential>();
|
||||
}
|
||||
result.add(ca.getCredential(getCredentialStore(), curi));
|
||||
result.add(c);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -40,7 +40,7 @@ import org.apache.commons.io.IOUtils;
|
||||
import org.archive.bdb.AutoKryo;
|
||||
import org.archive.io.ReplayInputStream;
|
||||
import org.archive.modules.CrawlURI;
|
||||
import org.archive.modules.credential.CredentialAvatar;
|
||||
import org.archive.modules.credential.Credential;
|
||||
import org.archive.modules.fetcher.FetchStats;
|
||||
import org.archive.net.UURI;
|
||||
import org.archive.net.UURIFactory;
|
||||
@@ -77,9 +77,9 @@ public class CrawlServer implements Serializable, FetchStats.HasFetchStats {
|
||||
protected int consecutiveConnectionErrors = 0;
|
||||
|
||||
/**
|
||||
* Set of credential avatars.
|
||||
* Set of credentials.
|
||||
*/
|
||||
private transient Set<CredentialAvatar> avatars = null;
|
||||
private transient Set<Credential> credentials = null;
|
||||
|
||||
/**
|
||||
* Creates a new CrawlServer object.
|
||||
@@ -226,15 +226,15 @@ public class CrawlServer implements Serializable, FetchStats.HasFetchStats {
|
||||
/**
|
||||
* @return Credential avatars for this server. Returns null if none.
|
||||
*/
|
||||
public Set<CredentialAvatar> getCredentialAvatars() {
|
||||
return this.avatars;
|
||||
public Set<Credential> getCredentials() {
|
||||
return this.credentials;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return True if there are avatars attached to this instance.
|
||||
*/
|
||||
public boolean hasCredentialAvatars() {
|
||||
return this.avatars != null && this.avatars.size() > 0;
|
||||
public boolean hasCredentials() {
|
||||
return this.credentials != null && this.credentials.size() > 0;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -242,11 +242,11 @@ public class CrawlServer implements Serializable, FetchStats.HasFetchStats {
|
||||
*
|
||||
* @param ca Credential avatar to add to set of avatars.
|
||||
*/
|
||||
public void addCredentialAvatar(CredentialAvatar ca) {
|
||||
if (this.avatars == null) {
|
||||
this.avatars = new HashSet<CredentialAvatar>();
|
||||
public void addCredential(Credential cred) {
|
||||
if (this.credentials == null) {
|
||||
this.credentials = new HashSet<Credential>();
|
||||
}
|
||||
this.avatars.add(ca);
|
||||
this.credentials.add(cred);
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
Reference in New Issue
Block a user