diff --git a/engine/src/main/java/org/archive/crawler/frontier/AbstractFrontier.java b/engine/src/main/java/org/archive/crawler/frontier/AbstractFrontier.java index a5f6a2ff..ab9981ba 100644 --- a/engine/src/main/java/org/archive/crawler/frontier/AbstractFrontier.java +++ b/engine/src/main/java/org/archive/crawler/frontier/AbstractFrontier.java @@ -1155,7 +1155,7 @@ public abstract class AbstractFrontier // rfc2617 credential present and if there, assume it got // loaded in FetchHTTP on expectation that we're to go around // again. If no rfc2617 loaded, we should not be here. - boolean loaded = curi.hasRfc2617CredentialAvatar(); + boolean loaded = curi.hasRfc2617Credential(); if (!loaded && logger.isLoggable(Level.FINE)) { logger.fine("Have 401 but no creds loaded " + curi); } diff --git a/engine/src/main/java/org/archive/crawler/prefetch/PreconditionEnforcer.java b/engine/src/main/java/org/archive/crawler/prefetch/PreconditionEnforcer.java index 18f3ed73..0638ab9a 100644 --- a/engine/src/main/java/org/archive/crawler/prefetch/PreconditionEnforcer.java +++ b/engine/src/main/java/org/archive/crawler/prefetch/PreconditionEnforcer.java @@ -35,7 +35,6 @@ import org.archive.modules.CrawlURI; import org.archive.modules.ProcessResult; import org.archive.modules.Processor; import org.archive.modules.credential.Credential; -import org.archive.modules.credential.CredentialAvatar; import org.archive.modules.credential.CredentialStore; import org.archive.modules.extractor.Hop; import org.archive.modules.extractor.Link; @@ -447,22 +446,19 @@ public class PreconditionEnforcer extends Processor { * @param curi CrawlURI. * @return True if already run. */ - private boolean authenticated(final Credential credential, - final CrawlURI curi) { - boolean result = false; + private boolean authenticated(final Credential credential, final CrawlURI curi) { CrawlServer server = serverCache.getServerFor(curi.getUURI()); - if (!server.hasCredentialAvatars()) { - return result; + if (!server.hasCredentials()) { + return false; } - Set avatars = server.getCredentialAvatars(); - for (CredentialAvatar ca: avatars) { - String key = null; - key = credential.getKey(); - if (ca.match(credential.getClass(), key)) { - result = true; + Set credentials = server.getCredentials(); + for (Credential cred: credentials) { + if (cred.getKey().equals(credential.getKey()) + && cred.getClass().isInstance(credential)) { + return true; } } - return result; + return false; } diff --git a/modules/src/main/java/org/archive/modules/CoreAttributeConstants.java b/modules/src/main/java/org/archive/modules/CoreAttributeConstants.java index b5def89f..361a8f73 100644 --- a/modules/src/main/java/org/archive/modules/CoreAttributeConstants.java +++ b/modules/src/main/java/org/archive/modules/CoreAttributeConstants.java @@ -72,8 +72,8 @@ public interface CoreAttributeConstants { /** * Key to get credential avatars from A_LIST. */ - public static final String A_CREDENTIAL_AVATARS_KEY = - "credential-avatars"; + public static final String A_CREDENTIALS_KEY = + "credentials"; /** a 'source' (usu. URI) that's inherited by discovered URIs */ public static String A_SOURCE_TAG = ModuleAttributeConstants.A_SOURCE_TAG; diff --git a/modules/src/main/java/org/archive/modules/CrawlURI.java b/modules/src/main/java/org/archive/modules/CrawlURI.java index 1f9b32d3..fa9d9cbd 100644 --- a/modules/src/main/java/org/archive/modules/CrawlURI.java +++ b/modules/src/main/java/org/archive/modules/CrawlURI.java @@ -20,7 +20,7 @@ package org.archive.modules; import static org.archive.modules.CoreAttributeConstants.A_ANNOTATIONS; -import static org.archive.modules.CoreAttributeConstants.A_CREDENTIAL_AVATARS_KEY; +import static org.archive.modules.CoreAttributeConstants.A_CREDENTIALS_KEY; import static org.archive.modules.CoreAttributeConstants.A_DNS_SERVER_IP_LABEL; import static org.archive.modules.CoreAttributeConstants.A_FETCH_COMPLETED_TIME; import static org.archive.modules.CoreAttributeConstants.A_FORCE_RETIRE; @@ -64,7 +64,6 @@ import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; import java.util.HashSet; -import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -78,7 +77,7 @@ import org.apache.commons.httpclient.methods.GetMethod; import org.apache.commons.httpclient.methods.PostMethod; import org.apache.commons.lang.StringUtils; import org.archive.bdb.AutoKryo; -import org.archive.modules.credential.CredentialAvatar; +import org.archive.modules.credential.Credential; import org.archive.modules.credential.HttpAuthenticationCredential; import org.archive.modules.extractor.HTMLLinkContext; import org.archive.modules.extractor.Hop; @@ -257,7 +256,7 @@ implements MultiReporter, Serializable, OverlayContext { */ private static final Collection persistentKeys = new CopyOnWriteArrayList( - new String [] {A_CREDENTIAL_AVATARS_KEY}); + new String [] {A_CREDENTIALS_KEY}); /** * A digest (hash, usually SHA1) of retrieved content-body. @@ -903,12 +902,12 @@ implements MultiReporter, Serializable, OverlayContext { /** * @return Credential avatars. Null if none set. */ - public Set getCredentialAvatars() { + public Set getCredentials() { @SuppressWarnings("unchecked") - Set r = (Set)getData().get(A_CREDENTIAL_AVATARS_KEY); + Set r = (Set)getData().get(A_CREDENTIALS_KEY); if (r == null) { - r = new HashSet(); - getData().put(A_CREDENTIAL_AVATARS_KEY, r); + r = new HashSet(); + getData().put(A_CREDENTIALS_KEY, r); } return r; } @@ -916,8 +915,8 @@ implements MultiReporter, Serializable, OverlayContext { /** * @return True if there are avatars attached to this instance. */ - public boolean hasCredentialAvatars() { - return containsDataKey(A_CREDENTIAL_AVATARS_KEY); + public boolean hasCredentials() { + return containsDataKey(A_CREDENTIALS_KEY); } @@ -942,7 +941,7 @@ implements MultiReporter, Serializable, OverlayContext { boolean result = false; int statusCode = this.fetchStatus; if (statusCode == HttpStatus.SC_UNAUTHORIZED && - hasRfc2617CredentialAvatar()) { + hasRfc2617Credential()) { result = false; } else { result = (statusCode > 0); @@ -961,22 +960,18 @@ implements MultiReporter, Serializable, OverlayContext { /** * @return True if we have an rfc2617 payload. */ - public boolean hasRfc2617CredentialAvatar() { - boolean result = false; - Set avatars = getCredentialAvatars(); - if (avatars != null && avatars.size() > 0) { - for (Iterator i = avatars.iterator(); i.hasNext();) { - if (((CredentialAvatar)i.next()). - match(HttpAuthenticationCredential.class)) { - result = true; - break; + public boolean hasRfc2617Credential() { + Set credentials = getCredentials(); + if (credentials != null && credentials.size() > 0) { + for (Credential credential : credentials) { + if(credential instanceof HttpAuthenticationCredential) { + return true; } } } - return result; + return false; } - /** * Set the retained content-digest value (usu. SHA1). * @@ -1824,6 +1819,8 @@ implements MultiReporter, Serializable, OverlayContext { kryo.autoregister(org.archive.modules.extractor.HTMLLinkContext.class); kryo.autoregister(org.archive.modules.extractor.LinkContext.SimpleLinkContext.class); kryo.autoregister(java.util.HashMap[].class); + kryo.autoregister(org.archive.modules.credential.HttpAuthenticationCredential.class); + kryo.autoregister(org.archive.modules.credential.HtmlFormCredential.class); kryo.setRegistrationOptional(true); } } diff --git a/modules/src/main/java/org/archive/modules/Processor.java b/modules/src/main/java/org/archive/modules/Processor.java index 9f4b632f..8f720293 100644 --- a/modules/src/main/java/org/archive/modules/Processor.java +++ b/modules/src/main/java/org/archive/modules/Processor.java @@ -26,7 +26,7 @@ import java.util.concurrent.atomic.AtomicLong; import org.apache.commons.httpclient.HttpStatus; import org.archive.checkpointing.Checkpoint; import org.archive.checkpointing.Checkpointable; -import org.archive.modules.credential.CredentialAvatar; +import org.archive.modules.credential.Credential; import org.archive.modules.credential.HttpAuthenticationCredential; import org.archive.modules.deciderules.AcceptDecideRule; import org.archive.modules.deciderules.DecideResult; @@ -211,7 +211,7 @@ implements HasKeyedProperties, boolean result = false; int statusCode = puri.getFetchStatus(); if (statusCode == HttpStatus.SC_UNAUTHORIZED && - hasHttpAuthenticationCredentialAvatar(puri)) { + hasHttpAuthenticationCredential(puri)) { result = false; } else { result = (statusCode > 0); @@ -232,10 +232,10 @@ implements HasKeyedProperties, /** * @return True if we have an HttpAuthentication (rfc2617) payload. */ - public static boolean hasHttpAuthenticationCredentialAvatar(CrawlURI puri) { - Set avatars = puri.getCredentialAvatars(); - for (CredentialAvatar ca: avatars) { - if (ca.match(HttpAuthenticationCredential.class)) { + public static boolean hasHttpAuthenticationCredential(CrawlURI puri) { + Set credentials = puri.getCredentials(); + for (Credential ca: credentials) { + if (ca instanceof HttpAuthenticationCredential) { return true; } } diff --git a/modules/src/main/java/org/archive/modules/credential/Credential.java b/modules/src/main/java/org/archive/modules/credential/Credential.java index 1c49b23d..9f71e85d 100644 --- a/modules/src/main/java/org/archive/modules/credential/Credential.java +++ b/modules/src/main/java/org/archive/modules/credential/Credential.java @@ -20,7 +20,6 @@ package org.archive.modules.credential; import java.io.Serializable; import java.util.Iterator; -import java.util.Set; import java.util.logging.Logger; import javax.management.AttributeNotFoundException; @@ -96,20 +95,7 @@ public abstract class Credential implements Serializable { * @param curi CrawlURI to load with credentials. */ public void attach(CrawlURI curi) { - attach(curi, null); - } - - /** - * Attach this credentials avatar to the passed curi . - * - * @param curi CrawlURI to load with credentials. - * @param payload Payload to carry in avatar. Usually credentials. - */ - public void attach(CrawlURI curi, String payload) { - CredentialAvatar ca = (payload == null )? - new CredentialAvatar(this.getClass(), getKey()): - new CredentialAvatar(this.getClass(), getKey(), payload); - curi.getCredentialAvatars().add(ca); + curi.getCredentials().add(this); } /** @@ -119,22 +105,7 @@ public abstract class Credential implements Serializable { * @return True if we detached a Credential reference. */ public boolean detach(CrawlURI curi) { - boolean result = false; - Set avatars = curi.getCredentialAvatars(); - if (avatars.isEmpty()) { - logger.severe("This curi " + curi + " has no cred when it should"); - } - - Iterator iter = avatars.iterator(); - while (iter.hasNext()) { - CredentialAvatar ca = iter.next(); - if (ca.match(getClass(), getKey())) { - iter.remove(); - result = true; - } - } - - return result; + return curi.getCredentials().remove(this); } /** @@ -145,15 +116,10 @@ public abstract class Credential implements Serializable { */ public boolean detachAll(CrawlURI curi) { boolean result = false; - Set avatars = curi.getCredentialAvatars(); - if (avatars.isEmpty()) { - logger.severe("This curi " + curi +" has no creds when it should."); - return false; - } - Iterator iter = avatars.iterator(); + Iterator iter = curi.getCredentials().iterator(); while (iter.hasNext()) { - CredentialAvatar ca = iter.next(); - if (ca.match(getClass())) { + Credential cred = iter.next(); + if (cred.getClass() == this.getClass()) { iter.remove(); result = true; } @@ -201,11 +167,10 @@ public abstract class Credential implements Serializable { * @param curi CrawlURI to as for context. * @param http Instance of httpclient. * @param method Method to populate. - * @param payload Avatar payload to use populating the method. * @return True if added a credentials. */ public abstract boolean populate(CrawlURI curi, HttpClient http, - HttpMethod method, String payload); + HttpMethod method); /** * @param curi CrawlURI to look at. diff --git a/modules/src/main/java/org/archive/modules/credential/CredentialAvatar.java b/modules/src/main/java/org/archive/modules/credential/CredentialAvatar.java deleted file mode 100644 index 1f4c8190..00000000 --- a/modules/src/main/java/org/archive/modules/credential/CredentialAvatar.java +++ /dev/null @@ -1,220 +0,0 @@ -/* - * This file is part of the Heritrix web crawler (crawler.archive.org). - * - * Licensed to the Internet Archive (IA) by one or more individual - * contributors. - * - * The IA licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.archive.modules.credential; - -import java.io.Serializable; -import java.util.Collection; -import java.util.Iterator; -import java.util.List; -import java.util.logging.Logger; - -import org.archive.modules.CrawlURI; - -/** - * A credential representation. - * - * Added to the CrawlServer upon successful authentication. Used as a marker - * of successful authentication event and for carrying credential - * payload to be used subsequently doing preemptive authentications (e.g. - * For case of RFC2617, needs to be offered everytime we're accessing inside - * a protected area). Also carried by the ProcessorURI when cycling through - * processing chain trying a credential to see if it will authenticate. - * - *

This class exists because its not safe to keep references - * to the settings derived Credential classes so instead of keeping references - * to credential classes, we carry around this avatar. - * - *

Scope for avatars is crawlserver. Only used within a CrawlServer - * scope. - * - *

Immutable. - * - * @author stack - * @version $Revision$, $Date$ - */ -public class CredentialAvatar -implements Serializable { - - private static final long serialVersionUID = 3L; - - private static final Logger logger = - Logger.getLogger(CredentialAvatar.class.getName()); - - /** - * Key for this credential avatar. - */ - private final String key; - - /** - * Type represented by this avatar. - */ - private final Class type; - - /** - * Data. - * - * May be null. - * - *

This used to be an Object and I used to store in here - * the httpclient AuthScheme but AuthScheme is not serializable - * and so there'd be trouble getting this payload to lie down - * in a bdb database. Changed it to String. That should be - * generic enough for credential purposes. - */ - private final String payload; - - - /** - * Constructor. - * @param type Type for this credential avatar. - * @param key Key for this credential avatar. - */ - public CredentialAvatar(Class type, String key) { - this(type, key, null); - } - - /** - * Constructor. - * @param type Type for this credential avatar. - * @param key Key for this credential avatar. - * @param payload Data credential needs rerunning or preempting. May be - * null and then just the presence is used as signifier of successful - * auth. - */ - public CredentialAvatar(Class type, String key, String payload) { - if (!checkType(type)) { - throw new IllegalArgumentException("Type is unrecognized: " + - type); - } - this.key = key; - this.type = type; - this.payload = payload; - } - - /** - * Shutdown default constructor. - */ - @SuppressWarnings("unused") - private CredentialAvatar() { - super(); - this.key = null; - this.type = null; - this.payload = null; - } - - /** - * @param candidateType Type to check. - * @return True if this is a known credential type. - */ - protected boolean checkType(Class candidateType) { - boolean result = false; - List> types = CredentialStore.getCredentialTypes(); - for (Iterator> i = types.iterator(); i.hasNext();) { - if (i.next().equals(candidateType)) { - result = true; - break; - } - } - return result; - } - - /** - * @return Returns the payload. May be null. - */ - public String getPayload() { - return this.payload; - } - - /** - * @return Returns the key. - */ - public String getKey() { - return this.key; - } - - /** - * @return Type represented by this avatar. - */ - public Class getType() { - return this.type; - } - - /** - * @param otherType Class to match. - * @return True if this credential avatar is of same type. - */ - public boolean match(Class otherType) { - return this.type.equals(otherType); - } - - /** - * @param otherType Credential to match. - * @param otherKey Key to test. - * @return True if this is avatar for passed credential. - */ - public boolean match(Class otherType, String otherKey) { - return match(otherType) && - (otherKey != null && this.key != null && - this.key.equals(otherKey)); - } - - public String toString() { - return getType() + "." + this.getKey(); - } - - /** - * @param handler Settings handler. - * @param curi ProcessorURI to use for context. - * @return The credential this avatar represents. - */ - public Credential getCredential(CredentialStore cs, CrawlURI curi) { - Credential result = null; - - if (cs == null) { - logger.severe("No credential store for " + curi); - return result; - } - - Collection all = cs.getAll(); - if (all == null) { - logger.severe("Have CredentialAvatar " + toString() + - " but no collection: " + curi); - return result; - } - - for (Credential c: all) { - if (!this.type.isInstance(c)) { - continue; - } - String credKey = c.getKey(); - if (credKey != null && credKey.equals(getKey())) { - result = c; - break; - } - } - - if (result == null) { - logger.severe("Have CredentialAvatar " + toString() + - " but no corresponding credential: " + curi); - } - - return result; - } -} diff --git a/modules/src/main/java/org/archive/modules/credential/CredentialStore.java b/modules/src/main/java/org/archive/modules/credential/CredentialStore.java index 336397bb..fdbf99b0 100644 --- a/modules/src/main/java/org/archive/modules/credential/CredentialStore.java +++ b/modules/src/main/java/org/archive/modules/credential/CredentialStore.java @@ -54,6 +54,7 @@ public class CredentialStore implements Serializable, HasKeyedProperties { private static final long serialVersionUID = 3L; + @SuppressWarnings("unused") private static Logger logger = Logger.getLogger( "org.archive.crawler.datamodel.CredentialStore"); @@ -179,35 +180,4 @@ public class CredentialStore implements Serializable, HasKeyedProperties { } return result; } - - - public Credential getCredential(CrawlURI curi, CredentialAvatar ca) { - Credential result = null; - - Collection all = getAll(); - if (all == null) { - logger.severe("Have CredentialAvatar " + toString() + - " but no collection: " + curi); - return result; - } - - for (Credential c: all) { - if (!ca.getType().isInstance(c)) { - continue; - } - String credKey = c.getKey(); - if (credKey != null && credKey.equals(ca.getKey())) { - result = c; - break; - } - } - - if (result == null) { - logger.severe("Have CredentialAvatar " + toString() + - " but no corresponding credential: " + curi); - } - - return result; - - } } diff --git a/modules/src/main/java/org/archive/modules/credential/HtmlFormCredential.java b/modules/src/main/java/org/archive/modules/credential/HtmlFormCredential.java index 5ce10b87..563ad1a4 100644 --- a/modules/src/main/java/org/archive/modules/credential/HtmlFormCredential.java +++ b/modules/src/main/java/org/archive/modules/credential/HtmlFormCredential.java @@ -133,10 +133,8 @@ public class HtmlFormCredential extends Credential { return false; } - public boolean populate(CrawlURI curi, HttpClient http, HttpMethod method, - String payload) { - // http is not used. - // payload is not used. + public boolean populate(CrawlURI curi, HttpClient http, HttpMethod method) { + // http is not used boolean result = false; Map formItems = getFormItems(); if (formItems == null || formItems.size() <= 0) { diff --git a/modules/src/main/java/org/archive/modules/credential/HttpAuthenticationCredential.java b/modules/src/main/java/org/archive/modules/credential/HttpAuthenticationCredential.java index e6afdbb0..3152444b 100644 --- a/modules/src/main/java/org/archive/modules/credential/HttpAuthenticationCredential.java +++ b/modules/src/main/java/org/archive/modules/credential/HttpAuthenticationCredential.java @@ -104,14 +104,9 @@ public class HttpAuthenticationCredential extends Credential { return true; } - public boolean populate(CrawlURI curi, HttpClient http, HttpMethod method, - String payload) { + public boolean populate(CrawlURI curi, HttpClient http, HttpMethod method) { boolean result = false; - String authRealm = payload; - if (authRealm == null) { - logger.severe("No authscheme though creds: " + curi); - return result; - } + // Always add the credential to HttpState. Doing this because no way of // removing the credential once added AND there is a bug in the @@ -130,8 +125,8 @@ public class HttpAuthenticationCredential extends Credential { upc = new UsernamePasswordCredentials(getLogin(), getPassword()); http.getState().setCredentials(new AuthScope(curi.getUURI().getHost(), - curi.getUURI().getPort(), authRealm), upc); - logger.fine("Credentials for realm " + authRealm + + curi.getUURI().getPort(), getRealm()), upc); + logger.fine("Credentials for realm " + getRealm() + " for CrawlURI " + curi.toString() + " added to request: " + result); result = true; diff --git a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java index 14aa97cb..28a53bbd 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java +++ b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java @@ -93,7 +93,6 @@ import org.archive.modules.CrawlURI; import org.archive.modules.ProcessResult; import org.archive.modules.Processor; import org.archive.modules.credential.Credential; -import org.archive.modules.credential.CredentialAvatar; import org.archive.modules.credential.CredentialStore; import org.archive.modules.credential.HttpAuthenticationCredential; import org.archive.modules.deciderules.AcceptDecideRule; @@ -1043,11 +1042,10 @@ public class FetchHTTP extends Processor implements Lifecycle { return false; } CrawlServer server = serverCache.getServerFor(serverKey); - if (server.hasCredentialAvatars()) { - for (CredentialAvatar ca : server.getCredentialAvatars()) { - Credential c = ca.getCredential(getCredentialStore(), curi); - if (c.isEveryTime()) { - c.populate(curi, this.http, method, ca.getPayload()); + if (server.hasCredentials()) { + for (Credential cred : server.getCredentials()) { + if (cred.isEveryTime()) { + cred.populate(curi, this.http, method); } } } @@ -1057,9 +1055,8 @@ public class FetchHTTP extends Processor implements Lifecycle { // Now look in the curi. The Curi will have credentials loaded either // by the handle401 method if its a rfc2617 or it'll have been set into // the curi by the preconditionenforcer as this login uri came through. - for (CredentialAvatar ca : curi.getCredentialAvatars()) { - Credential c = ca.getCredential(getCredentialStore(), curi); - if (c.populate(curi, this.http, method, ca.getPayload())) { + for (Credential c: curi.getCredentials()) { + if (c.populate(curi, this.http, method)) { result = true; } } @@ -1074,21 +1071,20 @@ public class FetchHTTP extends Processor implements Lifecycle { * CrawlURI whose credentials we are to promote. */ private void promoteCredentials(final CrawlURI curi) { - Set avatars = curi.getCredentialAvatars(); - for (Iterator i = avatars.iterator(); i.hasNext();) { - CredentialAvatar ca = i.next(); + Set credentials = curi.getCredentials(); + for (Iterator i = credentials.iterator(); i.hasNext();) { + Credential c = i.next(); i.remove(); // The server to attach too may not be the server that hosts // this passed curi. It might be of another subdomain. // The avatar needs to be added to the server that is dependent // on this precondition. Find it by name. Get the name from // the credential this avatar represents. - Credential c = getCredentialStore().getCredential(curi, ca); String cd = c.getDomain(); if (cd != null) { CrawlServer cs = serverCache.getServerFor(cd); if (cs != null) { - cs.addCredentialAvatar(ca); + cs.addCredential(c); } } } @@ -1163,7 +1159,7 @@ public class FetchHTTP extends Processor implements Lifecycle { logger.fine("No rfc2617 credentials for realm " + realm + " in " + curi); } else { - found.attach(curi, authscheme.getRealm()); + found.attach(curi); logger.fine("Found credential for realm " + realm + " in store for " + curi.toString()); } @@ -1253,15 +1249,13 @@ public class FetchHTTP extends Processor implements Lifecycle { private Set getCredentials(CrawlURI curi, Class type) { Set result = null; - if (curi.hasCredentialAvatars()) { - for (Iterator i = curi.getCredentialAvatars().iterator(); i - .hasNext();) { - CredentialAvatar ca = (CredentialAvatar) i.next(); - if (ca.match(type)) { + if (curi.hasCredentials()) { + for (Credential c : curi.getCredentials()) { + if (type.isInstance(c)) { if (result == null) { result = new HashSet(); } - result.add(ca.getCredential(getCredentialStore(), curi)); + result.add(c); } } } diff --git a/modules/src/main/java/org/archive/modules/net/CrawlServer.java b/modules/src/main/java/org/archive/modules/net/CrawlServer.java index 5d71ebde..27ba509b 100644 --- a/modules/src/main/java/org/archive/modules/net/CrawlServer.java +++ b/modules/src/main/java/org/archive/modules/net/CrawlServer.java @@ -40,7 +40,7 @@ import org.apache.commons.io.IOUtils; import org.archive.bdb.AutoKryo; import org.archive.io.ReplayInputStream; import org.archive.modules.CrawlURI; -import org.archive.modules.credential.CredentialAvatar; +import org.archive.modules.credential.Credential; import org.archive.modules.fetcher.FetchStats; import org.archive.net.UURI; import org.archive.net.UURIFactory; @@ -77,9 +77,9 @@ public class CrawlServer implements Serializable, FetchStats.HasFetchStats { protected int consecutiveConnectionErrors = 0; /** - * Set of credential avatars. + * Set of credentials. */ - private transient Set avatars = null; + private transient Set credentials = null; /** * Creates a new CrawlServer object. @@ -226,15 +226,15 @@ public class CrawlServer implements Serializable, FetchStats.HasFetchStats { /** * @return Credential avatars for this server. Returns null if none. */ - public Set getCredentialAvatars() { - return this.avatars; + public Set getCredentials() { + return this.credentials; } /** * @return True if there are avatars attached to this instance. */ - public boolean hasCredentialAvatars() { - return this.avatars != null && this.avatars.size() > 0; + public boolean hasCredentials() { + return this.credentials != null && this.credentials.size() > 0; } /** @@ -242,11 +242,11 @@ public class CrawlServer implements Serializable, FetchStats.HasFetchStats { * * @param ca Credential avatar to add to set of avatars. */ - public void addCredentialAvatar(CredentialAvatar ca) { - if (this.avatars == null) { - this.avatars = new HashSet(); + public void addCredential(Credential cred) { + if (this.credentials == null) { + this.credentials = new HashSet(); } - this.avatars.add(ca); + this.credentials.add(cred); } /**