mirror of
https://github.com/internetarchive/heritrix3.git
synced 2026-11-17 11:17:06 +00:00
working on form auth test
This commit is contained in:
1 parent
d71fbb3a1a
commit
864fea938b
5 files changed
+122
-20
No files matched your search
@@ -376,7 +376,7 @@ public class PreconditionEnforcer extends Processor {
|
||||
* be met before we can proceed. False if we can precede to process
|
||||
* this url.
|
||||
*/
|
||||
protected boolean credentialPrecondition(final CrawlURI curi) {
|
||||
public boolean credentialPrecondition(final CrawlURI curi) {
|
||||
|
||||
boolean result = false;
|
||||
|
||||
|
||||
@@ -1829,7 +1829,7 @@ implements Reporter, Serializable, OverlayContext {
|
||||
public CrawlURI markPrerequisite(String preq)
|
||||
throws URIException {
|
||||
UURI src = getUURI();
|
||||
UURI dest = UURIFactory.getInstance(preq);
|
||||
UURI dest = UURIFactory.getInstance(src, preq);
|
||||
LinkContext lc = LinkContext.PREREQ_MISC;
|
||||
Hop hop = Hop.PREREQ;
|
||||
Link link = new Link(src, dest, lc, hop);
|
||||
|
||||
@@ -64,7 +64,7 @@ public class HtmlFormCredential extends Credential {
|
||||
}
|
||||
|
||||
|
||||
enum Method {
|
||||
public enum Method {
|
||||
GET,
|
||||
POST
|
||||
}
|
||||
|
||||
@@ -22,9 +22,11 @@ import java.util.List;
|
||||
|
||||
import org.archive.modules.Processor;
|
||||
import org.archive.modules.credential.CredentialStore;
|
||||
import org.archive.modules.net.ServerCache;
|
||||
|
||||
abstract public class AbstractFetchHTTP extends Processor {
|
||||
abstract public void setAcceptHeaders(List<String> headers);
|
||||
abstract public void setIgnoreCookies(boolean ignoreCookies);
|
||||
abstract public CredentialStore getCredentialStore();
|
||||
abstract public ServerCache getServerCache();
|
||||
}
|
||||
@@ -27,7 +27,6 @@ import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.logging.Handler;
|
||||
import java.util.logging.Level;
|
||||
import java.util.logging.Logger;
|
||||
|
||||
@@ -38,9 +37,11 @@ import javax.servlet.http.HttpServletResponse;
|
||||
|
||||
import org.apache.commons.httpclient.URIException;
|
||||
import org.apache.commons.io.IOUtils;
|
||||
import org.archive.crawler.prefetch.PreconditionEnforcer;
|
||||
import org.archive.modules.CrawlURI;
|
||||
import org.archive.modules.CrawlURI.FetchType;
|
||||
import org.archive.modules.ProcessorTestBase;
|
||||
import org.archive.modules.credential.HtmlFormCredential;
|
||||
import org.archive.modules.credential.HttpAuthenticationCredential;
|
||||
import org.archive.net.UURI;
|
||||
import org.archive.net.UURIFactory;
|
||||
@@ -50,11 +51,16 @@ import org.archive.util.TmpDirTestCase;
|
||||
import org.mortbay.jetty.Request;
|
||||
import org.mortbay.jetty.Server;
|
||||
import org.mortbay.jetty.bio.SocketConnector;
|
||||
import org.mortbay.jetty.handler.AbstractHandler;
|
||||
import org.mortbay.jetty.security.Authenticator;
|
||||
import org.mortbay.jetty.security.BasicAuthenticator;
|
||||
import org.mortbay.jetty.security.Constraint;
|
||||
import org.mortbay.jetty.security.ConstraintMapping;
|
||||
import org.mortbay.jetty.security.DigestAuthenticator;
|
||||
import org.mortbay.jetty.security.FormAuthenticator;
|
||||
import org.mortbay.jetty.security.HashUserRealm;
|
||||
import org.mortbay.jetty.security.SecurityHandler;
|
||||
import org.mortbay.jetty.servlet.HashSessionManager;
|
||||
import org.mortbay.jetty.servlet.SessionHandler;
|
||||
import org.mortbay.log.Log;
|
||||
|
||||
public abstract class FetchHTTPTestBase extends ProcessorTestBase {
|
||||
@@ -62,15 +68,15 @@ public abstract class FetchHTTPTestBase extends ProcessorTestBase {
|
||||
private static Logger logger = Logger.getLogger(FetchHTTPTestBase.class.getName());
|
||||
static {
|
||||
Logger.getLogger("").setLevel(Level.ALL);
|
||||
for (Handler h: Logger.getLogger("").getHandlers()) {
|
||||
for (java.util.logging.Handler h: Logger.getLogger("").getHandlers()) {
|
||||
h.setLevel(Level.ALL);
|
||||
h.setFormatter(new OneLineSimpleLogger());
|
||||
}
|
||||
}
|
||||
|
||||
protected static final String BASIC_AUTH_REALM = "basic-auth-realm";
|
||||
protected static final String BASIC_AUTH_ROLE = "basic-auth-role";
|
||||
protected static final String BASIC_AUTH_LOGIN = "basic-auth-login";
|
||||
protected static final String BASIC_AUTH_REALM = "basic-auth-realm";
|
||||
protected static final String BASIC_AUTH_ROLE = "basic-auth-role";
|
||||
protected static final String BASIC_AUTH_LOGIN = "basic-auth-login";
|
||||
protected static final String BASIC_AUTH_PASSWORD = "basic-auth-password";
|
||||
|
||||
protected static final String DIGEST_AUTH_REALM = "digest-auth-realm";
|
||||
@@ -78,20 +84,50 @@ public abstract class FetchHTTPTestBase extends ProcessorTestBase {
|
||||
protected static final String DIGEST_AUTH_LOGIN = "digest-auth-login";
|
||||
protected static final String DIGEST_AUTH_PASSWORD = "digest-auth-password";
|
||||
|
||||
protected static final String DEFAULT_PAYLOAD_STRING = "abcdefghijklmnopqrstuvwxyz0123456789\n";
|
||||
protected static final String FORM_AUTH_REALM = "form-auth-realm";
|
||||
protected static final String FORM_AUTH_ROLE = "form-auth-role";
|
||||
protected static final String FORM_AUTH_LOGIN = "form-auth-login";
|
||||
protected static final String FORM_AUTH_PASSWORD = "form-auth-password";
|
||||
|
||||
protected static class TestHandler extends AbstractHandler {
|
||||
protected static final String DEFAULT_PAYLOAD_STRING = "abcdefghijklmnopqrstuvwxyz0123456789\n";
|
||||
|
||||
protected static final String LOGIN_HTML =
|
||||
"<html>" +
|
||||
"<head><title>Log In</title></head>" +
|
||||
"<body>" +
|
||||
"<form action='/j_security_check' method='post'>" +
|
||||
"<div> username: <input name='j_username' type='text'/> </div>" +
|
||||
"<div> password: <input name='j_password' type='password'/> </div>" +
|
||||
"<div> <input type='submit' /> </div>" +
|
||||
"</form>" +
|
||||
"</body>" +
|
||||
"</html>";
|
||||
|
||||
protected static class TestHandler extends SessionHandler {
|
||||
public TestHandler() {
|
||||
super();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void handle(String target, HttpServletRequest request,
|
||||
HttpServletResponse response, int dispatch) throws IOException,
|
||||
ServletException {
|
||||
|
||||
if (target.endsWith("/set-cookie")) {
|
||||
response.addCookie(new Cookie("test-cookie-name", "test-cookie-value"));
|
||||
}
|
||||
response.setContentType("text/plain;charset=US-ASCII");
|
||||
response.setStatus(HttpServletResponse.SC_OK);
|
||||
response.getOutputStream().write(DEFAULT_PAYLOAD_STRING.getBytes("US-ASCII"));
|
||||
((Request)request).setHandled(true);
|
||||
|
||||
if (target.equals("/login.html")) {
|
||||
response.setContentType("text/html;charset=US-ASCII");
|
||||
response.setStatus(HttpServletResponse.SC_OK);
|
||||
response.getOutputStream().write(LOGIN_HTML.getBytes("US-ASCII"));
|
||||
((Request)request).setHandled(true);
|
||||
} else {
|
||||
response.setContentType("text/plain;charset=US-ASCII");
|
||||
response.setStatus(HttpServletResponse.SC_OK);
|
||||
response.getOutputStream().write(DEFAULT_PAYLOAD_STRING.getBytes("US-ASCII"));
|
||||
((Request)request).setHandled(true);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -100,7 +136,7 @@ public abstract class FetchHTTPTestBase extends ProcessorTestBase {
|
||||
|
||||
abstract protected AbstractFetchHTTP makeModule() throws IOException;
|
||||
|
||||
protected static SecurityHandler makeAuthWrapper(String authMethod,
|
||||
protected static SecurityHandler makeAuthWrapper(Authenticator authenticator,
|
||||
final String role, String realm, final String login,
|
||||
final String password) {
|
||||
Constraint constraint = new Constraint();
|
||||
@@ -112,7 +148,8 @@ public abstract class FetchHTTPTestBase extends ProcessorTestBase {
|
||||
constraintMapping.setPathSpec("/auth/*");
|
||||
|
||||
SecurityHandler authWrapper = new SecurityHandler();
|
||||
authWrapper.setAuthMethod(authMethod);
|
||||
authWrapper.setAuthenticator(authenticator);
|
||||
|
||||
authWrapper.setConstraintMappings(new ConstraintMapping[] {constraintMapping});
|
||||
authWrapper.setUserRealm(new HashUserRealm(realm) {
|
||||
{
|
||||
@@ -142,7 +179,7 @@ public abstract class FetchHTTPTestBase extends ProcessorTestBase {
|
||||
sc.setPort(7777);
|
||||
server.addConnector(sc);
|
||||
|
||||
SecurityHandler authWrapper = makeAuthWrapper(Constraint.__BASIC_AUTH,
|
||||
SecurityHandler authWrapper = makeAuthWrapper(new BasicAuthenticator(),
|
||||
BASIC_AUTH_ROLE, BASIC_AUTH_REALM, BASIC_AUTH_LOGIN,
|
||||
BASIC_AUTH_PASSWORD);
|
||||
authWrapper.setHandler(new TestHandler());
|
||||
@@ -159,12 +196,35 @@ public abstract class FetchHTTPTestBase extends ProcessorTestBase {
|
||||
sc.setPort(7778);
|
||||
server.addConnector(sc);
|
||||
|
||||
authWrapper = makeAuthWrapper(Constraint.__DIGEST_AUTH,
|
||||
authWrapper = makeAuthWrapper(new DigestAuthenticator(),
|
||||
DIGEST_AUTH_ROLE, DIGEST_AUTH_REALM, DIGEST_AUTH_LOGIN,
|
||||
DIGEST_AUTH_PASSWORD);
|
||||
authWrapper.setHandler(new TestHandler());
|
||||
server.setHandler(authWrapper);
|
||||
|
||||
server.start();
|
||||
servers.put(sc.getPort(), server);
|
||||
|
||||
// server for form auth
|
||||
server = new Server();
|
||||
|
||||
sc = new SocketConnector();
|
||||
sc.setHost("127.0.0.1");
|
||||
sc.setPort(7779);
|
||||
server.addConnector(sc);
|
||||
|
||||
FormAuthenticator formAuthenticatrix = new FormAuthenticator();
|
||||
formAuthenticatrix.setLoginPage("/login.html");
|
||||
|
||||
authWrapper = makeAuthWrapper(formAuthenticatrix,
|
||||
FORM_AUTH_ROLE, FORM_AUTH_REALM, FORM_AUTH_LOGIN,
|
||||
FORM_AUTH_PASSWORD);
|
||||
authWrapper.setHandler(new TestHandler());
|
||||
SessionHandler sessionHandler = new SessionHandler();
|
||||
sessionHandler.setSessionManager(new HashSessionManager());
|
||||
sessionHandler.setHandler(authWrapper);
|
||||
server.setHandler(sessionHandler);
|
||||
|
||||
server.start();
|
||||
servers.put(sc.getPort(), server);
|
||||
|
||||
@@ -203,6 +263,7 @@ public abstract class FetchHTTPTestBase extends ProcessorTestBase {
|
||||
IOException {
|
||||
UURI uuri = UURIFactory.getInstance(uri);
|
||||
CrawlURI curi = new CrawlURI(uuri);
|
||||
curi.setSeed(true);
|
||||
curi.setRecorder(getRecorder());
|
||||
return curi;
|
||||
}
|
||||
@@ -386,7 +447,7 @@ public abstract class FetchHTTPTestBase extends ProcessorTestBase {
|
||||
// fetch origin again with the credentials
|
||||
getFetcher().process(curi);
|
||||
String httpRequestString = httpRequestString(curi);
|
||||
logger.info('\n' + httpRequestString + contentString(curi));
|
||||
// logger.info('\n' + httpRequestString + contentString(curi));
|
||||
assertTrue(httpRequestString.contains("Authorization: Digest"));
|
||||
// otherwise should be a normal 200 response
|
||||
runDefaultChecks(curi, new HashSet<String>(Arrays.asList("requestLine", "hostHeader")));
|
||||
@@ -399,6 +460,45 @@ public abstract class FetchHTTPTestBase extends ProcessorTestBase {
|
||||
// otherwise should be a normal 200 response
|
||||
runDefaultChecks(curi, new HashSet<String>(Arrays.asList("requestLine")));
|
||||
}
|
||||
|
||||
// server for form auth is at localhost:7779
|
||||
public void testFormAuth() throws Exception {
|
||||
ensureHttpServers();
|
||||
|
||||
HtmlFormCredential cred = new HtmlFormCredential();
|
||||
cred.setDomain("localhost:7779");
|
||||
cred.setLoginUri("/j_security_check");
|
||||
cred.setHttpMethod(HtmlFormCredential.Method.POST);
|
||||
HashMap<String, String> formItems = new HashMap<String,String>();
|
||||
formItems.put("j_username", FORM_AUTH_LOGIN);
|
||||
formItems.put("j_password", FORM_AUTH_PASSWORD);
|
||||
cred.setFormItems(formItems);
|
||||
|
||||
getFetcher().getCredentialStore().getCredentials().put("form-auth-credential",
|
||||
cred);
|
||||
|
||||
CrawlURI curi = makeCrawlURI("http://localhost:7779/");
|
||||
getFetcher().process(curi);
|
||||
|
||||
PreconditionEnforcer preconditionEnforcer = new PreconditionEnforcer();
|
||||
preconditionEnforcer.setServerCache(getFetcher().getServerCache());
|
||||
preconditionEnforcer.setCredentialStore(getFetcher().getCredentialStore());
|
||||
boolean result = preconditionEnforcer.credentialPrecondition(curi);
|
||||
assertTrue(result);
|
||||
|
||||
CrawlURI loginUri = curi.getPrerequisiteUri();
|
||||
assertEquals("http://localhost:7779/j_security_check", loginUri.toString());
|
||||
loginUri.setRecorder(getRecorder());
|
||||
getFetcher().process(loginUri);
|
||||
|
||||
getFetcher().process(curi);
|
||||
|
||||
curi = makeCrawlURI("http://localhost:7779/auth/1");
|
||||
getFetcher().process(curi);
|
||||
|
||||
curi = makeCrawlURI("http://localhost:7779/auth/1");
|
||||
getFetcher().process(curi);
|
||||
}
|
||||
|
||||
protected void checkSetCookieURI() throws URIException, IOException,
|
||||
InterruptedException, UnsupportedEncodingException {
|
||||
|
||||
Reference in new issue
Block a user