lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-09-22 21:55:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #53 from nlevitt/fix-form-login

Browse Source 
Fix form login
This commit is contained in:
vonrosen
2014-04-01 12:29:50 -07:00
parent 19e8cd5b66 552200b2fa
commit 4bd8045bfb
14 changed files with 348 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files
engine/src/test/java/org/archive/crawler/selftest/FormLoginSelfTest.java
+103
View File
@@ -0,0 +1,103 @@
/*
* This file is part of the Heritrix web crawler (crawler.archive.org).
*
* Licensed to the Internet Archive (IA) by one or more individual
* contributors.
*
* The IA licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.archive.crawler.selftest;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
import java.util.Set;
import org.mortbay.jetty.Handler;
import org.mortbay.jetty.Server;
import org.mortbay.jetty.bio.SocketConnector;
import org.mortbay.jetty.handler.DefaultHandler;
import org.mortbay.jetty.handler.HandlerList;
import org.mortbay.jetty.handler.ResourceHandler;
import org.mortbay.jetty.servlet.ServletHandler;
import org.mortbay.jetty.servlet.ServletHolder;
/**
* Test form-based authentication
*
* @contributor stack
* @contributor gojomo
*/
public class FormLoginSelfTest
extends SelfTestBase
{
/**
* Files to find as a list.
*/
final private static Set<String> EXPECTED = Collections.unmodifiableSet(
new HashSet<String>(Arrays.asList(new String[] {
"index.html", "login/login.html", "success.html", "robots.txt", "favicon.ico"
})));
@Override
protected void verify() throws Exception {
Set<String> found = this.filesInArcs();
assertEquals("wrong files in ARCs",EXPECTED,found);
}
@Override
protected void startHttpServer() throws Exception {
Server server = new Server();
SocketConnector sc = new SocketConnector();
sc.setHost("127.0.0.1");
sc.setPort(7777);
server.addConnector(sc);
ResourceHandler rhandler = new ResourceHandler();
rhandler.setResourceBase(getSrcHtdocs().getAbsolutePath());
ServletHandler servletHandler = new ServletHandler();
HandlerList handlers = new HandlerList();
handlers.setHandlers(new Handler[] {
rhandler,
servletHandler,
new DefaultHandler() });
server.setHandler(handlers);
ServletHolder holder = new ServletHolder(new FormAuthServlet());
servletHandler.addServletWithMapping(holder, "/login/*");
this.httpServer = server;
this.httpServer.start();
}
protected String getSeedsString() {
return "http://127.0.0.1:7777/index.html";
}
@Override
protected String changeGlobalConfig(String config) {
String formLoginConfig =
" <bean id='extractorForms' class='org.archive.modules.forms.ExtractorHTMLForms'/>\n"
+ " <bean id='formFiller' class='org.archive.modules.forms.FormLoginProcessor'>\n"
+ " <property name='loginUsername' value='Mr. Happy Pants' />\n"
+ " <property name='loginPassword' value='xyzzy' />\n"
+ " </bean>\n";
config = config.replace("<!--@@MORE_EXTRACTORS@@-->", formLoginConfig);
return super.changeGlobalConfig(config);
}
}
engine/testdata/selftest/FormLoginSelfTest/htdocs/failure.html
Vendored
+9
View File
@@ -0,0 +1,9 @@
<a href="index.html">index</a>
<a href="link1.html">Link 1</a>
<a href="link2.html"> Link 2</a>
<a href="link3.html"> Link 3</a>
<a href="basic/index.html">Secure</a>
engine/testdata/selftest/FormLoginSelfTest/htdocs/index.html
Vendored
+5
View File
@@ -0,0 +1,5 @@
<form action="login/login.html" method="post">
<input type="text" name="username"/>
<input type="password" name="password"/>
</form>
engine/testdata/selftest/FormLoginSelfTest/htdocs/link1.html
Vendored
+7
View File
@@ -0,0 +1,7 @@
<a href="index.html">index</a>
<a href="link1.html">Link 1</a>
<a href="link2.html"> Link 2</a>
<a href="link3.html"> Link 3</a>
engine/testdata/selftest/FormLoginSelfTest/htdocs/link2.html
Vendored
+7
View File
@@ -0,0 +1,7 @@
<a href="index.html">index</a>
<a href="link1.html">Link 1</a>
<a href="link2.html"> Link 2</a>
<a href="link3.html"> Link 3</a>
engine/testdata/selftest/FormLoginSelfTest/htdocs/link3.html
Vendored
+7
View File
@@ -0,0 +1,7 @@
<a href="index.html">index</a>
<a href="link1.html">Link 1</a>
<a href="link2.html"> Link 2</a>
<a href="link3.html"> Link 3</a>
engine/testdata/selftest/FormLoginSelfTest/htdocs/success.html
Vendored
+2
View File
@@ -0,0 +1,2 @@
<a href="index.html">index</a>
engine/testdata/selftest/conf/selftest-crawler-beans.cxml
Vendored
+1
View File
@@ -124,6 +124,7 @@ crawlController.pauseAtStart=false
<ref bean="extractorCss"/>
<ref bean="extractorJs"/>
<ref bean="extractorSwf"/>
<!--@@MORE_EXTRACTORS@@-->
<ref bean="arcWriterProcessor"/>
</list>
</property>
modules/src/main/java/org/archive/modules/fetcher/FetchHTTPRequest.java
+15 -3
View File
@@ -64,6 +64,7 @@ import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.AbstractExecutionAwareRequest;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.client.utils.URLEncodedUtils;
import org.apache.http.config.ConnectionConfig;
import org.apache.http.config.MessageConstraints;
import org.apache.http.config.Registry;
@@ -79,6 +80,8 @@ import org.apache.http.conn.socket.PlainConnectionSocketFactory;
import org.apache.http.conn.ssl.AllowAllHostnameVerifier;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
import org.apache.http.entity.ContentLengthStrategy;
import org.apache.http.entity.ContentType;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.DefaultBHttpClientConnection;
import org.apache.http.impl.client.BasicAuthCache;
import org.apache.http.impl.client.BasicCredentialsProvider;
@@ -93,6 +96,7 @@ import org.apache.http.message.BasicNameValuePair;
import org.apache.http.protocol.HTTP;
import org.apache.http.protocol.HttpContext;
import org.apache.http.util.Args;
import org.archive.modules.CoreAttributeConstants;
import org.archive.modules.CrawlURI;
import org.archive.modules.CrawlURI.FetchType;
import org.archive.modules.Processor;
@@ -175,8 +179,16 @@ class FetchHTTPRequest {
}
if (curi.getFetchType() == FetchType.HTTP_POST) {
this.request = new BasicExecutionAwareEntityEnclosingRequest("POST",
requestLineUri, httpVersion);
BasicExecutionAwareEntityEnclosingRequest postRequest = new BasicExecutionAwareEntityEnclosingRequest(
"POST", requestLineUri, httpVersion);
this.request = postRequest;
String submitData = (String) curi.getData().get(CoreAttributeConstants.A_SUBMIT_DATA);
if (submitData != null) {
// XXX brittle, doesn't support multipart form data etc
ContentType contentType = ContentType.create(URLEncodedUtils.CONTENT_TYPE, "UTF-8");
StringEntity formEntity = new StringEntity(submitData, contentType);
postRequest.setEntity(formEntity);
}
} else {
this.request = new BasicExecutionAwareRequest("GET",
requestLineUri, httpVersion);
@@ -196,7 +208,7 @@ class FetchHTTPRequest {
this.addedCredentials = populateTargetCredential();
populateHttpProxyCredential();
}
protected void configureRequestHeaders() {
if (fetcher.getAcceptCompression()) {
request.addHeader("Accept-Encoding", "gzip,deflate");
modules/src/main/java/org/archive/modules/forms/FormLoginProcessor.java
+30 -22
View File
@@ -21,14 +21,11 @@ package org.archive.modules.forms;
import static org.archive.modules.CoreAttributeConstants.A_WARC_RESPONSE_HEADERS;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.atomic.AtomicLong;
import java.util.logging.Level;
import java.util.logging.Logger;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import org.apache.commons.httpclient.URIException;
import org.apache.commons.lang.StringUtils;
import org.archive.checkpointing.Checkpointable;
@@ -45,6 +42,10 @@ import org.json.JSONException;
import org.json.JSONObject;
import org.springframework.beans.factory.annotation.Autowired;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
/**
* A step, post-ExtractorHTMLForms, where a followup CrawlURI to
* attempt a form submission may be synthesized.
@@ -119,24 +120,24 @@ public class FormLoginProcessor extends Processor implements Checkpointable {
Logger.getLogger(FormLoginProcessor.class.getName());
// formProvince (String) -> count
ConcurrentMap<String, AtomicLong> eligibleFormsSeenCount =
protected LoadingCache<String, AtomicLong> eligibleFormsSeenCount =
CacheBuilder.newBuilder()
.<String, AtomicLong>build(
new CacheLoader<String, AtomicLong>() {
public AtomicLong load(String arg0) {
return new AtomicLong(0L);
}
}).asMap();
});
// formProvince (String) -> count
ConcurrentMap<String, AtomicLong> eligibleFormsAttemptsCount =
protected LoadingCache<String, AtomicLong> eligibleFormsAttemptsCount =
CacheBuilder.newBuilder()
.<String, AtomicLong>build(
new CacheLoader<String, AtomicLong>() {
public AtomicLong load(String arg0) {
return new AtomicLong(0L);
}
}).asMap();
});
/**
* SURT prefix against which configured username/password is
@@ -219,13 +220,17 @@ public class FormLoginProcessor extends Processor implements Checkpointable {
for( Object formObject : curi.getDataList(ExtractorHTMLForms.A_HTML_FORM_OBJECTS)) {
HTMLForm form = (HTMLForm) formObject;
if(form.seemsLoginForm()) {
eligibleFormsSeenCount.get(formProvince).incrementAndGet();
if(eligibleFormsAttemptsCount.get(formProvince).get()<1) {
eligibleFormsAttemptsCount.get(formProvince).incrementAndGet();
createFormSubmissionAttempt(curi,form,formProvince);
} else {
// note decline-to-submit: in volume, may be signal of failed first login
curi.getAnnotations().add("nosubmit:"+submitStatusFor(formProvince));
try {
eligibleFormsSeenCount.get(formProvince).incrementAndGet();
if(eligibleFormsAttemptsCount.get(formProvince).get()<1) {
eligibleFormsAttemptsCount.get(formProvince).incrementAndGet();
createFormSubmissionAttempt(curi,form,formProvince);
} else {
// note decline-to-submit: in volume, may be signal of failed first login
curi.getAnnotations().add("nosubmit:"+submitStatusFor(formProvince));
}
} catch (ExecutionException e) {
throw new RuntimeException(e); // can't happen?
}
return;
}
@@ -260,10 +265,9 @@ public class FormLoginProcessor extends Processor implements Checkpointable {
submitCuri.setFetchType(FetchType.HTTP_POST);
submitCuri.getData().put(
CoreAttributeConstants.A_SUBMIT_DATA,
templateForm.asHttpClientDataWith(
templateForm.asFormDataString(
getLoginUsername(),
getLoginPassword()));
//submitCuri.setSchedulingDirective(Math.max(curi.getSchedulingDirective()-1, 0));
submitCuri.setSchedulingDirective(SchedulingConstants.HIGH);
submitCuri.setForceFetch(true);
curi.getOutCandidates().add(submitCuri);
@@ -278,9 +282,13 @@ public class FormLoginProcessor extends Processor implements Checkpointable {
}
protected String submitStatusFor(String formProvince) {
return eligibleFormsAttemptsCount.get(formProvince).get()
+","+eligibleFormsSeenCount.get(formProvince).get()
+","+formProvince;
try {
return eligibleFormsAttemptsCount.get(formProvince).get()
+","+eligibleFormsSeenCount.get(formProvince).get()
+","+formProvince;
} catch (ExecutionException e) {
throw new RuntimeException(e);
}
}
@Override
@@ -295,10 +303,10 @@ public class FormLoginProcessor extends Processor implements Checkpointable {
protected void fromCheckpointJson(JSONObject json) throws JSONException {
super.fromCheckpointJson(json);
JSONUtils.putAllAtomicLongs(
eligibleFormsAttemptsCount,
eligibleFormsAttemptsCount.asMap(),
json.getJSONObject("eligibleFormsAttemptsCount"));
JSONUtils.putAllAtomicLongs(
eligibleFormsSeenCount,
eligibleFormsSeenCount.asMap(),
json.getJSONObject("eligibleFormsSeenCount"));
}
}
modules/src/main/java/org/archive/modules/forms/HTMLForm.java
+19
View File
@@ -20,10 +20,12 @@
package org.archive.modules.forms;
import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
import org.apache.commons.httpclient.NameValuePair;
import org.apache.commons.lang.StringUtils;
import org.archive.util.TextUtils;
/**
* Simple representation of a discovered HTML Form.
@@ -104,6 +106,7 @@ public class HTMLForm {
* @param username
* @param password
* @return
* @deprecated specific to a particular FetchHTTP implementation based on commons-httpclient, use {@link #asFormDataString(String, String)}
*/
public NameValuePair[] asHttpClientDataWith(String username, String password) {
ArrayList<NameValuePair> data = new ArrayList<NameValuePair>(allInputs.size());
@@ -119,6 +122,22 @@ public class HTMLForm {
}
return data.toArray(new NameValuePair[data.size()]);
}
public String asFormDataString(String username, String password) {
List<String> nameVals = new LinkedList<String>();
for (FormInput input : allInputs) {
if(input == candidateUsernameInputs.get(0)) {
nameVals.add(TextUtils.urlEscape(input.name) + "=" + TextUtils.urlEscape(username));
} else if(input == candidatePasswordInputs.get(0)) {
nameVals.add(TextUtils.urlEscape(input.name) + "=" + TextUtils.urlEscape(password));
} else if (StringUtils.isNotEmpty(input.name) && StringUtils.isNotEmpty(input.value)) {
nameVals.add(TextUtils.urlEscape(input.name) + "=" + TextUtils.urlEscape(input.value));
}
}
return StringUtils.join(nameVals, '&');
}
public String toString() {
StringBuilder sb = new StringBuilder();
commons/src/main/java/org/archive/state/ModuleTestBase.java → modules/src/main/java/org/archive/state/ModuleTestBase.java
+28
View File
@@ -20,12 +20,19 @@ package org.archive.state;
import java.io.File;
import java.io.IOException;
import java.io.Serializable;
import java.util.Arrays;
import junit.framework.TestCase;
import org.apache.commons.httpclient.URIException;
import org.apache.commons.lang.SerializationUtils;
import org.archive.modules.CrawlURI;
import org.archive.net.UURI;
import org.archive.net.UURIFactory;
import org.archive.util.Recorder;
import org.archive.util.TmpDirTestCase;
/**
@@ -213,4 +220,25 @@ public abstract class ModuleTestBase extends TestCase {
throw t;
}
}
protected Recorder getRecorder() throws IOException {
if (Recorder.getHttpRecorder() == null) {
Recorder httpRecorder = new Recorder(TmpDirTestCase.tmpDir(),
getClass().getName(), 16 * 1024, 512 * 1024);
Recorder.setHttpRecorder(httpRecorder);
}
return Recorder.getHttpRecorder();
}
protected CrawlURI makeCrawlURI(String uri) throws URIException,
IOException {
UURI uuri = UURIFactory.getInstance(uri);
CrawlURI curi = new CrawlURI(uuri);
curi.setSeed(true);
curi.setRecorder(getRecorder());
return curi;
}
}
modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTests.java
+39 -25
View File
@@ -43,6 +43,7 @@ import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Set;
import java.util.logging.Level;
@@ -52,8 +53,12 @@ import javax.net.ssl.SSLException;
import org.apache.commons.httpclient.URIException;
import org.apache.commons.io.IOUtils;
import org.apache.http.NameValuePair;
import org.apache.http.NoHttpResponseException;
import org.apache.http.client.utils.URLEncodedUtils;
import org.apache.http.message.BasicNameValuePair;
import org.archive.httpclient.ConfigurableX509TrustManager.TrustLevel;
import org.archive.modules.CoreAttributeConstants;
import org.archive.modules.CrawlMetadata;
import org.archive.modules.CrawlURI;
import org.archive.modules.CrawlURI.FetchType;
@@ -62,9 +67,6 @@ import org.archive.modules.credential.HttpAuthenticationCredential;
import org.archive.modules.deciderules.RejectDecideRule;
import org.archive.modules.recrawl.FetchHistoryProcessor;
import org.archive.net.UURI;
import org.archive.net.UURIFactory;
import org.archive.util.Recorder;
import org.archive.util.TmpDirTestCase;
import org.jboss.netty.handler.codec.http.HttpRequest;
import org.littleshoot.proxy.DefaultHttpProxyServer;
import org.littleshoot.proxy.HttpFilter;
@@ -94,25 +96,6 @@ public class FetchHTTPTests extends ProcessorTestBase {
return getClass().getName();
}
protected Recorder getRecorder() throws IOException {
if (Recorder.getHttpRecorder() == null) {
Recorder httpRecorder = new Recorder(TmpDirTestCase.tmpDir(),
getClass().getName(), 16 * 1024, 512 * 1024);
Recorder.setHttpRecorder(httpRecorder);
}
return Recorder.getHttpRecorder();
}
protected CrawlURI makeCrawlURI(String uri) throws URIException,
IOException {
UURI uuri = UURIFactory.getInstance(uri);
CrawlURI curi = new CrawlURI(uuri);
curi.setSeed(true);
curi.setRecorder(getRecorder());
return curi;
}
protected void runDefaultChecks(CrawlURI curi, String... exclusionsArray)
throws IOException, UnsupportedEncodingException {
@@ -130,7 +113,9 @@ public class FetchHTTPTests extends ProcessorTestBase {
if (!exclusions.contains("hostHeader")) {
assertTrue(requestString.contains("Host: localhost:7777\r\n"));
}
assertTrue(requestString.endsWith("\r\n\r\n"));
if (!exclusions.contains("trailingCRLFCRLF")) {
assertTrue(requestString.endsWith("\r\n\r\n"));
}
// check sizes
assertEquals(DEFAULT_PAYLOAD_STRING.length(), curi.getContentLength());
@@ -147,7 +132,9 @@ public class FetchHTTPTests extends ProcessorTestBase {
if (!exclusions.contains("fetchStatus")) {
assertTrue(curi.getFetchStatus() == 200);
}
assertTrue(curi.getFetchType() == FetchType.HTTP_GET);
if (!exclusions.contains("fetchTypeGET")) {
assertTrue(curi.getFetchType() == FetchType.HTTP_GET);
}
// check message body, i.e. "raw, possibly chunked-transfer-encoded message contents not including the leading headers"
assertEquals(DEFAULT_PAYLOAD_STRING, messageBodyString(curi));
@@ -703,8 +690,9 @@ public class FetchHTTPTests extends ProcessorTestBase {
@Override
public void run() {
ServerSocket listeningSocket = null;
try {
ServerSocket listeningSocket = new ServerSocket(listenPort, 0, Inet4Address.getByName(listenAddress));
listeningSocket = new ServerSocket(listenPort, 0, Inet4Address.getByName(listenAddress));
listeningSocket.setSoTimeout(600);
while (!isTimeToBeDone) {
try {
@@ -718,6 +706,12 @@ public class FetchHTTPTests extends ProcessorTestBase {
// logger.warning("caught exception: " + e);
} finally {
// logger.info("all done suckers");
if (listeningSocket != null) {
try {
listeningSocket.close();
} catch (IOException e) {
}
}
}
}
@@ -829,6 +823,26 @@ public class FetchHTTPTests extends ProcessorTestBase {
fetcher().process(curi);
assertTrue(httpRequestString(curi).contains("Host: example.com\r\n"));
}
public void testHttpPost() throws Exception {
CrawlURI curi = makeCrawlURI("http://localhost:7777/");
curi.setFetchType(FetchType.HTTP_POST);
List<NameValuePair> params = new LinkedList<NameValuePair>();
params.add(new BasicNameValuePair("name1", "value1"));
params.add(new BasicNameValuePair("name1", "value2"));
params.add(new BasicNameValuePair("funky name 2", "whoa crazy\t && 🍺 🍻 \n crazier \rooo"));
String submitData = URLEncodedUtils.format(params, "UTF-8");
assertEquals("name1=value1&name1=value2&funky+name+2=whoa+crazy%09+%26%26+%F0%9F%8D%BA+%F0%9F%8D%BB+%0A+crazier+%0Dooo", submitData);
curi.getData().put(CoreAttributeConstants.A_SUBMIT_DATA, submitData);
fetcher().process(curi);
assertTrue(httpRequestString(curi).startsWith("POST / HTTP/1.0\r\n"));
assertTrue(httpRequestString(curi).endsWith("\r\n\r\nname1=value1&name1=value2&funky+name+2=whoa+crazy%09+%26%26+%F0%9F%8D%BA+%F0%9F%8D%BB+%0A+crazier+%0Dooo"));
assertEquals(FetchType.HTTP_POST, curi.getFetchType());
runDefaultChecks(curi, "requestLine", "trailingCRLFCRLF", "fetchTypeGET");
}
@Override
protected FetchHTTP makeModule() throws IOException {
modules/src/test/java/org/archive/modules/forms/FormLoginProcessorTest.java
+76
View File
@@ -0,0 +1,76 @@
/*
* This file is part of the Heritrix web crawler (crawler.archive.org).
*
* Licensed to the Internet Archive (IA) by one or more individual
* contributors.
*
* The IA licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.archive.modules.forms;
import static org.archive.modules.CoreAttributeConstants.A_WARC_RESPONSE_HEADERS;
import org.apache.commons.httpclient.NameValuePair;
import org.apache.commons.httpclient.util.EncodingUtil;
import org.archive.modules.CoreAttributeConstants;
import org.archive.modules.CrawlURI;
import org.archive.modules.ProcessorTestBase;
import org.archive.modules.CrawlURI.FetchType;
import org.archive.modules.forms.FormLoginProcessor;
public class FormLoginProcessorTest extends ProcessorTestBase {
public void testNoFormLogin() throws Exception {
CrawlURI curi = makeCrawlURI("http://example.com/");
FormLoginProcessor p = (FormLoginProcessor) makeModule();
p.setLoginUsername("jdoe");
p.setLoginPassword("********");
p.setApplicableSurtPrefix("http://(com,example,)");
p.process(curi);
assertEquals(1, curi.getDataList(A_WARC_RESPONSE_HEADERS).size());
assertEquals("WARC-Simple-Form-Province-Status: 0,0,http://(com,example,)", curi.getDataList(A_WARC_RESPONSE_HEADERS).get(0));
}
public void testFormLogin() throws Exception {
CrawlURI curi = makeCrawlURI("http://example.com/");
HTMLForm form = new HTMLForm();
form.addField("text", "username-form-field", "");
form.addField("password", "password-form-field", "");
form.setMethod("post");
form.setAction("/login");
curi.getDataList(ExtractorHTMLForms.A_HTML_FORM_OBJECTS).add(form);
FormLoginProcessor p = (FormLoginProcessor) makeModule();
p.setLoginUsername("jdoe");
p.setLoginPassword("********");
p.setApplicableSurtPrefix("http://(com,example,)");
p.process(curi);
assertEquals(1, curi.getDataList(A_WARC_RESPONSE_HEADERS).size());
assertEquals("WARC-Simple-Form-Province-Status: 0,0,http://(com,example,)", curi.getDataList(A_WARC_RESPONSE_HEADERS).get(0));
assertTrue(curi.getAnnotations().contains("submit:/login"));
assertEquals(1, curi.getOutCandidates().size());
CrawlURI submitCuri = curi.getOutCandidates().toArray(new CrawlURI[0])[0];
assertEquals("http://example.com/login", submitCuri.toString());
assertEquals(FetchType.HTTP_POST, submitCuri.getFetchType());
NameValuePair[] queryParams = (NameValuePair[]) submitCuri.getData().get(CoreAttributeConstants.A_SUBMIT_DATA);
String queryString = EncodingUtil.formUrlEncode(queryParams, "UTF-8");
assertEquals("username-form-field=jdoe&password-form-field=********", queryString);
}
}