diff --git a/modules/src/main/java/org/archive/modules/forms/HTMLForm.java b/modules/src/main/java/org/archive/modules/forms/HTMLForm.java index 0634176b..b1947ec7 100644 --- a/modules/src/main/java/org/archive/modules/forms/HTMLForm.java +++ b/modules/src/main/java/org/archive/modules/forms/HTMLForm.java @@ -23,7 +23,6 @@ import java.util.ArrayList; import java.util.LinkedList; import java.util.List; -import org.apache.commons.httpclient.NameValuePair; import org.apache.commons.lang.StringUtils; import org.archive.util.TextUtils; @@ -72,13 +71,14 @@ public class HTMLForm { } input.name = name; input.value = value; + input.checked = checked; allInputs.add(input); + if("text".equalsIgnoreCase(input.type) || "email".equalsIgnoreCase(input.type)) { candidateUsernameInputs.add(input); } else if ("password".equalsIgnoreCase(type)) { candidatePasswordInputs.add(input); } - input.checked = checked; } /** @@ -113,41 +113,39 @@ public class HTMLForm { */ public boolean seemsLoginForm() { return "post".equalsIgnoreCase(method) - && candidateUsernameInputs.size() == 1 - && candidatePasswordInputs.size() == 1; + && candidatePasswordInputs.size() == 1 + && presumedUsernameInput() != null; } - /** - * Create the NameValuePair array expected by HttpClient, merging - * username and password into the appropriate value slots. - * - * @param username - * @param password - * @return - * @deprecated specific to a particular FetchHTTP implementation based on commons-httpclient, use {@link #asFormDataString(String, String)} - */ - public NameValuePair[] asHttpClientDataWith(String username, String password) { - ArrayList data = new ArrayList(allInputs.size()); - - for (FormInput input : allInputs) { - if(input == candidateUsernameInputs.get(0)) { - data.add(new NameValuePair(input.name,username)); - } else if(input == candidatePasswordInputs.get(0)) { - data.add(new NameValuePair(input.name,password)); - } else if (StringUtils.isNotEmpty(input.name) && StringUtils.isNotEmpty(input.value)) { - data.add(new NameValuePair(input.name,input.value)); + protected FormInput presumedUsernameInput() { + if (candidateUsernameInputs.size() < 1) { + return null; + } else if (candidateUsernameInputs.size() == 1) { + return candidateUsernameInputs.get(0); + } else { + // more than one candidate; if there is exactly one whose name + // contains the string "username", choose that one + FormInput choice = null; + for (FormInput input: candidateUsernameInputs) { + if (input.name != null && input.name.toLowerCase().indexOf("username") != -1) { + if (choice == null) { + choice = input; + } else { + return null; + } + } } + return choice; } - return data.toArray(new NameValuePair[data.size()]); } - + public String asFormDataString(String username, String password) { List nameVals = new LinkedList(); for (FormInput input : allInputs) { - if(input == candidateUsernameInputs.get(0)) { + if (input == presumedUsernameInput()) { nameVals.add(TextUtils.urlEscape(input.name) + "=" + TextUtils.urlEscape(username)); - } else if(input == candidatePasswordInputs.get(0)) { + } else if (input == candidatePasswordInputs.get(0)) { nameVals.add(TextUtils.urlEscape(input.name) + "=" + TextUtils.urlEscape(password)); } else if (StringUtils.isNotEmpty(input.name) && StringUtils.isNotEmpty(input.value) diff --git a/modules/src/test/java/org/archive/modules/forms/FormLoginProcessorTest.java b/modules/src/test/java/org/archive/modules/forms/FormLoginProcessorTest.java index 0fc297b5..b411d5b2 100644 --- a/modules/src/test/java/org/archive/modules/forms/FormLoginProcessorTest.java +++ b/modules/src/test/java/org/archive/modules/forms/FormLoginProcessorTest.java @@ -69,4 +69,38 @@ public class FormLoginProcessorTest extends ProcessorTestBase { String queryString = (String) submitCuri.getData().get(CoreAttributeConstants.A_SUBMIT_DATA); assertEquals("username-form-field=jdoe&password-form-field=********", queryString); } + + public void testFormLoginExtraInputs() throws Exception { + CrawlURI curi = makeCrawlURI("http://example.com/"); + + HTMLForm form = new HTMLForm(); + form.addField("text", "username-form-field", ""); + form.addField("password", "password-form-field", ""); + form.addField("text", "some-other-form-field", "default value!"); + form.addField("hidden", "hidden-field", "hidden value!"); + form.addField("checkbox", "checkbox-field", "unchecked-value", false); + form.addField("checkbox", "checkbox-field", "checked-value", true); + form.addField("radio", "radio-field", "unchecked-value", false); + form.addField("checkbox", "radio-field", "checked-value", true); + form.setMethod("post"); + form.setAction("/login"); + curi.getDataList(ExtractorHTMLForms.A_HTML_FORM_OBJECTS).add(form); + + FormLoginProcessor p = (FormLoginProcessor) makeModule(); + p.setLoginUsername("jdoe"); + p.setLoginPassword("********"); + p.setApplicableSurtPrefix("http://(com,example,)"); + + p.process(curi); + assertEquals(1, curi.getDataList(A_WARC_RESPONSE_HEADERS).size()); + assertEquals("WARC-Simple-Form-Province-Status: 0,0,http://(com,example,)", curi.getDataList(A_WARC_RESPONSE_HEADERS).get(0)); + assertTrue(curi.getAnnotations().contains("submit:/login")); + + assertEquals(1, curi.getOutLinks().size()); + CrawlURI submitCuri = curi.getOutLinks().toArray(new CrawlURI[0])[0]; + assertEquals("http://example.com/login", submitCuri.toString()); + assertEquals(FetchType.HTTP_POST, submitCuri.getFetchType()); + String queryString = (String) submitCuri.getData().get(CoreAttributeConstants.A_SUBMIT_DATA); + assertEquals("username-form-field=jdoe&password-form-field=********&some-other-form-field=default+value%21&hidden-field=hidden+value%21&checkbox-field=checked-value&radio-field=checked-value", queryString); + } }