lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-09-23 22:26:01 +00:00
Code Issues Packages Projects Releases Wiki Activity

handle login forms with <input type="text"> fields in addition to username (for this to work there must be exactly one input field whose name contains the string "username")

Browse Source
This commit is contained in:
Noah Levitt
2016-01-14 18:02:54 -08:00
parent eeeea6f0c4
commit c1abbfa5bd
2 changed files with 59 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/src/main/java/org/archive/modules/forms/HTMLForm.java
+25 -27
View File
@@ -23,7 +23,6 @@ import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
import org.apache.commons.httpclient.NameValuePair;
import org.apache.commons.lang.StringUtils;
import org.archive.util.TextUtils;
@@ -72,13 +71,14 @@ public class HTMLForm {
}
input.name = name;
input.value = value;
input.checked = checked;
allInputs.add(input);
if("text".equalsIgnoreCase(input.type) || "email".equalsIgnoreCase(input.type)) {
candidateUsernameInputs.add(input);
} else if ("password".equalsIgnoreCase(type)) {
candidatePasswordInputs.add(input);
}
input.checked = checked;
}
/**
@@ -113,41 +113,39 @@ public class HTMLForm {
*/
public boolean seemsLoginForm() {
return "post".equalsIgnoreCase(method)
&& candidateUsernameInputs.size() == 1
&& candidatePasswordInputs.size() == 1;
&& candidatePasswordInputs.size() == 1
&& presumedUsernameInput() != null;
}
/**
* Create the NameValuePair array expected by HttpClient, merging
* username and password into the appropriate value slots.
*
* @param username
* @param password
* @return
* @deprecated specific to a particular FetchHTTP implementation based on commons-httpclient, use {@link #asFormDataString(String, String)}
*/
public NameValuePair[] asHttpClientDataWith(String username, String password) {
ArrayList<NameValuePair> data = new ArrayList<NameValuePair>(allInputs.size());
for (FormInput input : allInputs) {
if(input == candidateUsernameInputs.get(0)) {
data.add(new NameValuePair(input.name,username));
} else if(input == candidatePasswordInputs.get(0)) {
data.add(new NameValuePair(input.name,password));
} else if (StringUtils.isNotEmpty(input.name) && StringUtils.isNotEmpty(input.value)) {
data.add(new NameValuePair(input.name,input.value));
protected FormInput presumedUsernameInput() {
if (candidateUsernameInputs.size() < 1) {
return null;
} else if (candidateUsernameInputs.size() == 1) {
return candidateUsernameInputs.get(0);
} else {
// more than one candidate; if there is exactly one whose name
// contains the string "username", choose that one
FormInput choice = null;
for (FormInput input: candidateUsernameInputs) {
if (input.name != null && input.name.toLowerCase().indexOf("username") != -1) {
if (choice == null) {
choice = input;
} else {
return null;
}
}
}
return choice;
}
return data.toArray(new NameValuePair[data.size()]);
}
public String asFormDataString(String username, String password) {
List<String> nameVals = new LinkedList<String>();
for (FormInput input : allInputs) {
if(input == candidateUsernameInputs.get(0)) {
if (input == presumedUsernameInput()) {
nameVals.add(TextUtils.urlEscape(input.name) + "=" + TextUtils.urlEscape(username));
} else if(input == candidatePasswordInputs.get(0)) {
} else if (input == candidatePasswordInputs.get(0)) {
nameVals.add(TextUtils.urlEscape(input.name) + "=" + TextUtils.urlEscape(password));
} else if (StringUtils.isNotEmpty(input.name)
&& StringUtils.isNotEmpty(input.value)
modules/src/test/java/org/archive/modules/forms/FormLoginProcessorTest.java
+34
View File
@@ -69,4 +69,38 @@ public class FormLoginProcessorTest extends ProcessorTestBase {
String queryString = (String) submitCuri.getData().get(CoreAttributeConstants.A_SUBMIT_DATA);
assertEquals("username-form-field=jdoe&password-form-field=********", queryString);
}
public void testFormLoginExtraInputs() throws Exception {
CrawlURI curi = makeCrawlURI("http://example.com/");
HTMLForm form = new HTMLForm();
form.addField("text", "username-form-field", "");
form.addField("password", "password-form-field", "");
form.addField("text", "some-other-form-field", "default value!");
form.addField("hidden", "hidden-field", "hidden value!");
form.addField("checkbox", "checkbox-field", "unchecked-value", false);
form.addField("checkbox", "checkbox-field", "checked-value", true);
form.addField("radio", "radio-field", "unchecked-value", false);
form.addField("checkbox", "radio-field", "checked-value", true);
form.setMethod("post");
form.setAction("/login");
curi.getDataList(ExtractorHTMLForms.A_HTML_FORM_OBJECTS).add(form);
FormLoginProcessor p = (FormLoginProcessor) makeModule();
p.setLoginUsername("jdoe");
p.setLoginPassword("********");
p.setApplicableSurtPrefix("http://(com,example,)");
p.process(curi);
assertEquals(1, curi.getDataList(A_WARC_RESPONSE_HEADERS).size());
assertEquals("WARC-Simple-Form-Province-Status: 0,0,http://(com,example,)", curi.getDataList(A_WARC_RESPONSE_HEADERS).get(0));
assertTrue(curi.getAnnotations().contains("submit:/login"));
assertEquals(1, curi.getOutLinks().size());
CrawlURI submitCuri = curi.getOutLinks().toArray(new CrawlURI[0])[0];
assertEquals("http://example.com/login", submitCuri.toString());
assertEquals(FetchType.HTTP_POST, submitCuri.getFetchType());
String queryString = (String) submitCuri.getData().get(CoreAttributeConstants.A_SUBMIT_DATA);
assertEquals("username-form-field=jdoe&password-form-field=********&some-other-form-field=default+value%21&hidden-field=hidden+value%21&checkbox-field=checked-value&radio-field=checked-value", queryString);
}
}