From afa4f6535b38f8d46a5c9b8d9ef836bcb60a5e1e Mon Sep 17 00:00:00 2001 From: Hunter Stern Date: Mon, 7 Dec 2015 16:36:49 -0800 Subject: [PATCH 01/10] If multiple username inputs in login form, favor the one with the word 'login' in it. --- .../org/archive/modules/forms/HTMLForm.java | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/modules/src/main/java/org/archive/modules/forms/HTMLForm.java b/modules/src/main/java/org/archive/modules/forms/HTMLForm.java index 0634176b..a2b74c4e 100644 --- a/modules/src/main/java/org/archive/modules/forms/HTMLForm.java +++ b/modules/src/main/java/org/archive/modules/forms/HTMLForm.java @@ -112,9 +112,22 @@ public class HTMLForm { * @return boolean likely login form */ public boolean seemsLoginForm() { - return "post".equalsIgnoreCase(method) - && candidateUsernameInputs.size() == 1 - && candidatePasswordInputs.size() == 1; + if ("post".equalsIgnoreCase(method)) { + if (candidatePasswordInputs.size() == 1) { + if (candidateUsernameInputs.size() == 1) { + return true; + } + else if (candidateUsernameInputs.size() > 1) { + for (FormInput formInput : candidateUsernameInputs) { + if (formInput.name != null && formInput.name.toLowerCase().indexOf("login") > 0) { + return true; + } + } + } + } + } + + return false; } /** From 34cffd512a6c022f93ed9f7edac60975d7800d90 Mon Sep 17 00:00:00 2001 From: Hunter Stern Date: Thu, 17 Dec 2015 09:46:32 -0800 Subject: [PATCH 02/10] Add ability to submit forms with enctype="multipart/form-data" --- commons/pom.xml | 5 ++++ .../modules/fetcher/FetchHTTPRequest.java | 25 ++++++++++++++++--- .../modules/forms/ExtractorHTMLForms.java | 4 ++- .../org/archive/modules/forms/HTMLForm.java | 9 +++++++ 4 files changed, 38 insertions(+), 5 deletions(-) diff --git a/commons/pom.xml b/commons/pom.xml index 2b502b0c..722de719 100644 --- a/commons/pom.xml +++ b/commons/pom.xml @@ -27,6 +27,11 @@ org.apache.httpcomponents httpclient + + org.apache.httpcomponents + httpmime + 4.3.6 + com.sleepycat je diff --git a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTPRequest.java b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTPRequest.java index 19a47771..c0a59a59 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTPRequest.java +++ b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTPRequest.java @@ -45,6 +45,7 @@ import javax.net.ssl.SSLSocket; import org.apache.commons.httpclient.URIException; import org.apache.commons.lang.StringUtils; +import org.apache.http.HttpEntity; import org.apache.http.HttpEntityEnclosingRequest; import org.apache.http.HttpException; import org.apache.http.HttpHeaders; @@ -83,6 +84,7 @@ import org.apache.http.conn.ssl.SSLConnectionSocketFactory; import org.apache.http.entity.ContentLengthStrategy; import org.apache.http.entity.ContentType; import org.apache.http.entity.StringEntity; +import org.apache.http.entity.mime.MultipartEntityBuilder; import org.apache.http.impl.DefaultBHttpClientConnection; import org.apache.http.impl.client.BasicAuthCache; import org.apache.http.impl.client.BasicCredentialsProvider; @@ -105,6 +107,8 @@ import org.archive.modules.credential.Credential; import org.archive.modules.credential.HtmlFormCredential; import org.archive.modules.credential.HttpAuthenticationCredential; import org.archive.modules.extractor.LinkContext; +import org.archive.modules.forms.ExtractorHTMLForms; +import org.archive.modules.forms.HTMLForm; import org.archive.modules.net.CrawlHost; import org.archive.modules.net.CrawlServer; import org.archive.modules.net.ServerCache; @@ -185,10 +189,23 @@ class FetchHTTPRequest { this.request = postRequest; String submitData = (String) curi.getData().get(CoreAttributeConstants.A_SUBMIT_DATA); if (submitData != null) { - // XXX brittle, doesn't support multipart form data etc - ContentType contentType = ContentType.create(URLEncodedUtils.CONTENT_TYPE, "UTF-8"); - StringEntity formEntity = new StringEntity(submitData, contentType); - postRequest.setEntity(formEntity); + if (curi.getDataList(ExtractorHTMLForms.A_HTML_FORM_OBJECTS) != null + && "multipart/form-data" + .equals(((HTMLForm) curi.getDataList( + ExtractorHTMLForms.A_HTML_FORM_OBJECTS).get(0)) + .getEnctype())) { + HttpEntity multipartFormEntity = MultipartEntityBuilder + .create() + .addTextBody("submitData", submitData, + ContentType.MULTIPART_FORM_DATA).build(); + + postRequest.setEntity(multipartFormEntity); + } + else { + ContentType contentType = ContentType.create(URLEncodedUtils.CONTENT_TYPE, "UTF-8"); + StringEntity formEntity = new StringEntity(submitData, contentType); + postRequest.setEntity(formEntity); + } } } else { this.request = new BasicExecutionAwareRequest("GET", diff --git a/modules/src/main/java/org/archive/modules/forms/ExtractorHTMLForms.java b/modules/src/main/java/org/archive/modules/forms/ExtractorHTMLForms.java index 231a1e05..0c672102 100644 --- a/modules/src/main/java/org/archive/modules/forms/ExtractorHTMLForms.java +++ b/modules/src/main/java/org/archive/modules/forms/ExtractorHTMLForms.java @@ -145,9 +145,11 @@ public class ExtractorHTMLForms extends Extractor { CharSequence relevantSequence = cs.subSequence(offsetInt, cs.length()); String method = findAttributeValueGroup("(?i)^[^>]*\\smethod\\s*=\\s*([^>\\s]+)[^>]*>",1,relevantSequence); String action = findAttributeValueGroup("(?i)^[^>]*\\saction\\s*=\\s*([^>\\s]+)[^>]*>",1,relevantSequence); + String enctype = findAttributeValueGroup("(?i)^[^>]*\\senctype\\s*=\\s*([^>\\s]+)[^>]*>",1,relevantSequence); HTMLForm form = new HTMLForm(); form.setMethod(method); - form.setAction(action); + form.setAction(action); + form.setEnctype(enctype); for(CharSequence input : findGroups("(?i)(]*>)|()",1,relevantSequence)) { String type = findAttributeValueGroup("(?i)^[^>]*\\stype\\s*=\\s*([^>\\s]+)[^>]*>",1,input); String name = findAttributeValueGroup("(?i)^[^>]*\\sname\\s*=\\s*([^>\\s]+)[^>]*>",1,input); diff --git a/modules/src/main/java/org/archive/modules/forms/HTMLForm.java b/modules/src/main/java/org/archive/modules/forms/HTMLForm.java index a2b74c4e..333350b2 100644 --- a/modules/src/main/java/org/archive/modules/forms/HTMLForm.java +++ b/modules/src/main/java/org/archive/modules/forms/HTMLForm.java @@ -50,6 +50,7 @@ public class HTMLForm { String method; String action; + String enctype; List allInputs = new ArrayList(); List candidateUsernameInputs = new ArrayList(); @@ -103,6 +104,14 @@ public class HTMLForm { public void setAction(String action) { this.action = action; } + + public String getEnctype() { + return enctype; + } + + public void setEnctype(String enctype) { + this.enctype = enctype; + } /** * For now, we consider a POST form with only 1 password From ad329ceea8a31d4214e4af1adf8c490946dbcbb4 Mon Sep 17 00:00:00 2001 From: Hunter Stern Date: Thu, 17 Dec 2015 10:40:57 -0800 Subject: [PATCH 03/10] Save enctype of form in submission curi --- .../java/org/archive/modules/CoreAttributeConstants.java | 3 +++ .../java/org/archive/modules/fetcher/FetchHTTPRequest.java | 7 ++----- .../java/org/archive/modules/forms/FormLoginProcessor.java | 5 +++++ 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/modules/src/main/java/org/archive/modules/CoreAttributeConstants.java b/modules/src/main/java/org/archive/modules/CoreAttributeConstants.java index f0579b61..a86a0854 100644 --- a/modules/src/main/java/org/archive/modules/CoreAttributeConstants.java +++ b/modules/src/main/java/org/archive/modules/CoreAttributeConstants.java @@ -122,6 +122,9 @@ public interface CoreAttributeConstants { // FORMS support - a persistent member (survives frontier enqueue/dequeue/retries) public static final String A_SUBMIT_DATA = "submit-data"; + //the encoding type of the form + public static final String A_SUBMIT_DATA_ENCTYPE = "submit-data-enctype"; + // arbitrary additions to WARC response record headers public static final String A_WARC_RESPONSE_HEADERS = "warc-response-headers"; diff --git a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTPRequest.java b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTPRequest.java index c0a59a59..3d5101f3 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTPRequest.java +++ b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTPRequest.java @@ -189,11 +189,8 @@ class FetchHTTPRequest { this.request = postRequest; String submitData = (String) curi.getData().get(CoreAttributeConstants.A_SUBMIT_DATA); if (submitData != null) { - if (curi.getDataList(ExtractorHTMLForms.A_HTML_FORM_OBJECTS) != null - && "multipart/form-data" - .equals(((HTMLForm) curi.getDataList( - ExtractorHTMLForms.A_HTML_FORM_OBJECTS).get(0)) - .getEnctype())) { + if ("multipart/form-data".equals(curi.getData().get( + CoreAttributeConstants.A_SUBMIT_DATA_ENCTYPE))) { HttpEntity multipartFormEntity = MultipartEntityBuilder .create() .addTextBody("submitData", submitData, diff --git a/modules/src/main/java/org/archive/modules/forms/FormLoginProcessor.java b/modules/src/main/java/org/archive/modules/forms/FormLoginProcessor.java index 9a2d540d..1307cb75 100644 --- a/modules/src/main/java/org/archive/modules/forms/FormLoginProcessor.java +++ b/modules/src/main/java/org/archive/modules/forms/FormLoginProcessor.java @@ -270,6 +270,11 @@ public class FormLoginProcessor extends Processor implements Checkpointable { templateForm.asFormDataString( getLoginUsername(), getLoginPassword())); + submitCuri.getData().put( + CoreAttributeConstants.A_SUBMIT_DATA_ENCTYPE, + ((HTMLForm) curi.getDataList( + ExtractorHTMLForms.A_HTML_FORM_OBJECTS).get(0)) + .getEnctype()); submitCuri.setSchedulingDirective(SchedulingConstants.HIGH); submitCuri.setForceFetch(true); curi.getOutLinks().add(submitCuri); From 4997014b34e7ed212d9aa78319ef6d71389ca306 Mon Sep 17 00:00:00 2001 From: Hunter Stern Date: Mon, 28 Dec 2015 10:06:53 -0800 Subject: [PATCH 04/10] More changes --- .../modules/fetcher/FetchHTTPRequest.java | 18 ++++------ .../modules/forms/FormLoginProcessor.java | 2 +- .../org/archive/modules/forms/HTMLForm.java | 33 +++++++++++++++++++ 3 files changed, 41 insertions(+), 12 deletions(-) diff --git a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTPRequest.java b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTPRequest.java index 3d5101f3..9b3108a1 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTPRequest.java +++ b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTPRequest.java @@ -187,20 +187,16 @@ class FetchHTTPRequest { BasicExecutionAwareEntityEnclosingRequest postRequest = new BasicExecutionAwareEntityEnclosingRequest( "POST", requestLineUri, httpVersion); this.request = postRequest; - String submitData = (String) curi.getData().get(CoreAttributeConstants.A_SUBMIT_DATA); - if (submitData != null) { - if ("multipart/form-data".equals(curi.getData().get( - CoreAttributeConstants.A_SUBMIT_DATA_ENCTYPE))) { - HttpEntity multipartFormEntity = MultipartEntityBuilder - .create() - .addTextBody("submitData", submitData, - ContentType.MULTIPART_FORM_DATA).build(); + + Object submitData = curi.getData().get(CoreAttributeConstants.A_SUBMIT_DATA); - postRequest.setEntity(multipartFormEntity); + if (submitData != null) { + if (submitData instanceof HttpEntity) { + postRequest.setEntity((HttpEntity)submitData); } - else { + else if (submitData instanceof String) { ContentType contentType = ContentType.create(URLEncodedUtils.CONTENT_TYPE, "UTF-8"); - StringEntity formEntity = new StringEntity(submitData, contentType); + StringEntity formEntity = new StringEntity((String)submitData, contentType); postRequest.setEntity(formEntity); } } diff --git a/modules/src/main/java/org/archive/modules/forms/FormLoginProcessor.java b/modules/src/main/java/org/archive/modules/forms/FormLoginProcessor.java index 1307cb75..d268ac60 100644 --- a/modules/src/main/java/org/archive/modules/forms/FormLoginProcessor.java +++ b/modules/src/main/java/org/archive/modules/forms/FormLoginProcessor.java @@ -267,7 +267,7 @@ public class FormLoginProcessor extends Processor implements Checkpointable { submitCuri.setFetchType(FetchType.HTTP_POST); submitCuri.getData().put( CoreAttributeConstants.A_SUBMIT_DATA, - templateForm.asFormDataString( + templateForm.asFormDataMultiPartEntity( getLoginUsername(), getLoginPassword())); submitCuri.getData().put( diff --git a/modules/src/main/java/org/archive/modules/forms/HTMLForm.java b/modules/src/main/java/org/archive/modules/forms/HTMLForm.java index 333350b2..08a28624 100644 --- a/modules/src/main/java/org/archive/modules/forms/HTMLForm.java +++ b/modules/src/main/java/org/archive/modules/forms/HTMLForm.java @@ -25,6 +25,10 @@ import java.util.List; import org.apache.commons.httpclient.NameValuePair; import org.apache.commons.lang.StringUtils; +import org.apache.http.HttpEntity; +import org.apache.http.entity.ContentType; +import org.apache.http.entity.mime.MultipartEntityBuilder; +import org.apache.http.entity.mime.content.StringBody; import org.archive.util.TextUtils; /** @@ -163,6 +167,35 @@ public class HTMLForm { return data.toArray(new NameValuePair[data.size()]); } + public HttpEntity asFormDataMultiPartEntity(String username, String password) { + //List nameVals = new LinkedList(); + MultipartEntityBuilder multiPartEntityBuilder = MultipartEntityBuilder.create(); + + for (FormInput input : allInputs) { + if ("dnn$ctr577$AACC_Login$txtFirstName".equals(input.name) + || "dnn$ctr577$AACC_Login$txtLastName".equals(input.name) + || "dnn$ctr577$AACC_Login$cmdContinue".equals(input.name) + || "dnn$ctr577$AACC_Login$txtEmailId".equals(input.name) + || "dnn$ctr577$AACC_Login$ctlCaptcha".equals(input.name)) { + + continue; + } + + if(input == candidateUsernameInputs.get(0)) { + multiPartEntityBuilder.addPart(TextUtils.urlEscape(input.name), new StringBody(TextUtils.urlEscape(username), ContentType.MULTIPART_FORM_DATA)); + //nameVals.add(TextUtils.urlEscape(input.name) + "=" + TextUtils.urlEscape(username)); + } else if(input == candidatePasswordInputs.get(0)) { + multiPartEntityBuilder.addPart(TextUtils.urlEscape(input.name), new StringBody(TextUtils.urlEscape(password), ContentType.MULTIPART_FORM_DATA)); + //nameVals.add(TextUtils.urlEscape(input.name) + "=" + TextUtils.urlEscape(password)); + } else if (StringUtils.isNotEmpty(input.name)) { + multiPartEntityBuilder.addPart(TextUtils.urlEscape(input.name), new StringBody(TextUtils.urlEscape(StringUtils.isNotEmpty(input.value) ? input.value : ""), ContentType.MULTIPART_FORM_DATA)); + //nameVals.add(TextUtils.urlEscape(input.name) + "=" + TextUtils.urlEscape(input.value)); + } + } + + return multiPartEntityBuilder.build(); + } + public String asFormDataString(String username, String password) { List nameVals = new LinkedList(); From 93bca3d55cb774f02c2c3195b4e79f27e015b8b8 Mon Sep 17 00:00:00 2001 From: Hunter Stern Date: Thu, 7 Jan 2016 14:52:00 -0800 Subject: [PATCH 05/10] More changes for https://webarchive.jira.com/browse/ARI-4656 --- .../org/archive/modules/forms/HTMLForm.java | 56 +++++++++++++------ 1 file changed, 39 insertions(+), 17 deletions(-) diff --git a/modules/src/main/java/org/archive/modules/forms/HTMLForm.java b/modules/src/main/java/org/archive/modules/forms/HTMLForm.java index 08a28624..cdbb3213 100644 --- a/modules/src/main/java/org/archive/modules/forms/HTMLForm.java +++ b/modules/src/main/java/org/archive/modules/forms/HTMLForm.java @@ -27,6 +27,7 @@ import org.apache.commons.httpclient.NameValuePair; import org.apache.commons.lang.StringUtils; import org.apache.http.HttpEntity; import org.apache.http.entity.ContentType; +import org.apache.http.entity.mime.HttpMultipartMode; import org.apache.http.entity.mime.MultipartEntityBuilder; import org.apache.http.entity.mime.content.StringBody; import org.archive.util.TextUtils; @@ -71,10 +72,16 @@ public class HTMLForm { public void addField(String type, String name, String value, boolean checked) { FormInput input = new FormInput(); input.type = type; + + if (isMultipleFormSubmitInputs(type)) { + return; + } + // default input type is text per html standard if (input.type == null) { input.type = "text"; } + input.name = name; input.value = value; allInputs.add(input); @@ -96,6 +103,18 @@ public class HTMLForm { public void addField(String type, String name, String value) { addField(type, name, value, false); } + + public boolean isMultipleFormSubmitInputs(String type) { + if (!type.toLowerCase().equals("submit")) return false; + + for (FormInput input : allInputs) { + if (input.type.toLowerCase().equals("submit")) { + return true; + } + } + + return false; + } public void setMethod(String method) { this.method = method; @@ -166,30 +185,33 @@ public class HTMLForm { } return data.toArray(new NameValuePair[data.size()]); } + + public FormInput getLoginInputFromCandidates() { + if (candidateUsernameInputs == null) return null; + + if (candidateUsernameInputs.size() == 1) return candidateUsernameInputs.get(0); + + for (FormInput input : candidateUsernameInputs) { + + if (input.name != null && input.name.toLowerCase().indexOf("login") != -1) { + return input; + } + } + + return null; + } public HttpEntity asFormDataMultiPartEntity(String username, String password) { - //List nameVals = new LinkedList(); MultipartEntityBuilder multiPartEntityBuilder = MultipartEntityBuilder.create(); + multiPartEntityBuilder.setMode(HttpMultipartMode.BROWSER_COMPATIBLE); for (FormInput input : allInputs) { - if ("dnn$ctr577$AACC_Login$txtFirstName".equals(input.name) - || "dnn$ctr577$AACC_Login$txtLastName".equals(input.name) - || "dnn$ctr577$AACC_Login$cmdContinue".equals(input.name) - || "dnn$ctr577$AACC_Login$txtEmailId".equals(input.name) - || "dnn$ctr577$AACC_Login$ctlCaptcha".equals(input.name)) { - - continue; - } - - if(input == candidateUsernameInputs.get(0)) { - multiPartEntityBuilder.addPart(TextUtils.urlEscape(input.name), new StringBody(TextUtils.urlEscape(username), ContentType.MULTIPART_FORM_DATA)); - //nameVals.add(TextUtils.urlEscape(input.name) + "=" + TextUtils.urlEscape(username)); + if(input == getLoginInputFromCandidates()) { + multiPartEntityBuilder.addPart(input.name, new StringBody(username, ContentType.MULTIPART_FORM_DATA)); } else if(input == candidatePasswordInputs.get(0)) { - multiPartEntityBuilder.addPart(TextUtils.urlEscape(input.name), new StringBody(TextUtils.urlEscape(password), ContentType.MULTIPART_FORM_DATA)); - //nameVals.add(TextUtils.urlEscape(input.name) + "=" + TextUtils.urlEscape(password)); + multiPartEntityBuilder.addPart(input.name, new StringBody(password, ContentType.MULTIPART_FORM_DATA)); } else if (StringUtils.isNotEmpty(input.name)) { - multiPartEntityBuilder.addPart(TextUtils.urlEscape(input.name), new StringBody(TextUtils.urlEscape(StringUtils.isNotEmpty(input.value) ? input.value : ""), ContentType.MULTIPART_FORM_DATA)); - //nameVals.add(TextUtils.urlEscape(input.name) + "=" + TextUtils.urlEscape(input.value)); + multiPartEntityBuilder.addPart(input.name, new StringBody(StringUtils.isNotEmpty(input.value) ? input.value : "", ContentType.MULTIPART_FORM_DATA)); } } From a2a39fe052cc031f20eb3894a3c1f2367ac7f9c7 Mon Sep 17 00:00:00 2001 From: Hunter Stern Date: Thu, 7 Jan 2016 15:03:25 -0800 Subject: [PATCH 06/10] Make sure to check form encoding type when addign form to curi data --- .../archive/modules/forms/FormLoginProcessor.java | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/modules/src/main/java/org/archive/modules/forms/FormLoginProcessor.java b/modules/src/main/java/org/archive/modules/forms/FormLoginProcessor.java index d268ac60..843a2dfd 100644 --- a/modules/src/main/java/org/archive/modules/forms/FormLoginProcessor.java +++ b/modules/src/main/java/org/archive/modules/forms/FormLoginProcessor.java @@ -265,11 +265,15 @@ public class FormLoginProcessor extends Processor implements Checkpointable { CrawlURI submitCuri = curi.createCrawlURI(submitUrl, lc, Hop.SUBMIT); submitCuri.setFetchType(FetchType.HTTP_POST); - submitCuri.getData().put( - CoreAttributeConstants.A_SUBMIT_DATA, - templateForm.asFormDataMultiPartEntity( - getLoginUsername(), - getLoginPassword())); + submitCuri.getData() + .put(CoreAttributeConstants.A_SUBMIT_DATA, + "multipart/form-data".equals(templateForm + .getEnctype()) ? templateForm + .asFormDataMultiPartEntity( + getLoginUsername(), + getLoginPassword()) : templateForm + .asFormDataString(getLoginUsername(), + getLoginPassword())); submitCuri.getData().put( CoreAttributeConstants.A_SUBMIT_DATA_ENCTYPE, ((HTMLForm) curi.getDataList( From 704ec62487cbc30129020dc35f6e82a153371f1d Mon Sep 17 00:00:00 2001 From: Hunter Stern Date: Thu, 7 Jan 2016 15:08:02 -0800 Subject: [PATCH 07/10] Remove unneeded constant --- .../java/org/archive/modules/CoreAttributeConstants.java | 3 --- .../java/org/archive/modules/forms/FormLoginProcessor.java | 5 ----- 2 files changed, 8 deletions(-) diff --git a/modules/src/main/java/org/archive/modules/CoreAttributeConstants.java b/modules/src/main/java/org/archive/modules/CoreAttributeConstants.java index a86a0854..f0579b61 100644 --- a/modules/src/main/java/org/archive/modules/CoreAttributeConstants.java +++ b/modules/src/main/java/org/archive/modules/CoreAttributeConstants.java @@ -122,9 +122,6 @@ public interface CoreAttributeConstants { // FORMS support - a persistent member (survives frontier enqueue/dequeue/retries) public static final String A_SUBMIT_DATA = "submit-data"; - //the encoding type of the form - public static final String A_SUBMIT_DATA_ENCTYPE = "submit-data-enctype"; - // arbitrary additions to WARC response record headers public static final String A_WARC_RESPONSE_HEADERS = "warc-response-headers"; diff --git a/modules/src/main/java/org/archive/modules/forms/FormLoginProcessor.java b/modules/src/main/java/org/archive/modules/forms/FormLoginProcessor.java index 843a2dfd..86268538 100644 --- a/modules/src/main/java/org/archive/modules/forms/FormLoginProcessor.java +++ b/modules/src/main/java/org/archive/modules/forms/FormLoginProcessor.java @@ -274,11 +274,6 @@ public class FormLoginProcessor extends Processor implements Checkpointable { getLoginPassword()) : templateForm .asFormDataString(getLoginUsername(), getLoginPassword())); - submitCuri.getData().put( - CoreAttributeConstants.A_SUBMIT_DATA_ENCTYPE, - ((HTMLForm) curi.getDataList( - ExtractorHTMLForms.A_HTML_FORM_OBJECTS).get(0)) - .getEnctype()); submitCuri.setSchedulingDirective(SchedulingConstants.HIGH); submitCuri.setForceFetch(true); curi.getOutLinks().add(submitCuri); From 309847cc401330cbb55bb6f89481a144330c4bba Mon Sep 17 00:00:00 2001 From: Hunter Stern Date: Mon, 11 Jan 2016 16:40:28 -0800 Subject: [PATCH 08/10] Be sure to return at least one of the candidates --- modules/src/main/java/org/archive/modules/forms/HTMLForm.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/modules/src/main/java/org/archive/modules/forms/HTMLForm.java b/modules/src/main/java/org/archive/modules/forms/HTMLForm.java index cdbb3213..7eec35a9 100644 --- a/modules/src/main/java/org/archive/modules/forms/HTMLForm.java +++ b/modules/src/main/java/org/archive/modules/forms/HTMLForm.java @@ -192,13 +192,12 @@ public class HTMLForm { if (candidateUsernameInputs.size() == 1) return candidateUsernameInputs.get(0); for (FormInput input : candidateUsernameInputs) { - if (input.name != null && input.name.toLowerCase().indexOf("login") != -1) { return input; } } - return null; + return candidateUsernameInputs.get(0); } public HttpEntity asFormDataMultiPartEntity(String username, String password) { From 3b12a751e1693275e58277281a56cca4a4a66836 Mon Sep 17 00:00:00 2001 From: Noah Levitt Date: Thu, 14 Jan 2016 12:17:19 -0800 Subject: [PATCH 09/10] to make this branch only about multipart/form-data, undo the changes to handle more complicated forms with extra input fields; add unit test; store CrawlURI submit data in http-library agnostic way and remove httpcomponents-specific code from HTMLForm, refactor creation of post data accordingly --- commons/pom.xml | 5 +- .../modules/CoreAttributeConstants.java | 7 +- .../java/org/archive/modules/CrawlURI.java | 3 +- .../modules/fetcher/FetchHTTPRequest.java | 98 ++++++++++--- .../modules/forms/FormLoginProcessor.java | 16 +- .../org/archive/modules/forms/HTMLForm.java | 138 ++++-------------- .../modules/fetcher/FetchHTTPTests.java | 18 +-- .../modules/forms/FormLoginProcessorTest.java | 88 ++++++++++- pom.xml | 5 + 9 files changed, 214 insertions(+), 164 deletions(-) diff --git a/commons/pom.xml b/commons/pom.xml index 722de719..75c31503 100644 --- a/commons/pom.xml +++ b/commons/pom.xml @@ -28,9 +28,8 @@ httpclient - org.apache.httpcomponents - httpmime - 4.3.6 + org.apache.httpcomponents + httpmime com.sleepycat diff --git a/modules/src/main/java/org/archive/modules/CoreAttributeConstants.java b/modules/src/main/java/org/archive/modules/CoreAttributeConstants.java index f0579b61..f58c56e1 100644 --- a/modules/src/main/java/org/archive/modules/CoreAttributeConstants.java +++ b/modules/src/main/java/org/archive/modules/CoreAttributeConstants.java @@ -118,10 +118,11 @@ public interface CoreAttributeConstants { public static final String A_HTTP_RESPONSE_HEADERS = "http-response-headers"; public static final String A_HTTP_AUTH_CHALLENGES = "http-auth-challenges"; - - // FORMS support - a persistent member (survives frontier enqueue/dequeue/retries) + + // FORMS support - persistent members (survive frontier enqueue/dequeue/retries) public static final String A_SUBMIT_DATA = "submit-data"; - + public static final String A_SUBMIT_ENCTYPE = "submit-enctype"; + // arbitrary additions to WARC response record headers public static final String A_WARC_RESPONSE_HEADERS = "warc-response-headers"; diff --git a/modules/src/main/java/org/archive/modules/CrawlURI.java b/modules/src/main/java/org/archive/modules/CrawlURI.java index dc666061..6fbab82d 100644 --- a/modules/src/main/java/org/archive/modules/CrawlURI.java +++ b/modules/src/main/java/org/archive/modules/CrawlURI.java @@ -32,6 +32,7 @@ import static org.archive.modules.CoreAttributeConstants.A_NONFATAL_ERRORS; import static org.archive.modules.CoreAttributeConstants.A_PREREQUISITE_URI; import static org.archive.modules.CoreAttributeConstants.A_SOURCE_TAG; import static org.archive.modules.CoreAttributeConstants.A_SUBMIT_DATA; +import static org.archive.modules.CoreAttributeConstants.A_SUBMIT_ENCTYPE; import static org.archive.modules.CoreAttributeConstants.A_WARC_RESPONSE_HEADERS; import static org.archive.modules.SchedulingConstants.NORMAL; import static org.archive.modules.fetcher.FetchStatusCodes.S_BLOCKED_BY_CUSTOM_PROCESSOR; @@ -256,7 +257,7 @@ implements Reporter, Serializable, OverlayContext, Comparable { */ private static final Collection persistentKeys = new CopyOnWriteArrayList( - new String [] {A_CREDENTIALS_KEY, A_HTTP_AUTH_CHALLENGES, A_SUBMIT_DATA, A_WARC_RESPONSE_HEADERS, A_ANNOTATIONS}); + new String [] {A_CREDENTIALS_KEY, A_HTTP_AUTH_CHALLENGES, A_SUBMIT_DATA, A_WARC_RESPONSE_HEADERS, A_ANNOTATIONS, A_SUBMIT_ENCTYPE}); /** maximum length for pathFromSeed/hopsPath; longer truncated with leading counter **/ private static final int MAX_HOPS_DISPLAYED = 50; diff --git a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTPRequest.java b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTPRequest.java index 9b3108a1..92ede7d6 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTPRequest.java +++ b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTPRequest.java @@ -25,6 +25,7 @@ import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_STATUS; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.io.UnsupportedEncodingException; import java.net.InetAddress; import java.net.Socket; import java.net.UnknownHostException; @@ -34,6 +35,7 @@ import java.nio.charset.CharsetEncoder; import java.nio.charset.CodingErrorAction; import java.util.ArrayList; import java.util.HashMap; +import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Map.Entry; @@ -66,7 +68,6 @@ import org.apache.http.client.config.RequestConfig; import org.apache.http.client.entity.UrlEncodedFormEntity; import org.apache.http.client.methods.AbstractExecutionAwareRequest; import org.apache.http.client.protocol.HttpClientContext; -import org.apache.http.client.utils.URLEncodedUtils; import org.apache.http.config.ConnectionConfig; import org.apache.http.config.MessageConstraints; import org.apache.http.config.Registry; @@ -83,7 +84,7 @@ import org.apache.http.conn.ssl.AllowAllHostnameVerifier; import org.apache.http.conn.ssl.SSLConnectionSocketFactory; import org.apache.http.entity.ContentLengthStrategy; import org.apache.http.entity.ContentType; -import org.apache.http.entity.StringEntity; +import org.apache.http.entity.mime.HttpMultipartMode; import org.apache.http.entity.mime.MultipartEntityBuilder; import org.apache.http.impl.DefaultBHttpClientConnection; import org.apache.http.impl.client.BasicAuthCache; @@ -107,8 +108,7 @@ import org.archive.modules.credential.Credential; import org.archive.modules.credential.HtmlFormCredential; import org.archive.modules.credential.HttpAuthenticationCredential; import org.archive.modules.extractor.LinkContext; -import org.archive.modules.forms.ExtractorHTMLForms; -import org.archive.modules.forms.HTMLForm; +import org.archive.modules.forms.HTMLForm.NameValue; import org.archive.modules.net.CrawlHost; import org.archive.modules.net.CrawlServer; import org.archive.modules.net.ServerCache; @@ -117,7 +117,7 @@ import org.archive.util.Recorder; /** * @contributor nlevitt */ -class FetchHTTPRequest { +public class FetchHTTPRequest { /** * Implementation of {@link DnsResolver} that uses the server cache which is @@ -187,19 +187,8 @@ class FetchHTTPRequest { BasicExecutionAwareEntityEnclosingRequest postRequest = new BasicExecutionAwareEntityEnclosingRequest( "POST", requestLineUri, httpVersion); this.request = postRequest; - - Object submitData = curi.getData().get(CoreAttributeConstants.A_SUBMIT_DATA); - - if (submitData != null) { - if (submitData instanceof HttpEntity) { - postRequest.setEntity((HttpEntity)submitData); - } - else if (submitData instanceof String) { - ContentType contentType = ContentType.create(URLEncodedUtils.CONTENT_TYPE, "UTF-8"); - StringEntity formEntity = new StringEntity((String)submitData, contentType); - postRequest.setEntity(formEntity); - } - } + HttpEntity entity = buildPostRequestEntity(curi); + postRequest.setEntity(entity); } else { this.request = new BasicExecutionAwareRequest("GET", requestLineUri, httpVersion); @@ -220,6 +209,79 @@ class FetchHTTPRequest { populateHttpProxyCredential(); } + /** + * Returns a copy of the string with non-ascii characters replaced by their + * html numeric character reference in decimal (e.g. 〹). + * + *

+ * The purpose of this is to produce a multipart/formdata submission that + * any server should be able to handle, based on experiments using a modern + * browser (chromium 47.0.2526.106 for mac). What chromium posts depends on + * what it considers the character encoding of the page containing the form, + * and maybe other factors. It would be too complicated to try to simulate + * that behavior in heritrix. + * + *

+ * Instead what we do is approximately what the browser does when the form + * page is plain ascii. It html-escapes characters outside of the + * latin1/cp1252 range. Characters in the U+0080-U+00FF range are encoded in + * latin1/cp1252. That is the one way that we differ from chromium. We + * html-escape those characters (U+0080-U+00FF) as well. That way the http + * post is plain ascii, and should work regardless of which encoding the + * server expects. + * + *

+ * N.b. chromium doesn't indicate the encoding of the request in any way (no + * charset in the content-type or anything like that). Also of note is that + * when it considers the form page to be utf-8, it submits in utf-8. That's + * part of the complicated behavior we don't want to try to simulate. + */ + public static String escapeForMultipart(String str) { + StringBuilder buf = new StringBuilder(); + for (int i = 0; i < str.length(); ) { + int codepoint = str.codePointAt(i); + if (codepoint <= 0x7f) { + buf.appendCodePoint(codepoint); + } else { + buf.append("&#" + codepoint + ";"); + } + i += Character.charCount(codepoint); + } + return buf.toString(); + } + + protected HttpEntity buildPostRequestEntity(CrawlURI curi) { + String enctype = (String) curi.getData().get(CoreAttributeConstants.A_SUBMIT_ENCTYPE); + if (enctype == null) { + enctype = ContentType.APPLICATION_FORM_URLENCODED.getMimeType(); + } + + @SuppressWarnings("unchecked") + List submitData = (List) curi.getData().get(CoreAttributeConstants.A_SUBMIT_DATA); + + if (enctype.equals(ContentType.APPLICATION_FORM_URLENCODED.getMimeType())) { + LinkedList nvps = new LinkedList(); + for (NameValue nv: submitData) { + nvps.add(new BasicNameValuePair(nv.name, nv.value)); + } + try { + return new UrlEncodedFormEntity(nvps, "UTF-8"); + } catch (UnsupportedEncodingException e) { + throw new IllegalStateException(e); + } + } else if (enctype.equals(ContentType.MULTIPART_FORM_DATA.getMimeType())) { + MultipartEntityBuilder entityBuilder = MultipartEntityBuilder.create(); + entityBuilder.setMode(HttpMultipartMode.BROWSER_COMPATIBLE); + for (NameValue nv: submitData) { + entityBuilder.addTextBody(escapeForMultipart(nv.name), + escapeForMultipart(nv.value)); + } + return entityBuilder.build(); + } else { + throw new IllegalStateException("unsupported form submission enctype='" + enctype + "'"); + } + } + protected void configureRequestHeaders() { if (fetcher.getAcceptCompression()) { request.addHeader("Accept-Encoding", "gzip,deflate"); diff --git a/modules/src/main/java/org/archive/modules/forms/FormLoginProcessor.java b/modules/src/main/java/org/archive/modules/forms/FormLoginProcessor.java index 86268538..b6ee86a2 100644 --- a/modules/src/main/java/org/archive/modules/forms/FormLoginProcessor.java +++ b/modules/src/main/java/org/archive/modules/forms/FormLoginProcessor.java @@ -265,15 +265,13 @@ public class FormLoginProcessor extends Processor implements Checkpointable { CrawlURI submitCuri = curi.createCrawlURI(submitUrl, lc, Hop.SUBMIT); submitCuri.setFetchType(FetchType.HTTP_POST); - submitCuri.getData() - .put(CoreAttributeConstants.A_SUBMIT_DATA, - "multipart/form-data".equals(templateForm - .getEnctype()) ? templateForm - .asFormDataMultiPartEntity( - getLoginUsername(), - getLoginPassword()) : templateForm - .asFormDataString(getLoginUsername(), - getLoginPassword())); + submitCuri.getData().put( + CoreAttributeConstants.A_SUBMIT_DATA, + templateForm.formData( + getLoginUsername(), + getLoginPassword())); + submitCuri.getData().put(CoreAttributeConstants.A_SUBMIT_ENCTYPE, + templateForm.getEnctype()); submitCuri.setSchedulingDirective(SchedulingConstants.HIGH); submitCuri.setForceFetch(true); curi.getOutLinks().add(submitCuri); diff --git a/modules/src/main/java/org/archive/modules/forms/HTMLForm.java b/modules/src/main/java/org/archive/modules/forms/HTMLForm.java index 7eec35a9..198f5285 100644 --- a/modules/src/main/java/org/archive/modules/forms/HTMLForm.java +++ b/modules/src/main/java/org/archive/modules/forms/HTMLForm.java @@ -23,14 +23,7 @@ import java.util.ArrayList; import java.util.LinkedList; import java.util.List; -import org.apache.commons.httpclient.NameValuePair; import org.apache.commons.lang.StringUtils; -import org.apache.http.HttpEntity; -import org.apache.http.entity.ContentType; -import org.apache.http.entity.mime.HttpMultipartMode; -import org.apache.http.entity.mime.MultipartEntityBuilder; -import org.apache.http.entity.mime.content.StringBody; -import org.archive.util.TextUtils; /** * Simple representation of a discovered HTML Form. @@ -52,14 +45,14 @@ public class HTMLForm { return str; } } - - String method; - String action; - String enctype; - - List allInputs = new ArrayList(); - List candidateUsernameInputs = new ArrayList(); - List candidatePasswordInputs = new ArrayList(); + + protected String method; + protected String action; + protected String enctype; + + protected List allInputs = new ArrayList(); + protected List candidateUsernameInputs = new ArrayList(); + protected List candidatePasswordInputs = new ArrayList(); /** * Add a discovered INPUT, tracking it as potential @@ -72,16 +65,10 @@ public class HTMLForm { public void addField(String type, String name, String value, boolean checked) { FormInput input = new FormInput(); input.type = type; - - if (isMultipleFormSubmitInputs(type)) { - return; - } - // default input type is text per html standard if (input.type == null) { input.type = "text"; } - input.name = name; input.value = value; allInputs.add(input); @@ -103,18 +90,6 @@ public class HTMLForm { public void addField(String type, String name, String value) { addField(type, name, value, false); } - - public boolean isMultipleFormSubmitInputs(String type) { - if (!type.toLowerCase().equals("submit")) return false; - - for (FormInput input : allInputs) { - if (input.type.toLowerCase().equals("submit")) { - return true; - } - } - - return false; - } public void setMethod(String method) { this.method = method; @@ -123,11 +98,11 @@ public class HTMLForm { public String getAction() { return action; } - + public void setAction(String action) { this.action = action; } - + public String getEnctype() { return enctype; } @@ -144,97 +119,34 @@ public class HTMLForm { * @return boolean likely login form */ public boolean seemsLoginForm() { - if ("post".equalsIgnoreCase(method)) { - if (candidatePasswordInputs.size() == 1) { - if (candidateUsernameInputs.size() == 1) { - return true; - } - else if (candidateUsernameInputs.size() > 1) { - for (FormInput formInput : candidateUsernameInputs) { - if (formInput.name != null && formInput.name.toLowerCase().indexOf("login") > 0) { - return true; - } - } - } - } - } - - return false; + return "post".equalsIgnoreCase(method) + && candidateUsernameInputs.size() == 1 + && candidatePasswordInputs.size() == 1; } - /** - * Create the NameValuePair array expected by HttpClient, merging - * username and password into the appropriate value slots. - * - * @param username - * @param password - * @return - * @deprecated specific to a particular FetchHTTP implementation based on commons-httpclient, use {@link #asFormDataString(String, String)} - */ - public NameValuePair[] asHttpClientDataWith(String username, String password) { - ArrayList data = new ArrayList(allInputs.size()); - - for (FormInput input : allInputs) { - if(input == candidateUsernameInputs.get(0)) { - data.add(new NameValuePair(input.name,username)); - } else if(input == candidatePasswordInputs.get(0)) { - data.add(new NameValuePair(input.name,password)); - } else if (StringUtils.isNotEmpty(input.name) && StringUtils.isNotEmpty(input.value)) { - data.add(new NameValuePair(input.name,input.value)); - } + public static class NameValue { + public String name, value; + public NameValue(String name, String value) { + this.name = name; + this.value = value; } - return data.toArray(new NameValuePair[data.size()]); } - public FormInput getLoginInputFromCandidates() { - if (candidateUsernameInputs == null) return null; - - if (candidateUsernameInputs.size() == 1) return candidateUsernameInputs.get(0); - - for (FormInput input : candidateUsernameInputs) { - if (input.name != null && input.name.toLowerCase().indexOf("login") != -1) { - return input; - } - } - - return candidateUsernameInputs.get(0); - } - - public HttpEntity asFormDataMultiPartEntity(String username, String password) { - MultipartEntityBuilder multiPartEntityBuilder = MultipartEntityBuilder.create(); - multiPartEntityBuilder.setMode(HttpMultipartMode.BROWSER_COMPATIBLE); - + public LinkedList formData(String username, String password) { + LinkedList nameVals = new LinkedList(); for (FormInput input : allInputs) { - if(input == getLoginInputFromCandidates()) { - multiPartEntityBuilder.addPart(input.name, new StringBody(username, ContentType.MULTIPART_FORM_DATA)); + if (input == candidateUsernameInputs.get(0)) { + nameVals.add(new NameValue(input.name, username)); } else if(input == candidatePasswordInputs.get(0)) { - multiPartEntityBuilder.addPart(input.name, new StringBody(password, ContentType.MULTIPART_FORM_DATA)); - } else if (StringUtils.isNotEmpty(input.name)) { - multiPartEntityBuilder.addPart(input.name, new StringBody(StringUtils.isNotEmpty(input.value) ? input.value : "", ContentType.MULTIPART_FORM_DATA)); - } - } - - return multiPartEntityBuilder.build(); - } - - public String asFormDataString(String username, String password) { - List nameVals = new LinkedList(); - - for (FormInput input : allInputs) { - if(input == candidateUsernameInputs.get(0)) { - nameVals.add(TextUtils.urlEscape(input.name) + "=" + TextUtils.urlEscape(username)); - } else if(input == candidatePasswordInputs.get(0)) { - nameVals.add(TextUtils.urlEscape(input.name) + "=" + TextUtils.urlEscape(password)); + nameVals.add(new NameValue(input.name, password)); } else if (StringUtils.isNotEmpty(input.name) && StringUtils.isNotEmpty(input.value) && (!"radio".equalsIgnoreCase(input.type) && !"checkbox".equals(input.type) || input.checked)) { - nameVals.add(TextUtils.urlEscape(input.name) + "=" - + TextUtils.urlEscape(input.value)); + nameVals.add(new NameValue(input.name, input.value)); } } - - return StringUtils.join(nameVals, '&'); + return nameVals; } public String toString() { diff --git a/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTests.java b/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTests.java index 91449e6f..174cb89b 100644 --- a/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTests.java +++ b/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTests.java @@ -53,10 +53,7 @@ import javax.net.ssl.SSLException; import org.apache.commons.httpclient.URIException; import org.apache.commons.io.IOUtils; -import org.apache.http.NameValuePair; import org.apache.http.NoHttpResponseException; -import org.apache.http.client.utils.URLEncodedUtils; -import org.apache.http.message.BasicNameValuePair; import org.archive.httpclient.ConfigurableX509TrustManager.TrustLevel; import org.archive.modules.CoreAttributeConstants; import org.archive.modules.CrawlMetadata; @@ -65,6 +62,7 @@ import org.archive.modules.CrawlURI.FetchType; import org.archive.modules.ProcessorTestBase; import org.archive.modules.credential.HttpAuthenticationCredential; import org.archive.modules.deciderules.RejectDecideRule; +import org.archive.modules.forms.HTMLForm.NameValue; import org.archive.modules.recrawl.FetchHistoryProcessor; import org.archive.modules.revisit.ServerNotModifiedRevisit; import org.archive.net.UURI; @@ -856,16 +854,14 @@ public class FetchHTTPTests extends ProcessorTestBase { CrawlURI curi = makeCrawlURI("http://localhost:7777/"); curi.setFetchType(FetchType.HTTP_POST); - List params = new LinkedList(); - params.add(new BasicNameValuePair("name1", "value1")); - params.add(new BasicNameValuePair("name1", "value2")); - params.add(new BasicNameValuePair("funky name 2", "whoa crazy\t && 🍺 🍻 \n crazier \rooo")); - String submitData = URLEncodedUtils.format(params, "UTF-8"); - assertEquals("name1=value1&name1=value2&funky+name+2=whoa+crazy%09+%26%26+%F0%9F%8D%BA+%F0%9F%8D%BB+%0A+crazier+%0Dooo", submitData); + List params = new LinkedList(); + params.add(new NameValue("name1", "value1")); + params.add(new NameValue("name1", "value2")); + params.add(new NameValue("funky name 2", "whoa crazy\t && 🍺 🍻 \n crazier \rooo")); + curi.getData().put(CoreAttributeConstants.A_SUBMIT_DATA, params); - curi.getData().put(CoreAttributeConstants.A_SUBMIT_DATA, submitData); fetcher().process(curi); - + assertTrue(httpRequestString(curi).startsWith("POST / HTTP/1.0\r\n")); assertTrue(httpRequestString(curi).endsWith("\r\n\r\nname1=value1&name1=value2&funky+name+2=whoa+crazy%09+%26%26+%F0%9F%8D%BA+%F0%9F%8D%BB+%0A+crazier+%0Dooo")); assertEquals(FetchType.HTTP_POST, curi.getFetchType()); diff --git a/modules/src/test/java/org/archive/modules/forms/FormLoginProcessorTest.java b/modules/src/test/java/org/archive/modules/forms/FormLoginProcessorTest.java index 0fc297b5..84c9cd52 100644 --- a/modules/src/test/java/org/archive/modules/forms/FormLoginProcessorTest.java +++ b/modules/src/test/java/org/archive/modules/forms/FormLoginProcessorTest.java @@ -1,8 +1,8 @@ /* * This file is part of the Heritrix web crawler (crawler.archive.org). * - * Licensed to the Internet Archive (IA) by one or more individual - * contributors. + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. * * The IA licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with @@ -21,13 +21,32 @@ package org.archive.modules.forms; import static org.archive.modules.CoreAttributeConstants.A_WARC_RESPONSE_HEADERS; -import org.archive.modules.CoreAttributeConstants; +import java.io.ByteArrayOutputStream; + +import org.apache.commons.httpclient.URIException; +import org.apache.commons.lang.StringEscapeUtils; +import org.apache.http.HttpRequest; +import org.apache.http.client.entity.UrlEncodedFormEntity; +import org.apache.http.util.EntityUtils; import org.archive.modules.CrawlURI; import org.archive.modules.CrawlURI.FetchType; import org.archive.modules.ProcessorTestBase; +import org.archive.modules.fetcher.BasicExecutionAwareEntityEnclosingRequest; +import org.archive.modules.fetcher.FetchHTTP; +import org.archive.modules.fetcher.FetchHTTPRequest; +import org.archive.modules.fetcher.FetchHTTPTests; public class FormLoginProcessorTest extends ProcessorTestBase { + static class FetchHTTPRequestSpy extends FetchHTTPRequest { + public FetchHTTPRequestSpy(FetchHTTP fetcher, CrawlURI curi) throws URIException { + super(fetcher, curi); + } + public HttpRequest getRequest() { + return request; + } + } + public void testNoFormLogin() throws Exception { CrawlURI curi = makeCrawlURI("http://example.com/"); @@ -38,7 +57,7 @@ public class FormLoginProcessorTest extends ProcessorTestBase { p.process(curi); - assertEquals(1, curi.getDataList(A_WARC_RESPONSE_HEADERS).size()); + assertEquals(1, curi.getDataList(A_WARC_RESPONSE_HEADERS).size()); assertEquals("WARC-Simple-Form-Province-Status: 0,0,http://(com,example,)", curi.getDataList(A_WARC_RESPONSE_HEADERS).get(0)); } @@ -48,6 +67,7 @@ public class FormLoginProcessorTest extends ProcessorTestBase { HTMLForm form = new HTMLForm(); form.addField("text", "username-form-field", ""); form.addField("password", "password-form-field", ""); + form.addField("hidden", "crazy🐒monkey", "úhóh"); form.setMethod("post"); form.setAction("/login"); curi.getDataList(ExtractorHTMLForms.A_HTML_FORM_OBJECTS).add(form); @@ -57,6 +77,44 @@ public class FormLoginProcessorTest extends ProcessorTestBase { p.setLoginPassword("********"); p.setApplicableSurtPrefix("http://(com,example,)"); + p.process(curi); + assertEquals(1, curi.getDataList(A_WARC_RESPONSE_HEADERS).size()); + assertEquals("WARC-Simple-Form-Province-Status: 0,0,http://(com,example,)", curi.getDataList(A_WARC_RESPONSE_HEADERS).get(0)); + assertTrue(curi.getAnnotations().contains("submit:/login")); + + assertEquals(1, curi.getOutLinks().size()); + CrawlURI submitCuri = curi.getOutLinks().toArray(new CrawlURI[0])[0]; + assertEquals("http://example.com/login", submitCuri.toString()); + assertEquals(FetchType.HTTP_POST, submitCuri.getFetchType()); + + FetchHTTPRequestSpy reqSpy = new FetchHTTPRequestSpy( + FetchHTTPTests.newTestFetchHttp(getClass().getName()), + submitCuri); + assertTrue(reqSpy.getRequest() instanceof BasicExecutionAwareEntityEnclosingRequest); + BasicExecutionAwareEntityEnclosingRequest req = (BasicExecutionAwareEntityEnclosingRequest) reqSpy.getRequest(); + assertTrue(req.getEntity() instanceof UrlEncodedFormEntity); + assertTrue(req.toString().startsWith("POST /login")); + assertEquals("username-form-field=jdoe&password-form-field=********&crazy%F0%9F%90%92monkey=%C3%BAh%C3%B3h", + EntityUtils.toString(req.getEntity())); + } + + public void testMultipartFormLogin() throws Exception { + CrawlURI curi = makeCrawlURI("http://example.com/"); + + HTMLForm form = new HTMLForm(); + form.addField("text", "username-form-field", ""); + form.addField("password", "password-form-field", ""); + form.addField("hidden", "crazy🐒monkey", "úhóh"); + form.setMethod("post"); + form.setAction("/login"); + form.setEnctype("multipart/form-data"); + curi.getDataList(ExtractorHTMLForms.A_HTML_FORM_OBJECTS).add(form); + + FormLoginProcessor p = (FormLoginProcessor) makeModule(); + p.setLoginUsername("jdoe"); + p.setLoginPassword("********"); + p.setApplicableSurtPrefix("http://(com,example,)"); + p.process(curi); assertEquals(1, curi.getDataList(A_WARC_RESPONSE_HEADERS).size()); assertEquals("WARC-Simple-Form-Province-Status: 0,0,http://(com,example,)", curi.getDataList(A_WARC_RESPONSE_HEADERS).get(0)); @@ -66,7 +124,25 @@ public class FormLoginProcessorTest extends ProcessorTestBase { CrawlURI submitCuri = curi.getOutLinks().toArray(new CrawlURI[0])[0]; assertEquals("http://example.com/login", submitCuri.toString()); assertEquals(FetchType.HTTP_POST, submitCuri.getFetchType()); - String queryString = (String) submitCuri.getData().get(CoreAttributeConstants.A_SUBMIT_DATA); - assertEquals("username-form-field=jdoe&password-form-field=********", queryString); + + FetchHTTPRequestSpy reqSpy = new FetchHTTPRequestSpy( + FetchHTTPTests.newTestFetchHttp(getClass().getName()), + submitCuri); + assertTrue(reqSpy.getRequest() instanceof BasicExecutionAwareEntityEnclosingRequest); + BasicExecutionAwareEntityEnclosingRequest req = (BasicExecutionAwareEntityEnclosingRequest) reqSpy.getRequest(); + assertEquals("org.apache.http.entity.mime.MultipartFormEntity", req.getEntity().getClass().getName()); + assertTrue(req.toString().startsWith("POST /login")); + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + req.getEntity().writeTo(baos); + System.out.println(StringEscapeUtils.escapeJava(baos.toString("UTF-8"))); + // --e5XWkWxQ2EXBQAEPQ7n3yyvv9bI-8YIHok\r\nContent-Disposition: form-data; name=\"username-form-field\"\r\n\r\njdoe\r\n--e5XWkWxQ2EXBQAEPQ7n3yyvv9bI-8YIHok\r\nContent-Disposition: form-data; name=\"password-form-field\"\r\n\r\n********\r\n--e5XWkWxQ2EXBQAEPQ7n3yyvv9bI-8YIHok\r\nContent-Disposition: form-data; name=\"crazy🐒monkey\"\r\n\r\núhóh\r\n--e5XWkWxQ2EXBQAEPQ7n3yyvv9bI-8YIHok--\r\n + assertTrue(baos.toString("ascii").matches("--([a-zA-Z0-9_-]{30,41})\r\nContent-Disposition: form-data; name=\"username-form-field\"\r\n\r\njdoe\r\n--\\1\r\nContent-Disposition: form-data; name=\"password-form-field\"\r\n\r\n\\*\\*\\*\\*\\*\\*\\*\\*\r\n--\\1\r\nContent-Disposition: form-data; name=\"crazy🐒monkey\"\r\n\r\núhóh\r\n--\\1--\r\n")); + } + + public void testEscapeForMultipart() { + assertEquals("abcd", FetchHTTPRequest.escapeForMultipart("abcd")); + assertEquals("abcdé", FetchHTTPRequest.escapeForMultipart("abcdé")); + assertEquals("abcdé😬", FetchHTTPRequest.escapeForMultipart("abcdé😬")); } } diff --git a/pom.xml b/pom.xml index b5ae17ec..1dc9d97f 100644 --- a/pom.xml +++ b/pom.xml @@ -124,6 +124,11 @@ http://maven.apache.org/guides/mini/guide-m1-m2.html httpclient 4.3.6 + + org.apache.httpcomponents + httpmime + 4.3.6 + From 368d586eb1b7bc003f9ab982b0de935ca849cf35 Mon Sep 17 00:00:00 2001 From: Noah Levitt Date: Thu, 14 Jan 2016 16:00:08 -0800 Subject: [PATCH 10/10] fix NullPointerException when using old HtmlFormCredential login --- .../java/org/archive/modules/fetcher/FetchHTTPRequest.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTPRequest.java b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTPRequest.java index 92ede7d6..a9038e18 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTPRequest.java +++ b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTPRequest.java @@ -187,8 +187,10 @@ public class FetchHTTPRequest { BasicExecutionAwareEntityEnclosingRequest postRequest = new BasicExecutionAwareEntityEnclosingRequest( "POST", requestLineUri, httpVersion); this.request = postRequest; - HttpEntity entity = buildPostRequestEntity(curi); - postRequest.setEntity(entity); + if (curi.containsDataKey(CoreAttributeConstants.A_SUBMIT_DATA)) { + HttpEntity entity = buildPostRequestEntity(curi); + postRequest.setEntity(entity); + } } else { this.request = new BasicExecutionAwareRequest("GET", requestLineUri, httpVersion);