(
- new String [] {A_CREDENTIALS_KEY, A_HTTP_AUTH_CHALLENGES, A_SUBMIT_DATA, A_WARC_RESPONSE_HEADERS, A_ANNOTATIONS});
+ new String [] {A_CREDENTIALS_KEY, A_HTTP_AUTH_CHALLENGES, A_SUBMIT_DATA, A_WARC_RESPONSE_HEADERS, A_ANNOTATIONS, A_SUBMIT_ENCTYPE});
/** maximum length for pathFromSeed/hopsPath; longer truncated with leading counter **/
private static final int MAX_HOPS_DISPLAYED = 50;
diff --git a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTPRequest.java b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTPRequest.java
index 19a47771..a9038e18 100644
--- a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTPRequest.java
+++ b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTPRequest.java
@@ -25,6 +25,7 @@ import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_STATUS;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
+import java.io.UnsupportedEncodingException;
import java.net.InetAddress;
import java.net.Socket;
import java.net.UnknownHostException;
@@ -34,6 +35,7 @@ import java.nio.charset.CharsetEncoder;
import java.nio.charset.CodingErrorAction;
import java.util.ArrayList;
import java.util.HashMap;
+import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
@@ -45,6 +47,7 @@ import javax.net.ssl.SSLSocket;
import org.apache.commons.httpclient.URIException;
import org.apache.commons.lang.StringUtils;
+import org.apache.http.HttpEntity;
import org.apache.http.HttpEntityEnclosingRequest;
import org.apache.http.HttpException;
import org.apache.http.HttpHeaders;
@@ -65,7 +68,6 @@ import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.AbstractExecutionAwareRequest;
import org.apache.http.client.protocol.HttpClientContext;
-import org.apache.http.client.utils.URLEncodedUtils;
import org.apache.http.config.ConnectionConfig;
import org.apache.http.config.MessageConstraints;
import org.apache.http.config.Registry;
@@ -82,7 +84,8 @@ import org.apache.http.conn.ssl.AllowAllHostnameVerifier;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
import org.apache.http.entity.ContentLengthStrategy;
import org.apache.http.entity.ContentType;
-import org.apache.http.entity.StringEntity;
+import org.apache.http.entity.mime.HttpMultipartMode;
+import org.apache.http.entity.mime.MultipartEntityBuilder;
import org.apache.http.impl.DefaultBHttpClientConnection;
import org.apache.http.impl.client.BasicAuthCache;
import org.apache.http.impl.client.BasicCredentialsProvider;
@@ -105,6 +108,7 @@ import org.archive.modules.credential.Credential;
import org.archive.modules.credential.HtmlFormCredential;
import org.archive.modules.credential.HttpAuthenticationCredential;
import org.archive.modules.extractor.LinkContext;
+import org.archive.modules.forms.HTMLForm.NameValue;
import org.archive.modules.net.CrawlHost;
import org.archive.modules.net.CrawlServer;
import org.archive.modules.net.ServerCache;
@@ -113,7 +117,7 @@ import org.archive.util.Recorder;
/**
* @contributor nlevitt
*/
-class FetchHTTPRequest {
+public class FetchHTTPRequest {
/**
* Implementation of {@link DnsResolver} that uses the server cache which is
@@ -183,12 +187,9 @@ class FetchHTTPRequest {
BasicExecutionAwareEntityEnclosingRequest postRequest = new BasicExecutionAwareEntityEnclosingRequest(
"POST", requestLineUri, httpVersion);
this.request = postRequest;
- String submitData = (String) curi.getData().get(CoreAttributeConstants.A_SUBMIT_DATA);
- if (submitData != null) {
- // XXX brittle, doesn't support multipart form data etc
- ContentType contentType = ContentType.create(URLEncodedUtils.CONTENT_TYPE, "UTF-8");
- StringEntity formEntity = new StringEntity(submitData, contentType);
- postRequest.setEntity(formEntity);
+ if (curi.containsDataKey(CoreAttributeConstants.A_SUBMIT_DATA)) {
+ HttpEntity entity = buildPostRequestEntity(curi);
+ postRequest.setEntity(entity);
}
} else {
this.request = new BasicExecutionAwareRequest("GET",
@@ -210,6 +211,79 @@ class FetchHTTPRequest {
populateHttpProxyCredential();
}
+ /**
+ * Returns a copy of the string with non-ascii characters replaced by their
+ * html numeric character reference in decimal (e.g. 〹).
+ *
+ *
+ * The purpose of this is to produce a multipart/formdata submission that
+ * any server should be able to handle, based on experiments using a modern
+ * browser (chromium 47.0.2526.106 for mac). What chromium posts depends on
+ * what it considers the character encoding of the page containing the form,
+ * and maybe other factors. It would be too complicated to try to simulate
+ * that behavior in heritrix.
+ *
+ *
+ * Instead what we do is approximately what the browser does when the form
+ * page is plain ascii. It html-escapes characters outside of the
+ * latin1/cp1252 range. Characters in the U+0080-U+00FF range are encoded in
+ * latin1/cp1252. That is the one way that we differ from chromium. We
+ * html-escape those characters (U+0080-U+00FF) as well. That way the http
+ * post is plain ascii, and should work regardless of which encoding the
+ * server expects.
+ *
+ *
+ * N.b. chromium doesn't indicate the encoding of the request in any way (no
+ * charset in the content-type or anything like that). Also of note is that
+ * when it considers the form page to be utf-8, it submits in utf-8. That's
+ * part of the complicated behavior we don't want to try to simulate.
+ */
+ public static String escapeForMultipart(String str) {
+ StringBuilder buf = new StringBuilder();
+ for (int i = 0; i < str.length(); ) {
+ int codepoint = str.codePointAt(i);
+ if (codepoint <= 0x7f) {
+ buf.appendCodePoint(codepoint);
+ } else {
+ buf.append("&#" + codepoint + ";");
+ }
+ i += Character.charCount(codepoint);
+ }
+ return buf.toString();
+ }
+
+ protected HttpEntity buildPostRequestEntity(CrawlURI curi) {
+ String enctype = (String) curi.getData().get(CoreAttributeConstants.A_SUBMIT_ENCTYPE);
+ if (enctype == null) {
+ enctype = ContentType.APPLICATION_FORM_URLENCODED.getMimeType();
+ }
+
+ @SuppressWarnings("unchecked")
+ List submitData = (List) curi.getData().get(CoreAttributeConstants.A_SUBMIT_DATA);
+
+ if (enctype.equals(ContentType.APPLICATION_FORM_URLENCODED.getMimeType())) {
+ LinkedList nvps = new LinkedList();
+ for (NameValue nv: submitData) {
+ nvps.add(new BasicNameValuePair(nv.name, nv.value));
+ }
+ try {
+ return new UrlEncodedFormEntity(nvps, "UTF-8");
+ } catch (UnsupportedEncodingException e) {
+ throw new IllegalStateException(e);
+ }
+ } else if (enctype.equals(ContentType.MULTIPART_FORM_DATA.getMimeType())) {
+ MultipartEntityBuilder entityBuilder = MultipartEntityBuilder.create();
+ entityBuilder.setMode(HttpMultipartMode.BROWSER_COMPATIBLE);
+ for (NameValue nv: submitData) {
+ entityBuilder.addTextBody(escapeForMultipart(nv.name),
+ escapeForMultipart(nv.value));
+ }
+ return entityBuilder.build();
+ } else {
+ throw new IllegalStateException("unsupported form submission enctype='" + enctype + "'");
+ }
+ }
+
protected void configureRequestHeaders() {
if (fetcher.getAcceptCompression()) {
request.addHeader("Accept-Encoding", "gzip,deflate");
diff --git a/modules/src/main/java/org/archive/modules/forms/ExtractorHTMLForms.java b/modules/src/main/java/org/archive/modules/forms/ExtractorHTMLForms.java
index 231a1e05..0c672102 100644
--- a/modules/src/main/java/org/archive/modules/forms/ExtractorHTMLForms.java
+++ b/modules/src/main/java/org/archive/modules/forms/ExtractorHTMLForms.java
@@ -145,9 +145,11 @@ public class ExtractorHTMLForms extends Extractor {
CharSequence relevantSequence = cs.subSequence(offsetInt, cs.length());
String method = findAttributeValueGroup("(?i)^[^>]*\\smethod\\s*=\\s*([^>\\s]+)[^>]*>",1,relevantSequence);
String action = findAttributeValueGroup("(?i)^[^>]*\\saction\\s*=\\s*([^>\\s]+)[^>]*>",1,relevantSequence);
+ String enctype = findAttributeValueGroup("(?i)^[^>]*\\senctype\\s*=\\s*([^>\\s]+)[^>]*>",1,relevantSequence);
HTMLForm form = new HTMLForm();
form.setMethod(method);
- form.setAction(action);
+ form.setAction(action);
+ form.setEnctype(enctype);
for(CharSequence input : findGroups("(?i)(]*>)|()",1,relevantSequence)) {
String type = findAttributeValueGroup("(?i)^[^>]*\\stype\\s*=\\s*([^>\\s]+)[^>]*>",1,input);
String name = findAttributeValueGroup("(?i)^[^>]*\\sname\\s*=\\s*([^>\\s]+)[^>]*>",1,input);
diff --git a/modules/src/main/java/org/archive/modules/forms/FormLoginProcessor.java b/modules/src/main/java/org/archive/modules/forms/FormLoginProcessor.java
index 9a2d540d..b6ee86a2 100644
--- a/modules/src/main/java/org/archive/modules/forms/FormLoginProcessor.java
+++ b/modules/src/main/java/org/archive/modules/forms/FormLoginProcessor.java
@@ -266,10 +266,12 @@ public class FormLoginProcessor extends Processor implements Checkpointable {
CrawlURI submitCuri = curi.createCrawlURI(submitUrl, lc, Hop.SUBMIT);
submitCuri.setFetchType(FetchType.HTTP_POST);
submitCuri.getData().put(
- CoreAttributeConstants.A_SUBMIT_DATA,
- templateForm.asFormDataString(
+ CoreAttributeConstants.A_SUBMIT_DATA,
+ templateForm.formData(
getLoginUsername(),
getLoginPassword()));
+ submitCuri.getData().put(CoreAttributeConstants.A_SUBMIT_ENCTYPE,
+ templateForm.getEnctype());
submitCuri.setSchedulingDirective(SchedulingConstants.HIGH);
submitCuri.setForceFetch(true);
curi.getOutLinks().add(submitCuri);
diff --git a/modules/src/main/java/org/archive/modules/forms/HTMLForm.java b/modules/src/main/java/org/archive/modules/forms/HTMLForm.java
index 0634176b..198f5285 100644
--- a/modules/src/main/java/org/archive/modules/forms/HTMLForm.java
+++ b/modules/src/main/java/org/archive/modules/forms/HTMLForm.java
@@ -23,9 +23,7 @@ import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
-import org.apache.commons.httpclient.NameValuePair;
import org.apache.commons.lang.StringUtils;
-import org.archive.util.TextUtils;
/**
* Simple representation of a discovered HTML Form.
@@ -47,13 +45,14 @@ public class HTMLForm {
return str;
}
}
-
- String method;
- String action;
-
- List allInputs = new ArrayList();
- List candidateUsernameInputs = new ArrayList();
- List candidatePasswordInputs = new ArrayList();
+
+ protected String method;
+ protected String action;
+ protected String enctype;
+
+ protected List allInputs = new ArrayList();
+ protected List candidateUsernameInputs = new ArrayList();
+ protected List candidatePasswordInputs = new ArrayList();
/**
* Add a discovered INPUT, tracking it as potential
@@ -99,11 +98,19 @@ public class HTMLForm {
public String getAction() {
return action;
}
-
+
public void setAction(String action) {
this.action = action;
}
+ public String getEnctype() {
+ return enctype;
+ }
+
+ public void setEnctype(String enctype) {
+ this.enctype = enctype;
+ }
+
/**
* For now, we consider a POST form with only 1 password
* field and 1 potential username field (type text or email)
@@ -117,48 +124,29 @@ public class HTMLForm {
&& candidatePasswordInputs.size() == 1;
}
- /**
- * Create the NameValuePair array expected by HttpClient, merging
- * username and password into the appropriate value slots.
- *
- * @param username
- * @param password
- * @return
- * @deprecated specific to a particular FetchHTTP implementation based on commons-httpclient, use {@link #asFormDataString(String, String)}
- */
- public NameValuePair[] asHttpClientDataWith(String username, String password) {
- ArrayList data = new ArrayList(allInputs.size());
-
- for (FormInput input : allInputs) {
- if(input == candidateUsernameInputs.get(0)) {
- data.add(new NameValuePair(input.name,username));
- } else if(input == candidatePasswordInputs.get(0)) {
- data.add(new NameValuePair(input.name,password));
- } else if (StringUtils.isNotEmpty(input.name) && StringUtils.isNotEmpty(input.value)) {
- data.add(new NameValuePair(input.name,input.value));
- }
+ public static class NameValue {
+ public String name, value;
+ public NameValue(String name, String value) {
+ this.name = name;
+ this.value = value;
}
- return data.toArray(new NameValuePair[data.size()]);
}
-
- public String asFormDataString(String username, String password) {
- List nameVals = new LinkedList();
+ public LinkedList formData(String username, String password) {
+ LinkedList nameVals = new LinkedList();
for (FormInput input : allInputs) {
- if(input == candidateUsernameInputs.get(0)) {
- nameVals.add(TextUtils.urlEscape(input.name) + "=" + TextUtils.urlEscape(username));
+ if (input == candidateUsernameInputs.get(0)) {
+ nameVals.add(new NameValue(input.name, username));
} else if(input == candidatePasswordInputs.get(0)) {
- nameVals.add(TextUtils.urlEscape(input.name) + "=" + TextUtils.urlEscape(password));
+ nameVals.add(new NameValue(input.name, password));
} else if (StringUtils.isNotEmpty(input.name)
&& StringUtils.isNotEmpty(input.value)
&& (!"radio".equalsIgnoreCase(input.type)
&& !"checkbox".equals(input.type) || input.checked)) {
- nameVals.add(TextUtils.urlEscape(input.name) + "="
- + TextUtils.urlEscape(input.value));
+ nameVals.add(new NameValue(input.name, input.value));
}
}
-
- return StringUtils.join(nameVals, '&');
+ return nameVals;
}
public String toString() {
diff --git a/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTests.java b/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTests.java
index 91449e6f..174cb89b 100644
--- a/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTests.java
+++ b/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTests.java
@@ -53,10 +53,7 @@ import javax.net.ssl.SSLException;
import org.apache.commons.httpclient.URIException;
import org.apache.commons.io.IOUtils;
-import org.apache.http.NameValuePair;
import org.apache.http.NoHttpResponseException;
-import org.apache.http.client.utils.URLEncodedUtils;
-import org.apache.http.message.BasicNameValuePair;
import org.archive.httpclient.ConfigurableX509TrustManager.TrustLevel;
import org.archive.modules.CoreAttributeConstants;
import org.archive.modules.CrawlMetadata;
@@ -65,6 +62,7 @@ import org.archive.modules.CrawlURI.FetchType;
import org.archive.modules.ProcessorTestBase;
import org.archive.modules.credential.HttpAuthenticationCredential;
import org.archive.modules.deciderules.RejectDecideRule;
+import org.archive.modules.forms.HTMLForm.NameValue;
import org.archive.modules.recrawl.FetchHistoryProcessor;
import org.archive.modules.revisit.ServerNotModifiedRevisit;
import org.archive.net.UURI;
@@ -856,16 +854,14 @@ public class FetchHTTPTests extends ProcessorTestBase {
CrawlURI curi = makeCrawlURI("http://localhost:7777/");
curi.setFetchType(FetchType.HTTP_POST);
- List params = new LinkedList();
- params.add(new BasicNameValuePair("name1", "value1"));
- params.add(new BasicNameValuePair("name1", "value2"));
- params.add(new BasicNameValuePair("funky name 2", "whoa crazy\t && 🍺 🍻 \n crazier \rooo"));
- String submitData = URLEncodedUtils.format(params, "UTF-8");
- assertEquals("name1=value1&name1=value2&funky+name+2=whoa+crazy%09+%26%26+%F0%9F%8D%BA+%F0%9F%8D%BB+%0A+crazier+%0Dooo", submitData);
+ List params = new LinkedList();
+ params.add(new NameValue("name1", "value1"));
+ params.add(new NameValue("name1", "value2"));
+ params.add(new NameValue("funky name 2", "whoa crazy\t && 🍺 🍻 \n crazier \rooo"));
+ curi.getData().put(CoreAttributeConstants.A_SUBMIT_DATA, params);
- curi.getData().put(CoreAttributeConstants.A_SUBMIT_DATA, submitData);
fetcher().process(curi);
-
+
assertTrue(httpRequestString(curi).startsWith("POST / HTTP/1.0\r\n"));
assertTrue(httpRequestString(curi).endsWith("\r\n\r\nname1=value1&name1=value2&funky+name+2=whoa+crazy%09+%26%26+%F0%9F%8D%BA+%F0%9F%8D%BB+%0A+crazier+%0Dooo"));
assertEquals(FetchType.HTTP_POST, curi.getFetchType());
diff --git a/modules/src/test/java/org/archive/modules/forms/FormLoginProcessorTest.java b/modules/src/test/java/org/archive/modules/forms/FormLoginProcessorTest.java
index 0fc297b5..84c9cd52 100644
--- a/modules/src/test/java/org/archive/modules/forms/FormLoginProcessorTest.java
+++ b/modules/src/test/java/org/archive/modules/forms/FormLoginProcessorTest.java
@@ -1,8 +1,8 @@
/*
* This file is part of the Heritrix web crawler (crawler.archive.org).
*
- * Licensed to the Internet Archive (IA) by one or more individual
- * contributors.
+ * Licensed to the Internet Archive (IA) by one or more individual
+ * contributors.
*
* The IA licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
@@ -21,13 +21,32 @@ package org.archive.modules.forms;
import static org.archive.modules.CoreAttributeConstants.A_WARC_RESPONSE_HEADERS;
-import org.archive.modules.CoreAttributeConstants;
+import java.io.ByteArrayOutputStream;
+
+import org.apache.commons.httpclient.URIException;
+import org.apache.commons.lang.StringEscapeUtils;
+import org.apache.http.HttpRequest;
+import org.apache.http.client.entity.UrlEncodedFormEntity;
+import org.apache.http.util.EntityUtils;
import org.archive.modules.CrawlURI;
import org.archive.modules.CrawlURI.FetchType;
import org.archive.modules.ProcessorTestBase;
+import org.archive.modules.fetcher.BasicExecutionAwareEntityEnclosingRequest;
+import org.archive.modules.fetcher.FetchHTTP;
+import org.archive.modules.fetcher.FetchHTTPRequest;
+import org.archive.modules.fetcher.FetchHTTPTests;
public class FormLoginProcessorTest extends ProcessorTestBase {
+ static class FetchHTTPRequestSpy extends FetchHTTPRequest {
+ public FetchHTTPRequestSpy(FetchHTTP fetcher, CrawlURI curi) throws URIException {
+ super(fetcher, curi);
+ }
+ public HttpRequest getRequest() {
+ return request;
+ }
+ }
+
public void testNoFormLogin() throws Exception {
CrawlURI curi = makeCrawlURI("http://example.com/");
@@ -38,7 +57,7 @@ public class FormLoginProcessorTest extends ProcessorTestBase {
p.process(curi);
- assertEquals(1, curi.getDataList(A_WARC_RESPONSE_HEADERS).size());
+ assertEquals(1, curi.getDataList(A_WARC_RESPONSE_HEADERS).size());
assertEquals("WARC-Simple-Form-Province-Status: 0,0,http://(com,example,)", curi.getDataList(A_WARC_RESPONSE_HEADERS).get(0));
}
@@ -48,6 +67,7 @@ public class FormLoginProcessorTest extends ProcessorTestBase {
HTMLForm form = new HTMLForm();
form.addField("text", "username-form-field", "");
form.addField("password", "password-form-field", "");
+ form.addField("hidden", "crazy🐒monkey", "úhóh");
form.setMethod("post");
form.setAction("/login");
curi.getDataList(ExtractorHTMLForms.A_HTML_FORM_OBJECTS).add(form);
@@ -57,6 +77,44 @@ public class FormLoginProcessorTest extends ProcessorTestBase {
p.setLoginPassword("********");
p.setApplicableSurtPrefix("http://(com,example,)");
+ p.process(curi);
+ assertEquals(1, curi.getDataList(A_WARC_RESPONSE_HEADERS).size());
+ assertEquals("WARC-Simple-Form-Province-Status: 0,0,http://(com,example,)", curi.getDataList(A_WARC_RESPONSE_HEADERS).get(0));
+ assertTrue(curi.getAnnotations().contains("submit:/login"));
+
+ assertEquals(1, curi.getOutLinks().size());
+ CrawlURI submitCuri = curi.getOutLinks().toArray(new CrawlURI[0])[0];
+ assertEquals("http://example.com/login", submitCuri.toString());
+ assertEquals(FetchType.HTTP_POST, submitCuri.getFetchType());
+
+ FetchHTTPRequestSpy reqSpy = new FetchHTTPRequestSpy(
+ FetchHTTPTests.newTestFetchHttp(getClass().getName()),
+ submitCuri);
+ assertTrue(reqSpy.getRequest() instanceof BasicExecutionAwareEntityEnclosingRequest);
+ BasicExecutionAwareEntityEnclosingRequest req = (BasicExecutionAwareEntityEnclosingRequest) reqSpy.getRequest();
+ assertTrue(req.getEntity() instanceof UrlEncodedFormEntity);
+ assertTrue(req.toString().startsWith("POST /login"));
+ assertEquals("username-form-field=jdoe&password-form-field=********&crazy%F0%9F%90%92monkey=%C3%BAh%C3%B3h",
+ EntityUtils.toString(req.getEntity()));
+ }
+
+ public void testMultipartFormLogin() throws Exception {
+ CrawlURI curi = makeCrawlURI("http://example.com/");
+
+ HTMLForm form = new HTMLForm();
+ form.addField("text", "username-form-field", "");
+ form.addField("password", "password-form-field", "");
+ form.addField("hidden", "crazy🐒monkey", "úhóh");
+ form.setMethod("post");
+ form.setAction("/login");
+ form.setEnctype("multipart/form-data");
+ curi.getDataList(ExtractorHTMLForms.A_HTML_FORM_OBJECTS).add(form);
+
+ FormLoginProcessor p = (FormLoginProcessor) makeModule();
+ p.setLoginUsername("jdoe");
+ p.setLoginPassword("********");
+ p.setApplicableSurtPrefix("http://(com,example,)");
+
p.process(curi);
assertEquals(1, curi.getDataList(A_WARC_RESPONSE_HEADERS).size());
assertEquals("WARC-Simple-Form-Province-Status: 0,0,http://(com,example,)", curi.getDataList(A_WARC_RESPONSE_HEADERS).get(0));
@@ -66,7 +124,25 @@ public class FormLoginProcessorTest extends ProcessorTestBase {
CrawlURI submitCuri = curi.getOutLinks().toArray(new CrawlURI[0])[0];
assertEquals("http://example.com/login", submitCuri.toString());
assertEquals(FetchType.HTTP_POST, submitCuri.getFetchType());
- String queryString = (String) submitCuri.getData().get(CoreAttributeConstants.A_SUBMIT_DATA);
- assertEquals("username-form-field=jdoe&password-form-field=********", queryString);
+
+ FetchHTTPRequestSpy reqSpy = new FetchHTTPRequestSpy(
+ FetchHTTPTests.newTestFetchHttp(getClass().getName()),
+ submitCuri);
+ assertTrue(reqSpy.getRequest() instanceof BasicExecutionAwareEntityEnclosingRequest);
+ BasicExecutionAwareEntityEnclosingRequest req = (BasicExecutionAwareEntityEnclosingRequest) reqSpy.getRequest();
+ assertEquals("org.apache.http.entity.mime.MultipartFormEntity", req.getEntity().getClass().getName());
+ assertTrue(req.toString().startsWith("POST /login"));
+
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ req.getEntity().writeTo(baos);
+ System.out.println(StringEscapeUtils.escapeJava(baos.toString("UTF-8")));
+ // --e5XWkWxQ2EXBQAEPQ7n3yyvv9bI-8YIHok\r\nContent-Disposition: form-data; name=\"username-form-field\"\r\n\r\njdoe\r\n--e5XWkWxQ2EXBQAEPQ7n3yyvv9bI-8YIHok\r\nContent-Disposition: form-data; name=\"password-form-field\"\r\n\r\n********\r\n--e5XWkWxQ2EXBQAEPQ7n3yyvv9bI-8YIHok\r\nContent-Disposition: form-data; name=\"crazy🐒monkey\"\r\n\r\núhóh\r\n--e5XWkWxQ2EXBQAEPQ7n3yyvv9bI-8YIHok--\r\n
+ assertTrue(baos.toString("ascii").matches("--([a-zA-Z0-9_-]{30,41})\r\nContent-Disposition: form-data; name=\"username-form-field\"\r\n\r\njdoe\r\n--\\1\r\nContent-Disposition: form-data; name=\"password-form-field\"\r\n\r\n\\*\\*\\*\\*\\*\\*\\*\\*\r\n--\\1\r\nContent-Disposition: form-data; name=\"crazy🐒monkey\"\r\n\r\núhóh\r\n--\\1--\r\n"));
+ }
+
+ public void testEscapeForMultipart() {
+ assertEquals("abcd", FetchHTTPRequest.escapeForMultipart("abcd"));
+ assertEquals("abcdé", FetchHTTPRequest.escapeForMultipart("abcdé"));
+ assertEquals("abcdé😬", FetchHTTPRequest.escapeForMultipart("abcdé😬"));
}
}
diff --git a/pom.xml b/pom.xml
index b5ae17ec..1dc9d97f 100644
--- a/pom.xml
+++ b/pom.xml
@@ -124,6 +124,11 @@ http://maven.apache.org/guides/mini/guide-m1-m2.html
httpclient
4.3.6