lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-09-17 03:05:46 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #142 from nlevitt/form-login-multipart

Browse Source 
Form login multipart
This commit is contained in:
Hunter
2016-01-15 08:19:23 -08:00
parent eeeea6f0c4 368d586eb1
commit 02e3a554f0
10 changed files with 223 additions and 74 deletions
Download Patch File Download Diff File Expand all files Collapse all files
commons/pom.xml
+4
View File
@@ -27,6 +27,10 @@
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpmime</artifactId>
</dependency>
<dependency>
<groupId>com.sleepycat</groupId>
<artifactId>je</artifactId>
modules/src/main/java/org/archive/modules/CoreAttributeConstants.java
+4 -3
View File
@@ -118,10 +118,11 @@ public interface CoreAttributeConstants {
public static final String A_HTTP_RESPONSE_HEADERS = "http-response-headers";
public static final String A_HTTP_AUTH_CHALLENGES = "http-auth-challenges";
// FORMS support - a persistent member (survives frontier enqueue/dequeue/retries)
// FORMS support - persistent members (survive frontier enqueue/dequeue/retries)
public static final String A_SUBMIT_DATA = "submit-data";
public static final String A_SUBMIT_ENCTYPE = "submit-enctype";
// arbitrary additions to WARC response record headers
public static final String A_WARC_RESPONSE_HEADERS = "warc-response-headers";
modules/src/main/java/org/archive/modules/CrawlURI.java
+2 -1
View File
@@ -32,6 +32,7 @@ import static org.archive.modules.CoreAttributeConstants.A_NONFATAL_ERRORS;
import static org.archive.modules.CoreAttributeConstants.A_PREREQUISITE_URI;
import static org.archive.modules.CoreAttributeConstants.A_SOURCE_TAG;
import static org.archive.modules.CoreAttributeConstants.A_SUBMIT_DATA;
import static org.archive.modules.CoreAttributeConstants.A_SUBMIT_ENCTYPE;
import static org.archive.modules.CoreAttributeConstants.A_WARC_RESPONSE_HEADERS;
import static org.archive.modules.SchedulingConstants.NORMAL;
import static org.archive.modules.fetcher.FetchStatusCodes.S_BLOCKED_BY_CUSTOM_PROCESSOR;
@@ -256,7 +257,7 @@ implements Reporter, Serializable, OverlayContext, Comparable<CrawlURI> {
*/
private static final Collection<String> persistentKeys
= new CopyOnWriteArrayList<String>(
new String [] {A_CREDENTIALS_KEY, A_HTTP_AUTH_CHALLENGES, A_SUBMIT_DATA, A_WARC_RESPONSE_HEADERS, A_ANNOTATIONS});
new String [] {A_CREDENTIALS_KEY, A_HTTP_AUTH_CHALLENGES, A_SUBMIT_DATA, A_WARC_RESPONSE_HEADERS, A_ANNOTATIONS, A_SUBMIT_ENCTYPE});
/** maximum length for pathFromSeed/hopsPath; longer truncated with leading counter **/
private static final int MAX_HOPS_DISPLAYED = 50;
modules/src/main/java/org/archive/modules/fetcher/FetchHTTPRequest.java
+83 -9
View File
@@ -25,6 +25,7 @@ import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_STATUS;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.UnsupportedEncodingException;
import java.net.InetAddress;
import java.net.Socket;
import java.net.UnknownHostException;
@@ -34,6 +35,7 @@ import java.nio.charset.CharsetEncoder;
import java.nio.charset.CodingErrorAction;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
@@ -45,6 +47,7 @@ import javax.net.ssl.SSLSocket;
import org.apache.commons.httpclient.URIException;
import org.apache.commons.lang.StringUtils;
import org.apache.http.HttpEntity;
import org.apache.http.HttpEntityEnclosingRequest;
import org.apache.http.HttpException;
import org.apache.http.HttpHeaders;
@@ -65,7 +68,6 @@ import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.AbstractExecutionAwareRequest;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.client.utils.URLEncodedUtils;
import org.apache.http.config.ConnectionConfig;
import org.apache.http.config.MessageConstraints;
import org.apache.http.config.Registry;
@@ -82,7 +84,8 @@ import org.apache.http.conn.ssl.AllowAllHostnameVerifier;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
import org.apache.http.entity.ContentLengthStrategy;
import org.apache.http.entity.ContentType;
import org.apache.http.entity.StringEntity;
import org.apache.http.entity.mime.HttpMultipartMode;
import org.apache.http.entity.mime.MultipartEntityBuilder;
import org.apache.http.impl.DefaultBHttpClientConnection;
import org.apache.http.impl.client.BasicAuthCache;
import org.apache.http.impl.client.BasicCredentialsProvider;
@@ -105,6 +108,7 @@ import org.archive.modules.credential.Credential;
import org.archive.modules.credential.HtmlFormCredential;
import org.archive.modules.credential.HttpAuthenticationCredential;
import org.archive.modules.extractor.LinkContext;
import org.archive.modules.forms.HTMLForm.NameValue;
import org.archive.modules.net.CrawlHost;
import org.archive.modules.net.CrawlServer;
import org.archive.modules.net.ServerCache;
@@ -113,7 +117,7 @@ import org.archive.util.Recorder;
/**
* @contributor nlevitt
*/
class FetchHTTPRequest {
public class FetchHTTPRequest {
/**
* Implementation of {@link DnsResolver} that uses the server cache which is
@@ -183,12 +187,9 @@ class FetchHTTPRequest {
BasicExecutionAwareEntityEnclosingRequest postRequest = new BasicExecutionAwareEntityEnclosingRequest(
"POST", requestLineUri, httpVersion);
this.request = postRequest;
String submitData = (String) curi.getData().get(CoreAttributeConstants.A_SUBMIT_DATA);
if (submitData != null) {
// XXX brittle, doesn't support multipart form data etc
ContentType contentType = ContentType.create(URLEncodedUtils.CONTENT_TYPE, "UTF-8");
StringEntity formEntity = new StringEntity(submitData, contentType);
postRequest.setEntity(formEntity);
if (curi.containsDataKey(CoreAttributeConstants.A_SUBMIT_DATA)) {
HttpEntity entity = buildPostRequestEntity(curi);
postRequest.setEntity(entity);
}
} else {
this.request = new BasicExecutionAwareRequest("GET",
@@ -210,6 +211,79 @@ class FetchHTTPRequest {
populateHttpProxyCredential();
}
/**
* Returns a copy of the string with non-ascii characters replaced by their
* html numeric character reference in decimal (e.g. &amp;#12345;).
*
* <p>
* The purpose of this is to produce a multipart/formdata submission that
* any server should be able to handle, based on experiments using a modern
* browser (chromium 47.0.2526.106 for mac). What chromium posts depends on
* what it considers the character encoding of the page containing the form,
* and maybe other factors. It would be too complicated to try to simulate
* that behavior in heritrix.
*
* <p>
* Instead what we do is approximately what the browser does when the form
* page is plain ascii. It html-escapes characters outside of the
* latin1/cp1252 range. Characters in the U+0080-U+00FF range are encoded in
* latin1/cp1252. That is the one way that we differ from chromium. We
* html-escape those characters (U+0080-U+00FF) as well. That way the http
* post is plain ascii, and should work regardless of which encoding the
* server expects.
*
* <p>
* N.b. chromium doesn't indicate the encoding of the request in any way (no
* charset in the content-type or anything like that). Also of note is that
* when it considers the form page to be utf-8, it submits in utf-8. That's
* part of the complicated behavior we don't want to try to simulate.
*/
public static String escapeForMultipart(String str) {
StringBuilder buf = new StringBuilder();
for (int i = 0; i < str.length(); ) {
int codepoint = str.codePointAt(i);
if (codepoint <= 0x7f) {
buf.appendCodePoint(codepoint);
} else {
buf.append("&#" + codepoint + ";");
}
i += Character.charCount(codepoint);
}
return buf.toString();
}
protected HttpEntity buildPostRequestEntity(CrawlURI curi) {
String enctype = (String) curi.getData().get(CoreAttributeConstants.A_SUBMIT_ENCTYPE);
if (enctype == null) {
enctype = ContentType.APPLICATION_FORM_URLENCODED.getMimeType();
}
@SuppressWarnings("unchecked")
List<NameValue> submitData = (List<NameValue>) curi.getData().get(CoreAttributeConstants.A_SUBMIT_DATA);
if (enctype.equals(ContentType.APPLICATION_FORM_URLENCODED.getMimeType())) {
LinkedList<NameValuePair> nvps = new LinkedList<NameValuePair>();
for (NameValue nv: submitData) {
nvps.add(new BasicNameValuePair(nv.name, nv.value));
}
try {
return new UrlEncodedFormEntity(nvps, "UTF-8");
} catch (UnsupportedEncodingException e) {
throw new IllegalStateException(e);
}
} else if (enctype.equals(ContentType.MULTIPART_FORM_DATA.getMimeType())) {
MultipartEntityBuilder entityBuilder = MultipartEntityBuilder.create();
entityBuilder.setMode(HttpMultipartMode.BROWSER_COMPATIBLE);
for (NameValue nv: submitData) {
entityBuilder.addTextBody(escapeForMultipart(nv.name),
escapeForMultipart(nv.value));
}
return entityBuilder.build();
} else {
throw new IllegalStateException("unsupported form submission enctype='" + enctype + "'");
}
}
protected void configureRequestHeaders() {
if (fetcher.getAcceptCompression()) {
request.addHeader("Accept-Encoding", "gzip,deflate");
modules/src/main/java/org/archive/modules/forms/ExtractorHTMLForms.java
+3 -1
View File
@@ -145,9 +145,11 @@ public class ExtractorHTMLForms extends Extractor {
CharSequence relevantSequence = cs.subSequence(offsetInt, cs.length());
String method = findAttributeValueGroup("(?i)^[^>]*\\smethod\\s*=\\s*([^>\\s]+)[^>]*>",1,relevantSequence);
String action = findAttributeValueGroup("(?i)^[^>]*\\saction\\s*=\\s*([^>\\s]+)[^>]*>",1,relevantSequence);
String enctype = findAttributeValueGroup("(?i)^[^>]*\\senctype\\s*=\\s*([^>\\s]+)[^>]*>",1,relevantSequence);
HTMLForm form = new HTMLForm();
form.setMethod(method);
form.setAction(action);
form.setAction(action);
form.setEnctype(enctype);
for(CharSequence input : findGroups("(?i)(<input\\s[^>]*>)|(</?form>)",1,relevantSequence)) {
String type = findAttributeValueGroup("(?i)^[^>]*\\stype\\s*=\\s*([^>\\s]+)[^>]*>",1,input);
String name = findAttributeValueGroup("(?i)^[^>]*\\sname\\s*=\\s*([^>\\s]+)[^>]*>",1,input);
modules/src/main/java/org/archive/modules/forms/FormLoginProcessor.java
+4 -2
View File
@@ -266,10 +266,12 @@ public class FormLoginProcessor extends Processor implements Checkpointable {
CrawlURI submitCuri = curi.createCrawlURI(submitUrl, lc, Hop.SUBMIT);
submitCuri.setFetchType(FetchType.HTTP_POST);
submitCuri.getData().put(
CoreAttributeConstants.A_SUBMIT_DATA,
templateForm.asFormDataString(
CoreAttributeConstants.A_SUBMIT_DATA,
templateForm.formData(
getLoginUsername(),
getLoginPassword()));
submitCuri.getData().put(CoreAttributeConstants.A_SUBMIT_ENCTYPE,
templateForm.getEnctype());
submitCuri.setSchedulingDirective(SchedulingConstants.HIGH);
submitCuri.setForceFetch(true);
curi.getOutLinks().add(submitCuri);
modules/src/main/java/org/archive/modules/forms/HTMLForm.java
+29 -41
View File
@@ -23,9 +23,7 @@ import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
import org.apache.commons.httpclient.NameValuePair;
import org.apache.commons.lang.StringUtils;
import org.archive.util.TextUtils;
/**
* Simple representation of a discovered HTML Form.
@@ -47,13 +45,14 @@ public class HTMLForm {
return str;
}
}
String method;
String action;
List<FormInput> allInputs = new ArrayList<FormInput>();
List<FormInput> candidateUsernameInputs = new ArrayList<FormInput>();
List<FormInput> candidatePasswordInputs = new ArrayList<FormInput>();
protected String method;
protected String action;
protected String enctype;
protected List<FormInput> allInputs = new ArrayList<FormInput>();
protected List<FormInput> candidateUsernameInputs = new ArrayList<FormInput>();
protected List<FormInput> candidatePasswordInputs = new ArrayList<FormInput>();
/**
* Add a discovered INPUT, tracking it as potential
@@ -99,11 +98,19 @@ public class HTMLForm {
public String getAction() {
return action;
}
public void setAction(String action) {
this.action = action;
}
public String getEnctype() {
return enctype;
}
public void setEnctype(String enctype) {
this.enctype = enctype;
}
/**
* For now, we consider a POST form with only 1 password
* field and 1 potential username field (type text or email)
@@ -117,48 +124,29 @@ public class HTMLForm {
&& candidatePasswordInputs.size() == 1;
}
/**
* Create the NameValuePair array expected by HttpClient, merging
* username and password into the appropriate value slots.
*
* @param username
* @param password
* @return
* @deprecated specific to a particular FetchHTTP implementation based on commons-httpclient, use {@link #asFormDataString(String, String)}
*/
public NameValuePair[] asHttpClientDataWith(String username, String password) {
ArrayList<NameValuePair> data = new ArrayList<NameValuePair>(allInputs.size());
for (FormInput input : allInputs) {
if(input == candidateUsernameInputs.get(0)) {
data.add(new NameValuePair(input.name,username));
} else if(input == candidatePasswordInputs.get(0)) {
data.add(new NameValuePair(input.name,password));
} else if (StringUtils.isNotEmpty(input.name) && StringUtils.isNotEmpty(input.value)) {
data.add(new NameValuePair(input.name,input.value));
}
public static class NameValue {
public String name, value;
public NameValue(String name, String value) {
this.name = name;
this.value = value;
}
return data.toArray(new NameValuePair[data.size()]);
}
public String asFormDataString(String username, String password) {
List<String> nameVals = new LinkedList<String>();
public LinkedList<NameValue> formData(String username, String password) {
LinkedList<NameValue> nameVals = new LinkedList<NameValue>();
for (FormInput input : allInputs) {
if(input == candidateUsernameInputs.get(0)) {
nameVals.add(TextUtils.urlEscape(input.name) + "=" + TextUtils.urlEscape(username));
if (input == candidateUsernameInputs.get(0)) {
nameVals.add(new NameValue(input.name, username));
} else if(input == candidatePasswordInputs.get(0)) {
nameVals.add(TextUtils.urlEscape(input.name) + "=" + TextUtils.urlEscape(password));
nameVals.add(new NameValue(input.name, password));
} else if (StringUtils.isNotEmpty(input.name)
&& StringUtils.isNotEmpty(input.value)
&& (!"radio".equalsIgnoreCase(input.type)
&& !"checkbox".equals(input.type) || input.checked)) {
nameVals.add(TextUtils.urlEscape(input.name) + "="
+ TextUtils.urlEscape(input.value));
nameVals.add(new NameValue(input.name, input.value));
}
}
return StringUtils.join(nameVals, '&');
return nameVals;
}
public String toString() {
modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTests.java
+7 -11
View File
@@ -53,10 +53,7 @@ import javax.net.ssl.SSLException;
import org.apache.commons.httpclient.URIException;
import org.apache.commons.io.IOUtils;
import org.apache.http.NameValuePair;
import org.apache.http.NoHttpResponseException;
import org.apache.http.client.utils.URLEncodedUtils;
import org.apache.http.message.BasicNameValuePair;
import org.archive.httpclient.ConfigurableX509TrustManager.TrustLevel;
import org.archive.modules.CoreAttributeConstants;
import org.archive.modules.CrawlMetadata;
@@ -65,6 +62,7 @@ import org.archive.modules.CrawlURI.FetchType;
import org.archive.modules.ProcessorTestBase;
import org.archive.modules.credential.HttpAuthenticationCredential;
import org.archive.modules.deciderules.RejectDecideRule;
import org.archive.modules.forms.HTMLForm.NameValue;
import org.archive.modules.recrawl.FetchHistoryProcessor;
import org.archive.modules.revisit.ServerNotModifiedRevisit;
import org.archive.net.UURI;
@@ -856,16 +854,14 @@ public class FetchHTTPTests extends ProcessorTestBase {
CrawlURI curi = makeCrawlURI("http://localhost:7777/");
curi.setFetchType(FetchType.HTTP_POST);
List<NameValuePair> params = new LinkedList<NameValuePair>();
params.add(new BasicNameValuePair("name1", "value1"));
params.add(new BasicNameValuePair("name1", "value2"));
params.add(new BasicNameValuePair("funky name 2", "whoa crazy\t && 🍺 🍻 \n crazier \rooo"));
String submitData = URLEncodedUtils.format(params, "UTF-8");
assertEquals("name1=value1&name1=value2&funky+name+2=whoa+crazy%09+%26%26+%F0%9F%8D%BA+%F0%9F%8D%BB+%0A+crazier+%0Dooo", submitData);
List<NameValue> params = new LinkedList<NameValue>();
params.add(new NameValue("name1", "value1"));
params.add(new NameValue("name1", "value2"));
params.add(new NameValue("funky name 2", "whoa crazy\t && 🍺 🍻 \n crazier \rooo"));
curi.getData().put(CoreAttributeConstants.A_SUBMIT_DATA, params);
curi.getData().put(CoreAttributeConstants.A_SUBMIT_DATA, submitData);
fetcher().process(curi);
assertTrue(httpRequestString(curi).startsWith("POST / HTTP/1.0\r\n"));
assertTrue(httpRequestString(curi).endsWith("\r\n\r\nname1=value1&name1=value2&funky+name+2=whoa+crazy%09+%26%26+%F0%9F%8D%BA+%F0%9F%8D%BB+%0A+crazier+%0Dooo"));
assertEquals(FetchType.HTTP_POST, curi.getFetchType());
modules/src/test/java/org/archive/modules/forms/FormLoginProcessorTest.java
+82 -6
View File
@@ -1,8 +1,8 @@
/*
* This file is part of the Heritrix web crawler (crawler.archive.org).
*
* Licensed to the Internet Archive (IA) by one or more individual
* contributors.
* Licensed to the Internet Archive (IA) by one or more individual
* contributors.
*
* The IA licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
@@ -21,13 +21,32 @@ package org.archive.modules.forms;
import static org.archive.modules.CoreAttributeConstants.A_WARC_RESPONSE_HEADERS;
import org.archive.modules.CoreAttributeConstants;
import java.io.ByteArrayOutputStream;
import org.apache.commons.httpclient.URIException;
import org.apache.commons.lang.StringEscapeUtils;
import org.apache.http.HttpRequest;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.util.EntityUtils;
import org.archive.modules.CrawlURI;
import org.archive.modules.CrawlURI.FetchType;
import org.archive.modules.ProcessorTestBase;
import org.archive.modules.fetcher.BasicExecutionAwareEntityEnclosingRequest;
import org.archive.modules.fetcher.FetchHTTP;
import org.archive.modules.fetcher.FetchHTTPRequest;
import org.archive.modules.fetcher.FetchHTTPTests;
public class FormLoginProcessorTest extends ProcessorTestBase {
static class FetchHTTPRequestSpy extends FetchHTTPRequest {
public FetchHTTPRequestSpy(FetchHTTP fetcher, CrawlURI curi) throws URIException {
super(fetcher, curi);
}
public HttpRequest getRequest() {
return request;
}
}
public void testNoFormLogin() throws Exception {
CrawlURI curi = makeCrawlURI("http://example.com/");
@@ -38,7 +57,7 @@ public class FormLoginProcessorTest extends ProcessorTestBase {
p.process(curi);
assertEquals(1, curi.getDataList(A_WARC_RESPONSE_HEADERS).size());
assertEquals(1, curi.getDataList(A_WARC_RESPONSE_HEADERS).size());
assertEquals("WARC-Simple-Form-Province-Status: 0,0,http://(com,example,)", curi.getDataList(A_WARC_RESPONSE_HEADERS).get(0));
}
@@ -48,6 +67,7 @@ public class FormLoginProcessorTest extends ProcessorTestBase {
HTMLForm form = new HTMLForm();
form.addField("text", "username-form-field", "");
form.addField("password", "password-form-field", "");
form.addField("hidden", "crazy🐒monkey", "úhóh");
form.setMethod("post");
form.setAction("/login");
curi.getDataList(ExtractorHTMLForms.A_HTML_FORM_OBJECTS).add(form);
@@ -57,6 +77,44 @@ public class FormLoginProcessorTest extends ProcessorTestBase {
p.setLoginPassword("********");
p.setApplicableSurtPrefix("http://(com,example,)");
p.process(curi);
assertEquals(1, curi.getDataList(A_WARC_RESPONSE_HEADERS).size());
assertEquals("WARC-Simple-Form-Province-Status: 0,0,http://(com,example,)", curi.getDataList(A_WARC_RESPONSE_HEADERS).get(0));
assertTrue(curi.getAnnotations().contains("submit:/login"));
assertEquals(1, curi.getOutLinks().size());
CrawlURI submitCuri = curi.getOutLinks().toArray(new CrawlURI[0])[0];
assertEquals("http://example.com/login", submitCuri.toString());
assertEquals(FetchType.HTTP_POST, submitCuri.getFetchType());
FetchHTTPRequestSpy reqSpy = new FetchHTTPRequestSpy(
FetchHTTPTests.newTestFetchHttp(getClass().getName()),
submitCuri);
assertTrue(reqSpy.getRequest() instanceof BasicExecutionAwareEntityEnclosingRequest);
BasicExecutionAwareEntityEnclosingRequest req = (BasicExecutionAwareEntityEnclosingRequest) reqSpy.getRequest();
assertTrue(req.getEntity() instanceof UrlEncodedFormEntity);
assertTrue(req.toString().startsWith("POST /login"));
assertEquals("username-form-field=jdoe&password-form-field=********&crazy%F0%9F%90%92monkey=%C3%BAh%C3%B3h",
EntityUtils.toString(req.getEntity()));
}
public void testMultipartFormLogin() throws Exception {
CrawlURI curi = makeCrawlURI("http://example.com/");
HTMLForm form = new HTMLForm();
form.addField("text", "username-form-field", "");
form.addField("password", "password-form-field", "");
form.addField("hidden", "crazy🐒monkey", "úhóh");
form.setMethod("post");
form.setAction("/login");
form.setEnctype("multipart/form-data");
curi.getDataList(ExtractorHTMLForms.A_HTML_FORM_OBJECTS).add(form);
FormLoginProcessor p = (FormLoginProcessor) makeModule();
p.setLoginUsername("jdoe");
p.setLoginPassword("********");
p.setApplicableSurtPrefix("http://(com,example,)");
p.process(curi);
assertEquals(1, curi.getDataList(A_WARC_RESPONSE_HEADERS).size());
assertEquals("WARC-Simple-Form-Province-Status: 0,0,http://(com,example,)", curi.getDataList(A_WARC_RESPONSE_HEADERS).get(0));
@@ -66,7 +124,25 @@ public class FormLoginProcessorTest extends ProcessorTestBase {
CrawlURI submitCuri = curi.getOutLinks().toArray(new CrawlURI[0])[0];
assertEquals("http://example.com/login", submitCuri.toString());
assertEquals(FetchType.HTTP_POST, submitCuri.getFetchType());
String queryString = (String) submitCuri.getData().get(CoreAttributeConstants.A_SUBMIT_DATA);
assertEquals("username-form-field=jdoe&password-form-field=********", queryString);
FetchHTTPRequestSpy reqSpy = new FetchHTTPRequestSpy(
FetchHTTPTests.newTestFetchHttp(getClass().getName()),
submitCuri);
assertTrue(reqSpy.getRequest() instanceof BasicExecutionAwareEntityEnclosingRequest);
BasicExecutionAwareEntityEnclosingRequest req = (BasicExecutionAwareEntityEnclosingRequest) reqSpy.getRequest();
assertEquals("org.apache.http.entity.mime.MultipartFormEntity", req.getEntity().getClass().getName());
assertTrue(req.toString().startsWith("POST /login"));
ByteArrayOutputStream baos = new ByteArrayOutputStream();
req.getEntity().writeTo(baos);
System.out.println(StringEscapeUtils.escapeJava(baos.toString("UTF-8")));
// --e5XWkWxQ2EXBQAEPQ7n3yyvv9bI-8YIHok\r\nContent-Disposition: form-data; name=\"username-form-field\"\r\n\r\njdoe\r\n--e5XWkWxQ2EXBQAEPQ7n3yyvv9bI-8YIHok\r\nContent-Disposition: form-data; name=\"password-form-field\"\r\n\r\n********\r\n--e5XWkWxQ2EXBQAEPQ7n3yyvv9bI-8YIHok\r\nContent-Disposition: form-data; name=\"crazy&#128018;monkey\"\r\n\r\n&#250;h&#243;h\r\n--e5XWkWxQ2EXBQAEPQ7n3yyvv9bI-8YIHok--\r\n
assertTrue(baos.toString("ascii").matches("--([a-zA-Z0-9_-]{30,41})\r\nContent-Disposition: form-data; name=\"username-form-field\"\r\n\r\njdoe\r\n--\\1\r\nContent-Disposition: form-data; name=\"password-form-field\"\r\n\r\n\\*\\*\\*\\*\\*\\*\\*\\*\r\n--\\1\r\nContent-Disposition: form-data; name=\"crazy&#128018;monkey\"\r\n\r\n&#250;h&#243;h\r\n--\\1--\r\n"));
}
public void testEscapeForMultipart() {
assertEquals("abcd", FetchHTTPRequest.escapeForMultipart("abcd"));
assertEquals("abcd&#233;", FetchHTTPRequest.escapeForMultipart("abcdé"));
assertEquals("abcd&#233;&#128556;", FetchHTTPRequest.escapeForMultipart("abcdé😬"));
}
}
pom.xml
+5
View File
@@ -124,6 +124,11 @@ http://maven.apache.org/guides/mini/guide-m1-m2.html
<artifactId>httpclient</artifactId>
<version>4.3.6</version>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpmime</artifactId>
<version>4.3.6</version>
</dependency>
</dependencies>
</dependencyManagement>