Save enctype of form in submission curi

This commit is contained in:
Hunter Stern
2016-01-14 12:05:38 -08:00
committed by Noah Levitt
parent 34cffd512a
commit ad329ceea8
3 changed files with 10 additions and 5 deletions
@@ -122,6 +122,9 @@ public interface CoreAttributeConstants {
// FORMS support - a persistent member (survives frontier enqueue/dequeue/retries)
public static final String A_SUBMIT_DATA = "submit-data";
//the encoding type of the form
public static final String A_SUBMIT_DATA_ENCTYPE = "submit-data-enctype";
// arbitrary additions to WARC response record headers
public static final String A_WARC_RESPONSE_HEADERS = "warc-response-headers";
@@ -189,11 +189,8 @@ class FetchHTTPRequest {
this.request = postRequest;
String submitData = (String) curi.getData().get(CoreAttributeConstants.A_SUBMIT_DATA);
if (submitData != null) {
if (curi.getDataList(ExtractorHTMLForms.A_HTML_FORM_OBJECTS) != null
&& "multipart/form-data"
.equals(((HTMLForm) curi.getDataList(
ExtractorHTMLForms.A_HTML_FORM_OBJECTS).get(0))
.getEnctype())) {
if ("multipart/form-data".equals(curi.getData().get(
CoreAttributeConstants.A_SUBMIT_DATA_ENCTYPE))) {
HttpEntity multipartFormEntity = MultipartEntityBuilder
.create()
.addTextBody("submitData", submitData,
@@ -270,6 +270,11 @@ public class FormLoginProcessor extends Processor implements Checkpointable {
templateForm.asFormDataString(
getLoginUsername(),
getLoginPassword()));
submitCuri.getData().put(
CoreAttributeConstants.A_SUBMIT_DATA_ENCTYPE,
((HTMLForm) curi.getDataList(
ExtractorHTMLForms.A_HTML_FORM_OBJECTS).get(0))
.getEnctype());
submitCuri.setSchedulingDirective(SchedulingConstants.HIGH);
submitCuri.setForceFetch(true);
curi.getOutLinks().add(submitCuri);