diff --git a/engine/src/test/java/org/archive/crawler/selftest/FormLoginSelfTest.java b/engine/src/test/java/org/archive/crawler/selftest/FormLoginSelfTest.java new file mode 100644 index 00000000..b5c83687 --- /dev/null +++ b/engine/src/test/java/org/archive/crawler/selftest/FormLoginSelfTest.java @@ -0,0 +1,103 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.crawler.selftest; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; + +import org.mortbay.jetty.Handler; +import org.mortbay.jetty.Server; +import org.mortbay.jetty.bio.SocketConnector; +import org.mortbay.jetty.handler.DefaultHandler; +import org.mortbay.jetty.handler.HandlerList; +import org.mortbay.jetty.handler.ResourceHandler; +import org.mortbay.jetty.servlet.ServletHandler; +import org.mortbay.jetty.servlet.ServletHolder; + +/** + * Test form-based authentication + * + * @contributor stack + * @contributor gojomo + */ +public class FormLoginSelfTest + extends SelfTestBase +{ + /** + * Files to find as a list. + */ + final private static Set EXPECTED = Collections.unmodifiableSet( + new HashSet(Arrays.asList(new String[] { + "index.html", "login/login.html", "success.html", "robots.txt", "favicon.ico" + }))); + + @Override + protected void verify() throws Exception { + Set found = this.filesInArcs(); + assertEquals("wrong files in ARCs",EXPECTED,found); + } + + @Override + protected void startHttpServer() throws Exception { + Server server = new Server(); + + SocketConnector sc = new SocketConnector(); + sc.setHost("127.0.0.1"); + sc.setPort(7777); + server.addConnector(sc); + ResourceHandler rhandler = new ResourceHandler(); + rhandler.setResourceBase(getSrcHtdocs().getAbsolutePath()); + + ServletHandler servletHandler = new ServletHandler(); + + HandlerList handlers = new HandlerList(); + handlers.setHandlers(new Handler[] { + rhandler, + servletHandler, + new DefaultHandler() }); + server.setHandler(handlers); + + ServletHolder holder = new ServletHolder(new FormAuthServlet()); + servletHandler.addServletWithMapping(holder, "/login/*"); + + this.httpServer = server; + this.httpServer.start(); + } + + protected String getSeedsString() { + return "http://127.0.0.1:7777/index.html"; + } + + @Override + protected String changeGlobalConfig(String config) { + String formLoginConfig = + " \n" + + " \n" + + " \n" + + " \n" + + " \n"; + config = config.replace("", formLoginConfig); + return super.changeGlobalConfig(config); + } + +} + diff --git a/engine/testdata/selftest/FormLoginSelfTest/htdocs/failure.html b/engine/testdata/selftest/FormLoginSelfTest/htdocs/failure.html new file mode 100644 index 00000000..891cbb5a --- /dev/null +++ b/engine/testdata/selftest/FormLoginSelfTest/htdocs/failure.html @@ -0,0 +1,9 @@ +index + +Link 1 + + Link 2 + + Link 3 + +Secure \ No newline at end of file diff --git a/engine/testdata/selftest/FormLoginSelfTest/htdocs/index.html b/engine/testdata/selftest/FormLoginSelfTest/htdocs/index.html new file mode 100644 index 00000000..ed045598 --- /dev/null +++ b/engine/testdata/selftest/FormLoginSelfTest/htdocs/index.html @@ -0,0 +1,5 @@ +
+ + +
+ diff --git a/engine/testdata/selftest/FormLoginSelfTest/htdocs/link1.html b/engine/testdata/selftest/FormLoginSelfTest/htdocs/link1.html new file mode 100644 index 00000000..e9521a25 --- /dev/null +++ b/engine/testdata/selftest/FormLoginSelfTest/htdocs/link1.html @@ -0,0 +1,7 @@ +index + +Link 1 + + Link 2 + + Link 3 \ No newline at end of file diff --git a/engine/testdata/selftest/FormLoginSelfTest/htdocs/link2.html b/engine/testdata/selftest/FormLoginSelfTest/htdocs/link2.html new file mode 100644 index 00000000..e9521a25 --- /dev/null +++ b/engine/testdata/selftest/FormLoginSelfTest/htdocs/link2.html @@ -0,0 +1,7 @@ +index + +Link 1 + + Link 2 + + Link 3 \ No newline at end of file diff --git a/engine/testdata/selftest/FormLoginSelfTest/htdocs/link3.html b/engine/testdata/selftest/FormLoginSelfTest/htdocs/link3.html new file mode 100644 index 00000000..e9521a25 --- /dev/null +++ b/engine/testdata/selftest/FormLoginSelfTest/htdocs/link3.html @@ -0,0 +1,7 @@ +index + +Link 1 + + Link 2 + + Link 3 \ No newline at end of file diff --git a/engine/testdata/selftest/FormLoginSelfTest/htdocs/success.html b/engine/testdata/selftest/FormLoginSelfTest/htdocs/success.html new file mode 100644 index 00000000..52fc82b8 --- /dev/null +++ b/engine/testdata/selftest/FormLoginSelfTest/htdocs/success.html @@ -0,0 +1,2 @@ +index + diff --git a/engine/testdata/selftest/conf/selftest-crawler-beans.cxml b/engine/testdata/selftest/conf/selftest-crawler-beans.cxml index 91beb82e..55a20702 100644 --- a/engine/testdata/selftest/conf/selftest-crawler-beans.cxml +++ b/engine/testdata/selftest/conf/selftest-crawler-beans.cxml @@ -124,6 +124,7 @@ crawlController.pauseAtStart=false + diff --git a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTPRequest.java b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTPRequest.java index 7f48ae9f..b8c2c786 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTPRequest.java +++ b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTPRequest.java @@ -64,6 +64,7 @@ import org.apache.http.client.config.RequestConfig; import org.apache.http.client.entity.UrlEncodedFormEntity; import org.apache.http.client.methods.AbstractExecutionAwareRequest; import org.apache.http.client.protocol.HttpClientContext; +import org.apache.http.client.utils.URLEncodedUtils; import org.apache.http.config.ConnectionConfig; import org.apache.http.config.MessageConstraints; import org.apache.http.config.Registry; @@ -79,6 +80,8 @@ import org.apache.http.conn.socket.PlainConnectionSocketFactory; import org.apache.http.conn.ssl.AllowAllHostnameVerifier; import org.apache.http.conn.ssl.SSLConnectionSocketFactory; import org.apache.http.entity.ContentLengthStrategy; +import org.apache.http.entity.ContentType; +import org.apache.http.entity.StringEntity; import org.apache.http.impl.DefaultBHttpClientConnection; import org.apache.http.impl.client.BasicAuthCache; import org.apache.http.impl.client.BasicCredentialsProvider; @@ -93,6 +96,7 @@ import org.apache.http.message.BasicNameValuePair; import org.apache.http.protocol.HTTP; import org.apache.http.protocol.HttpContext; import org.apache.http.util.Args; +import org.archive.modules.CoreAttributeConstants; import org.archive.modules.CrawlURI; import org.archive.modules.CrawlURI.FetchType; import org.archive.modules.Processor; @@ -175,8 +179,16 @@ class FetchHTTPRequest { } if (curi.getFetchType() == FetchType.HTTP_POST) { - this.request = new BasicExecutionAwareEntityEnclosingRequest("POST", - requestLineUri, httpVersion); + BasicExecutionAwareEntityEnclosingRequest postRequest = new BasicExecutionAwareEntityEnclosingRequest( + "POST", requestLineUri, httpVersion); + this.request = postRequest; + String submitData = (String) curi.getData().get(CoreAttributeConstants.A_SUBMIT_DATA); + if (submitData != null) { + // XXX brittle, doesn't support multipart form data etc + ContentType contentType = ContentType.create(URLEncodedUtils.CONTENT_TYPE, "UTF-8"); + StringEntity formEntity = new StringEntity(submitData, contentType); + postRequest.setEntity(formEntity); + } } else { this.request = new BasicExecutionAwareRequest("GET", requestLineUri, httpVersion); @@ -196,7 +208,7 @@ class FetchHTTPRequest { this.addedCredentials = populateTargetCredential(); populateHttpProxyCredential(); } - + protected void configureRequestHeaders() { if (fetcher.getAcceptCompression()) { request.addHeader("Accept-Encoding", "gzip,deflate"); diff --git a/modules/src/main/java/org/archive/modules/forms/FormLoginProcessor.java b/modules/src/main/java/org/archive/modules/forms/FormLoginProcessor.java index f2292f77..3ea7dc90 100644 --- a/modules/src/main/java/org/archive/modules/forms/FormLoginProcessor.java +++ b/modules/src/main/java/org/archive/modules/forms/FormLoginProcessor.java @@ -21,14 +21,11 @@ package org.archive.modules.forms; import static org.archive.modules.CoreAttributeConstants.A_WARC_RESPONSE_HEADERS; -import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ExecutionException; import java.util.concurrent.atomic.AtomicLong; import java.util.logging.Level; import java.util.logging.Logger; -import com.google.common.cache.CacheBuilder; -import com.google.common.cache.CacheLoader; -import com.google.common.cache.LoadingCache; import org.apache.commons.httpclient.URIException; import org.apache.commons.lang.StringUtils; import org.archive.checkpointing.Checkpointable; @@ -45,6 +42,10 @@ import org.json.JSONException; import org.json.JSONObject; import org.springframework.beans.factory.annotation.Autowired; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; + /** * A step, post-ExtractorHTMLForms, where a followup CrawlURI to * attempt a form submission may be synthesized. @@ -119,24 +120,24 @@ public class FormLoginProcessor extends Processor implements Checkpointable { Logger.getLogger(FormLoginProcessor.class.getName()); // formProvince (String) -> count - ConcurrentMap eligibleFormsSeenCount = + protected LoadingCache eligibleFormsSeenCount = CacheBuilder.newBuilder() .build( new CacheLoader() { public AtomicLong load(String arg0) { return new AtomicLong(0L); } - }).asMap(); + }); // formProvince (String) -> count - ConcurrentMap eligibleFormsAttemptsCount = + protected LoadingCache eligibleFormsAttemptsCount = CacheBuilder.newBuilder() .build( new CacheLoader() { public AtomicLong load(String arg0) { return new AtomicLong(0L); } - }).asMap(); + }); /** * SURT prefix against which configured username/password is @@ -219,13 +220,17 @@ public class FormLoginProcessor extends Processor implements Checkpointable { for( Object formObject : curi.getDataList(ExtractorHTMLForms.A_HTML_FORM_OBJECTS)) { HTMLForm form = (HTMLForm) formObject; if(form.seemsLoginForm()) { - eligibleFormsSeenCount.get(formProvince).incrementAndGet(); - if(eligibleFormsAttemptsCount.get(formProvince).get()<1) { - eligibleFormsAttemptsCount.get(formProvince).incrementAndGet(); - createFormSubmissionAttempt(curi,form,formProvince); - } else { - // note decline-to-submit: in volume, may be signal of failed first login - curi.getAnnotations().add("nosubmit:"+submitStatusFor(formProvince)); + try { + eligibleFormsSeenCount.get(formProvince).incrementAndGet(); + if(eligibleFormsAttemptsCount.get(formProvince).get()<1) { + eligibleFormsAttemptsCount.get(formProvince).incrementAndGet(); + createFormSubmissionAttempt(curi,form,formProvince); + } else { + // note decline-to-submit: in volume, may be signal of failed first login + curi.getAnnotations().add("nosubmit:"+submitStatusFor(formProvince)); + } + } catch (ExecutionException e) { + throw new RuntimeException(e); // can't happen? } return; } @@ -260,10 +265,9 @@ public class FormLoginProcessor extends Processor implements Checkpointable { submitCuri.setFetchType(FetchType.HTTP_POST); submitCuri.getData().put( CoreAttributeConstants.A_SUBMIT_DATA, - templateForm.asHttpClientDataWith( + templateForm.asFormDataString( getLoginUsername(), getLoginPassword())); - //submitCuri.setSchedulingDirective(Math.max(curi.getSchedulingDirective()-1, 0)); submitCuri.setSchedulingDirective(SchedulingConstants.HIGH); submitCuri.setForceFetch(true); curi.getOutCandidates().add(submitCuri); @@ -278,9 +282,13 @@ public class FormLoginProcessor extends Processor implements Checkpointable { } protected String submitStatusFor(String formProvince) { - return eligibleFormsAttemptsCount.get(formProvince).get() - +","+eligibleFormsSeenCount.get(formProvince).get() - +","+formProvince; + try { + return eligibleFormsAttemptsCount.get(formProvince).get() + +","+eligibleFormsSeenCount.get(formProvince).get() + +","+formProvince; + } catch (ExecutionException e) { + throw new RuntimeException(e); + } } @Override @@ -295,10 +303,10 @@ public class FormLoginProcessor extends Processor implements Checkpointable { protected void fromCheckpointJson(JSONObject json) throws JSONException { super.fromCheckpointJson(json); JSONUtils.putAllAtomicLongs( - eligibleFormsAttemptsCount, + eligibleFormsAttemptsCount.asMap(), json.getJSONObject("eligibleFormsAttemptsCount")); JSONUtils.putAllAtomicLongs( - eligibleFormsSeenCount, + eligibleFormsSeenCount.asMap(), json.getJSONObject("eligibleFormsSeenCount")); } } diff --git a/modules/src/main/java/org/archive/modules/forms/HTMLForm.java b/modules/src/main/java/org/archive/modules/forms/HTMLForm.java index f2719e22..4c2c2147 100644 --- a/modules/src/main/java/org/archive/modules/forms/HTMLForm.java +++ b/modules/src/main/java/org/archive/modules/forms/HTMLForm.java @@ -20,10 +20,12 @@ package org.archive.modules.forms; import java.util.ArrayList; +import java.util.LinkedList; import java.util.List; import org.apache.commons.httpclient.NameValuePair; import org.apache.commons.lang.StringUtils; +import org.archive.util.TextUtils; /** * Simple representation of a discovered HTML Form. @@ -104,6 +106,7 @@ public class HTMLForm { * @param username * @param password * @return + * @deprecated specific to a particular FetchHTTP implementation based on commons-httpclient, use {@link #asFormDataString(String, String)} */ public NameValuePair[] asHttpClientDataWith(String username, String password) { ArrayList data = new ArrayList(allInputs.size()); @@ -119,6 +122,22 @@ public class HTMLForm { } return data.toArray(new NameValuePair[data.size()]); } + + public String asFormDataString(String username, String password) { + List nameVals = new LinkedList(); + + for (FormInput input : allInputs) { + if(input == candidateUsernameInputs.get(0)) { + nameVals.add(TextUtils.urlEscape(input.name) + "=" + TextUtils.urlEscape(username)); + } else if(input == candidatePasswordInputs.get(0)) { + nameVals.add(TextUtils.urlEscape(input.name) + "=" + TextUtils.urlEscape(password)); + } else if (StringUtils.isNotEmpty(input.name) && StringUtils.isNotEmpty(input.value)) { + nameVals.add(TextUtils.urlEscape(input.name) + "=" + TextUtils.urlEscape(input.value)); + } + } + + return StringUtils.join(nameVals, '&'); + } public String toString() { StringBuilder sb = new StringBuilder(); diff --git a/commons/src/main/java/org/archive/state/ModuleTestBase.java b/modules/src/main/java/org/archive/state/ModuleTestBase.java similarity index 89% rename from commons/src/main/java/org/archive/state/ModuleTestBase.java rename to modules/src/main/java/org/archive/state/ModuleTestBase.java index 835d6ada..884daa4b 100644 --- a/commons/src/main/java/org/archive/state/ModuleTestBase.java +++ b/modules/src/main/java/org/archive/state/ModuleTestBase.java @@ -20,12 +20,19 @@ package org.archive.state; import java.io.File; +import java.io.IOException; import java.io.Serializable; import java.util.Arrays; import junit.framework.TestCase; +import org.apache.commons.httpclient.URIException; import org.apache.commons.lang.SerializationUtils; +import org.archive.modules.CrawlURI; +import org.archive.net.UURI; +import org.archive.net.UURIFactory; +import org.archive.util.Recorder; +import org.archive.util.TmpDirTestCase; /** @@ -213,4 +220,25 @@ public abstract class ModuleTestBase extends TestCase { throw t; } } + + + protected Recorder getRecorder() throws IOException { + if (Recorder.getHttpRecorder() == null) { + Recorder httpRecorder = new Recorder(TmpDirTestCase.tmpDir(), + getClass().getName(), 16 * 1024, 512 * 1024); + Recorder.setHttpRecorder(httpRecorder); + } + + return Recorder.getHttpRecorder(); + } + + + protected CrawlURI makeCrawlURI(String uri) throws URIException, + IOException { + UURI uuri = UURIFactory.getInstance(uri); + CrawlURI curi = new CrawlURI(uuri); + curi.setSeed(true); + curi.setRecorder(getRecorder()); + return curi; + } } diff --git a/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTests.java b/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTests.java index 5602c650..385258f1 100644 --- a/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTests.java +++ b/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTests.java @@ -43,6 +43,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedList; import java.util.List; import java.util.Set; import java.util.logging.Level; @@ -52,8 +53,12 @@ import javax.net.ssl.SSLException; import org.apache.commons.httpclient.URIException; import org.apache.commons.io.IOUtils; +import org.apache.http.NameValuePair; import org.apache.http.NoHttpResponseException; +import org.apache.http.client.utils.URLEncodedUtils; +import org.apache.http.message.BasicNameValuePair; import org.archive.httpclient.ConfigurableX509TrustManager.TrustLevel; +import org.archive.modules.CoreAttributeConstants; import org.archive.modules.CrawlMetadata; import org.archive.modules.CrawlURI; import org.archive.modules.CrawlURI.FetchType; @@ -62,9 +67,6 @@ import org.archive.modules.credential.HttpAuthenticationCredential; import org.archive.modules.deciderules.RejectDecideRule; import org.archive.modules.recrawl.FetchHistoryProcessor; import org.archive.net.UURI; -import org.archive.net.UURIFactory; -import org.archive.util.Recorder; -import org.archive.util.TmpDirTestCase; import org.jboss.netty.handler.codec.http.HttpRequest; import org.littleshoot.proxy.DefaultHttpProxyServer; import org.littleshoot.proxy.HttpFilter; @@ -94,25 +96,6 @@ public class FetchHTTPTests extends ProcessorTestBase { return getClass().getName(); } - protected Recorder getRecorder() throws IOException { - if (Recorder.getHttpRecorder() == null) { - Recorder httpRecorder = new Recorder(TmpDirTestCase.tmpDir(), - getClass().getName(), 16 * 1024, 512 * 1024); - Recorder.setHttpRecorder(httpRecorder); - } - - return Recorder.getHttpRecorder(); - } - - protected CrawlURI makeCrawlURI(String uri) throws URIException, - IOException { - UURI uuri = UURIFactory.getInstance(uri); - CrawlURI curi = new CrawlURI(uuri); - curi.setSeed(true); - curi.setRecorder(getRecorder()); - return curi; - } - protected void runDefaultChecks(CrawlURI curi, String... exclusionsArray) throws IOException, UnsupportedEncodingException { @@ -130,7 +113,9 @@ public class FetchHTTPTests extends ProcessorTestBase { if (!exclusions.contains("hostHeader")) { assertTrue(requestString.contains("Host: localhost:7777\r\n")); } - assertTrue(requestString.endsWith("\r\n\r\n")); + if (!exclusions.contains("trailingCRLFCRLF")) { + assertTrue(requestString.endsWith("\r\n\r\n")); + } // check sizes assertEquals(DEFAULT_PAYLOAD_STRING.length(), curi.getContentLength()); @@ -147,7 +132,9 @@ public class FetchHTTPTests extends ProcessorTestBase { if (!exclusions.contains("fetchStatus")) { assertTrue(curi.getFetchStatus() == 200); } - assertTrue(curi.getFetchType() == FetchType.HTTP_GET); + if (!exclusions.contains("fetchTypeGET")) { + assertTrue(curi.getFetchType() == FetchType.HTTP_GET); + } // check message body, i.e. "raw, possibly chunked-transfer-encoded message contents not including the leading headers" assertEquals(DEFAULT_PAYLOAD_STRING, messageBodyString(curi)); @@ -703,8 +690,9 @@ public class FetchHTTPTests extends ProcessorTestBase { @Override public void run() { + ServerSocket listeningSocket = null; try { - ServerSocket listeningSocket = new ServerSocket(listenPort, 0, Inet4Address.getByName(listenAddress)); + listeningSocket = new ServerSocket(listenPort, 0, Inet4Address.getByName(listenAddress)); listeningSocket.setSoTimeout(600); while (!isTimeToBeDone) { try { @@ -718,6 +706,12 @@ public class FetchHTTPTests extends ProcessorTestBase { // logger.warning("caught exception: " + e); } finally { // logger.info("all done suckers"); + if (listeningSocket != null) { + try { + listeningSocket.close(); + } catch (IOException e) { + } + } } } @@ -829,6 +823,26 @@ public class FetchHTTPTests extends ProcessorTestBase { fetcher().process(curi); assertTrue(httpRequestString(curi).contains("Host: example.com\r\n")); } + + public void testHttpPost() throws Exception { + CrawlURI curi = makeCrawlURI("http://localhost:7777/"); + curi.setFetchType(FetchType.HTTP_POST); + + List params = new LinkedList(); + params.add(new BasicNameValuePair("name1", "value1")); + params.add(new BasicNameValuePair("name1", "value2")); + params.add(new BasicNameValuePair("funky name 2", "whoa crazy\t && 🍺 🍻 \n crazier \rooo")); + String submitData = URLEncodedUtils.format(params, "UTF-8"); + assertEquals("name1=value1&name1=value2&funky+name+2=whoa+crazy%09+%26%26+%F0%9F%8D%BA+%F0%9F%8D%BB+%0A+crazier+%0Dooo", submitData); + + curi.getData().put(CoreAttributeConstants.A_SUBMIT_DATA, submitData); + fetcher().process(curi); + + assertTrue(httpRequestString(curi).startsWith("POST / HTTP/1.0\r\n")); + assertTrue(httpRequestString(curi).endsWith("\r\n\r\nname1=value1&name1=value2&funky+name+2=whoa+crazy%09+%26%26+%F0%9F%8D%BA+%F0%9F%8D%BB+%0A+crazier+%0Dooo")); + assertEquals(FetchType.HTTP_POST, curi.getFetchType()); + runDefaultChecks(curi, "requestLine", "trailingCRLFCRLF", "fetchTypeGET"); + } @Override protected FetchHTTP makeModule() throws IOException { diff --git a/modules/src/test/java/org/archive/modules/forms/FormLoginProcessorTest.java b/modules/src/test/java/org/archive/modules/forms/FormLoginProcessorTest.java new file mode 100644 index 00000000..3e0201fb --- /dev/null +++ b/modules/src/test/java/org/archive/modules/forms/FormLoginProcessorTest.java @@ -0,0 +1,76 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.modules.forms; + +import static org.archive.modules.CoreAttributeConstants.A_WARC_RESPONSE_HEADERS; + +import org.apache.commons.httpclient.NameValuePair; +import org.apache.commons.httpclient.util.EncodingUtil; +import org.archive.modules.CoreAttributeConstants; +import org.archive.modules.CrawlURI; +import org.archive.modules.ProcessorTestBase; +import org.archive.modules.CrawlURI.FetchType; +import org.archive.modules.forms.FormLoginProcessor; + +public class FormLoginProcessorTest extends ProcessorTestBase { + + public void testNoFormLogin() throws Exception { + CrawlURI curi = makeCrawlURI("http://example.com/"); + + FormLoginProcessor p = (FormLoginProcessor) makeModule(); + p.setLoginUsername("jdoe"); + p.setLoginPassword("********"); + p.setApplicableSurtPrefix("http://(com,example,)"); + + p.process(curi); + + assertEquals(1, curi.getDataList(A_WARC_RESPONSE_HEADERS).size()); + assertEquals("WARC-Simple-Form-Province-Status: 0,0,http://(com,example,)", curi.getDataList(A_WARC_RESPONSE_HEADERS).get(0)); + } + + public void testFormLogin() throws Exception { + CrawlURI curi = makeCrawlURI("http://example.com/"); + + HTMLForm form = new HTMLForm(); + form.addField("text", "username-form-field", ""); + form.addField("password", "password-form-field", ""); + form.setMethod("post"); + form.setAction("/login"); + curi.getDataList(ExtractorHTMLForms.A_HTML_FORM_OBJECTS).add(form); + + FormLoginProcessor p = (FormLoginProcessor) makeModule(); + p.setLoginUsername("jdoe"); + p.setLoginPassword("********"); + p.setApplicableSurtPrefix("http://(com,example,)"); + + p.process(curi); + assertEquals(1, curi.getDataList(A_WARC_RESPONSE_HEADERS).size()); + assertEquals("WARC-Simple-Form-Province-Status: 0,0,http://(com,example,)", curi.getDataList(A_WARC_RESPONSE_HEADERS).get(0)); + assertTrue(curi.getAnnotations().contains("submit:/login")); + + assertEquals(1, curi.getOutCandidates().size()); + CrawlURI submitCuri = curi.getOutCandidates().toArray(new CrawlURI[0])[0]; + assertEquals("http://example.com/login", submitCuri.toString()); + assertEquals(FetchType.HTTP_POST, submitCuri.getFetchType()); + NameValuePair[] queryParams = (NameValuePair[]) submitCuri.getData().get(CoreAttributeConstants.A_SUBMIT_DATA); + String queryString = EncodingUtil.formUrlEncode(queryParams, "UTF-8"); + assertEquals("username-form-field=jdoe&password-form-field=********", queryString); + } +}