From 0d7f633fdba10cb8216bd9fbe2d07232365f432e Mon Sep 17 00:00:00 2001 From: Noah Levitt Date: Mon, 31 Mar 2014 13:36:44 -0700 Subject: [PATCH 1/7] pull methods getRecorder() and makeCrawlURI(String) into ModuleTestBase, and move ModuleTestBase into heritrix-modules so that CrawlURI is available --- .../org/archive/state/ModuleTestBase.java | 244 ++++++++++++++++++ .../modules/fetcher/FetchHTTPTests.java | 22 -- 2 files changed, 244 insertions(+), 22 deletions(-) create mode 100644 modules/src/main/java/org/archive/state/ModuleTestBase.java diff --git a/modules/src/main/java/org/archive/state/ModuleTestBase.java b/modules/src/main/java/org/archive/state/ModuleTestBase.java new file mode 100644 index 00000000..884daa4b --- /dev/null +++ b/modules/src/main/java/org/archive/state/ModuleTestBase.java @@ -0,0 +1,244 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.archive.state; + + +import java.io.File; +import java.io.IOException; +import java.io.Serializable; +import java.util.Arrays; + +import junit.framework.TestCase; + +import org.apache.commons.httpclient.URIException; +import org.apache.commons.lang.SerializationUtils; +import org.archive.modules.CrawlURI; +import org.archive.net.UURI; +import org.archive.net.UURIFactory; +import org.archive.util.Recorder; +import org.archive.util.TmpDirTestCase; + + +/** + * Base class for unit testing Module implementations. + * + * @author pjack + */ +public abstract class ModuleTestBase extends TestCase { + + + /** + * Magical constructor that attempts to auto-create static key field + * descriptions for your module class. + * + *

If {@link #getSourceCodeDir} and {@link #getResourceDir} both return + * non-null values, then the constructor will look in the resources + * directory for an English resource file for the class. If it finds + * one, nothing magical happens. + * + *

Otherwise, the source code for the module being tested is loaded, + * and parsed to extract the JavaDoc descriptions for the static key + * fields. The results are stored in the appropriate English locale file + * in the resource directory. + * + *

Note the parsing is naive; at minimum, you should load the resulting + * locale file and remove any HTML markup. + */ + public ModuleTestBase() { + getSourceCodeDir(); + getResourceDir(); + } + + + /** + * Returns the location of the source code directory for your project. + * This defaults to "src/main/java", which is the standard for projects + * built with maven2. If you use a different source code directory, + * you should override this method. + * + *

If you want to disable automatic key description generation, + * return null from this method. + * + * @return the source code directory for the project + */ + protected File getSourceCodeDir() { + return getProjectDir("src/main/java"); + } + + + /** + * Returns the location of the Java resources directory for your project. + * This defaults to "src/resources/java", which is the standard for projects + * built with maven2. If you use a different source code directory -- + * for instance, if your resources directory is the same as your source + * code directory -- you should override this method. + * + *

If you want to disable automatic key description generation, + * return null from this method. + * + * @return the source code directory for the project + */ + protected File getResourceDir() { + return getProjectDir("src/main/resources"); + } + + + /** + * Returns a project directory for a Heritrix subproject. This is here + * so that the src and resources directories can be found whether the + * unit test is run using maven2 or using Eclipse. The two build systems + * use different working directories. + * + * @param path the path the path to find + * @return the found path + */ + private File getProjectDir(String path) { + File r = new File(path); + if (r.exists()) { + return r; + } + String cname = getClass().getName(); + if (cname.startsWith("org.archive.processors")) { + return new File("modules/" + path); + } + if (cname.startsWith("org.archive.deciderules")) { + return new File("modules/" + path); + } + if (cname.startsWith("org.archive.crawler")) { + return new File("engine/" + path); + } + return null; + } + + /** + * Returns the class of the module to test. Deduces from + * test class name if possible. + * + * @return the class of the module to test + */ + protected Class getModuleClass() { + String myClassName = this.getClass().getCanonicalName(); + if(!myClassName.endsWith("Test")) { + throw new UnsupportedOperationException( + "Cannot get module class of "+myClassName); + } + String moduleClassName = myClassName.substring(0,myClassName.length()-4); + try { + return Class.forName(moduleClassName); + } catch (ClassNotFoundException e) { + throw new RuntimeException(e); + } + } + + /** + * Return an example instance of the module. This is used by + * testSerialization to ensure the module can be serialized. + * + * @return an example instance of the module + * @throws Exception if the module cannot be constructed for any reason + */ + protected Object makeModule() throws Exception { + return getModuleClass().newInstance(); + } + + /** + * Tests that the module can be serialized. The value returned by + * {@link #makeModule} is serialized to a byte array, and then + * deserialized, and then serialized to a second byte array. The results + * are passed to {@link #verifySerialization}, which will simply compare + * the two byte arrays for equality. (That won't always work; see + * that method for details). + * + *

If nothing else, this test is useful for catching NotSerializable + * exceptions for your module or classes it depends on. + * + * @throws Exception if the module cannot be serialized + */ + public void testSerializationIfAppropriate() throws Exception { + Object first = makeModule(); + if(!(first instanceof Serializable)) { + return; + } + byte[] firstBytes = SerializationUtils.serialize((Serializable)first); + + Object second = SerializationUtils.deserialize(firstBytes); + byte[] secondBytes = SerializationUtils.serialize((Serializable)second); + + Object third = SerializationUtils.deserialize(secondBytes); + byte[] thirdBytes = SerializationUtils.serialize((Serializable)third); + + // HashMap serialization reverses order of items in linked buckets + // each roundtrip -- so don't check one roundtrip, check two +// verifySerialization(first, firstBytes, second, secondBytes); + verifySerialization(first, firstBytes, third, thirdBytes); + } + + /** + * Verifies that serialization was successful. + * + *

By default, this method simply compares the first and second byte + * arrays for equality. That may not work if you use custom serialization + * -- for instance, if you're serializing a timestamp. If that's the case + * you should override this method to compare the given objects, or to + * simply do nothing. (If this method does nothing, then the + * {@link #testSerialization} test is still useful for catching + * NotSerializable problems). + * + * @param first the first object that was serialized + * @param firstBytes the byte array the first object was serialized to + * @param second the second object that was serialized + * @param secondBytes the byte array the second object was serialized to + * @throws Exception if anyt problem occurs + */ + protected void verifySerialization(Object first, byte[] firstBytes, + Object second, byte[] secondBytes) throws Exception { + assertTrue(Arrays.equals(firstBytes, secondBytes)); + } + + @Override + protected void runTest() throws Throwable { + try { + super.runTest(); + } catch (Throwable t) { + t.printStackTrace(); + throw t; + } + } + + + protected Recorder getRecorder() throws IOException { + if (Recorder.getHttpRecorder() == null) { + Recorder httpRecorder = new Recorder(TmpDirTestCase.tmpDir(), + getClass().getName(), 16 * 1024, 512 * 1024); + Recorder.setHttpRecorder(httpRecorder); + } + + return Recorder.getHttpRecorder(); + } + + + protected CrawlURI makeCrawlURI(String uri) throws URIException, + IOException { + UURI uuri = UURIFactory.getInstance(uri); + CrawlURI curi = new CrawlURI(uuri); + curi.setSeed(true); + curi.setRecorder(getRecorder()); + return curi; + } +} diff --git a/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTests.java b/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTests.java index 5602c650..17696c54 100644 --- a/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTests.java +++ b/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTests.java @@ -62,9 +62,6 @@ import org.archive.modules.credential.HttpAuthenticationCredential; import org.archive.modules.deciderules.RejectDecideRule; import org.archive.modules.recrawl.FetchHistoryProcessor; import org.archive.net.UURI; -import org.archive.net.UURIFactory; -import org.archive.util.Recorder; -import org.archive.util.TmpDirTestCase; import org.jboss.netty.handler.codec.http.HttpRequest; import org.littleshoot.proxy.DefaultHttpProxyServer; import org.littleshoot.proxy.HttpFilter; @@ -94,25 +91,6 @@ public class FetchHTTPTests extends ProcessorTestBase { return getClass().getName(); } - protected Recorder getRecorder() throws IOException { - if (Recorder.getHttpRecorder() == null) { - Recorder httpRecorder = new Recorder(TmpDirTestCase.tmpDir(), - getClass().getName(), 16 * 1024, 512 * 1024); - Recorder.setHttpRecorder(httpRecorder); - } - - return Recorder.getHttpRecorder(); - } - - protected CrawlURI makeCrawlURI(String uri) throws URIException, - IOException { - UURI uuri = UURIFactory.getInstance(uri); - CrawlURI curi = new CrawlURI(uuri); - curi.setSeed(true); - curi.setRecorder(getRecorder()); - return curi; - } - protected void runDefaultChecks(CrawlURI curi, String... exclusionsArray) throws IOException, UnsupportedEncodingException { From 26a577ff6e4f98ecd87f7190d31706684706e222 Mon Sep 17 00:00:00 2001 From: Noah Levitt Date: Mon, 31 Mar 2014 13:38:39 -0700 Subject: [PATCH 2/7] basic unit tests for FormLoginProcessor --- .../modules/forms/FormLoginProcessorTest.java | 76 +++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 modules/src/test/java/org/archive/modules/forms/FormLoginProcessorTest.java diff --git a/modules/src/test/java/org/archive/modules/forms/FormLoginProcessorTest.java b/modules/src/test/java/org/archive/modules/forms/FormLoginProcessorTest.java new file mode 100644 index 00000000..3e0201fb --- /dev/null +++ b/modules/src/test/java/org/archive/modules/forms/FormLoginProcessorTest.java @@ -0,0 +1,76 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.modules.forms; + +import static org.archive.modules.CoreAttributeConstants.A_WARC_RESPONSE_HEADERS; + +import org.apache.commons.httpclient.NameValuePair; +import org.apache.commons.httpclient.util.EncodingUtil; +import org.archive.modules.CoreAttributeConstants; +import org.archive.modules.CrawlURI; +import org.archive.modules.ProcessorTestBase; +import org.archive.modules.CrawlURI.FetchType; +import org.archive.modules.forms.FormLoginProcessor; + +public class FormLoginProcessorTest extends ProcessorTestBase { + + public void testNoFormLogin() throws Exception { + CrawlURI curi = makeCrawlURI("http://example.com/"); + + FormLoginProcessor p = (FormLoginProcessor) makeModule(); + p.setLoginUsername("jdoe"); + p.setLoginPassword("********"); + p.setApplicableSurtPrefix("http://(com,example,)"); + + p.process(curi); + + assertEquals(1, curi.getDataList(A_WARC_RESPONSE_HEADERS).size()); + assertEquals("WARC-Simple-Form-Province-Status: 0,0,http://(com,example,)", curi.getDataList(A_WARC_RESPONSE_HEADERS).get(0)); + } + + public void testFormLogin() throws Exception { + CrawlURI curi = makeCrawlURI("http://example.com/"); + + HTMLForm form = new HTMLForm(); + form.addField("text", "username-form-field", ""); + form.addField("password", "password-form-field", ""); + form.setMethod("post"); + form.setAction("/login"); + curi.getDataList(ExtractorHTMLForms.A_HTML_FORM_OBJECTS).add(form); + + FormLoginProcessor p = (FormLoginProcessor) makeModule(); + p.setLoginUsername("jdoe"); + p.setLoginPassword("********"); + p.setApplicableSurtPrefix("http://(com,example,)"); + + p.process(curi); + assertEquals(1, curi.getDataList(A_WARC_RESPONSE_HEADERS).size()); + assertEquals("WARC-Simple-Form-Province-Status: 0,0,http://(com,example,)", curi.getDataList(A_WARC_RESPONSE_HEADERS).get(0)); + assertTrue(curi.getAnnotations().contains("submit:/login")); + + assertEquals(1, curi.getOutCandidates().size()); + CrawlURI submitCuri = curi.getOutCandidates().toArray(new CrawlURI[0])[0]; + assertEquals("http://example.com/login", submitCuri.toString()); + assertEquals(FetchType.HTTP_POST, submitCuri.getFetchType()); + NameValuePair[] queryParams = (NameValuePair[]) submitCuri.getData().get(CoreAttributeConstants.A_SUBMIT_DATA); + String queryString = EncodingUtil.formUrlEncode(queryParams, "UTF-8"); + assertEquals("username-form-field=jdoe&password-form-field=********", queryString); + } +} From cc838747af367a8749f37107d912b2573db43946 Mon Sep 17 00:00:00 2001 From: Noah Levitt Date: Mon, 31 Mar 2014 13:38:54 -0700 Subject: [PATCH 3/7] pull methods getRecorder() and makeCrawlURI(String) into ModuleTestBase, and move ModuleTestBase into heritrix-modules so that CrawlURI is available --- .../org/archive/state/ModuleTestBase.java | 216 ------------------ 1 file changed, 216 deletions(-) delete mode 100644 commons/src/main/java/org/archive/state/ModuleTestBase.java diff --git a/commons/src/main/java/org/archive/state/ModuleTestBase.java b/commons/src/main/java/org/archive/state/ModuleTestBase.java deleted file mode 100644 index 835d6ada..00000000 --- a/commons/src/main/java/org/archive/state/ModuleTestBase.java +++ /dev/null @@ -1,216 +0,0 @@ -/* - * This file is part of the Heritrix web crawler (crawler.archive.org). - * - * Licensed to the Internet Archive (IA) by one or more individual - * contributors. - * - * The IA licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.archive.state; - - -import java.io.File; -import java.io.Serializable; -import java.util.Arrays; - -import junit.framework.TestCase; - -import org.apache.commons.lang.SerializationUtils; - - -/** - * Base class for unit testing Module implementations. - * - * @author pjack - */ -public abstract class ModuleTestBase extends TestCase { - - - /** - * Magical constructor that attempts to auto-create static key field - * descriptions for your module class. - * - *

If {@link #getSourceCodeDir} and {@link #getResourceDir} both return - * non-null values, then the constructor will look in the resources - * directory for an English resource file for the class. If it finds - * one, nothing magical happens. - * - *

Otherwise, the source code for the module being tested is loaded, - * and parsed to extract the JavaDoc descriptions for the static key - * fields. The results are stored in the appropriate English locale file - * in the resource directory. - * - *

Note the parsing is naive; at minimum, you should load the resulting - * locale file and remove any HTML markup. - */ - public ModuleTestBase() { - getSourceCodeDir(); - getResourceDir(); - } - - - /** - * Returns the location of the source code directory for your project. - * This defaults to "src/main/java", which is the standard for projects - * built with maven2. If you use a different source code directory, - * you should override this method. - * - *

If you want to disable automatic key description generation, - * return null from this method. - * - * @return the source code directory for the project - */ - protected File getSourceCodeDir() { - return getProjectDir("src/main/java"); - } - - - /** - * Returns the location of the Java resources directory for your project. - * This defaults to "src/resources/java", which is the standard for projects - * built with maven2. If you use a different source code directory -- - * for instance, if your resources directory is the same as your source - * code directory -- you should override this method. - * - *

If you want to disable automatic key description generation, - * return null from this method. - * - * @return the source code directory for the project - */ - protected File getResourceDir() { - return getProjectDir("src/main/resources"); - } - - - /** - * Returns a project directory for a Heritrix subproject. This is here - * so that the src and resources directories can be found whether the - * unit test is run using maven2 or using Eclipse. The two build systems - * use different working directories. - * - * @param path the path the path to find - * @return the found path - */ - private File getProjectDir(String path) { - File r = new File(path); - if (r.exists()) { - return r; - } - String cname = getClass().getName(); - if (cname.startsWith("org.archive.processors")) { - return new File("modules/" + path); - } - if (cname.startsWith("org.archive.deciderules")) { - return new File("modules/" + path); - } - if (cname.startsWith("org.archive.crawler")) { - return new File("engine/" + path); - } - return null; - } - - /** - * Returns the class of the module to test. Deduces from - * test class name if possible. - * - * @return the class of the module to test - */ - protected Class getModuleClass() { - String myClassName = this.getClass().getCanonicalName(); - if(!myClassName.endsWith("Test")) { - throw new UnsupportedOperationException( - "Cannot get module class of "+myClassName); - } - String moduleClassName = myClassName.substring(0,myClassName.length()-4); - try { - return Class.forName(moduleClassName); - } catch (ClassNotFoundException e) { - throw new RuntimeException(e); - } - } - - /** - * Return an example instance of the module. This is used by - * testSerialization to ensure the module can be serialized. - * - * @return an example instance of the module - * @throws Exception if the module cannot be constructed for any reason - */ - protected Object makeModule() throws Exception { - return getModuleClass().newInstance(); - } - - /** - * Tests that the module can be serialized. The value returned by - * {@link #makeModule} is serialized to a byte array, and then - * deserialized, and then serialized to a second byte array. The results - * are passed to {@link #verifySerialization}, which will simply compare - * the two byte arrays for equality. (That won't always work; see - * that method for details). - * - *

If nothing else, this test is useful for catching NotSerializable - * exceptions for your module or classes it depends on. - * - * @throws Exception if the module cannot be serialized - */ - public void testSerializationIfAppropriate() throws Exception { - Object first = makeModule(); - if(!(first instanceof Serializable)) { - return; - } - byte[] firstBytes = SerializationUtils.serialize((Serializable)first); - - Object second = SerializationUtils.deserialize(firstBytes); - byte[] secondBytes = SerializationUtils.serialize((Serializable)second); - - Object third = SerializationUtils.deserialize(secondBytes); - byte[] thirdBytes = SerializationUtils.serialize((Serializable)third); - - // HashMap serialization reverses order of items in linked buckets - // each roundtrip -- so don't check one roundtrip, check two -// verifySerialization(first, firstBytes, second, secondBytes); - verifySerialization(first, firstBytes, third, thirdBytes); - } - - /** - * Verifies that serialization was successful. - * - *

By default, this method simply compares the first and second byte - * arrays for equality. That may not work if you use custom serialization - * -- for instance, if you're serializing a timestamp. If that's the case - * you should override this method to compare the given objects, or to - * simply do nothing. (If this method does nothing, then the - * {@link #testSerialization} test is still useful for catching - * NotSerializable problems). - * - * @param first the first object that was serialized - * @param firstBytes the byte array the first object was serialized to - * @param second the second object that was serialized - * @param secondBytes the byte array the second object was serialized to - * @throws Exception if anyt problem occurs - */ - protected void verifySerialization(Object first, byte[] firstBytes, - Object second, byte[] secondBytes) throws Exception { - assertTrue(Arrays.equals(firstBytes, secondBytes)); - } - - @Override - protected void runTest() throws Throwable { - try { - super.runTest(); - } catch (Throwable t) { - t.printStackTrace(); - throw t; - } - } -} From 23e8d41651c2bebdf238ec2858dfc658c425acec Mon Sep 17 00:00:00 2001 From: Noah Levitt Date: Mon, 31 Mar 2014 13:40:29 -0700 Subject: [PATCH 4/7] avoid NPE; root cause was assumption that LoadingCache.asMap().get() would call CacheLoader.load() but it doesnt, see https://code.google.com/p/guava-libraries/wiki/MapMakerMigration --- .../modules/forms/FormLoginProcessor.java | 49 +++++++++++-------- 1 file changed, 29 insertions(+), 20 deletions(-) diff --git a/modules/src/main/java/org/archive/modules/forms/FormLoginProcessor.java b/modules/src/main/java/org/archive/modules/forms/FormLoginProcessor.java index f2292f77..d6aa637e 100644 --- a/modules/src/main/java/org/archive/modules/forms/FormLoginProcessor.java +++ b/modules/src/main/java/org/archive/modules/forms/FormLoginProcessor.java @@ -21,14 +21,11 @@ package org.archive.modules.forms; import static org.archive.modules.CoreAttributeConstants.A_WARC_RESPONSE_HEADERS; -import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ExecutionException; import java.util.concurrent.atomic.AtomicLong; import java.util.logging.Level; import java.util.logging.Logger; -import com.google.common.cache.CacheBuilder; -import com.google.common.cache.CacheLoader; -import com.google.common.cache.LoadingCache; import org.apache.commons.httpclient.URIException; import org.apache.commons.lang.StringUtils; import org.archive.checkpointing.Checkpointable; @@ -45,6 +42,10 @@ import org.json.JSONException; import org.json.JSONObject; import org.springframework.beans.factory.annotation.Autowired; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; + /** * A step, post-ExtractorHTMLForms, where a followup CrawlURI to * attempt a form submission may be synthesized. @@ -119,24 +120,24 @@ public class FormLoginProcessor extends Processor implements Checkpointable { Logger.getLogger(FormLoginProcessor.class.getName()); // formProvince (String) -> count - ConcurrentMap eligibleFormsSeenCount = + protected LoadingCache eligibleFormsSeenCount = CacheBuilder.newBuilder() .build( new CacheLoader() { public AtomicLong load(String arg0) { return new AtomicLong(0L); } - }).asMap(); + }); // formProvince (String) -> count - ConcurrentMap eligibleFormsAttemptsCount = + protected LoadingCache eligibleFormsAttemptsCount = CacheBuilder.newBuilder() .build( new CacheLoader() { public AtomicLong load(String arg0) { return new AtomicLong(0L); } - }).asMap(); + }); /** * SURT prefix against which configured username/password is @@ -219,13 +220,17 @@ public class FormLoginProcessor extends Processor implements Checkpointable { for( Object formObject : curi.getDataList(ExtractorHTMLForms.A_HTML_FORM_OBJECTS)) { HTMLForm form = (HTMLForm) formObject; if(form.seemsLoginForm()) { - eligibleFormsSeenCount.get(formProvince).incrementAndGet(); - if(eligibleFormsAttemptsCount.get(formProvince).get()<1) { - eligibleFormsAttemptsCount.get(formProvince).incrementAndGet(); - createFormSubmissionAttempt(curi,form,formProvince); - } else { - // note decline-to-submit: in volume, may be signal of failed first login - curi.getAnnotations().add("nosubmit:"+submitStatusFor(formProvince)); + try { + eligibleFormsSeenCount.get(formProvince).incrementAndGet(); + if(eligibleFormsAttemptsCount.get(formProvince).get()<1) { + eligibleFormsAttemptsCount.get(formProvince).incrementAndGet(); + createFormSubmissionAttempt(curi,form,formProvince); + } else { + // note decline-to-submit: in volume, may be signal of failed first login + curi.getAnnotations().add("nosubmit:"+submitStatusFor(formProvince)); + } + } catch (ExecutionException e) { + throw new RuntimeException(e); // can't happen? } return; } @@ -278,9 +283,13 @@ public class FormLoginProcessor extends Processor implements Checkpointable { } protected String submitStatusFor(String formProvince) { - return eligibleFormsAttemptsCount.get(formProvince).get() - +","+eligibleFormsSeenCount.get(formProvince).get() - +","+formProvince; + try { + return eligibleFormsAttemptsCount.get(formProvince).get() + +","+eligibleFormsSeenCount.get(formProvince).get() + +","+formProvince; + } catch (ExecutionException e) { + throw new RuntimeException(e); + } } @Override @@ -295,10 +304,10 @@ public class FormLoginProcessor extends Processor implements Checkpointable { protected void fromCheckpointJson(JSONObject json) throws JSONException { super.fromCheckpointJson(json); JSONUtils.putAllAtomicLongs( - eligibleFormsAttemptsCount, + eligibleFormsAttemptsCount.asMap(), json.getJSONObject("eligibleFormsAttemptsCount")); JSONUtils.putAllAtomicLongs( - eligibleFormsSeenCount, + eligibleFormsSeenCount.asMap(), json.getJSONObject("eligibleFormsSeenCount")); } } From a82886b74769ccffe9ff20eceb443649284318c0 Mon Sep 17 00:00:00 2001 From: Noah Levitt Date: Mon, 31 Mar 2014 18:09:23 -0700 Subject: [PATCH 5/7] restore support for sending http post data, add unit test --- .../modules/fetcher/FetchHTTPRequest.java | 18 ++++++-- .../modules/fetcher/FetchHTTPTests.java | 42 +++++++++++++++++-- 2 files changed, 54 insertions(+), 6 deletions(-) diff --git a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTPRequest.java b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTPRequest.java index 7f48ae9f..b8c2c786 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTPRequest.java +++ b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTPRequest.java @@ -64,6 +64,7 @@ import org.apache.http.client.config.RequestConfig; import org.apache.http.client.entity.UrlEncodedFormEntity; import org.apache.http.client.methods.AbstractExecutionAwareRequest; import org.apache.http.client.protocol.HttpClientContext; +import org.apache.http.client.utils.URLEncodedUtils; import org.apache.http.config.ConnectionConfig; import org.apache.http.config.MessageConstraints; import org.apache.http.config.Registry; @@ -79,6 +80,8 @@ import org.apache.http.conn.socket.PlainConnectionSocketFactory; import org.apache.http.conn.ssl.AllowAllHostnameVerifier; import org.apache.http.conn.ssl.SSLConnectionSocketFactory; import org.apache.http.entity.ContentLengthStrategy; +import org.apache.http.entity.ContentType; +import org.apache.http.entity.StringEntity; import org.apache.http.impl.DefaultBHttpClientConnection; import org.apache.http.impl.client.BasicAuthCache; import org.apache.http.impl.client.BasicCredentialsProvider; @@ -93,6 +96,7 @@ import org.apache.http.message.BasicNameValuePair; import org.apache.http.protocol.HTTP; import org.apache.http.protocol.HttpContext; import org.apache.http.util.Args; +import org.archive.modules.CoreAttributeConstants; import org.archive.modules.CrawlURI; import org.archive.modules.CrawlURI.FetchType; import org.archive.modules.Processor; @@ -175,8 +179,16 @@ class FetchHTTPRequest { } if (curi.getFetchType() == FetchType.HTTP_POST) { - this.request = new BasicExecutionAwareEntityEnclosingRequest("POST", - requestLineUri, httpVersion); + BasicExecutionAwareEntityEnclosingRequest postRequest = new BasicExecutionAwareEntityEnclosingRequest( + "POST", requestLineUri, httpVersion); + this.request = postRequest; + String submitData = (String) curi.getData().get(CoreAttributeConstants.A_SUBMIT_DATA); + if (submitData != null) { + // XXX brittle, doesn't support multipart form data etc + ContentType contentType = ContentType.create(URLEncodedUtils.CONTENT_TYPE, "UTF-8"); + StringEntity formEntity = new StringEntity(submitData, contentType); + postRequest.setEntity(formEntity); + } } else { this.request = new BasicExecutionAwareRequest("GET", requestLineUri, httpVersion); @@ -196,7 +208,7 @@ class FetchHTTPRequest { this.addedCredentials = populateTargetCredential(); populateHttpProxyCredential(); } - + protected void configureRequestHeaders() { if (fetcher.getAcceptCompression()) { request.addHeader("Accept-Encoding", "gzip,deflate"); diff --git a/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTests.java b/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTests.java index 17696c54..385258f1 100644 --- a/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTests.java +++ b/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTests.java @@ -43,6 +43,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedList; import java.util.List; import java.util.Set; import java.util.logging.Level; @@ -52,8 +53,12 @@ import javax.net.ssl.SSLException; import org.apache.commons.httpclient.URIException; import org.apache.commons.io.IOUtils; +import org.apache.http.NameValuePair; import org.apache.http.NoHttpResponseException; +import org.apache.http.client.utils.URLEncodedUtils; +import org.apache.http.message.BasicNameValuePair; import org.archive.httpclient.ConfigurableX509TrustManager.TrustLevel; +import org.archive.modules.CoreAttributeConstants; import org.archive.modules.CrawlMetadata; import org.archive.modules.CrawlURI; import org.archive.modules.CrawlURI.FetchType; @@ -108,7 +113,9 @@ public class FetchHTTPTests extends ProcessorTestBase { if (!exclusions.contains("hostHeader")) { assertTrue(requestString.contains("Host: localhost:7777\r\n")); } - assertTrue(requestString.endsWith("\r\n\r\n")); + if (!exclusions.contains("trailingCRLFCRLF")) { + assertTrue(requestString.endsWith("\r\n\r\n")); + } // check sizes assertEquals(DEFAULT_PAYLOAD_STRING.length(), curi.getContentLength()); @@ -125,7 +132,9 @@ public class FetchHTTPTests extends ProcessorTestBase { if (!exclusions.contains("fetchStatus")) { assertTrue(curi.getFetchStatus() == 200); } - assertTrue(curi.getFetchType() == FetchType.HTTP_GET); + if (!exclusions.contains("fetchTypeGET")) { + assertTrue(curi.getFetchType() == FetchType.HTTP_GET); + } // check message body, i.e. "raw, possibly chunked-transfer-encoded message contents not including the leading headers" assertEquals(DEFAULT_PAYLOAD_STRING, messageBodyString(curi)); @@ -681,8 +690,9 @@ public class FetchHTTPTests extends ProcessorTestBase { @Override public void run() { + ServerSocket listeningSocket = null; try { - ServerSocket listeningSocket = new ServerSocket(listenPort, 0, Inet4Address.getByName(listenAddress)); + listeningSocket = new ServerSocket(listenPort, 0, Inet4Address.getByName(listenAddress)); listeningSocket.setSoTimeout(600); while (!isTimeToBeDone) { try { @@ -696,6 +706,12 @@ public class FetchHTTPTests extends ProcessorTestBase { // logger.warning("caught exception: " + e); } finally { // logger.info("all done suckers"); + if (listeningSocket != null) { + try { + listeningSocket.close(); + } catch (IOException e) { + } + } } } @@ -807,6 +823,26 @@ public class FetchHTTPTests extends ProcessorTestBase { fetcher().process(curi); assertTrue(httpRequestString(curi).contains("Host: example.com\r\n")); } + + public void testHttpPost() throws Exception { + CrawlURI curi = makeCrawlURI("http://localhost:7777/"); + curi.setFetchType(FetchType.HTTP_POST); + + List params = new LinkedList(); + params.add(new BasicNameValuePair("name1", "value1")); + params.add(new BasicNameValuePair("name1", "value2")); + params.add(new BasicNameValuePair("funky name 2", "whoa crazy\t && 🍺 🍻 \n crazier \rooo")); + String submitData = URLEncodedUtils.format(params, "UTF-8"); + assertEquals("name1=value1&name1=value2&funky+name+2=whoa+crazy%09+%26%26+%F0%9F%8D%BA+%F0%9F%8D%BB+%0A+crazier+%0Dooo", submitData); + + curi.getData().put(CoreAttributeConstants.A_SUBMIT_DATA, submitData); + fetcher().process(curi); + + assertTrue(httpRequestString(curi).startsWith("POST / HTTP/1.0\r\n")); + assertTrue(httpRequestString(curi).endsWith("\r\n\r\nname1=value1&name1=value2&funky+name+2=whoa+crazy%09+%26%26+%F0%9F%8D%BA+%F0%9F%8D%BB+%0A+crazier+%0Dooo")); + assertEquals(FetchType.HTTP_POST, curi.getFetchType()); + runDefaultChecks(curi, "requestLine", "trailingCRLFCRLF", "fetchTypeGET"); + } @Override protected FetchHTTP makeModule() throws IOException { From 20a5f31e1944f299f8f128a4a38f96e85750a649 Mon Sep 17 00:00:00 2001 From: Noah Levitt Date: Mon, 31 Mar 2014 18:57:33 -0700 Subject: [PATCH 6/7] ExtractorHTMLForms+FormLoginProcessor full crawl job integration test (not passing yet) --- .../crawler/selftest/FormLoginSelfTest.java | 103 ++++++++++++++++++ .../FormLoginSelfTest/htdocs/failure.html | 9 ++ .../FormLoginSelfTest/htdocs/index.html | 15 +++ .../FormLoginSelfTest/htdocs/link1.html | 7 ++ .../FormLoginSelfTest/htdocs/link2.html | 7 ++ .../FormLoginSelfTest/htdocs/link3.html | 7 ++ .../FormLoginSelfTest/htdocs/success.html | 9 ++ .../selftest/conf/selftest-crawler-beans.cxml | 1 + 8 files changed, 158 insertions(+) create mode 100644 engine/src/test/java/org/archive/crawler/selftest/FormLoginSelfTest.java create mode 100644 engine/testdata/selftest/FormLoginSelfTest/htdocs/failure.html create mode 100644 engine/testdata/selftest/FormLoginSelfTest/htdocs/index.html create mode 100644 engine/testdata/selftest/FormLoginSelfTest/htdocs/link1.html create mode 100644 engine/testdata/selftest/FormLoginSelfTest/htdocs/link2.html create mode 100644 engine/testdata/selftest/FormLoginSelfTest/htdocs/link3.html create mode 100644 engine/testdata/selftest/FormLoginSelfTest/htdocs/success.html diff --git a/engine/src/test/java/org/archive/crawler/selftest/FormLoginSelfTest.java b/engine/src/test/java/org/archive/crawler/selftest/FormLoginSelfTest.java new file mode 100644 index 00000000..b5c83687 --- /dev/null +++ b/engine/src/test/java/org/archive/crawler/selftest/FormLoginSelfTest.java @@ -0,0 +1,103 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.crawler.selftest; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; + +import org.mortbay.jetty.Handler; +import org.mortbay.jetty.Server; +import org.mortbay.jetty.bio.SocketConnector; +import org.mortbay.jetty.handler.DefaultHandler; +import org.mortbay.jetty.handler.HandlerList; +import org.mortbay.jetty.handler.ResourceHandler; +import org.mortbay.jetty.servlet.ServletHandler; +import org.mortbay.jetty.servlet.ServletHolder; + +/** + * Test form-based authentication + * + * @contributor stack + * @contributor gojomo + */ +public class FormLoginSelfTest + extends SelfTestBase +{ + /** + * Files to find as a list. + */ + final private static Set EXPECTED = Collections.unmodifiableSet( + new HashSet(Arrays.asList(new String[] { + "index.html", "login/login.html", "success.html", "robots.txt", "favicon.ico" + }))); + + @Override + protected void verify() throws Exception { + Set found = this.filesInArcs(); + assertEquals("wrong files in ARCs",EXPECTED,found); + } + + @Override + protected void startHttpServer() throws Exception { + Server server = new Server(); + + SocketConnector sc = new SocketConnector(); + sc.setHost("127.0.0.1"); + sc.setPort(7777); + server.addConnector(sc); + ResourceHandler rhandler = new ResourceHandler(); + rhandler.setResourceBase(getSrcHtdocs().getAbsolutePath()); + + ServletHandler servletHandler = new ServletHandler(); + + HandlerList handlers = new HandlerList(); + handlers.setHandlers(new Handler[] { + rhandler, + servletHandler, + new DefaultHandler() }); + server.setHandler(handlers); + + ServletHolder holder = new ServletHolder(new FormAuthServlet()); + servletHandler.addServletWithMapping(holder, "/login/*"); + + this.httpServer = server; + this.httpServer.start(); + } + + protected String getSeedsString() { + return "http://127.0.0.1:7777/index.html"; + } + + @Override + protected String changeGlobalConfig(String config) { + String formLoginConfig = + " \n" + + " \n" + + " \n" + + " \n" + + " \n"; + config = config.replace("", formLoginConfig); + return super.changeGlobalConfig(config); + } + +} + diff --git a/engine/testdata/selftest/FormLoginSelfTest/htdocs/failure.html b/engine/testdata/selftest/FormLoginSelfTest/htdocs/failure.html new file mode 100644 index 00000000..891cbb5a --- /dev/null +++ b/engine/testdata/selftest/FormLoginSelfTest/htdocs/failure.html @@ -0,0 +1,9 @@ +index + +Link 1 + + Link 2 + + Link 3 + +Secure \ No newline at end of file diff --git a/engine/testdata/selftest/FormLoginSelfTest/htdocs/index.html b/engine/testdata/selftest/FormLoginSelfTest/htdocs/index.html new file mode 100644 index 00000000..d3a083ce --- /dev/null +++ b/engine/testdata/selftest/FormLoginSelfTest/htdocs/index.html @@ -0,0 +1,15 @@ +index + +Link 1 + + Link 2 + + Link 3 + +Secure + +

+ + +
+ diff --git a/engine/testdata/selftest/FormLoginSelfTest/htdocs/link1.html b/engine/testdata/selftest/FormLoginSelfTest/htdocs/link1.html new file mode 100644 index 00000000..e9521a25 --- /dev/null +++ b/engine/testdata/selftest/FormLoginSelfTest/htdocs/link1.html @@ -0,0 +1,7 @@ +index + +Link 1 + + Link 2 + + Link 3 \ No newline at end of file diff --git a/engine/testdata/selftest/FormLoginSelfTest/htdocs/link2.html b/engine/testdata/selftest/FormLoginSelfTest/htdocs/link2.html new file mode 100644 index 00000000..e9521a25 --- /dev/null +++ b/engine/testdata/selftest/FormLoginSelfTest/htdocs/link2.html @@ -0,0 +1,7 @@ +index + +Link 1 + + Link 2 + + Link 3 \ No newline at end of file diff --git a/engine/testdata/selftest/FormLoginSelfTest/htdocs/link3.html b/engine/testdata/selftest/FormLoginSelfTest/htdocs/link3.html new file mode 100644 index 00000000..e9521a25 --- /dev/null +++ b/engine/testdata/selftest/FormLoginSelfTest/htdocs/link3.html @@ -0,0 +1,7 @@ +index + +Link 1 + + Link 2 + + Link 3 \ No newline at end of file diff --git a/engine/testdata/selftest/FormLoginSelfTest/htdocs/success.html b/engine/testdata/selftest/FormLoginSelfTest/htdocs/success.html new file mode 100644 index 00000000..891cbb5a --- /dev/null +++ b/engine/testdata/selftest/FormLoginSelfTest/htdocs/success.html @@ -0,0 +1,9 @@ +index + +Link 1 + + Link 2 + + Link 3 + +Secure \ No newline at end of file diff --git a/engine/testdata/selftest/conf/selftest-crawler-beans.cxml b/engine/testdata/selftest/conf/selftest-crawler-beans.cxml index 91beb82e..55a20702 100644 --- a/engine/testdata/selftest/conf/selftest-crawler-beans.cxml +++ b/engine/testdata/selftest/conf/selftest-crawler-beans.cxml @@ -124,6 +124,7 @@ crawlController.pauseAtStart=false + From 552200b2fabe15d06b6af8c3cdc7b823b504f2e0 Mon Sep 17 00:00:00 2001 From: Noah Levitt Date: Mon, 31 Mar 2014 20:38:17 -0700 Subject: [PATCH 7/7] fix form login processor by saving login post data as url-encoded query string, which FetchHTTP now expects; tweaks to make FormLoginSelfTest pass --- .../FormLoginSelfTest/htdocs/index.html | 10 ---------- .../FormLoginSelfTest/htdocs/success.html | 7 ------- .../modules/forms/FormLoginProcessor.java | 3 +-- .../org/archive/modules/forms/HTMLForm.java | 19 +++++++++++++++++++ 4 files changed, 20 insertions(+), 19 deletions(-) diff --git a/engine/testdata/selftest/FormLoginSelfTest/htdocs/index.html b/engine/testdata/selftest/FormLoginSelfTest/htdocs/index.html index d3a083ce..ed045598 100644 --- a/engine/testdata/selftest/FormLoginSelfTest/htdocs/index.html +++ b/engine/testdata/selftest/FormLoginSelfTest/htdocs/index.html @@ -1,13 +1,3 @@ -index - -Link 1 - - Link 2 - - Link 3 - -Secure -
diff --git a/engine/testdata/selftest/FormLoginSelfTest/htdocs/success.html b/engine/testdata/selftest/FormLoginSelfTest/htdocs/success.html index 891cbb5a..52fc82b8 100644 --- a/engine/testdata/selftest/FormLoginSelfTest/htdocs/success.html +++ b/engine/testdata/selftest/FormLoginSelfTest/htdocs/success.html @@ -1,9 +1,2 @@ index -Link 1 - - Link 2 - - Link 3 - -Secure \ No newline at end of file diff --git a/modules/src/main/java/org/archive/modules/forms/FormLoginProcessor.java b/modules/src/main/java/org/archive/modules/forms/FormLoginProcessor.java index d6aa637e..3ea7dc90 100644 --- a/modules/src/main/java/org/archive/modules/forms/FormLoginProcessor.java +++ b/modules/src/main/java/org/archive/modules/forms/FormLoginProcessor.java @@ -265,10 +265,9 @@ public class FormLoginProcessor extends Processor implements Checkpointable { submitCuri.setFetchType(FetchType.HTTP_POST); submitCuri.getData().put( CoreAttributeConstants.A_SUBMIT_DATA, - templateForm.asHttpClientDataWith( + templateForm.asFormDataString( getLoginUsername(), getLoginPassword())); - //submitCuri.setSchedulingDirective(Math.max(curi.getSchedulingDirective()-1, 0)); submitCuri.setSchedulingDirective(SchedulingConstants.HIGH); submitCuri.setForceFetch(true); curi.getOutCandidates().add(submitCuri); diff --git a/modules/src/main/java/org/archive/modules/forms/HTMLForm.java b/modules/src/main/java/org/archive/modules/forms/HTMLForm.java index f2719e22..4c2c2147 100644 --- a/modules/src/main/java/org/archive/modules/forms/HTMLForm.java +++ b/modules/src/main/java/org/archive/modules/forms/HTMLForm.java @@ -20,10 +20,12 @@ package org.archive.modules.forms; import java.util.ArrayList; +import java.util.LinkedList; import java.util.List; import org.apache.commons.httpclient.NameValuePair; import org.apache.commons.lang.StringUtils; +import org.archive.util.TextUtils; /** * Simple representation of a discovered HTML Form. @@ -104,6 +106,7 @@ public class HTMLForm { * @param username * @param password * @return + * @deprecated specific to a particular FetchHTTP implementation based on commons-httpclient, use {@link #asFormDataString(String, String)} */ public NameValuePair[] asHttpClientDataWith(String username, String password) { ArrayList data = new ArrayList(allInputs.size()); @@ -119,6 +122,22 @@ public class HTMLForm { } return data.toArray(new NameValuePair[data.size()]); } + + public String asFormDataString(String username, String password) { + List nameVals = new LinkedList(); + + for (FormInput input : allInputs) { + if(input == candidateUsernameInputs.get(0)) { + nameVals.add(TextUtils.urlEscape(input.name) + "=" + TextUtils.urlEscape(username)); + } else if(input == candidatePasswordInputs.get(0)) { + nameVals.add(TextUtils.urlEscape(input.name) + "=" + TextUtils.urlEscape(password)); + } else if (StringUtils.isNotEmpty(input.name) && StringUtils.isNotEmpty(input.value)) { + nameVals.add(TextUtils.urlEscape(input.name) + "=" + TextUtils.urlEscape(input.value)); + } + } + + return StringUtils.join(nameVals, '&'); + } public String toString() { StringBuilder sb = new StringBuilder();