lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-09-21 05:05:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

test 304 not modified revist profile in FetchHTTPTest

Browse Source
This commit is contained in:
Noah Levitt
2014-07-10 21:30:54 -07:00
parent 2292ea9287
commit 77492802b3
4 changed files with 63 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/src/main/java/org/archive/modules/revisit/ServerNotModifiedRevisit.java
+2 -2
View File
@@ -40,12 +40,12 @@ public class ServerNotModifiedRevisit extends AbstractProfile {
return headers;
}
public String geteTag() {
public String getETag() {
return eTag;
}
public void seteTag(String eTag) {
public void setETag(String eTag) {
this.eTag = eTag;
}
modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTest.java
+20 -10
View File
@@ -173,35 +173,45 @@ public class FetchHTTPTest extends ProcessorTestBase {
} else if (target.equals("/401-no-challenge")) {
response.setStatus(401);
response.setContentType("text/plain;charset=US-ASCII");
response.setDateHeader("Last-Modified", 0);
response.setHeader("ETag", ETAG_TEST_VALUE);
response.getOutputStream().write(DEFAULT_PAYLOAD_STRING.getBytes("US-ASCII"));
((Request)request).setHandled(true);
} else if (target.equals("/cp1251")) {
response.setContentType("text/plain;charset=cp1251");
response.setDateHeader("Last-Modified", 0);
response.setHeader("ETag", ETAG_TEST_VALUE);
response.setStatus(HttpServletResponse.SC_OK);
response.getOutputStream().write(CP1251_PAYLOAD);
((Request)request).setHandled(true);
} else if (target.equals("/unsupported-charset")) {
response.setContentType("text/plain;charset=UNSUPPORTED-CHARSET");
response.setDateHeader("Last-Modified", 0);
response.setHeader("ETag", ETAG_TEST_VALUE);
response.setStatus(HttpServletResponse.SC_OK);
response.getOutputStream().write(DEFAULT_PAYLOAD_STRING.getBytes("US-ASCII"));
((Request)request).setHandled(true);
} else if (target.equals("/invalid-charset")) {
response.setContentType("text/plain;charset=%%INVALID-CHARSET%%");
response.setDateHeader("Last-Modified", 0);
response.setHeader("ETag", ETAG_TEST_VALUE);
response.setStatus(HttpServletResponse.SC_OK);
response.getOutputStream().write(DEFAULT_PAYLOAD_STRING.getBytes("US-ASCII"));
((Request)request).setHandled(true);
} else if (target.equals("/if-modified-since")) {
if (request.getHeader("if-modified-since") != null) {
response.setStatus(304);
((Request)request).setHandled(true);
} else {
response.setContentType("text/plain;charset=US-ASCII");
response.setStatus(HttpServletResponse.SC_OK);
response.getOutputStream().write(DEFAULT_PAYLOAD_STRING.getBytes("US-ASCII"));
((Request)request).setHandled(true);
}
} else if (target.equals("/if-none-match")) {
if (request.getHeader("if-none-match") != null) {
response.setStatus(304);
((Request)request).setHandled(true);
} else {
response.setContentType("text/plain;charset=US-ASCII");
response.setStatus(HttpServletResponse.SC_OK);
response.getOutputStream().write(DEFAULT_PAYLOAD_STRING.getBytes("US-ASCII"));
((Request)request).setHandled(true);
}
} else {
response.setContentType("text/plain;charset=US-ASCII");
response.setDateHeader("Last-Modified", 0);
response.setHeader("ETag", ETAG_TEST_VALUE);
response.setStatus(HttpServletResponse.SC_OK);
response.getOutputStream().write(DEFAULT_PAYLOAD_STRING.getBytes("US-ASCII"));
((Request)request).setHandled(true);
modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTests.java
+40 -10
View File
@@ -66,7 +66,9 @@ import org.archive.modules.ProcessorTestBase;
import org.archive.modules.credential.HttpAuthenticationCredential;
import org.archive.modules.deciderules.RejectDecideRule;
import org.archive.modules.recrawl.FetchHistoryProcessor;
import org.archive.modules.revisit.ServerNotModifiedRevisit;
import org.archive.net.UURI;
import org.archive.util.OneLineSimpleLogger;
import org.jboss.netty.handler.codec.http.HttpRequest;
import org.littleshoot.proxy.DefaultHttpProxyServer;
import org.littleshoot.proxy.HttpFilter;
@@ -81,6 +83,17 @@ import org.littleshoot.proxy.ProxyAuthorizationHandler;
* FetchHTTPTest suite.
*/
public class FetchHTTPTests extends ProcessorTestBase {
// private static Logger logger = Logger.getLogger(FetchHTTPTests.class.getName());
// static {
// Logger.getLogger("").setLevel(Level.FINE);
// for (java.util.logging.Handler h : Logger.getLogger("").getHandlers()) {
// h.setLevel(Level.ALL);
// h.setFormatter(new OneLineSimpleLogger());
// }
// }
protected FetchHTTP fetcher;
@@ -528,11 +541,11 @@ public class FetchHTTPTests extends ProcessorTestBase {
public void testSendIfModifiedSince() throws Exception {
fetcher().setSendIfModifiedSince(true);
CrawlURI curi = makeCrawlURI("http://localhost:7777/");
CrawlURI curi = makeCrawlURI("http://localhost:7777/if-modified-since");
fetcher().process(curi);
assertFalse(httpRequestString(curi).toLowerCase().contains("if-modified-since"));
assertFalse(httpRequestString(curi).toLowerCase().contains("if-modified-since: "));
assertTrue(curi.getHttpResponseHeader("last-modified").equals("Thu, 01 Jan 1970 00:00:00 GMT"));
runDefaultChecks(curi);
runDefaultChecks(curi, "requestLine");
// logger.info("before FetchHistoryProcessor fetchHistory=" + Arrays.toString(curi.getFetchHistory()));
FetchHistoryProcessor fetchHistoryProcessor = new FetchHistoryProcessor();
@@ -541,27 +554,44 @@ public class FetchHTTPTests extends ProcessorTestBase {
fetcher().process(curi);
// logger.info("\n" + httpRequestString(curi));
// logger.info("\n" + rawResponseString(curi));
assertTrue(httpRequestString(curi).contains("If-Modified-Since: Thu, 01 Jan 1970 00:00:00 GMT\r\n"));
runDefaultChecks(curi);
// XXX make server send 304 not-modified and check for it here?
assertTrue(curi.getFetchStatus() == 304);
assertNull(curi.getRevisitProfile());
fetchHistoryProcessor.process(curi);
assertNotNull(curi.getRevisitProfile());
assertTrue(curi.getRevisitProfile() instanceof ServerNotModifiedRevisit);
ServerNotModifiedRevisit revisit = (ServerNotModifiedRevisit) curi.getRevisitProfile();
assertEquals("Thu, 01 Jan 1970 00:00:00 GMT", revisit.getLastModified());
assertNull(revisit.getETag());
}
public void testSendIfNoneMatch() throws Exception {
fetcher().setSendIfNoneMatch(true);
CrawlURI curi = makeCrawlURI("http://localhost:7777/");
CrawlURI curi = makeCrawlURI("http://localhost:7777/if-none-match");
fetcher().process(curi);
assertFalse(httpRequestString(curi).toLowerCase().contains("if-none-match"));
assertFalse(httpRequestString(curi).toLowerCase().contains("if-none-match: "));
assertTrue(curi.getHttpResponseHeader("etag").equals(ETAG_TEST_VALUE));
runDefaultChecks(curi);
runDefaultChecks(curi, "requestLine");
FetchHistoryProcessor fetchHistoryProcessor = new FetchHistoryProcessor();
fetchHistoryProcessor.process(curi);
fetcher().process(curi);
// logger.info("\n" + httpRequestString(curi));
// logger.info("\n" + rawResponseString(curi));
assertTrue(httpRequestString(curi).contains("If-None-Match: " + ETAG_TEST_VALUE + "\r\n"));
runDefaultChecks(curi);
// XXX make server send 304 not-modified and check for it here?
assertNull(curi.getRevisitProfile());
fetchHistoryProcessor.process(curi);
assertNotNull(curi.getRevisitProfile());
assertTrue(curi.getRevisitProfile() instanceof ServerNotModifiedRevisit);
ServerNotModifiedRevisit revisit = (ServerNotModifiedRevisit) curi.getRevisitProfile();
assertEquals(ETAG_TEST_VALUE, revisit.getETag());
assertNull(revisit.getLastModified());
}
public void testShouldFetchBodyRule() throws Exception {
modules/src/test/java/org/archive/modules/recrawl/ContentDigestHistoryTest.java
+1 -1
View File
@@ -259,7 +259,7 @@ public class ContentDigestHistoryTest extends TmpDirTestCase {
assertEquals(1, curi2.getContentDigestHistory().get(A_CONTENT_DIGEST_COUNT));
warcWriter.process(curi2);
assertTrue(curi2.getAnnotations().contains("warcRevisit:digest"));
assertTrue(curi2.getAnnotations().contains("duplicate:digest"));
assertEquals(curi1.getUURI().toString(), curi2.getContentDigestHistory().get(A_ORIGINAL_URL));
assertNotSame(curi2.getUURI().toString(), curi2.getContentDigestHistory().get(A_ORIGINAL_URL));
assertEquals(2, curi2.getContentDigestHistory().get(A_CONTENT_DIGEST_COUNT));