diff --git a/modules/src/main/java/org/archive/modules/revisit/ServerNotModifiedRevisit.java b/modules/src/main/java/org/archive/modules/revisit/ServerNotModifiedRevisit.java index 844facf1..0ec25804 100644 --- a/modules/src/main/java/org/archive/modules/revisit/ServerNotModifiedRevisit.java +++ b/modules/src/main/java/org/archive/modules/revisit/ServerNotModifiedRevisit.java @@ -40,12 +40,12 @@ public class ServerNotModifiedRevisit extends AbstractProfile { return headers; } - public String geteTag() { + public String getETag() { return eTag; } - public void seteTag(String eTag) { + public void setETag(String eTag) { this.eTag = eTag; } diff --git a/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTest.java b/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTest.java index 76154bb8..23a99f75 100644 --- a/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTest.java +++ b/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTest.java @@ -173,35 +173,45 @@ public class FetchHTTPTest extends ProcessorTestBase { } else if (target.equals("/401-no-challenge")) { response.setStatus(401); response.setContentType("text/plain;charset=US-ASCII"); - response.setDateHeader("Last-Modified", 0); - response.setHeader("ETag", ETAG_TEST_VALUE); response.getOutputStream().write(DEFAULT_PAYLOAD_STRING.getBytes("US-ASCII")); ((Request)request).setHandled(true); } else if (target.equals("/cp1251")) { response.setContentType("text/plain;charset=cp1251"); - response.setDateHeader("Last-Modified", 0); - response.setHeader("ETag", ETAG_TEST_VALUE); response.setStatus(HttpServletResponse.SC_OK); response.getOutputStream().write(CP1251_PAYLOAD); ((Request)request).setHandled(true); } else if (target.equals("/unsupported-charset")) { response.setContentType("text/plain;charset=UNSUPPORTED-CHARSET"); - response.setDateHeader("Last-Modified", 0); - response.setHeader("ETag", ETAG_TEST_VALUE); response.setStatus(HttpServletResponse.SC_OK); response.getOutputStream().write(DEFAULT_PAYLOAD_STRING.getBytes("US-ASCII")); ((Request)request).setHandled(true); } else if (target.equals("/invalid-charset")) { response.setContentType("text/plain;charset=%%INVALID-CHARSET%%"); - response.setDateHeader("Last-Modified", 0); - response.setHeader("ETag", ETAG_TEST_VALUE); response.setStatus(HttpServletResponse.SC_OK); response.getOutputStream().write(DEFAULT_PAYLOAD_STRING.getBytes("US-ASCII")); ((Request)request).setHandled(true); + } else if (target.equals("/if-modified-since")) { + if (request.getHeader("if-modified-since") != null) { + response.setStatus(304); + ((Request)request).setHandled(true); + } else { + response.setContentType("text/plain;charset=US-ASCII"); + response.setStatus(HttpServletResponse.SC_OK); + response.getOutputStream().write(DEFAULT_PAYLOAD_STRING.getBytes("US-ASCII")); + ((Request)request).setHandled(true); + } + } else if (target.equals("/if-none-match")) { + if (request.getHeader("if-none-match") != null) { + response.setStatus(304); + ((Request)request).setHandled(true); + } else { + response.setContentType("text/plain;charset=US-ASCII"); + response.setStatus(HttpServletResponse.SC_OK); + response.getOutputStream().write(DEFAULT_PAYLOAD_STRING.getBytes("US-ASCII")); + ((Request)request).setHandled(true); + } } else { response.setContentType("text/plain;charset=US-ASCII"); - response.setDateHeader("Last-Modified", 0); - response.setHeader("ETag", ETAG_TEST_VALUE); response.setStatus(HttpServletResponse.SC_OK); response.getOutputStream().write(DEFAULT_PAYLOAD_STRING.getBytes("US-ASCII")); ((Request)request).setHandled(true); diff --git a/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTests.java b/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTests.java index 385258f1..4201715d 100644 --- a/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTests.java +++ b/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTests.java @@ -66,7 +66,9 @@ import org.archive.modules.ProcessorTestBase; import org.archive.modules.credential.HttpAuthenticationCredential; import org.archive.modules.deciderules.RejectDecideRule; import org.archive.modules.recrawl.FetchHistoryProcessor; +import org.archive.modules.revisit.ServerNotModifiedRevisit; import org.archive.net.UURI; +import org.archive.util.OneLineSimpleLogger; import org.jboss.netty.handler.codec.http.HttpRequest; import org.littleshoot.proxy.DefaultHttpProxyServer; import org.littleshoot.proxy.HttpFilter; @@ -81,6 +83,17 @@ import org.littleshoot.proxy.ProxyAuthorizationHandler; * FetchHTTPTest suite. */ public class FetchHTTPTests extends ProcessorTestBase { + + // private static Logger logger = Logger.getLogger(FetchHTTPTests.class.getName()); + + // static { + // Logger.getLogger("").setLevel(Level.FINE); + // for (java.util.logging.Handler h : Logger.getLogger("").getHandlers()) { + // h.setLevel(Level.ALL); + // h.setFormatter(new OneLineSimpleLogger()); + // } + // } + protected FetchHTTP fetcher; @@ -528,11 +541,11 @@ public class FetchHTTPTests extends ProcessorTestBase { public void testSendIfModifiedSince() throws Exception { fetcher().setSendIfModifiedSince(true); - CrawlURI curi = makeCrawlURI("http://localhost:7777/"); + CrawlURI curi = makeCrawlURI("http://localhost:7777/if-modified-since"); fetcher().process(curi); - assertFalse(httpRequestString(curi).toLowerCase().contains("if-modified-since")); + assertFalse(httpRequestString(curi).toLowerCase().contains("if-modified-since: ")); assertTrue(curi.getHttpResponseHeader("last-modified").equals("Thu, 01 Jan 1970 00:00:00 GMT")); - runDefaultChecks(curi); + runDefaultChecks(curi, "requestLine"); // logger.info("before FetchHistoryProcessor fetchHistory=" + Arrays.toString(curi.getFetchHistory())); FetchHistoryProcessor fetchHistoryProcessor = new FetchHistoryProcessor(); @@ -541,27 +554,44 @@ public class FetchHTTPTests extends ProcessorTestBase { fetcher().process(curi); // logger.info("\n" + httpRequestString(curi)); + // logger.info("\n" + rawResponseString(curi)); assertTrue(httpRequestString(curi).contains("If-Modified-Since: Thu, 01 Jan 1970 00:00:00 GMT\r\n")); - runDefaultChecks(curi); - // XXX make server send 304 not-modified and check for it here? + assertTrue(curi.getFetchStatus() == 304); + + assertNull(curi.getRevisitProfile()); + fetchHistoryProcessor.process(curi); + assertNotNull(curi.getRevisitProfile()); + assertTrue(curi.getRevisitProfile() instanceof ServerNotModifiedRevisit); + ServerNotModifiedRevisit revisit = (ServerNotModifiedRevisit) curi.getRevisitProfile(); + assertEquals("Thu, 01 Jan 1970 00:00:00 GMT", revisit.getLastModified()); + assertNull(revisit.getETag()); } public void testSendIfNoneMatch() throws Exception { fetcher().setSendIfNoneMatch(true); - CrawlURI curi = makeCrawlURI("http://localhost:7777/"); + CrawlURI curi = makeCrawlURI("http://localhost:7777/if-none-match"); fetcher().process(curi); - assertFalse(httpRequestString(curi).toLowerCase().contains("if-none-match")); + assertFalse(httpRequestString(curi).toLowerCase().contains("if-none-match: ")); assertTrue(curi.getHttpResponseHeader("etag").equals(ETAG_TEST_VALUE)); - runDefaultChecks(curi); + runDefaultChecks(curi, "requestLine"); FetchHistoryProcessor fetchHistoryProcessor = new FetchHistoryProcessor(); fetchHistoryProcessor.process(curi); fetcher().process(curi); + // logger.info("\n" + httpRequestString(curi)); + // logger.info("\n" + rawResponseString(curi)); assertTrue(httpRequestString(curi).contains("If-None-Match: " + ETAG_TEST_VALUE + "\r\n")); - runDefaultChecks(curi); - // XXX make server send 304 not-modified and check for it here? + + assertNull(curi.getRevisitProfile()); + fetchHistoryProcessor.process(curi); + assertNotNull(curi.getRevisitProfile()); + assertTrue(curi.getRevisitProfile() instanceof ServerNotModifiedRevisit); + ServerNotModifiedRevisit revisit = (ServerNotModifiedRevisit) curi.getRevisitProfile(); + assertEquals(ETAG_TEST_VALUE, revisit.getETag()); + assertNull(revisit.getLastModified()); + } public void testShouldFetchBodyRule() throws Exception { diff --git a/modules/src/test/java/org/archive/modules/recrawl/ContentDigestHistoryTest.java b/modules/src/test/java/org/archive/modules/recrawl/ContentDigestHistoryTest.java index 27e7db3d..9212e35b 100644 --- a/modules/src/test/java/org/archive/modules/recrawl/ContentDigestHistoryTest.java +++ b/modules/src/test/java/org/archive/modules/recrawl/ContentDigestHistoryTest.java @@ -259,7 +259,7 @@ public class ContentDigestHistoryTest extends TmpDirTestCase { assertEquals(1, curi2.getContentDigestHistory().get(A_CONTENT_DIGEST_COUNT)); warcWriter.process(curi2); - assertTrue(curi2.getAnnotations().contains("warcRevisit:digest")); + assertTrue(curi2.getAnnotations().contains("duplicate:digest")); assertEquals(curi1.getUURI().toString(), curi2.getContentDigestHistory().get(A_ORIGINAL_URL)); assertNotSame(curi2.getUURI().toString(), curi2.getContentDigestHistory().get(A_ORIGINAL_URL)); assertEquals(2, curi2.getContentDigestHistory().get(A_CONTENT_DIGEST_COUNT));