diff --git a/engine/pom.xml b/engine/pom.xml index 9e572e00..293aeb7d 100644 --- a/engine/pom.xml +++ b/engine/pom.xml @@ -31,8 +31,8 @@ ${jetty.version} - org.eclipse.jetty - jetty-servlet + org.eclipse.jetty.ee10 + jetty-ee10-servlet ${jetty.version} test @@ -46,9 +46,11 @@ org.restlet.ext.jetty ${restlet.version} + - org.eclipse.jetty - jetty-client + org.mortbay.jetty.quiche + jetty-quiche-native @@ -109,6 +111,6 @@ UTF-8 - 2.5.0 + 2.6.0-m2 diff --git a/engine/src/main/java/org/archive/crawler/Heritrix.java b/engine/src/main/java/org/archive/crawler/Heritrix.java index 13adae33..d8aced31 100644 --- a/engine/src/main/java/org/archive/crawler/Heritrix.java +++ b/engine/src/main/java/org/archive/crawler/Heritrix.java @@ -49,14 +49,14 @@ import org.apache.commons.lang.StringUtils; import org.archive.crawler.framework.CrawlJob; import org.archive.crawler.framework.Engine; import org.archive.crawler.restlet.EngineApplication; +import org.archive.crawler.restlet.NoSniHostCheckHttpsServerHelper; import org.archive.crawler.restlet.RateLimitGuard; import org.archive.util.ArchiveUtils; import org.archive.util.KeyTool; import org.restlet.Component; +import org.restlet.Context; import org.restlet.Server; -import org.restlet.data.ChallengeScheme; import org.restlet.data.Protocol; -import org.restlet.security.ChallengeAuthenticator; import org.restlet.security.MapVerifier; @@ -154,6 +154,7 @@ public class Heritrix { "to use for crawling."); options.addOption(null, "proxy-port", true, "Global http(s) proxy port " + "to use for crawling."); + options.addOption(null, "sni-host-check", false, "Validates SNI hostname against the SSL certificate"); return options; } @@ -349,13 +350,21 @@ public class Heritrix { engine = new Engine(jobsDir); component = new Component(); + // disable SNI host check by default for backwards compatibility with existing ad-hoc certificates + String helperClass = null; + if (!cl.hasOption("sni-host-check")) { + org.restlet.engine.Engine.getInstance().getRegisteredServers().add( + new NoSniHostCheckHttpsServerHelper(null)); + helperClass = NoSniHostCheckHttpsServerHelper.class.getName(); + } + if(bindHosts.isEmpty()) { // listen all addresses - setupServer(component, port, null, keystorePath, keystorePassword, keyPassword); + setupServer(component, port, null, keystorePath, keystorePassword, keyPassword, helperClass); } else { // bind only to declared addresses, or just 'localhost' for(String address : bindHosts) { - setupServer(component, port, address, keystorePath, keystorePassword, keyPassword); + setupServer(component, port, address, keystorePath, keystorePassword, keyPassword, helperClass); } } component.getClients().add(Protocol.FILE); @@ -504,8 +513,10 @@ public class Heritrix { * @param keystorePassword * @param keyPassword */ - protected void setupServer(Component component, int port, String address, String keystorePath, String keystorePassword, String keyPassword) { - Server server = component.getServers().add(Protocol.HTTPS, address, port); + protected void setupServer(Component component, int port, String address, String keystorePath, String keystorePassword, String keyPassword, String helperClass) { + Context serverContext = component.getServers().getContext().createChildContext(); + Server server = new Server(serverContext, List.of(Protocol.HTTPS), address, port, null, helperClass); + component.getServers().add(server); server.getContext().getParameters().add("keystorePath", keystorePath); server.getContext().getParameters().add("keystorePassword", keystorePassword); server.getContext().getParameters().add("keyPassword", keyPassword); diff --git a/engine/src/main/java/org/archive/crawler/restlet/JobResource.java b/engine/src/main/java/org/archive/crawler/restlet/JobResource.java index 8ad04377..ebcd1ceb 100644 --- a/engine/src/main/java/org/archive/crawler/restlet/JobResource.java +++ b/engine/src/main/java/org/archive/crawler/restlet/JobResource.java @@ -135,7 +135,8 @@ public class JobResource extends BaseResource { // conditional on whether /anypath/ service is present? String fullPath = f.getAbsolutePath(); fullPath = fullPath.replace(File.separatorChar, '/'); - return "../../anypath/" + fullPath; + if (!fullPath.startsWith("/")) fullPath = "/" + fullPath; + return "../../anypath" + fullPath; } @Override diff --git a/engine/src/main/java/org/archive/crawler/restlet/NoSniHostCheckHttpsServerHelper.java b/engine/src/main/java/org/archive/crawler/restlet/NoSniHostCheckHttpsServerHelper.java new file mode 100644 index 00000000..dd28d893 --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/restlet/NoSniHostCheckHttpsServerHelper.java @@ -0,0 +1,38 @@ +package org.archive.crawler.restlet; + +import org.eclipse.jetty.server.HttpConnectionFactory; +import org.eclipse.jetty.server.SecureRequestCustomizer; +import org.eclipse.jetty.server.ServerConnector; +import org.restlet.Server; +import org.restlet.ext.jetty.HttpsServerHelper; + +/** + * Subclass of HttpServerHelper which disables the SNI host check. This is to main backwards + * compatibility with the existing Heritrix ad-hoc certificates that don't include a hostname. + */ +public class NoSniHostCheckHttpsServerHelper extends HttpsServerHelper { + public NoSniHostCheckHttpsServerHelper(Server server) { + super(server); + } + + @Override + protected org.eclipse.jetty.server.Server getWrappedServer() { + org.eclipse.jetty.server.Server wrappedServer = super.getWrappedServer(); + disableSniHostCheck(wrappedServer); + return wrappedServer; + } + + private static void disableSniHostCheck(org.eclipse.jetty.server.Server jettyServer) { + for (var connector : jettyServer.getConnectors()) { + if (connector instanceof ServerConnector serverConnector) { + var connectionFactory = serverConnector.getConnectionFactory(HttpConnectionFactory.class); + if (connectionFactory != null) { + var secureRequestCustomizer = connectionFactory.getHttpConfiguration().getCustomizer(SecureRequestCustomizer.class); + if (secureRequestCustomizer != null) { + secureRequestCustomizer.setSniHostCheck(false); + } + } + } + } + } +} diff --git a/engine/src/main/java/org/archive/crawler/restlet/models/CrawlJobModel.java b/engine/src/main/java/org/archive/crawler/restlet/models/CrawlJobModel.java index 929a9cf1..fc3bc44f 100644 --- a/engine/src/main/java/org/archive/crawler/restlet/models/CrawlJobModel.java +++ b/engine/src/main/java/org/archive/crawler/restlet/models/CrawlJobModel.java @@ -17,6 +17,7 @@ import org.archive.checkpointing.Checkpoint; import org.archive.crawler.framework.CrawlController.State; import org.archive.crawler.framework.CrawlJob; import org.archive.crawler.reporting.Report; +import org.archive.crawler.restlet.JobResource; import org.archive.spring.ConfigPath; import org.archive.util.ArchiveUtils; import org.archive.util.FileUtils; @@ -134,10 +135,16 @@ public class CrawlJobModel extends LinkedHashMap implements Seri } } this.put("checkpointFiles",checkpointFiles); - if (crawlJob.hasApplicationContext()) - this.put("alertLogFilePath",crawlJob.getCrawlController().getLoggerModule().getAlertsLogPath().getFile().getAbsolutePath()); - if(crawlJob.isRunning() || (crawlJob.hasApplicationContext() && !crawlJob.isLaunchable())) - this.put("crawlLogFilePath",crawlJob.getCrawlController().getLoggerModule().getCrawlLogPath().getFile().getAbsolutePath()); + if (crawlJob.hasApplicationContext()) { + File file = crawlJob.getCrawlController().getLoggerModule().getAlertsLogPath().getFile(); + this.put("alertLogFilePath", file.getAbsolutePath()); + this.put("alertLogFileUrl", JobResource.getHrefPath(file, crawlJob)); + } + if(crawlJob.isRunning() || (crawlJob.hasApplicationContext() && !crawlJob.isLaunchable())) { + File file = crawlJob.getCrawlController().getLoggerModule().getCrawlLogPath().getFile(); + this.put("crawlLogFilePath", file.getAbsolutePath()); + this.put("crawlLogFileUrl", JobResource.getHrefPath(file, crawlJob)); + } this.put("reports", generateReports()); } public String formatBytes(Long bytes){ @@ -209,7 +216,7 @@ public class CrawlJobModel extends LinkedHashMap implements Seri configMap.put("key", key); configMap.put("name", cp.getName()); configMap.put("path",FileUtils.tryToCanonicalize(cp.getFile()).getAbsolutePath()); - configMap.put("url",baseRef+"engine/anypath/"+configMap.get("path")); + configMap.put("url", baseRef + JobResource.getHrefPath(cp.getFile(), crawlJob)); configMap.put("editable", EDIT_FILTER.accept(cp.getFile())); referencedPaths.add(configMap); } diff --git a/engine/src/main/resources/org/archive/crawler/restlet/Job.ftl b/engine/src/main/resources/org/archive/crawler/restlet/Job.ftl index c8afc265..ba5aed6d 100644 --- a/engine/src/main/resources/org/archive/crawler/restlet/Job.ftl +++ b/engine/src/main/resources/org/archive/crawler/restlet/Job.ftl @@ -190,7 +190,7 @@ none <#else> ${job.alertCount} - tail alert log... + tail alert log... @@ -272,7 +272,7 @@ <#if (job.isRunning || (job.hasApplicationContext && !job.isLaunchable))>
-

Crawl Log more

+

Crawl Log more

@@ -306,7 +306,7 @@ <#if config.path??> - ${config.path}<#if config.editable> [edit] + ${config.path}<#if config.editable> [edit] <#else> unset diff --git a/engine/src/test/java/org/archive/crawler/selftest/CheckpointSelfTest.java b/engine/src/test/java/org/archive/crawler/selftest/CheckpointSelfTest.java index 7f7d9e6b..4179b96b 100644 --- a/engine/src/test/java/org/archive/crawler/selftest/CheckpointSelfTest.java +++ b/engine/src/test/java/org/archive/crawler/selftest/CheckpointSelfTest.java @@ -22,10 +22,11 @@ package org.archive.crawler.selftest; import java.io.IOException; import org.archive.crawler.framework.CrawlJob; +import org.eclipse.jetty.ee10.servlet.ServletContextHandler; +import org.eclipse.jetty.ee10.servlet.ServletHandler; +import org.eclipse.jetty.ee10.servlet.ServletHolder; import org.eclipse.jetty.server.Server; import org.eclipse.jetty.server.ServerConnector; -import org.eclipse.jetty.servlet.ServletHandler; -import org.eclipse.jetty.servlet.ServletHolder; /** @@ -89,8 +90,6 @@ public class CheckpointSelfTest extends SelfTestBase { sc.setHost(HOST); sc.setPort(port); server.addConnector(sc); - ServletHandler servletHandler = new ServletHandler(); - server.setHandler(servletHandler); RandomServlet random = new RandomServlet(); random.setHost(HOST); @@ -99,8 +98,10 @@ public class CheckpointSelfTest extends SelfTestBase { random.setMaxHops(MAX_HOPS); random.setPathRoot("random"); - ServletHolder holder = new ServletHolder(random); - servletHandler.addServletWithMapping(holder, "/random/*"); + ServletContextHandler contextHandler = new ServletContextHandler(ServletContextHandler.SESSIONS); + contextHandler.addServlet(random, "/random/*"); + server.setHandler(contextHandler); + server.start(); return server; } diff --git a/engine/src/test/java/org/archive/crawler/selftest/FormAuthSelfTest.java b/engine/src/test/java/org/archive/crawler/selftest/FormAuthSelfTest.java index 6e04f56a..32d055b2 100644 --- a/engine/src/test/java/org/archive/crawler/selftest/FormAuthSelfTest.java +++ b/engine/src/test/java/org/archive/crawler/selftest/FormAuthSelfTest.java @@ -19,14 +19,15 @@ package org.archive.crawler.selftest; +import org.eclipse.jetty.ee10.servlet.ServletContextHandler; +import org.eclipse.jetty.ee10.servlet.ServletHandler; +import org.eclipse.jetty.ee10.servlet.ServletHolder; import org.eclipse.jetty.server.Handler; import org.eclipse.jetty.server.Server; import org.eclipse.jetty.server.ServerConnector; import org.eclipse.jetty.server.handler.DefaultHandler; -import org.eclipse.jetty.server.handler.HandlerList; import org.eclipse.jetty.server.handler.ResourceHandler; -import org.eclipse.jetty.servlet.ServletHandler; -import org.eclipse.jetty.servlet.ServletHolder; +import org.eclipse.jetty.util.resource.ResourceFactory; import java.util.Arrays; import java.util.Collections; @@ -66,19 +67,16 @@ public class FormAuthSelfTest sc.setPort(7777); server.addConnector(sc); ResourceHandler rhandler = new ResourceHandler(); - rhandler.setResourceBase(getSrcHtdocs().getAbsolutePath()); - - ServletHandler servletHandler = new ServletHandler(); - - HandlerList handlers = new HandlerList(); - handlers.setHandlers(new Handler[] { - rhandler, - servletHandler, - new DefaultHandler() }); - server.setHandler(handlers); - - ServletHolder holder = new ServletHolder(new FormAuthServlet()); - servletHandler.addServletWithMapping(holder, "/login/*"); + ResourceFactory resourceFactory = ResourceFactory.of(server); + rhandler.setBaseResource(resourceFactory.newResource(getSrcHtdocs().toPath().toAbsolutePath())); + + ServletContextHandler contextHandler = new ServletContextHandler(ServletContextHandler.SESSIONS); + contextHandler.addServlet(FormAuthServlet.class, "/login/*"); + + server.setHandler(new Handler.Sequence( + rhandler, + contextHandler, + new DefaultHandler())); this.httpServer = server; this.httpServer.start(); diff --git a/engine/src/test/java/org/archive/crawler/selftest/FormAuthServlet.java b/engine/src/test/java/org/archive/crawler/selftest/FormAuthServlet.java index fb35d877..fa4ff59e 100644 --- a/engine/src/test/java/org/archive/crawler/selftest/FormAuthServlet.java +++ b/engine/src/test/java/org/archive/crawler/selftest/FormAuthServlet.java @@ -18,12 +18,12 @@ */ package org.archive.crawler.selftest; -import java.io.IOException; +import jakarta.servlet.ServletException; +import jakarta.servlet.http.HttpServlet; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; -import javax.servlet.ServletException; -import javax.servlet.http.HttpServlet; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; +import java.io.IOException; /** @@ -36,8 +36,8 @@ public class FormAuthServlet extends HttpServlet { @Override - protected void doPost(HttpServletRequest req, HttpServletResponse resp) - throws ServletException, IOException { + protected void doPost(HttpServletRequest req, HttpServletResponse resp) + throws IOException { String username = req.getParameter("username"); String password = req.getParameter("password"); if (username.equals("Mr. Happy Pants") && password.equals("xyzzy")) { diff --git a/engine/src/test/java/org/archive/crawler/selftest/FormLoginSelfTest.java b/engine/src/test/java/org/archive/crawler/selftest/FormLoginSelfTest.java index 332f6835..5d17c42f 100644 --- a/engine/src/test/java/org/archive/crawler/selftest/FormLoginSelfTest.java +++ b/engine/src/test/java/org/archive/crawler/selftest/FormLoginSelfTest.java @@ -19,14 +19,13 @@ package org.archive.crawler.selftest; +import org.eclipse.jetty.ee10.servlet.ServletContextHandler; import org.eclipse.jetty.server.Handler; import org.eclipse.jetty.server.Server; import org.eclipse.jetty.server.ServerConnector; import org.eclipse.jetty.server.handler.DefaultHandler; -import org.eclipse.jetty.server.handler.HandlerList; import org.eclipse.jetty.server.handler.ResourceHandler; -import org.eclipse.jetty.servlet.ServletHandler; -import org.eclipse.jetty.servlet.ServletHolder; +import org.eclipse.jetty.util.resource.ResourceFactory; import java.util.Arrays; import java.util.Collections; @@ -65,20 +64,14 @@ public class FormLoginSelfTest sc.setPort(7777); server.addConnector(sc); ResourceHandler rhandler = new ResourceHandler(); - rhandler.setResourceBase(getSrcHtdocs().getAbsolutePath()); - - ServletHandler servletHandler = new ServletHandler(); - - HandlerList handlers = new HandlerList(); - handlers.setHandlers(new Handler[] { - rhandler, - servletHandler, - new DefaultHandler() }); - server.setHandler(handlers); - - ServletHolder holder = new ServletHolder(new FormAuthServlet()); - servletHandler.addServletWithMapping(holder, "/login/*"); + ResourceFactory resourceFactory = ResourceFactory.of(server); + rhandler.setBaseResource(resourceFactory.newResource(getSrcHtdocs().toPath().toAbsolutePath())); + ServletContextHandler contextHandler = new ServletContextHandler(ServletContextHandler.SESSIONS); + contextHandler.addServlet(FormAuthServlet.class, "/login/*"); + + server.setHandler(new Handler.Sequence(rhandler, contextHandler, new DefaultHandler())); + this.httpServer = server; this.httpServer.start(); } diff --git a/engine/src/test/java/org/archive/crawler/selftest/HttpAuthSelfTest.java b/engine/src/test/java/org/archive/crawler/selftest/HttpAuthSelfTest.java index 6ad5ebd2..10a0d887 100644 --- a/engine/src/test/java/org/archive/crawler/selftest/HttpAuthSelfTest.java +++ b/engine/src/test/java/org/archive/crawler/selftest/HttpAuthSelfTest.java @@ -24,15 +24,16 @@ import java.util.Collections; import java.util.HashSet; import java.util.Set; +import org.eclipse.jetty.ee10.servlet.ServletHandler; +import org.eclipse.jetty.ee10.servlet.security.ConstraintMapping; +import org.eclipse.jetty.ee10.servlet.security.ConstraintSecurityHandler; import org.eclipse.jetty.security.*; import org.eclipse.jetty.server.Handler; import org.eclipse.jetty.server.Server; import org.eclipse.jetty.server.ServerConnector; import org.eclipse.jetty.server.handler.DefaultHandler; -import org.eclipse.jetty.server.handler.HandlerList; import org.eclipse.jetty.server.handler.ResourceHandler; -import org.eclipse.jetty.servlet.ServletHandler; -import org.eclipse.jetty.util.security.Constraint; +import org.eclipse.jetty.util.resource.ResourceFactory; import org.eclipse.jetty.util.security.Password; /** @@ -64,10 +65,7 @@ public class HttpAuthSelfTest protected void startHttpServer() throws Exception { Server server = new Server(); - Constraint constraint = new Constraint(); - constraint.setName(Constraint.__BASIC_AUTH);; - constraint.setRoles(new String[]{"user","admin","moderator"}); - constraint.setAuthenticate(true); + Constraint constraint = Constraint.from("user","admin","moderator"); ConstraintMapping cm = new ConstraintMapping(); cm.setConstraint(constraint); @@ -87,17 +85,13 @@ public class HttpAuthSelfTest sc.setPort(7777); server.addConnector(sc); ResourceHandler rhandler = new ResourceHandler(); - rhandler.setResourceBase(getSrcHtdocs().getAbsolutePath()); + ResourceFactory resourceFactory = ResourceFactory.of(server); + rhandler.setBaseResource(resourceFactory.newResource(getSrcHtdocs().toPath().toAbsolutePath())); - ServletHandler servletHandler = new ServletHandler(); - - HandlerList handlers = new HandlerList(); - handlers.setHandlers(new Handler[] { - securityHandler, - rhandler, - servletHandler, - new DefaultHandler() }); - server.setHandler(handlers); + server.setHandler(new Handler.Sequence( + securityHandler, + rhandler, + new DefaultHandler())); this.httpServer = server; this.httpServer.start(); diff --git a/engine/src/test/java/org/archive/crawler/selftest/RandomServlet.java b/engine/src/test/java/org/archive/crawler/selftest/RandomServlet.java index 65f98d76..fefa9833 100644 --- a/engine/src/test/java/org/archive/crawler/selftest/RandomServlet.java +++ b/engine/src/test/java/org/archive/crawler/selftest/RandomServlet.java @@ -18,15 +18,15 @@ */ package org.archive.crawler.selftest; +import jakarta.servlet.ServletException; +import jakarta.servlet.http.HttpServlet; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; + import java.io.IOException; import java.io.Writer; import java.util.Random; -import javax.servlet.ServletException; -import javax.servlet.http.HttpServlet; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - /** * @author pjack @@ -104,8 +104,8 @@ public class RandomServlet extends HttpServlet { @Override - protected void doGet(HttpServletRequest req, HttpServletResponse resp) - throws ServletException, IOException { + protected void doGet(HttpServletRequest req, HttpServletResponse resp) + throws IOException { resp.setContentType("text/html"); RandomServletLinkWriter rslw = new RandomServletLinkWriter(); diff --git a/engine/src/test/java/org/archive/crawler/selftest/SelfTestBase.java b/engine/src/test/java/org/archive/crawler/selftest/SelfTestBase.java index 7f09dfbe..edba319c 100644 --- a/engine/src/test/java/org/archive/crawler/selftest/SelfTestBase.java +++ b/engine/src/test/java/org/archive/crawler/selftest/SelfTestBase.java @@ -41,9 +41,11 @@ import org.archive.util.TmpDirTestCase; import org.eclipse.jetty.server.Handler; import org.eclipse.jetty.server.Server; import org.eclipse.jetty.server.ServerConnector; +import org.eclipse.jetty.server.handler.ContextHandlerCollection; import org.eclipse.jetty.server.handler.DefaultHandler; -import org.eclipse.jetty.server.handler.HandlerList; import org.eclipse.jetty.server.handler.ResourceHandler; +import org.eclipse.jetty.util.resource.Resource; +import org.eclipse.jetty.util.resource.ResourceFactory; /** * Base class for 'self tests', integrations tests formatted as unit @@ -187,11 +189,10 @@ public abstract class SelfTestBase extends TmpDirTestCase { sc.setPort(7777); server.addConnector(sc); ResourceHandler rhandler = new ResourceHandler(); - rhandler.setResourceBase(getSrcHtdocs().getAbsolutePath()); - - HandlerList handlers = new HandlerList(); - handlers.setHandlers(new Handler[] { rhandler, new DefaultHandler() }); - server.setHandler(handlers); + ResourceFactory resourceFactory = ResourceFactory.of(server); + rhandler.setBaseResource(resourceFactory.newResource(getSrcHtdocs().toPath().toAbsolutePath())); + + server.setHandler(new Handler.Sequence(rhandler, new DefaultHandler())); this.httpServer = server; server.start(); diff --git a/engine/src/test/java/org/archive/crawler/selftest/StatisticsSelfTest.java b/engine/src/test/java/org/archive/crawler/selftest/StatisticsSelfTest.java index d9a2c5fa..f193e972 100644 --- a/engine/src/test/java/org/archive/crawler/selftest/StatisticsSelfTest.java +++ b/engine/src/test/java/org/archive/crawler/selftest/StatisticsSelfTest.java @@ -48,12 +48,12 @@ public class StatisticsSelfTest extends SelfTestBase { StatisticsTracker stats = heritrix.getEngine().getJob("selftest-job").getCrawlController().getStatisticsTracker(); assertNotNull(stats); assertEquals(13, (long) stats.getCrawledBytes().get(CrawledBytesHistotable.WARC_NOVEL_URLS)); - assertEquals(7999, (long) stats.getCrawledBytes().get(CrawledBytesHistotable.WARC_NOVEL_CONTENT_BYTES) - stats.getBytesPerHost("dns:")); + assertEquals(7960, (long) stats.getCrawledBytes().get(CrawledBytesHistotable.WARC_NOVEL_CONTENT_BYTES) - stats.getBytesPerHost("dns:")); assertEquals(3, (long) stats.getServerCache().getHostFor("127.0.0.1").getSubstats().get(CrawledBytesHistotable.WARC_NOVEL_URLS)); - assertEquals(2216, (long) stats.getServerCache().getHostFor("127.0.0.1").getSubstats().get(CrawledBytesHistotable.WARC_NOVEL_CONTENT_BYTES)); + assertEquals(2202, (long) stats.getServerCache().getHostFor("127.0.0.1").getSubstats().get(CrawledBytesHistotable.WARC_NOVEL_CONTENT_BYTES)); assertEquals(10, (long) stats.getServerCache().getHostFor("localhost").getSubstats().get(CrawledBytesHistotable.WARC_NOVEL_URLS)); - assertEquals(5783, (long) stats.getServerCache().getHostFor("localhost").getSubstats().get(CrawledBytesHistotable.WARC_NOVEL_CONTENT_BYTES)); + assertEquals(5758, (long) stats.getServerCache().getHostFor("localhost").getSubstats().get(CrawledBytesHistotable.WARC_NOVEL_CONTENT_BYTES)); assertEquals(0, (long) stats.getServerCache().getHostFor("dns:").getSubstats().get(CrawledBytesHistotable.WARC_NOVEL_URLS)); } @@ -66,17 +66,17 @@ public class StatisticsSelfTest extends SelfTestBase { sourceStats = stats.getSourceStats("http://127.0.0.1:7777/a.html"); assertNotNull(sourceStats); assertEquals(4, sourceStats.keySet().size()); - assertEquals(2216l, (long) sourceStats.get("novel")); + assertEquals(2202l, (long) sourceStats.get("novel")); assertEquals(3l, (long) sourceStats.get("novelCount")); - assertEquals(2216l, (long) sourceStats.get("warcNovelContentBytes")); + assertEquals(2202l, (long) sourceStats.get("warcNovelContentBytes")); assertEquals(3l, (long) sourceStats.get("warcNovelUrls")); sourceStats = stats.getSourceStats("http://localhost:7777/b.html"); assertNotNull(sourceStats); assertEquals(4, sourceStats.keySet().size()); - assertEquals(5783l, (long) sourceStats.get("novel") - stats.getBytesPerHost("dns:")); + assertEquals(5758l, (long) sourceStats.get("novel") - stats.getBytesPerHost("dns:")); assertEquals(11l, (long) sourceStats.get("novelCount")); - assertEquals(5783l, (long) sourceStats.get("warcNovelContentBytes") - stats.getBytesPerHost("dns:")); + assertEquals(5758l, (long) sourceStats.get("warcNovelContentBytes") - stats.getBytesPerHost("dns:")); assertEquals(10l, (long) sourceStats.get("warcNovelUrls")); } diff --git a/engine/src/test/java/org/archive/crawler/selftest/UserAgentSelfTest.java b/engine/src/test/java/org/archive/crawler/selftest/UserAgentSelfTest.java index d70ddc0b..e23d41fc 100644 --- a/engine/src/test/java/org/archive/crawler/selftest/UserAgentSelfTest.java +++ b/engine/src/test/java/org/archive/crawler/selftest/UserAgentSelfTest.java @@ -20,14 +20,15 @@ package org.archive.crawler.selftest; import org.archive.util.ArchiveUtils; +import org.eclipse.jetty.ee10.servlet.ServletContextHandler; +import org.eclipse.jetty.ee10.servlet.ServletHandler; +import org.eclipse.jetty.ee10.servlet.ServletHolder; import org.eclipse.jetty.server.Handler; import org.eclipse.jetty.server.Server; import org.eclipse.jetty.server.ServerConnector; import org.eclipse.jetty.server.handler.DefaultHandler; -import org.eclipse.jetty.server.handler.HandlerList; import org.eclipse.jetty.server.handler.ResourceHandler; -import org.eclipse.jetty.servlet.ServletHandler; -import org.eclipse.jetty.servlet.ServletHolder; +import org.eclipse.jetty.util.resource.ResourceFactory; /** * @author pjack @@ -59,20 +60,16 @@ public class UserAgentSelfTest extends SelfTestBase { sc.setPort(7777); server.addConnector(sc); ResourceHandler rhandler = new ResourceHandler(); - rhandler.setResourceBase(getSrcHtdocs().getAbsolutePath()); + ResourceFactory resourceFactory = ResourceFactory.of(server); + rhandler.setBaseResource(resourceFactory.newResource(getSrcHtdocs().toPath().toAbsolutePath())); - ServletHandler servletHandler = new ServletHandler(); - - HandlerList handlers = new HandlerList(); - handlers.setHandlers(new Handler[] { - rhandler, - servletHandler, - new DefaultHandler() }); - server.setHandler(handlers); - this.servlet = new UserAgentServlet(); - ServletHolder holder = new ServletHolder(servlet); - servletHandler.addServletWithMapping(holder, "/*"); + ServletContextHandler contextHandler = new ServletContextHandler(ServletContextHandler.SESSIONS); + contextHandler.addServlet(servlet, "/*"); + + server.setHandler(new Handler.Sequence(rhandler, + contextHandler, + new DefaultHandler())); this.httpServer = server; this.httpServer.start(); diff --git a/engine/src/test/java/org/archive/crawler/selftest/UserAgentServlet.java b/engine/src/test/java/org/archive/crawler/selftest/UserAgentServlet.java index ba4c7231..602df58a 100644 --- a/engine/src/test/java/org/archive/crawler/selftest/UserAgentServlet.java +++ b/engine/src/test/java/org/archive/crawler/selftest/UserAgentServlet.java @@ -19,14 +19,14 @@ package org.archive.crawler.selftest; +import jakarta.servlet.ServletException; +import jakarta.servlet.http.HttpServlet; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; + import java.io.IOException; import java.util.Enumeration; -import javax.servlet.ServletException; -import javax.servlet.http.HttpServlet; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - /** * @author pjack */ @@ -37,8 +37,8 @@ public class UserAgentServlet extends HttpServlet { private String from; @Override - protected void doGet(HttpServletRequest req, HttpServletResponse resp) - throws ServletException, IOException { + protected void doGet(HttpServletRequest req, HttpServletResponse resp) + throws IOException { Enumeration e = req.getHeaderNames(); while (e.hasMoreElements()) { String name = (String)e.nextElement(); diff --git a/engine/src/test/java/org/archive/modules/fetcher/FormAuthTest.java b/engine/src/test/java/org/archive/modules/fetcher/FormAuthTest.java index 39ca0bb6..04677ce2 100644 --- a/engine/src/test/java/org/archive/modules/fetcher/FormAuthTest.java +++ b/engine/src/test/java/org/archive/modules/fetcher/FormAuthTest.java @@ -27,10 +27,10 @@ import java.util.HashSet; import java.util.Set; import java.util.logging.Logger; -import javax.servlet.ServletException; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - +import jakarta.servlet.http.Cookie; +import jakarta.servlet.http.HttpServlet; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; import junit.framework.TestCase; import org.apache.commons.httpclient.URIException; @@ -44,19 +44,18 @@ import org.archive.net.UURI; import org.archive.net.UURIFactory; import org.archive.util.Recorder; import org.archive.util.TmpDirTestCase; +import org.eclipse.jetty.ee10.servlet.ServletContextHandler; +import org.eclipse.jetty.ee10.servlet.SessionHandler; +import org.eclipse.jetty.ee10.servlet.security.ConstraintMapping; +import org.eclipse.jetty.ee10.servlet.security.ConstraintSecurityHandler; import org.eclipse.jetty.security.*; import org.eclipse.jetty.security.authentication.FormAuthenticator; -import org.eclipse.jetty.server.NCSARequestLog; import org.eclipse.jetty.server.Request; import org.eclipse.jetty.server.Server; import org.eclipse.jetty.server.ServerConnector; -import org.eclipse.jetty.server.handler.HandlerCollection; -import org.eclipse.jetty.server.handler.RequestLogHandler; -import org.eclipse.jetty.server.session.DefaultSessionCache; -import org.eclipse.jetty.server.session.NullSessionDataStore; -import org.eclipse.jetty.server.session.SessionCache; -import org.eclipse.jetty.server.session.SessionHandler; -import org.eclipse.jetty.util.security.Constraint; +import org.eclipse.jetty.session.DefaultSessionCache; +import org.eclipse.jetty.session.NullSessionDataStore; +import org.eclipse.jetty.session.SessionCache; import org.eclipse.jetty.util.security.Password; /* Somewhat redundant to org.archive.crawler.selftest.FormAuthSelfTest, but @@ -169,7 +168,7 @@ public class FormAuthTest extends TestCase { getFetcher().process(curi); logger.info('\n' + httpRequestString(curi) + "\n\n" + rawResponseString(curi)); assertEquals(302, curi.getFetchStatus()); - assertTrue(curi.getHttpResponseHeader("Location").startsWith("http://localhost:7779/login.html")); + assertTrue(curi.getHttpResponseHeader("Location").startsWith("/login.html")); PreconditionEnforcer preconditionEnforcer = new PreconditionEnforcer(); preconditionEnforcer.setServerCache(getFetcher().getServerCache()); @@ -188,7 +187,7 @@ public class FormAuthTest extends TestCase { getFetcher().process(loginUri); logger.info('\n' + httpRequestString(loginUri) + "\n\n" + rawResponseString(loginUri)); assertEquals(302, loginUri.getFetchStatus()); // 302 on successful login - assertEquals("http://localhost:7779/auth/1", loginUri.getHttpResponseHeader("location")); + assertEquals("/auth/1", loginUri.getHttpResponseHeader("location")); curi = makeCrawlURI("http://localhost:7779/auth/1"); getFetcher().process(curi); @@ -206,29 +205,28 @@ public class FormAuthTest extends TestCase { + "
" + "" + "" + ""; - protected static class FormAuthTestHandler extends SessionHandler { + protected static class FormAuthTestServlet extends HttpServlet { - public FormAuthTestHandler() { + public FormAuthTestServlet() { super(); } @Override - public void doHandle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException { + protected void doGet(HttpServletRequest request, HttpServletResponse response) throws IOException { + String target = request.getRequestURI(); if (target.endsWith("/set-cookie")) { - response.addCookie(new javax.servlet.http.Cookie("test-cookie-name", "test-cookie-value")); + response.addCookie(new Cookie("test-cookie-name", "test-cookie-value")); } if (target.equals("/login.html")) { response.setContentType("text/html;charset=US-ASCII"); response.setStatus(HttpServletResponse.SC_OK); response.getOutputStream().write(LOGIN_HTML.getBytes("US-ASCII")); - ((Request)request).setHandled(true); } else { response.setContentType("text/plain;charset=US-ASCII"); response.setDateHeader("Last-Modified", 0); response.setStatus(HttpServletResponse.SC_OK); response.getOutputStream().write(DEFAULT_PAYLOAD_STRING.getBytes("US-ASCII")); - ((Request)request).setHandled(true); } } } @@ -236,9 +234,7 @@ public class FormAuthTest extends TestCase { protected static SecurityHandler makeAuthWrapper(Authenticator authenticator, final String role, String realm, final String login, final String password) { - Constraint constraint = new Constraint(); - constraint.setRoles(new String[] { role }); - constraint.setAuthenticate(true); + Constraint constraint = Constraint.from(role); ConstraintMapping constraintMapping = new ConstraintMapping(); constraintMapping.setConstraint(constraint); @@ -266,27 +262,19 @@ public class FormAuthTest extends TestCase { sc.setHost("127.0.0.1"); sc.setPort(7779); server.addConnector(sc); - - HandlerCollection handlers = new HandlerCollection(); - handlers.addHandler(new FormAuthTestHandler()); - RequestLogHandler requestLogHandler = new RequestLogHandler(); - NCSARequestLog requestLog = new NCSARequestLog(); - requestLogHandler.setRequestLog(requestLog); - handlers.addHandler(requestLogHandler); - + + ServletContextHandler context = new ServletContextHandler(ServletContextHandler.SESSIONS); + context.setContextPath("/"); + context.addServlet(FormAuthTestServlet.class, "/"); + FormAuthenticator formAuthenticatrix = new FormAuthenticator("/login.html", null, false); SecurityHandler authWrapper = makeAuthWrapper(formAuthenticatrix, FORM_AUTH_ROLE, FORM_AUTH_REALM, FORM_AUTH_LOGIN, FORM_AUTH_PASSWORD); - authWrapper.setHandler(handlers); - SessionHandler sessionHandler = new SessionHandler(); - SessionCache cache = new DefaultSessionCache(sessionHandler); - cache.setSessionDataStore(new NullSessionDataStore()); - sessionHandler.setSessionCache(cache); - sessionHandler.setHandler(authWrapper); - server.setHandler(sessionHandler); + context.setSecurityHandler(authWrapper); + server.setHandler(context); server.start(); } diff --git a/modules/pom.xml b/modules/pom.xml index 59cec7cd..6716b198 100644 --- a/modules/pom.xml +++ b/modules/pom.xml @@ -49,8 +49,14 @@ ${jetty.version} - org.eclipse.jetty - jetty-servlet + org.eclipse.jetty.ee10 + jetty-ee10-servlet + ${jetty.version} + test + + + org.eclipse.jetty.ee10 + jetty-ee10-proxy ${jetty.version} test diff --git a/modules/src/test/java/org/archive/modules/fetcher/CookieFetchHTTPIntegrationTest.java b/modules/src/test/java/org/archive/modules/fetcher/CookieFetchHTTPIntegrationTest.java index 3256e2d8..e87729c4 100644 --- a/modules/src/test/java/org/archive/modules/fetcher/CookieFetchHTTPIntegrationTest.java +++ b/modules/src/test/java/org/archive/modules/fetcher/CookieFetchHTTPIntegrationTest.java @@ -25,11 +25,11 @@ import java.net.UnknownHostException; import java.util.Set; import java.util.concurrent.TimeUnit; -import javax.servlet.ServletException; -import javax.servlet.http.Cookie; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - +import jakarta.servlet.ServletException; +import jakarta.servlet.http.Cookie; +import jakarta.servlet.http.HttpServlet; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; import org.apache.commons.collections.Closure; import org.apache.commons.httpclient.URIException; import org.apache.commons.io.FileUtils; @@ -44,16 +44,14 @@ import org.archive.spring.ConfigFile; import org.archive.spring.ConfigPath; import org.archive.util.KeyTool; import org.archive.util.TmpDirTestCase; +import org.eclipse.jetty.ee10.servlet.ServletContextHandler; import org.eclipse.jetty.http.HttpVersion; import org.eclipse.jetty.server.HttpConfiguration; import org.eclipse.jetty.server.HttpConnectionFactory; -import org.eclipse.jetty.server.Request; import org.eclipse.jetty.server.SecureRequestCustomizer; import org.eclipse.jetty.server.Server; import org.eclipse.jetty.server.ServerConnector; import org.eclipse.jetty.server.SslConnectionFactory; -import org.eclipse.jetty.server.session.SessionHandler; -import org.eclipse.jetty.util.log.Log; import org.eclipse.jetty.util.ssl.SslContextFactory; import com.google.common.io.Files; @@ -64,15 +62,15 @@ import junit.framework.TestSuite; public class CookieFetchHTTPIntegrationTest extends ProcessorTestBase { - protected static class TestHandler extends SessionHandler { - public TestHandler() { + protected static class TestServlet extends HttpServlet { + public TestServlet() { super(); } @Override - public void doHandle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException { + protected void doGet(HttpServletRequest request, HttpServletResponse response) throws IOException { if (request.getParameter("name") != null) { - Cookie cookie = new javax.servlet.http.Cookie(request.getParameter("name"), + Cookie cookie = new Cookie(request.getParameter("name"), request.getParameter("value")); if (request.getParameter("domain") != null) { cookie.setDomain(request.getParameter("domain")); @@ -86,12 +84,6 @@ public class CookieFetchHTTPIntegrationTest extends ProcessorTestBase { if (request.getParameter("secure") != null) { cookie.setSecure(request.getParameter("secure").equals(1)); } - if (request.getParameter("comment") != null) { - cookie.setComment(request.getParameter("comment")); - } - if (request.getParameter("version") != null) { - cookie.setVersion(Integer.valueOf(request.getParameter("version"))); - } response.addCookie(cookie); } @@ -106,17 +98,16 @@ public class CookieFetchHTTPIntegrationTest extends ProcessorTestBase { } else { response.getOutputStream().println("0 cookies received"); } - - ((Request)request).setHandled(true); } } public static Server startHttpServer() throws Exception { - Log.getLogger(Server.class.getCanonicalName()).setDebugEnabled(true); - Server server = new Server(); - server.setHandler(new TestHandler()); + ServletContextHandler context = new ServletContextHandler(ServletContextHandler.SESSIONS); + context.setContextPath("/"); + context.addServlet(TestServlet.class, "/"); + server.setHandler(context); ServerConnector sc = new ServerConnector(server); sc.setHost("127.0.0.1"); @@ -137,7 +128,7 @@ public class CookieFetchHTTPIntegrationTest extends ProcessorTestBase { "-dname", "CN=127.0.0.1", "-validity","3650"}); // 10 yr validity - SslContextFactory sslContextFactory = new SslContextFactory(); + SslContextFactory.Server sslContextFactory = new SslContextFactory.Server(); sslContextFactory.setKeyStorePassword(KEYSTORE_PASSWORD); sslContextFactory.setKeyStorePath(keystoreFile.getPath()); diff --git a/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTest.java b/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTest.java index d1a251b4..ea4e2966 100644 --- a/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTest.java +++ b/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTest.java @@ -41,10 +41,10 @@ import java.util.logging.Logger; import javax.net.ServerSocketFactory; import javax.net.ssl.SSLException; -import javax.servlet.ServletException; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; +import jakarta.servlet.ServletException; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; import org.apache.commons.httpclient.URIException; import org.apache.commons.io.IOUtils; import org.apache.http.NoHttpResponseException; @@ -63,12 +63,12 @@ import org.archive.net.UURIFactory; import org.archive.util.Recorder; import org.archive.util.TmpDirTestCase; import org.bbottema.javasocksproxyserver.SocksServer; -import org.eclipse.jetty.client.api.Response; -import org.eclipse.jetty.proxy.ConnectHandler; -import org.eclipse.jetty.proxy.ProxyServlet; +import org.eclipse.jetty.client.Response; +import org.eclipse.jetty.ee10.proxy.ProxyServlet; +import org.eclipse.jetty.ee10.servlet.ServletContextHandler; +import org.eclipse.jetty.ee10.servlet.ServletHolder; import org.eclipse.jetty.server.Server; -import org.eclipse.jetty.servlet.ServletContextHandler; -import org.eclipse.jetty.servlet.ServletHolder; +import org.eclipse.jetty.server.handler.ConnectHandler; import org.junit.*; public class FetchHTTPTest { @@ -270,7 +270,7 @@ public class FetchHTTPTest { // check that we got the expected response and the fetcher did its thing assertEquals(401, curi.getFetchStatus()); - assertEquals("basic realm=\"basic-auth-realm\"", curi.getHttpResponseHeader("WWW-Authenticate")); + assertEquals("Basic realm=\"basic-auth-realm\"", curi.getHttpResponseHeader("WWW-Authenticate")); assertTrue(curi.getCredentials().contains(basicAuthCredential)); assertTrue(curi.getHttpAuthChallenges() != null && curi.getHttpAuthChallenges().containsKey("basic")); @@ -461,8 +461,9 @@ public class FetchHTTPTest { Server httpProxyServer = new Server(new InetSocketAddress("localhost", 7877)); ConnectHandler connectHandler = new ConnectHandler(); httpProxyServer.setHandler(connectHandler); - ServletContextHandler context = new ServletContextHandler(connectHandler, "/", - ServletContextHandler.SESSIONS); + ServletContextHandler context = new ServletContextHandler( "/", ServletContextHandler.SESSIONS); + connectHandler.setHandler(context); + ServletHolder proxyServlet = new ServletHolder(TestProxyServlet.class); context.addServlet(proxyServlet, "/*"); context.setAttribute("proxy-user", user); diff --git a/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTestServers.java b/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTestServers.java index 986adaf7..84cee56a 100644 --- a/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTestServers.java +++ b/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTestServers.java @@ -23,22 +23,22 @@ import java.io.IOException; import java.util.HashMap; import java.util.Map; -import javax.servlet.ServletException; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - +import jakarta.servlet.ServletException; +import jakarta.servlet.http.Cookie; +import jakarta.servlet.http.HttpServlet; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; import org.archive.util.KeyTool; import org.archive.util.TmpDirTestCase; +import org.eclipse.jetty.ee10.servlet.ServletContextHandler; +import org.eclipse.jetty.ee10.servlet.security.ConstraintMapping; +import org.eclipse.jetty.ee10.servlet.security.ConstraintSecurityHandler; import org.eclipse.jetty.http.HttpVersion; import org.eclipse.jetty.security.*; import org.eclipse.jetty.security.authentication.BasicAuthenticator; import org.eclipse.jetty.security.authentication.DigestAuthenticator; import org.eclipse.jetty.server.*; -import org.eclipse.jetty.server.handler.HandlerCollection; -import org.eclipse.jetty.server.session.SessionHandler; -import org.eclipse.jetty.util.log.Log; -import org.eclipse.jetty.util.security.Constraint; import org.eclipse.jetty.util.security.Password; import org.eclipse.jetty.util.ssl.SslContextFactory; @@ -112,20 +112,20 @@ public class FetchHTTPTestServers { protected static final byte[] EIGHTY_BYTE_LINE = "1234567890123456789012345678901234567890123456789012345678901234567890123456789\n".getBytes(); - protected static class TestHandler extends SessionHandler { + protected static class TestServlet extends HttpServlet { - public TestHandler() { + public TestServlet() { super(); } @Override - public void doHandle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException { - - // echo the remote host back to the client so tests can reference it + protected void service(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { + // echo the remote host back to the client so tests can reference it response.setHeader("Client-Host", request.getRemoteHost()); + String target = request.getRequestURI(); if (target.endsWith("/set-cookie")) { - response.addCookie(new javax.servlet.http.Cookie("test-cookie-name", "test-cookie-value")); + response.addCookie(new Cookie("test-cookie-name", "test-cookie-value")); } if (target.equals("/200k")) { @@ -135,7 +135,6 @@ public class FetchHTTPTestServers { for (int i = 0; i < 200000 / EIGHTY_BYTE_LINE.length; i++) { response.getOutputStream().write(EIGHTY_BYTE_LINE); } - ((Request)request).setHandled(true); } else if (target.equals("/slow.txt")) { response.setContentType("text/plain;charset=US-ASCII"); response.setStatus(200); @@ -147,80 +146,66 @@ public class FetchHTTPTestServers { } catch (InterruptedException e) { } } - ((Request)request).setHandled(true); } else if (target.equals("/chunked.txt")) { response.setContentType("text/plain;charset=US-ASCII"); response.setStatus(200); // response.setContentLength(HttpTokens.CHUNKED_CONTENT); response.getOutputStream().write(DEFAULT_PAYLOAD_STRING.getBytes("US-ASCII")); response.getOutputStream().flush(); - ((Request)request).setHandled(true); } else if (request.getHeader("Accept-Encoding") != null && request.getHeader("Accept-Encoding").contains("gzip")) { response.setHeader("Content-Encoding", "gzip"); response.setContentType("text/plain;charset=US-ASCII"); response.setStatus(200); response.getOutputStream().write(DEFAULT_GZIPPED_PAYLOAD); - ((Request)request).setHandled(true); } else if (target.equals("/401-no-challenge")) { response.setStatus(401); response.setContentType("text/plain;charset=US-ASCII"); response.getOutputStream().write(DEFAULT_PAYLOAD_STRING.getBytes("US-ASCII")); - ((Request)request).setHandled(true); } else if (target.equals("/cp1251")) { response.setContentType("text/plain;charset=cp1251"); response.setStatus(200); response.getOutputStream().write(CP1251_PAYLOAD); - ((Request)request).setHandled(true); } else if (target.equals("/unsupported-charset")) { response.setContentType("text/plain;charset=UNSUPPORTED-CHARSET"); response.setStatus(200); response.getOutputStream().write(DEFAULT_PAYLOAD_STRING.getBytes("US-ASCII")); - ((Request)request).setHandled(true); } else if (target.equals("/invalid-charset")) { response.setContentType("text/plain;charset=%%INVALID-CHARSET%%"); response.setStatus(200); response.getOutputStream().write(DEFAULT_PAYLOAD_STRING.getBytes("US-ASCII")); - ((Request)request).setHandled(true); } else if (target.equals("/if-modified-since")) { if (request.getHeader("if-modified-since") != null) { response.setStatus(304); - ((Request)request).setHandled(true); } else { response.setContentType("text/plain;charset=US-ASCII"); response.setDateHeader("Last-Modified", 0); response.setStatus(200); response.getOutputStream().write(DEFAULT_PAYLOAD_STRING.getBytes("US-ASCII")); - ((Request)request).setHandled(true); } } else if (target.equals("/if-none-match")) { if (request.getHeader("if-none-match") != null) { response.setStatus(304); - ((Request)request).setHandled(true); } else { response.setContentType("text/plain;charset=US-ASCII"); response.setHeader("ETag", ETAG_TEST_VALUE); response.setStatus(200); response.getOutputStream().write(DEFAULT_PAYLOAD_STRING.getBytes("US-ASCII")); - ((Request)request).setHandled(true); } } else { response.setContentType("text/plain;charset=US-ASCII"); response.setStatus(200); response.getOutputStream().write(DEFAULT_PAYLOAD_STRING.getBytes("US-ASCII")); - ((Request)request).setHandled(true); } } } protected static Map httpServers; - protected static SecurityHandler makeAuthWrapper(Authenticator authenticator, + protected static ConstraintSecurityHandler makeAuthWrapper(Authenticator authenticator, final String role, String realm, final String login, final String password) { - Constraint constraint = new Constraint(); - constraint.setRoles(new String[] { role }); - constraint.setAuthenticate(true); + Constraint constraint = Constraint.from(role); ConstraintMapping constraintMapping = new ConstraintMapping(); constraintMapping.setConstraint(constraint); @@ -228,7 +213,6 @@ public class FetchHTTPTestServers { ConstraintSecurityHandler authWrapper = new ConstraintSecurityHandler(); authWrapper.setAuthenticator(authenticator); - authWrapper.setConstraintMappings(new ConstraintMapping[] {constraintMapping}); UserStore userStore = new UserStore(); userStore.addUser(login, new Password(password), new String[] {role}); @@ -245,8 +229,6 @@ public class FetchHTTPTestServers { * @return map(port->server) */ public static Map startHttpServers() throws Exception { - Log.getLogger(Server.class.getCanonicalName()).setDebugEnabled(true); - HashMap servers = new HashMap(); // server for basic auth @@ -257,14 +239,17 @@ public class FetchHTTPTestServers { sc.setPort(7777); server.addConnector(sc); - SecurityHandler authWrapper = makeAuthWrapper(new BasicAuthenticator(), + ConstraintSecurityHandler authWrapper = makeAuthWrapper(new BasicAuthenticator(), BASIC_AUTH_ROLE, BASIC_AUTH_REALM, BASIC_AUTH_LOGIN, BASIC_AUTH_PASSWORD); - HandlerCollection handlers = new HandlerCollection(); - handlers.addHandler(new TestHandler()); - authWrapper.setHandler(handlers); - server.setHandler(authWrapper); - + + ServletContextHandler context = new ServletContextHandler(ServletContextHandler.SESSIONS); + context.setContextPath("/"); + context.addServlet(TestServlet.class, "/"); + context.setSecurityHandler(authWrapper); + + server.setHandler(context); + servers.put(sc.getPort(), server); File keystoreFile = new File(TmpDirTestCase.tmpDir(), "keystore"); @@ -282,12 +267,14 @@ public class FetchHTTPTestServers { "-dname", "CN=127.0.0.1", "-validity","3650"}); // 10 yr validity - SslContextFactory sslContextFactory = new SslContextFactory(); + SslContextFactory.Server sslContextFactory = new SslContextFactory.Server(); sslContextFactory.setKeyStorePassword(KEYSTORE_PASSWORD); sslContextFactory.setKeyStorePath(keystoreFile.getPath()); HttpConfiguration httpsConfig = new HttpConfiguration(); - httpsConfig.addCustomizer(new SecureRequestCustomizer()); + SecureRequestCustomizer customizer = new SecureRequestCustomizer(); + customizer.setSniHostCheck(false); // disable SNI host check because it fails on localhost + httpsConfig.addCustomizer(customizer); ServerConnector ssc = new ServerConnector(server, new SslConnectionFactory(sslContextFactory, HttpVersion.HTTP_1_1.asString()), @@ -310,11 +297,12 @@ public class FetchHTTPTestServers { authWrapper = makeAuthWrapper(new DigestAuthenticator(), DIGEST_AUTH_ROLE, DIGEST_AUTH_REALM, DIGEST_AUTH_LOGIN, DIGEST_AUTH_PASSWORD); - HandlerCollection handlers2 = new HandlerCollection(); - handlers2.addHandler(new TestHandler()); - authWrapper.setHandler(handlers2); - server.setHandler(authWrapper); - + ServletContextHandler context2 = new ServletContextHandler(ServletContextHandler.SESSIONS); + context2.setContextPath("/"); + context2.addServlet(TestServlet.class, "/"); + context2.setSecurityHandler(authWrapper); + server.setHandler(context2); + server.start(); servers.put(sc.getPort(), server); diff --git a/modules/src/test/java/org/archive/modules/recrawl/ContentDigestHistoryTest.java b/modules/src/test/java/org/archive/modules/recrawl/ContentDigestHistoryTest.java index b955b753..b280d0e5 100644 --- a/modules/src/test/java/org/archive/modules/recrawl/ContentDigestHistoryTest.java +++ b/modules/src/test/java/org/archive/modules/recrawl/ContentDigestHistoryTest.java @@ -47,10 +47,6 @@ import java.util.Iterator; import java.util.List; import java.util.logging.Logger; -import javax.servlet.ServletException; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - import org.apache.commons.httpclient.URIException; import org.apache.commons.io.FileUtils; import org.archive.bdb.BdbModule; @@ -69,11 +65,11 @@ import org.archive.spring.ConfigPath; import org.archive.util.Base32; import org.archive.util.Recorder; import org.archive.util.TmpDirTestCase; -import org.eclipse.jetty.server.Request; -import org.eclipse.jetty.server.Server; -import org.eclipse.jetty.server.ServerConnector; -import org.eclipse.jetty.server.handler.HandlerCollection; -import org.eclipse.jetty.server.session.SessionHandler; +import org.eclipse.jetty.http.HttpHeader; +import org.eclipse.jetty.http.HttpStatus; +import org.eclipse.jetty.io.Content; +import org.eclipse.jetty.server.*; +import org.eclipse.jetty.util.Callback; public class ContentDigestHistoryTest extends TmpDirTestCase { @@ -367,19 +363,16 @@ public class ContentDigestHistoryTest extends TmpDirTestCase { protected static final String DEFAULT_PAYLOAD_STRING = "abcdefghijklmnopqrstuvwxyz0123456789\n"; protected Server newHttpServer() throws Exception { - HandlerCollection handlers = new HandlerCollection(); - handlers.addHandler(new SessionHandler(){ + Server server = new Server(); + server.setHandler(new Handler.Abstract.NonBlocking() { @Override - public void doHandle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException { - response.setContentType("text/plain;charset=US-ASCII"); - response.setStatus(HttpServletResponse.SC_OK); - response.getOutputStream().write(DEFAULT_PAYLOAD_STRING.getBytes("US-ASCII")); - ((Request)request).setHandled(true); + public boolean handle(Request request, Response response, Callback callback) throws Exception { + response.getHeaders().put(HttpHeader.CONTENT_TYPE, "text/plain;charset=US-ASCII"); + response.setStatus(HttpStatus.OK_200); + Content.Sink.write(response, true, DEFAULT_PAYLOAD_STRING, callback); + return true; } }); - - Server server = new Server(); - server.setHandler(handlers); ServerConnector sc = new ServerConnector(server); sc.setHost("127.0.0.1"); diff --git a/pom.xml b/pom.xml index 88e46623..47c1f1c4 100644 --- a/pom.xml +++ b/pom.xml @@ -372,7 +372,7 @@ http://maven.apache.org/guides/mini/guide-m1-m2.html none -Xdoclint:none 4.0.26 - 9.4.57.v20241219 + 12.0.16 2.0.17 17 17