lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-09-28 16:45:41 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #648 from internetarchive/jetty-12

Browse Source 
Update to Jetty 12.0.16 and Restlet 2.6.0-M2
This commit is contained in:
Alex Osborne
2025-04-29 12:47:25 +09:00
committed by GitHub
parent d73e2f6b87 db16b60960
commit bc9b21e2ee
23 changed files with 276 additions and 266 deletions
Download Patch File Download Diff File Expand all files Collapse all files
engine/pom.xml
+7 -5
View File
@@ -31,8 +31,8 @@
<version>${jetty.version}</version>
</dependency>
<dependency>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-servlet</artifactId>
<groupId>org.eclipse.jetty.ee10</groupId>
<artifactId>jetty-ee10-servlet</artifactId>
<version>${jetty.version}</version>
<scope>test</scope>
</dependency>
@@ -46,9 +46,11 @@
<artifactId>org.restlet.ext.jetty</artifactId>
<version>${restlet.version}</version>
<exclusions>
<!-- exclude quiche native library as it's very large and we don't use Restlet's
HTTP/3 support currently -->
<exclusion>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-client</artifactId>
<groupId>org.mortbay.jetty.quiche</groupId>
<artifactId>jetty-quiche-native</artifactId>
</exclusion>
</exclusions>
</dependency>
@@ -109,6 +111,6 @@
</build>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<restlet.version>2.5.0</restlet.version>
<restlet.version>2.6.0-m2</restlet.version>
</properties>
</project>
engine/src/main/java/org/archive/crawler/Heritrix.java
+17 -6
View File
@@ -49,14 +49,14 @@ import org.apache.commons.lang.StringUtils;
import org.archive.crawler.framework.CrawlJob;
import org.archive.crawler.framework.Engine;
import org.archive.crawler.restlet.EngineApplication;
import org.archive.crawler.restlet.NoSniHostCheckHttpsServerHelper;
import org.archive.crawler.restlet.RateLimitGuard;
import org.archive.util.ArchiveUtils;
import org.archive.util.KeyTool;
import org.restlet.Component;
import org.restlet.Context;
import org.restlet.Server;
import org.restlet.data.ChallengeScheme;
import org.restlet.data.Protocol;
import org.restlet.security.ChallengeAuthenticator;
import org.restlet.security.MapVerifier;
@@ -154,6 +154,7 @@ public class Heritrix {
"to use for crawling.");
options.addOption(null, "proxy-port", true, "Global http(s) proxy port " +
"to use for crawling.");
options.addOption(null, "sni-host-check", false, "Validates SNI hostname against the SSL certificate");
return options;
}
@@ -349,13 +350,21 @@ public class Heritrix {
engine = new Engine(jobsDir);
component = new Component();
// disable SNI host check by default for backwards compatibility with existing ad-hoc certificates
String helperClass = null;
if (!cl.hasOption("sni-host-check")) {
org.restlet.engine.Engine.getInstance().getRegisteredServers().add(
new NoSniHostCheckHttpsServerHelper(null));
helperClass = NoSniHostCheckHttpsServerHelper.class.getName();
}
if(bindHosts.isEmpty()) {
// listen all addresses
setupServer(component, port, null, keystorePath, keystorePassword, keyPassword);
setupServer(component, port, null, keystorePath, keystorePassword, keyPassword, helperClass);
} else {
// bind only to declared addresses, or just 'localhost'
for(String address : bindHosts) {
setupServer(component, port, address, keystorePath, keystorePassword, keyPassword);
setupServer(component, port, address, keystorePath, keystorePassword, keyPassword, helperClass);
}
}
component.getClients().add(Protocol.FILE);
@@ -504,8 +513,10 @@ public class Heritrix {
* @param keystorePassword
* @param keyPassword
*/
protected void setupServer(Component component, int port, String address, String keystorePath, String keystorePassword, String keyPassword) {
Server server = component.getServers().add(Protocol.HTTPS, address, port);
protected void setupServer(Component component, int port, String address, String keystorePath, String keystorePassword, String keyPassword, String helperClass) {
Context serverContext = component.getServers().getContext().createChildContext();
Server server = new Server(serverContext, List.of(Protocol.HTTPS), address, port, null, helperClass);
component.getServers().add(server);
server.getContext().getParameters().add("keystorePath", keystorePath);
server.getContext().getParameters().add("keystorePassword", keystorePassword);
server.getContext().getParameters().add("keyPassword", keyPassword);
engine/src/main/java/org/archive/crawler/restlet/JobResource.java
+2 -1
View File
@@ -135,7 +135,8 @@ public class JobResource extends BaseResource {
// conditional on whether /anypath/ service is present?
String fullPath = f.getAbsolutePath();
fullPath = fullPath.replace(File.separatorChar, '/');
return "../../anypath/" + fullPath;
if (!fullPath.startsWith("/")) fullPath = "/" + fullPath;
return "../../anypath" + fullPath;
}
@Override
engine/src/main/java/org/archive/crawler/restlet/NoSniHostCheckHttpsServerHelper.java
+38
View File
@@ -0,0 +1,38 @@
package org.archive.crawler.restlet;
import org.eclipse.jetty.server.HttpConnectionFactory;
import org.eclipse.jetty.server.SecureRequestCustomizer;
import org.eclipse.jetty.server.ServerConnector;
import org.restlet.Server;
import org.restlet.ext.jetty.HttpsServerHelper;
/**
* Subclass of HttpServerHelper which disables the SNI host check. This is to main backwards
* compatibility with the existing Heritrix ad-hoc certificates that don't include a hostname.
*/
public class NoSniHostCheckHttpsServerHelper extends HttpsServerHelper {
public NoSniHostCheckHttpsServerHelper(Server server) {
super(server);
}
@Override
protected org.eclipse.jetty.server.Server getWrappedServer() {
org.eclipse.jetty.server.Server wrappedServer = super.getWrappedServer();
disableSniHostCheck(wrappedServer);
return wrappedServer;
}
private static void disableSniHostCheck(org.eclipse.jetty.server.Server jettyServer) {
for (var connector : jettyServer.getConnectors()) {
if (connector instanceof ServerConnector serverConnector) {
var connectionFactory = serverConnector.getConnectionFactory(HttpConnectionFactory.class);
if (connectionFactory != null) {
var secureRequestCustomizer = connectionFactory.getHttpConfiguration().getCustomizer(SecureRequestCustomizer.class);
if (secureRequestCustomizer != null) {
secureRequestCustomizer.setSniHostCheck(false);
}
}
}
}
}
}
engine/src/main/java/org/archive/crawler/restlet/models/CrawlJobModel.java
+12 -5
View File
@@ -17,6 +17,7 @@ import org.archive.checkpointing.Checkpoint;
import org.archive.crawler.framework.CrawlController.State;
import org.archive.crawler.framework.CrawlJob;
import org.archive.crawler.reporting.Report;
import org.archive.crawler.restlet.JobResource;
import org.archive.spring.ConfigPath;
import org.archive.util.ArchiveUtils;
import org.archive.util.FileUtils;
@@ -134,10 +135,16 @@ public class CrawlJobModel extends LinkedHashMap<String, Object> implements Seri
}
}
this.put("checkpointFiles",checkpointFiles);
if (crawlJob.hasApplicationContext())
this.put("alertLogFilePath",crawlJob.getCrawlController().getLoggerModule().getAlertsLogPath().getFile().getAbsolutePath());
if(crawlJob.isRunning() || (crawlJob.hasApplicationContext() && !crawlJob.isLaunchable()))
this.put("crawlLogFilePath",crawlJob.getCrawlController().getLoggerModule().getCrawlLogPath().getFile().getAbsolutePath());
if (crawlJob.hasApplicationContext()) {
File file = crawlJob.getCrawlController().getLoggerModule().getAlertsLogPath().getFile();
this.put("alertLogFilePath", file.getAbsolutePath());
this.put("alertLogFileUrl", JobResource.getHrefPath(file, crawlJob));
}
if(crawlJob.isRunning() || (crawlJob.hasApplicationContext() && !crawlJob.isLaunchable())) {
File file = crawlJob.getCrawlController().getLoggerModule().getCrawlLogPath().getFile();
this.put("crawlLogFilePath", file.getAbsolutePath());
this.put("crawlLogFileUrl", JobResource.getHrefPath(file, crawlJob));
}
this.put("reports", generateReports());
}
public String formatBytes(Long bytes){
@@ -209,7 +216,7 @@ public class CrawlJobModel extends LinkedHashMap<String, Object> implements Seri
configMap.put("key", key);
configMap.put("name", cp.getName());
configMap.put("path",FileUtils.tryToCanonicalize(cp.getFile()).getAbsolutePath());
configMap.put("url",baseRef+"engine/anypath/"+configMap.get("path"));
configMap.put("url", baseRef + JobResource.getHrefPath(cp.getFile(), crawlJob));
configMap.put("editable", EDIT_FILTER.accept(cp.getFile()));
referencedPaths.add(configMap);
}
engine/src/main/resources/org/archive/crawler/restlet/Job.ftl
+3 -3
View File
@@ -190,7 +190,7 @@
<i>none</i>
<#else>
${job.alertCount}
<a href="/engine/anypath/${job.alertLogFilePath}?format=paged&amp;pos=-1&amp;lines=-128">tail alert log...</a></li>
<a href="${job.alertLogFileUrl}?format=paged&amp;pos=-1&amp;lines=-128">tail alert log...</a></li>
</#if>
</td>
</tr>
@@ -272,7 +272,7 @@
<#if (job.isRunning || (job.hasApplicationContext && !job.isLaunchable))>
<div class="row">
<div class="large-12 columns">
<h3>Crawl Log <a href="/engine/anypath/${job.crawlLogFilePath}?format=paged&amp;pos=-1&amp;lines=-128&amp;reverse=y">more</a></h3>
<h3>Crawl Log <a href="${job.crawlLogFileUrl}?format=paged&amp;pos=-1&amp;lines=-128&amp;reverse=y">more</a></h3>
<div class="row">
<div class="large-12 columns">
<div class="log-viewer" >
@@ -306,7 +306,7 @@
</td>
<td>
<#if config.path??>
<a href='/engine/anypath/${config.path}<#if config.path?ends_with("log")>?format=paged&amp;pos=-1&amp;lines=-128&amp;reverse=y</#if>'>${config.path}</a><#if config.editable> [<a href="/engine/anypath/${config.path}?format=textedit">edit</a>]</#if>
<a href='${config.url}<#if config.path?ends_with("log")>?format=paged&amp;pos=-1&amp;lines=-128&amp;reverse=y</#if>'>${config.path}</a><#if config.editable> [<a href="${config.url}?format=textedit">edit</a>]</#if>
<#else>
<i>unset</i>
</#if>
engine/src/test/java/org/archive/crawler/selftest/CheckpointSelfTest.java
+7 -6
View File
@@ -22,10 +22,11 @@ package org.archive.crawler.selftest;
import java.io.IOException;
import org.archive.crawler.framework.CrawlJob;
import org.eclipse.jetty.ee10.servlet.ServletContextHandler;
import org.eclipse.jetty.ee10.servlet.ServletHandler;
import org.eclipse.jetty.ee10.servlet.ServletHolder;
import org.eclipse.jetty.server.Server;
import org.eclipse.jetty.server.ServerConnector;
import org.eclipse.jetty.servlet.ServletHandler;
import org.eclipse.jetty.servlet.ServletHolder;
/**
@@ -89,8 +90,6 @@ public class CheckpointSelfTest extends SelfTestBase {
sc.setHost(HOST);
sc.setPort(port);
server.addConnector(sc);
ServletHandler servletHandler = new ServletHandler();
server.setHandler(servletHandler);
RandomServlet random = new RandomServlet();
random.setHost(HOST);
@@ -99,8 +98,10 @@ public class CheckpointSelfTest extends SelfTestBase {
random.setMaxHops(MAX_HOPS);
random.setPathRoot("random");
ServletHolder holder = new ServletHolder(random);
servletHandler.addServletWithMapping(holder, "/random/*");
ServletContextHandler contextHandler = new ServletContextHandler(ServletContextHandler.SESSIONS);
contextHandler.addServlet(random, "/random/*");
server.setHandler(contextHandler);
server.start();
return server;
}
engine/src/test/java/org/archive/crawler/selftest/FormAuthSelfTest.java
+14 -16
View File
@@ -19,14 +19,15 @@
package org.archive.crawler.selftest;
import org.eclipse.jetty.ee10.servlet.ServletContextHandler;
import org.eclipse.jetty.ee10.servlet.ServletHandler;
import org.eclipse.jetty.ee10.servlet.ServletHolder;
import org.eclipse.jetty.server.Handler;
import org.eclipse.jetty.server.Server;
import org.eclipse.jetty.server.ServerConnector;
import org.eclipse.jetty.server.handler.DefaultHandler;
import org.eclipse.jetty.server.handler.HandlerList;
import org.eclipse.jetty.server.handler.ResourceHandler;
import org.eclipse.jetty.servlet.ServletHandler;
import org.eclipse.jetty.servlet.ServletHolder;
import org.eclipse.jetty.util.resource.ResourceFactory;
import java.util.Arrays;
import java.util.Collections;
@@ -66,19 +67,16 @@ public class FormAuthSelfTest
sc.setPort(7777);
server.addConnector(sc);
ResourceHandler rhandler = new ResourceHandler();
rhandler.setResourceBase(getSrcHtdocs().getAbsolutePath());
ServletHandler servletHandler = new ServletHandler();
HandlerList handlers = new HandlerList();
handlers.setHandlers(new Handler[] {
rhandler,
servletHandler,
new DefaultHandler() });
server.setHandler(handlers);
ServletHolder holder = new ServletHolder(new FormAuthServlet());
servletHandler.addServletWithMapping(holder, "/login/*");
ResourceFactory resourceFactory = ResourceFactory.of(server);
rhandler.setBaseResource(resourceFactory.newResource(getSrcHtdocs().toPath().toAbsolutePath()));
ServletContextHandler contextHandler = new ServletContextHandler(ServletContextHandler.SESSIONS);
contextHandler.addServlet(FormAuthServlet.class, "/login/*");
server.setHandler(new Handler.Sequence(
rhandler,
contextHandler,
new DefaultHandler()));
this.httpServer = server;
this.httpServer.start();
engine/src/test/java/org/archive/crawler/selftest/FormAuthServlet.java
+7 -7
View File
@@ -18,12 +18,12 @@
*/
package org.archive.crawler.selftest;
import java.io.IOException;
import jakarta.servlet.ServletException;
import jakarta.servlet.http.HttpServlet;
import jakarta.servlet.http.HttpServletRequest;
import jakarta.servlet.http.HttpServletResponse;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.io.IOException;
/**
@@ -36,8 +36,8 @@ public class FormAuthServlet extends HttpServlet {
@Override
protected void doPost(HttpServletRequest req, HttpServletResponse resp)
throws ServletException, IOException {
protected void doPost(HttpServletRequest req, HttpServletResponse resp)
throws IOException {
String username = req.getParameter("username");
String password = req.getParameter("password");
if (username.equals("Mr. Happy Pants") && password.equals("xyzzy")) {
engine/src/test/java/org/archive/crawler/selftest/FormLoginSelfTest.java
+9 -16
View File
@@ -19,14 +19,13 @@
package org.archive.crawler.selftest;
import org.eclipse.jetty.ee10.servlet.ServletContextHandler;
import org.eclipse.jetty.server.Handler;
import org.eclipse.jetty.server.Server;
import org.eclipse.jetty.server.ServerConnector;
import org.eclipse.jetty.server.handler.DefaultHandler;
import org.eclipse.jetty.server.handler.HandlerList;
import org.eclipse.jetty.server.handler.ResourceHandler;
import org.eclipse.jetty.servlet.ServletHandler;
import org.eclipse.jetty.servlet.ServletHolder;
import org.eclipse.jetty.util.resource.ResourceFactory;
import java.util.Arrays;
import java.util.Collections;
@@ -65,20 +64,14 @@ public class FormLoginSelfTest
sc.setPort(7777);
server.addConnector(sc);
ResourceHandler rhandler = new ResourceHandler();
rhandler.setResourceBase(getSrcHtdocs().getAbsolutePath());
ServletHandler servletHandler = new ServletHandler();
HandlerList handlers = new HandlerList();
handlers.setHandlers(new Handler[] {
rhandler,
servletHandler,
new DefaultHandler() });
server.setHandler(handlers);
ServletHolder holder = new ServletHolder(new FormAuthServlet());
servletHandler.addServletWithMapping(holder, "/login/*");
ResourceFactory resourceFactory = ResourceFactory.of(server);
rhandler.setBaseResource(resourceFactory.newResource(getSrcHtdocs().toPath().toAbsolutePath()));
ServletContextHandler contextHandler = new ServletContextHandler(ServletContextHandler.SESSIONS);
contextHandler.addServlet(FormAuthServlet.class, "/login/*");
server.setHandler(new Handler.Sequence(rhandler, contextHandler, new DefaultHandler()));
this.httpServer = server;
this.httpServer.start();
}
engine/src/test/java/org/archive/crawler/selftest/HttpAuthSelfTest.java
+11 -17
View File
@@ -24,15 +24,16 @@ import java.util.Collections;
import java.util.HashSet;
import java.util.Set;
import org.eclipse.jetty.ee10.servlet.ServletHandler;
import org.eclipse.jetty.ee10.servlet.security.ConstraintMapping;
import org.eclipse.jetty.ee10.servlet.security.ConstraintSecurityHandler;
import org.eclipse.jetty.security.*;
import org.eclipse.jetty.server.Handler;
import org.eclipse.jetty.server.Server;
import org.eclipse.jetty.server.ServerConnector;
import org.eclipse.jetty.server.handler.DefaultHandler;
import org.eclipse.jetty.server.handler.HandlerList;
import org.eclipse.jetty.server.handler.ResourceHandler;
import org.eclipse.jetty.servlet.ServletHandler;
import org.eclipse.jetty.util.security.Constraint;
import org.eclipse.jetty.util.resource.ResourceFactory;
import org.eclipse.jetty.util.security.Password;
/**
@@ -64,10 +65,7 @@ public class HttpAuthSelfTest
protected void startHttpServer() throws Exception {
Server server = new Server();
Constraint constraint = new Constraint();
constraint.setName(Constraint.__BASIC_AUTH);;
constraint.setRoles(new String[]{"user","admin","moderator"});
constraint.setAuthenticate(true);
Constraint constraint = Constraint.from("user","admin","moderator");
ConstraintMapping cm = new ConstraintMapping();
cm.setConstraint(constraint);
@@ -87,17 +85,13 @@ public class HttpAuthSelfTest
sc.setPort(7777);
server.addConnector(sc);
ResourceHandler rhandler = new ResourceHandler();
rhandler.setResourceBase(getSrcHtdocs().getAbsolutePath());
ResourceFactory resourceFactory = ResourceFactory.of(server);
rhandler.setBaseResource(resourceFactory.newResource(getSrcHtdocs().toPath().toAbsolutePath()));
ServletHandler servletHandler = new ServletHandler();
HandlerList handlers = new HandlerList();
handlers.setHandlers(new Handler[] {
securityHandler,
rhandler,
servletHandler,
new DefaultHandler() });
server.setHandler(handlers);
server.setHandler(new Handler.Sequence(
securityHandler,
rhandler,
new DefaultHandler()));
this.httpServer = server;
this.httpServer.start();
engine/src/test/java/org/archive/crawler/selftest/RandomServlet.java
+7 -7
View File
@@ -18,15 +18,15 @@
*/
package org.archive.crawler.selftest;
import jakarta.servlet.ServletException;
import jakarta.servlet.http.HttpServlet;
import jakarta.servlet.http.HttpServletRequest;
import jakarta.servlet.http.HttpServletResponse;
import java.io.IOException;
import java.io.Writer;
import java.util.Random;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
/**
* @author pjack
@@ -104,8 +104,8 @@ public class RandomServlet extends HttpServlet {
@Override
protected void doGet(HttpServletRequest req, HttpServletResponse resp)
throws ServletException, IOException {
protected void doGet(HttpServletRequest req, HttpServletResponse resp)
throws IOException {
resp.setContentType("text/html");
RandomServletLinkWriter rslw = new RandomServletLinkWriter();
engine/src/test/java/org/archive/crawler/selftest/SelfTestBase.java
+7 -6
View File
@@ -41,9 +41,11 @@ import org.archive.util.TmpDirTestCase;
import org.eclipse.jetty.server.Handler;
import org.eclipse.jetty.server.Server;
import org.eclipse.jetty.server.ServerConnector;
import org.eclipse.jetty.server.handler.ContextHandlerCollection;
import org.eclipse.jetty.server.handler.DefaultHandler;
import org.eclipse.jetty.server.handler.HandlerList;
import org.eclipse.jetty.server.handler.ResourceHandler;
import org.eclipse.jetty.util.resource.Resource;
import org.eclipse.jetty.util.resource.ResourceFactory;
/**
* Base class for 'self tests', integrations tests formatted as unit
@@ -187,11 +189,10 @@ public abstract class SelfTestBase extends TmpDirTestCase {
sc.setPort(7777);
server.addConnector(sc);
ResourceHandler rhandler = new ResourceHandler();
rhandler.setResourceBase(getSrcHtdocs().getAbsolutePath());
HandlerList handlers = new HandlerList();
handlers.setHandlers(new Handler[] { rhandler, new DefaultHandler() });
server.setHandler(handlers);
ResourceFactory resourceFactory = ResourceFactory.of(server);
rhandler.setBaseResource(resourceFactory.newResource(getSrcHtdocs().toPath().toAbsolutePath()));
server.setHandler(new Handler.Sequence(rhandler, new DefaultHandler()));
this.httpServer = server;
server.start();
engine/src/test/java/org/archive/crawler/selftest/StatisticsSelfTest.java
+7 -7
View File
@@ -48,12 +48,12 @@ public class StatisticsSelfTest extends SelfTestBase {
StatisticsTracker stats = heritrix.getEngine().getJob("selftest-job").getCrawlController().getStatisticsTracker();
assertNotNull(stats);
assertEquals(13, (long) stats.getCrawledBytes().get(CrawledBytesHistotable.WARC_NOVEL_URLS));
assertEquals(7999, (long) stats.getCrawledBytes().get(CrawledBytesHistotable.WARC_NOVEL_CONTENT_BYTES) - stats.getBytesPerHost("dns:"));
assertEquals(7960, (long) stats.getCrawledBytes().get(CrawledBytesHistotable.WARC_NOVEL_CONTENT_BYTES) - stats.getBytesPerHost("dns:"));
assertEquals(3, (long) stats.getServerCache().getHostFor("127.0.0.1").getSubstats().get(CrawledBytesHistotable.WARC_NOVEL_URLS));
assertEquals(2216, (long) stats.getServerCache().getHostFor("127.0.0.1").getSubstats().get(CrawledBytesHistotable.WARC_NOVEL_CONTENT_BYTES));
assertEquals(2202, (long) stats.getServerCache().getHostFor("127.0.0.1").getSubstats().get(CrawledBytesHistotable.WARC_NOVEL_CONTENT_BYTES));
assertEquals(10, (long) stats.getServerCache().getHostFor("localhost").getSubstats().get(CrawledBytesHistotable.WARC_NOVEL_URLS));
assertEquals(5783, (long) stats.getServerCache().getHostFor("localhost").getSubstats().get(CrawledBytesHistotable.WARC_NOVEL_CONTENT_BYTES));
assertEquals(5758, (long) stats.getServerCache().getHostFor("localhost").getSubstats().get(CrawledBytesHistotable.WARC_NOVEL_CONTENT_BYTES));
assertEquals(0, (long) stats.getServerCache().getHostFor("dns:").getSubstats().get(CrawledBytesHistotable.WARC_NOVEL_URLS));
}
@@ -66,17 +66,17 @@ public class StatisticsSelfTest extends SelfTestBase {
sourceStats = stats.getSourceStats("http://127.0.0.1:7777/a.html");
assertNotNull(sourceStats);
assertEquals(4, sourceStats.keySet().size());
assertEquals(2216l, (long) sourceStats.get("novel"));
assertEquals(2202l, (long) sourceStats.get("novel"));
assertEquals(3l, (long) sourceStats.get("novelCount"));
assertEquals(2216l, (long) sourceStats.get("warcNovelContentBytes"));
assertEquals(2202l, (long) sourceStats.get("warcNovelContentBytes"));
assertEquals(3l, (long) sourceStats.get("warcNovelUrls"));
sourceStats = stats.getSourceStats("http://localhost:7777/b.html");
assertNotNull(sourceStats);
assertEquals(4, sourceStats.keySet().size());
assertEquals(5783l, (long) sourceStats.get("novel") - stats.getBytesPerHost("dns:"));
assertEquals(5758l, (long) sourceStats.get("novel") - stats.getBytesPerHost("dns:"));
assertEquals(11l, (long) sourceStats.get("novelCount"));
assertEquals(5783l, (long) sourceStats.get("warcNovelContentBytes") - stats.getBytesPerHost("dns:"));
assertEquals(5758l, (long) sourceStats.get("warcNovelContentBytes") - stats.getBytesPerHost("dns:"));
assertEquals(10l, (long) sourceStats.get("warcNovelUrls"));
}
engine/src/test/java/org/archive/crawler/selftest/UserAgentSelfTest.java
+12 -15
View File
@@ -20,14 +20,15 @@
package org.archive.crawler.selftest;
import org.archive.util.ArchiveUtils;
import org.eclipse.jetty.ee10.servlet.ServletContextHandler;
import org.eclipse.jetty.ee10.servlet.ServletHandler;
import org.eclipse.jetty.ee10.servlet.ServletHolder;
import org.eclipse.jetty.server.Handler;
import org.eclipse.jetty.server.Server;
import org.eclipse.jetty.server.ServerConnector;
import org.eclipse.jetty.server.handler.DefaultHandler;
import org.eclipse.jetty.server.handler.HandlerList;
import org.eclipse.jetty.server.handler.ResourceHandler;
import org.eclipse.jetty.servlet.ServletHandler;
import org.eclipse.jetty.servlet.ServletHolder;
import org.eclipse.jetty.util.resource.ResourceFactory;
/**
* @author pjack
@@ -59,20 +60,16 @@ public class UserAgentSelfTest extends SelfTestBase {
sc.setPort(7777);
server.addConnector(sc);
ResourceHandler rhandler = new ResourceHandler();
rhandler.setResourceBase(getSrcHtdocs().getAbsolutePath());
ResourceFactory resourceFactory = ResourceFactory.of(server);
rhandler.setBaseResource(resourceFactory.newResource(getSrcHtdocs().toPath().toAbsolutePath()));
ServletHandler servletHandler = new ServletHandler();
HandlerList handlers = new HandlerList();
handlers.setHandlers(new Handler[] {
rhandler,
servletHandler,
new DefaultHandler() });
server.setHandler(handlers);
this.servlet = new UserAgentServlet();
ServletHolder holder = new ServletHolder(servlet);
servletHandler.addServletWithMapping(holder, "/*");
ServletContextHandler contextHandler = new ServletContextHandler(ServletContextHandler.SESSIONS);
contextHandler.addServlet(servlet, "/*");
server.setHandler(new Handler.Sequence(rhandler,
contextHandler,
new DefaultHandler()));
this.httpServer = server;
this.httpServer.start();
engine/src/test/java/org/archive/crawler/selftest/UserAgentServlet.java
+7 -7
View File
@@ -19,14 +19,14 @@
package org.archive.crawler.selftest;
import jakarta.servlet.ServletException;
import jakarta.servlet.http.HttpServlet;
import jakarta.servlet.http.HttpServletRequest;
import jakarta.servlet.http.HttpServletResponse;
import java.io.IOException;
import java.util.Enumeration;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
/**
* @author pjack
*/
@@ -37,8 +37,8 @@ public class UserAgentServlet extends HttpServlet {
private String from;
@Override
protected void doGet(HttpServletRequest req, HttpServletResponse resp)
throws ServletException, IOException {
protected void doGet(HttpServletRequest req, HttpServletResponse resp)
throws IOException {
Enumeration<?> e = req.getHeaderNames();
while (e.hasMoreElements()) {
String name = (String)e.nextElement();
engine/src/test/java/org/archive/modules/fetcher/FormAuthTest.java
+26 -38
View File
@@ -27,10 +27,10 @@ import java.util.HashSet;
import java.util.Set;
import java.util.logging.Logger;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import jakarta.servlet.http.Cookie;
import jakarta.servlet.http.HttpServlet;
import jakarta.servlet.http.HttpServletRequest;
import jakarta.servlet.http.HttpServletResponse;
import junit.framework.TestCase;
import org.apache.commons.httpclient.URIException;
@@ -44,19 +44,18 @@ import org.archive.net.UURI;
import org.archive.net.UURIFactory;
import org.archive.util.Recorder;
import org.archive.util.TmpDirTestCase;
import org.eclipse.jetty.ee10.servlet.ServletContextHandler;
import org.eclipse.jetty.ee10.servlet.SessionHandler;
import org.eclipse.jetty.ee10.servlet.security.ConstraintMapping;
import org.eclipse.jetty.ee10.servlet.security.ConstraintSecurityHandler;
import org.eclipse.jetty.security.*;
import org.eclipse.jetty.security.authentication.FormAuthenticator;
import org.eclipse.jetty.server.NCSARequestLog;
import org.eclipse.jetty.server.Request;
import org.eclipse.jetty.server.Server;
import org.eclipse.jetty.server.ServerConnector;
import org.eclipse.jetty.server.handler.HandlerCollection;
import org.eclipse.jetty.server.handler.RequestLogHandler;
import org.eclipse.jetty.server.session.DefaultSessionCache;
import org.eclipse.jetty.server.session.NullSessionDataStore;
import org.eclipse.jetty.server.session.SessionCache;
import org.eclipse.jetty.server.session.SessionHandler;
import org.eclipse.jetty.util.security.Constraint;
import org.eclipse.jetty.session.DefaultSessionCache;
import org.eclipse.jetty.session.NullSessionDataStore;
import org.eclipse.jetty.session.SessionCache;
import org.eclipse.jetty.util.security.Password;
/* Somewhat redundant to org.archive.crawler.selftest.FormAuthSelfTest, but
@@ -169,7 +168,7 @@ public class FormAuthTest extends TestCase {
getFetcher().process(curi);
logger.info('\n' + httpRequestString(curi) + "\n\n" + rawResponseString(curi));
assertEquals(302, curi.getFetchStatus());
assertTrue(curi.getHttpResponseHeader("Location").startsWith("http://localhost:7779/login.html"));
assertTrue(curi.getHttpResponseHeader("Location").startsWith("/login.html"));
PreconditionEnforcer preconditionEnforcer = new PreconditionEnforcer();
preconditionEnforcer.setServerCache(getFetcher().getServerCache());
@@ -188,7 +187,7 @@ public class FormAuthTest extends TestCase {
getFetcher().process(loginUri);
logger.info('\n' + httpRequestString(loginUri) + "\n\n" + rawResponseString(loginUri));
assertEquals(302, loginUri.getFetchStatus()); // 302 on successful login
assertEquals("http://localhost:7779/auth/1", loginUri.getHttpResponseHeader("location"));
assertEquals("/auth/1", loginUri.getHttpResponseHeader("location"));
curi = makeCrawlURI("http://localhost:7779/auth/1");
getFetcher().process(curi);
@@ -206,29 +205,28 @@ public class FormAuthTest extends TestCase {
+ "<div> <input type='submit' /> </div>" + "</form>" + "</body>"
+ "</html>";
protected static class FormAuthTestHandler extends SessionHandler {
protected static class FormAuthTestServlet extends HttpServlet {
public FormAuthTestHandler() {
public FormAuthTestServlet() {
super();
}
@Override
public void doHandle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException {
protected void doGet(HttpServletRequest request, HttpServletResponse response) throws IOException {
String target = request.getRequestURI();
if (target.endsWith("/set-cookie")) {
response.addCookie(new javax.servlet.http.Cookie("test-cookie-name", "test-cookie-value"));
response.addCookie(new Cookie("test-cookie-name", "test-cookie-value"));
}
if (target.equals("/login.html")) {
response.setContentType("text/html;charset=US-ASCII");
response.setStatus(HttpServletResponse.SC_OK);
response.getOutputStream().write(LOGIN_HTML.getBytes("US-ASCII"));
((Request)request).setHandled(true);
} else {
response.setContentType("text/plain;charset=US-ASCII");
response.setDateHeader("Last-Modified", 0);
response.setStatus(HttpServletResponse.SC_OK);
response.getOutputStream().write(DEFAULT_PAYLOAD_STRING.getBytes("US-ASCII"));
((Request)request).setHandled(true);
}
}
}
@@ -236,9 +234,7 @@ public class FormAuthTest extends TestCase {
protected static SecurityHandler makeAuthWrapper(Authenticator authenticator,
final String role, String realm, final String login,
final String password) {
Constraint constraint = new Constraint();
constraint.setRoles(new String[] { role });
constraint.setAuthenticate(true);
Constraint constraint = Constraint.from(role);
ConstraintMapping constraintMapping = new ConstraintMapping();
constraintMapping.setConstraint(constraint);
@@ -266,27 +262,19 @@ public class FormAuthTest extends TestCase {
sc.setHost("127.0.0.1");
sc.setPort(7779);
server.addConnector(sc);
HandlerCollection handlers = new HandlerCollection();
handlers.addHandler(new FormAuthTestHandler());
RequestLogHandler requestLogHandler = new RequestLogHandler();
NCSARequestLog requestLog = new NCSARequestLog();
requestLogHandler.setRequestLog(requestLog);
handlers.addHandler(requestLogHandler);
ServletContextHandler context = new ServletContextHandler(ServletContextHandler.SESSIONS);
context.setContextPath("/");
context.addServlet(FormAuthTestServlet.class, "/");
FormAuthenticator formAuthenticatrix = new FormAuthenticator("/login.html", null, false);
SecurityHandler authWrapper = makeAuthWrapper(formAuthenticatrix,
FORM_AUTH_ROLE, FORM_AUTH_REALM, FORM_AUTH_LOGIN,
FORM_AUTH_PASSWORD);
authWrapper.setHandler(handlers);
SessionHandler sessionHandler = new SessionHandler();
SessionCache cache = new DefaultSessionCache(sessionHandler);
cache.setSessionDataStore(new NullSessionDataStore());
sessionHandler.setSessionCache(cache);
sessionHandler.setHandler(authWrapper);
server.setHandler(sessionHandler);
context.setSecurityHandler(authWrapper);
server.setHandler(context);
server.start();
}
modules/pom.xml
+8 -2
View File
@@ -49,8 +49,14 @@
<version>${jetty.version}</version>
</dependency>
<dependency>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-servlet</artifactId>
<groupId>org.eclipse.jetty.ee10</groupId>
<artifactId>jetty-ee10-servlet</artifactId>
<version>${jetty.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.eclipse.jetty.ee10</groupId>
<artifactId>jetty-ee10-proxy</artifactId>
<version>${jetty.version}</version>
<scope>test</scope>
</dependency>
modules/src/test/java/org/archive/modules/fetcher/CookieFetchHTTPIntegrationTest.java
+15 -24
View File
@@ -25,11 +25,11 @@ import java.net.UnknownHostException;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import javax.servlet.ServletException;
import javax.servlet.http.Cookie;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import jakarta.servlet.ServletException;
import jakarta.servlet.http.Cookie;
import jakarta.servlet.http.HttpServlet;
import jakarta.servlet.http.HttpServletRequest;
import jakarta.servlet.http.HttpServletResponse;
import org.apache.commons.collections.Closure;
import org.apache.commons.httpclient.URIException;
import org.apache.commons.io.FileUtils;
@@ -44,16 +44,14 @@ import org.archive.spring.ConfigFile;
import org.archive.spring.ConfigPath;
import org.archive.util.KeyTool;
import org.archive.util.TmpDirTestCase;
import org.eclipse.jetty.ee10.servlet.ServletContextHandler;
import org.eclipse.jetty.http.HttpVersion;
import org.eclipse.jetty.server.HttpConfiguration;
import org.eclipse.jetty.server.HttpConnectionFactory;
import org.eclipse.jetty.server.Request;
import org.eclipse.jetty.server.SecureRequestCustomizer;
import org.eclipse.jetty.server.Server;
import org.eclipse.jetty.server.ServerConnector;
import org.eclipse.jetty.server.SslConnectionFactory;
import org.eclipse.jetty.server.session.SessionHandler;
import org.eclipse.jetty.util.log.Log;
import org.eclipse.jetty.util.ssl.SslContextFactory;
import com.google.common.io.Files;
@@ -64,15 +62,15 @@ import junit.framework.TestSuite;
public class CookieFetchHTTPIntegrationTest extends ProcessorTestBase {
protected static class TestHandler extends SessionHandler {
public TestHandler() {
protected static class TestServlet extends HttpServlet {
public TestServlet() {
super();
}
@Override
public void doHandle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException {
protected void doGet(HttpServletRequest request, HttpServletResponse response) throws IOException {
if (request.getParameter("name") != null) {
Cookie cookie = new javax.servlet.http.Cookie(request.getParameter("name"),
Cookie cookie = new Cookie(request.getParameter("name"),
request.getParameter("value"));
if (request.getParameter("domain") != null) {
cookie.setDomain(request.getParameter("domain"));
@@ -86,12 +84,6 @@ public class CookieFetchHTTPIntegrationTest extends ProcessorTestBase {
if (request.getParameter("secure") != null) {
cookie.setSecure(request.getParameter("secure").equals(1));
}
if (request.getParameter("comment") != null) {
cookie.setComment(request.getParameter("comment"));
}
if (request.getParameter("version") != null) {
cookie.setVersion(Integer.valueOf(request.getParameter("version")));
}
response.addCookie(cookie);
}
@@ -106,17 +98,16 @@ public class CookieFetchHTTPIntegrationTest extends ProcessorTestBase {
} else {
response.getOutputStream().println("0 cookies received");
}
((Request)request).setHandled(true);
}
}
public static Server startHttpServer() throws Exception {
Log.getLogger(Server.class.getCanonicalName()).setDebugEnabled(true);
Server server = new Server();
server.setHandler(new TestHandler());
ServletContextHandler context = new ServletContextHandler(ServletContextHandler.SESSIONS);
context.setContextPath("/");
context.addServlet(TestServlet.class, "/");
server.setHandler(context);
ServerConnector sc = new ServerConnector(server);
sc.setHost("127.0.0.1");
@@ -137,7 +128,7 @@ public class CookieFetchHTTPIntegrationTest extends ProcessorTestBase {
"-dname", "CN=127.0.0.1",
"-validity","3650"}); // 10 yr validity
SslContextFactory sslContextFactory = new SslContextFactory();
SslContextFactory.Server sslContextFactory = new SslContextFactory.Server();
sslContextFactory.setKeyStorePassword(KEYSTORE_PASSWORD);
sslContextFactory.setKeyStorePath(keystoreFile.getPath());
modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTest.java
+12 -11
View File
@@ -41,10 +41,10 @@ import java.util.logging.Logger;
import javax.net.ServerSocketFactory;
import javax.net.ssl.SSLException;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import jakarta.servlet.ServletException;
import jakarta.servlet.http.HttpServletRequest;
import jakarta.servlet.http.HttpServletResponse;
import org.apache.commons.httpclient.URIException;
import org.apache.commons.io.IOUtils;
import org.apache.http.NoHttpResponseException;
@@ -63,12 +63,12 @@ import org.archive.net.UURIFactory;
import org.archive.util.Recorder;
import org.archive.util.TmpDirTestCase;
import org.bbottema.javasocksproxyserver.SocksServer;
import org.eclipse.jetty.client.api.Response;
import org.eclipse.jetty.proxy.ConnectHandler;
import org.eclipse.jetty.proxy.ProxyServlet;
import org.eclipse.jetty.client.Response;
import org.eclipse.jetty.ee10.proxy.ProxyServlet;
import org.eclipse.jetty.ee10.servlet.ServletContextHandler;
import org.eclipse.jetty.ee10.servlet.ServletHolder;
import org.eclipse.jetty.server.Server;
import org.eclipse.jetty.servlet.ServletContextHandler;
import org.eclipse.jetty.servlet.ServletHolder;
import org.eclipse.jetty.server.handler.ConnectHandler;
import org.junit.*;
public class FetchHTTPTest {
@@ -270,7 +270,7 @@ public class FetchHTTPTest {
// check that we got the expected response and the fetcher did its thing
assertEquals(401, curi.getFetchStatus());
assertEquals("basic realm=\"basic-auth-realm\"", curi.getHttpResponseHeader("WWW-Authenticate"));
assertEquals("Basic realm=\"basic-auth-realm\"", curi.getHttpResponseHeader("WWW-Authenticate"));
assertTrue(curi.getCredentials().contains(basicAuthCredential));
assertTrue(curi.getHttpAuthChallenges() != null && curi.getHttpAuthChallenges().containsKey("basic"));
@@ -461,8 +461,9 @@ public class FetchHTTPTest {
Server httpProxyServer = new Server(new InetSocketAddress("localhost", 7877));
ConnectHandler connectHandler = new ConnectHandler();
httpProxyServer.setHandler(connectHandler);
ServletContextHandler context = new ServletContextHandler(connectHandler, "/",
ServletContextHandler.SESSIONS);
ServletContextHandler context = new ServletContextHandler( "/", ServletContextHandler.SESSIONS);
connectHandler.setHandler(context);
ServletHolder proxyServlet = new ServletHolder(TestProxyServlet.class);
context.addServlet(proxyServlet, "/*");
context.setAttribute("proxy-user", user);
modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTestServers.java
+35 -47
View File
@@ -23,22 +23,22 @@ import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import jakarta.servlet.ServletException;
import jakarta.servlet.http.Cookie;
import jakarta.servlet.http.HttpServlet;
import jakarta.servlet.http.HttpServletRequest;
import jakarta.servlet.http.HttpServletResponse;
import org.archive.util.KeyTool;
import org.archive.util.TmpDirTestCase;
import org.eclipse.jetty.ee10.servlet.ServletContextHandler;
import org.eclipse.jetty.ee10.servlet.security.ConstraintMapping;
import org.eclipse.jetty.ee10.servlet.security.ConstraintSecurityHandler;
import org.eclipse.jetty.http.HttpVersion;
import org.eclipse.jetty.security.*;
import org.eclipse.jetty.security.authentication.BasicAuthenticator;
import org.eclipse.jetty.security.authentication.DigestAuthenticator;
import org.eclipse.jetty.server.*;
import org.eclipse.jetty.server.handler.HandlerCollection;
import org.eclipse.jetty.server.session.SessionHandler;
import org.eclipse.jetty.util.log.Log;
import org.eclipse.jetty.util.security.Constraint;
import org.eclipse.jetty.util.security.Password;
import org.eclipse.jetty.util.ssl.SslContextFactory;
@@ -112,20 +112,20 @@ public class FetchHTTPTestServers {
protected static final byte[] EIGHTY_BYTE_LINE = "1234567890123456789012345678901234567890123456789012345678901234567890123456789\n".getBytes();
protected static class TestHandler extends SessionHandler {
protected static class TestServlet extends HttpServlet {
public TestHandler() {
public TestServlet() {
super();
}
@Override
public void doHandle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException {
// echo the remote host back to the client so tests can reference it
protected void service(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
// echo the remote host back to the client so tests can reference it
response.setHeader("Client-Host", request.getRemoteHost());
String target = request.getRequestURI();
if (target.endsWith("/set-cookie")) {
response.addCookie(new javax.servlet.http.Cookie("test-cookie-name", "test-cookie-value"));
response.addCookie(new Cookie("test-cookie-name", "test-cookie-value"));
}
if (target.equals("/200k")) {
@@ -135,7 +135,6 @@ public class FetchHTTPTestServers {
for (int i = 0; i < 200000 / EIGHTY_BYTE_LINE.length; i++) {
response.getOutputStream().write(EIGHTY_BYTE_LINE);
}
((Request)request).setHandled(true);
} else if (target.equals("/slow.txt")) {
response.setContentType("text/plain;charset=US-ASCII");
response.setStatus(200);
@@ -147,80 +146,66 @@ public class FetchHTTPTestServers {
} catch (InterruptedException e) {
}
}
((Request)request).setHandled(true);
} else if (target.equals("/chunked.txt")) {
response.setContentType("text/plain;charset=US-ASCII");
response.setStatus(200);
// response.setContentLength(HttpTokens.CHUNKED_CONTENT);
response.getOutputStream().write(DEFAULT_PAYLOAD_STRING.getBytes("US-ASCII"));
response.getOutputStream().flush();
((Request)request).setHandled(true);
} else if (request.getHeader("Accept-Encoding") != null
&& request.getHeader("Accept-Encoding").contains("gzip")) {
response.setHeader("Content-Encoding", "gzip");
response.setContentType("text/plain;charset=US-ASCII");
response.setStatus(200);
response.getOutputStream().write(DEFAULT_GZIPPED_PAYLOAD);
((Request)request).setHandled(true);
} else if (target.equals("/401-no-challenge")) {
response.setStatus(401);
response.setContentType("text/plain;charset=US-ASCII");
response.getOutputStream().write(DEFAULT_PAYLOAD_STRING.getBytes("US-ASCII"));
((Request)request).setHandled(true);
} else if (target.equals("/cp1251")) {
response.setContentType("text/plain;charset=cp1251");
response.setStatus(200);
response.getOutputStream().write(CP1251_PAYLOAD);
((Request)request).setHandled(true);
} else if (target.equals("/unsupported-charset")) {
response.setContentType("text/plain;charset=UNSUPPORTED-CHARSET");
response.setStatus(200);
response.getOutputStream().write(DEFAULT_PAYLOAD_STRING.getBytes("US-ASCII"));
((Request)request).setHandled(true);
} else if (target.equals("/invalid-charset")) {
response.setContentType("text/plain;charset=%%INVALID-CHARSET%%");
response.setStatus(200);
response.getOutputStream().write(DEFAULT_PAYLOAD_STRING.getBytes("US-ASCII"));
((Request)request).setHandled(true);
} else if (target.equals("/if-modified-since")) {
if (request.getHeader("if-modified-since") != null) {
response.setStatus(304);
((Request)request).setHandled(true);
} else {
response.setContentType("text/plain;charset=US-ASCII");
response.setDateHeader("Last-Modified", 0);
response.setStatus(200);
response.getOutputStream().write(DEFAULT_PAYLOAD_STRING.getBytes("US-ASCII"));
((Request)request).setHandled(true);
}
} else if (target.equals("/if-none-match")) {
if (request.getHeader("if-none-match") != null) {
response.setStatus(304);
((Request)request).setHandled(true);
} else {
response.setContentType("text/plain;charset=US-ASCII");
response.setHeader("ETag", ETAG_TEST_VALUE);
response.setStatus(200);
response.getOutputStream().write(DEFAULT_PAYLOAD_STRING.getBytes("US-ASCII"));
((Request)request).setHandled(true);
}
} else {
response.setContentType("text/plain;charset=US-ASCII");
response.setStatus(200);
response.getOutputStream().write(DEFAULT_PAYLOAD_STRING.getBytes("US-ASCII"));
((Request)request).setHandled(true);
}
}
}
protected static Map<Integer, Server> httpServers;
protected static SecurityHandler makeAuthWrapper(Authenticator authenticator,
protected static ConstraintSecurityHandler makeAuthWrapper(Authenticator authenticator,
final String role, String realm, final String login,
final String password) {
Constraint constraint = new Constraint();
constraint.setRoles(new String[] { role });
constraint.setAuthenticate(true);
Constraint constraint = Constraint.from(role);
ConstraintMapping constraintMapping = new ConstraintMapping();
constraintMapping.setConstraint(constraint);
@@ -228,7 +213,6 @@ public class FetchHTTPTestServers {
ConstraintSecurityHandler authWrapper = new ConstraintSecurityHandler();
authWrapper.setAuthenticator(authenticator);
authWrapper.setConstraintMappings(new ConstraintMapping[] {constraintMapping});
UserStore userStore = new UserStore();
userStore.addUser(login, new Password(password), new String[] {role});
@@ -245,8 +229,6 @@ public class FetchHTTPTestServers {
* @return map(port->server)
*/
public static Map<Integer,Server> startHttpServers() throws Exception {
Log.getLogger(Server.class.getCanonicalName()).setDebugEnabled(true);
HashMap<Integer, Server> servers = new HashMap<Integer,Server>();
// server for basic auth
@@ -257,14 +239,17 @@ public class FetchHTTPTestServers {
sc.setPort(7777);
server.addConnector(sc);
SecurityHandler authWrapper = makeAuthWrapper(new BasicAuthenticator(),
ConstraintSecurityHandler authWrapper = makeAuthWrapper(new BasicAuthenticator(),
BASIC_AUTH_ROLE, BASIC_AUTH_REALM, BASIC_AUTH_LOGIN,
BASIC_AUTH_PASSWORD);
HandlerCollection handlers = new HandlerCollection();
handlers.addHandler(new TestHandler());
authWrapper.setHandler(handlers);
server.setHandler(authWrapper);
ServletContextHandler context = new ServletContextHandler(ServletContextHandler.SESSIONS);
context.setContextPath("/");
context.addServlet(TestServlet.class, "/");
context.setSecurityHandler(authWrapper);
server.setHandler(context);
servers.put(sc.getPort(), server);
File keystoreFile = new File(TmpDirTestCase.tmpDir(), "keystore");
@@ -282,12 +267,14 @@ public class FetchHTTPTestServers {
"-dname", "CN=127.0.0.1",
"-validity","3650"}); // 10 yr validity
SslContextFactory sslContextFactory = new SslContextFactory();
SslContextFactory.Server sslContextFactory = new SslContextFactory.Server();
sslContextFactory.setKeyStorePassword(KEYSTORE_PASSWORD);
sslContextFactory.setKeyStorePath(keystoreFile.getPath());
HttpConfiguration httpsConfig = new HttpConfiguration();
httpsConfig.addCustomizer(new SecureRequestCustomizer());
SecureRequestCustomizer customizer = new SecureRequestCustomizer();
customizer.setSniHostCheck(false); // disable SNI host check because it fails on localhost
httpsConfig.addCustomizer(customizer);
ServerConnector ssc = new ServerConnector(server,
new SslConnectionFactory(sslContextFactory, HttpVersion.HTTP_1_1.asString()),
@@ -310,11 +297,12 @@ public class FetchHTTPTestServers {
authWrapper = makeAuthWrapper(new DigestAuthenticator(),
DIGEST_AUTH_ROLE, DIGEST_AUTH_REALM, DIGEST_AUTH_LOGIN,
DIGEST_AUTH_PASSWORD);
HandlerCollection handlers2 = new HandlerCollection();
handlers2.addHandler(new TestHandler());
authWrapper.setHandler(handlers2);
server.setHandler(authWrapper);
ServletContextHandler context2 = new ServletContextHandler(ServletContextHandler.SESSIONS);
context2.setContextPath("/");
context2.addServlet(TestServlet.class, "/");
context2.setSecurityHandler(authWrapper);
server.setHandler(context2);
server.start();
servers.put(sc.getPort(), server);
modules/src/test/java/org/archive/modules/recrawl/ContentDigestHistoryTest.java
+12 -19
View File
@@ -47,10 +47,6 @@ import java.util.Iterator;
import java.util.List;
import java.util.logging.Logger;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.apache.commons.httpclient.URIException;
import org.apache.commons.io.FileUtils;
import org.archive.bdb.BdbModule;
@@ -69,11 +65,11 @@ import org.archive.spring.ConfigPath;
import org.archive.util.Base32;
import org.archive.util.Recorder;
import org.archive.util.TmpDirTestCase;
import org.eclipse.jetty.server.Request;
import org.eclipse.jetty.server.Server;
import org.eclipse.jetty.server.ServerConnector;
import org.eclipse.jetty.server.handler.HandlerCollection;
import org.eclipse.jetty.server.session.SessionHandler;
import org.eclipse.jetty.http.HttpHeader;
import org.eclipse.jetty.http.HttpStatus;
import org.eclipse.jetty.io.Content;
import org.eclipse.jetty.server.*;
import org.eclipse.jetty.util.Callback;
public class ContentDigestHistoryTest extends TmpDirTestCase {
@@ -367,19 +363,16 @@ public class ContentDigestHistoryTest extends TmpDirTestCase {
protected static final String DEFAULT_PAYLOAD_STRING = "abcdefghijklmnopqrstuvwxyz0123456789\n";
protected Server newHttpServer() throws Exception {
HandlerCollection handlers = new HandlerCollection();
handlers.addHandler(new SessionHandler(){
Server server = new Server();
server.setHandler(new Handler.Abstract.NonBlocking() {
@Override
public void doHandle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException {
response.setContentType("text/plain;charset=US-ASCII");
response.setStatus(HttpServletResponse.SC_OK);
response.getOutputStream().write(DEFAULT_PAYLOAD_STRING.getBytes("US-ASCII"));
((Request)request).setHandled(true);
public boolean handle(Request request, Response response, Callback callback) throws Exception {
response.getHeaders().put(HttpHeader.CONTENT_TYPE, "text/plain;charset=US-ASCII");
response.setStatus(HttpStatus.OK_200);
Content.Sink.write(response, true, DEFAULT_PAYLOAD_STRING, callback);
return true;
}
});
Server server = new Server();
server.setHandler(handlers);
ServerConnector sc = new ServerConnector(server);
sc.setHost("127.0.0.1");
pom.xml
+1 -1
View File
@@ -372,7 +372,7 @@ http://maven.apache.org/guides/mini/guide-m1-m2.html
<doclint>none</doclint>
<additionalparam>-Xdoclint:none</additionalparam>
<groovy.version>4.0.26</groovy.version>
<jetty.version>9.4.57.v20241219</jetty.version>
<jetty.version>12.0.16</jetty.version>
<slf4j.version>2.0.17</slf4j.version>
<maven.compiler.target>17</maven.compiler.target>
<maven.compiler.source>17</maven.compiler.source>