From c79798559007704bff320d24f110fd7c024ced19 Mon Sep 17 00:00:00 2001 From: Alex Osborne Date: Tue, 15 Oct 2024 19:29:50 +0900 Subject: [PATCH 1/3] Bump webarchive-commons from 1.1.8 to 1.1.10 We need to add log4j as a dependency temporarily as webarchive-commons no longer pulls it in. --- commons/pom.xml | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/commons/pom.xml b/commons/pom.xml index 3f47bcb0..f2b13c99 100644 --- a/commons/pom.xml +++ b/commons/pom.xml @@ -151,14 +151,16 @@ org.netpreserve.commons webarchive-commons - 1.1.8 - - - org.apache.hadoop - hadoop-core - - + 1.1.10 + + + + log4j + log4j + 1.2.17 + + com.jcraft jsch From 3d1b7e37ca517cbadd87f3877c36c0a55785c9fd Mon Sep 17 00:00:00 2001 From: Alex Osborne Date: Tue, 15 Oct 2024 20:08:53 +0900 Subject: [PATCH 2/3] Remove log4j --- commons/pom.xml | 7 --- .../org/archive/util/OneLineSimpleLayout.java | 54 ------------------- commons/src/test/resources/log4j.xml | 23 -------- contrib/src/main/resources/log4j.xml | 27 ---------- contrib/src/test/resources/log4j.xml | 27 ---------- dist/src/test/resources/log4j.xml | 23 -------- docs/operating.rst | 3 +- engine/src/test/resources/log4j.xml | 23 -------- modules/src/test/resources/log4j.xml | 23 -------- 9 files changed, 1 insertion(+), 209 deletions(-) delete mode 100644 commons/src/main/java/org/archive/util/OneLineSimpleLayout.java delete mode 100644 commons/src/test/resources/log4j.xml delete mode 100644 contrib/src/main/resources/log4j.xml delete mode 100644 contrib/src/test/resources/log4j.xml delete mode 100644 dist/src/test/resources/log4j.xml delete mode 100644 engine/src/test/resources/log4j.xml delete mode 100644 modules/src/test/resources/log4j.xml diff --git a/commons/pom.xml b/commons/pom.xml index f2b13c99..a5e76309 100644 --- a/commons/pom.xml +++ b/commons/pom.xml @@ -154,13 +154,6 @@ 1.1.10 - - - log4j - log4j - 1.2.17 - - com.jcraft jsch diff --git a/commons/src/main/java/org/archive/util/OneLineSimpleLayout.java b/commons/src/main/java/org/archive/util/OneLineSimpleLayout.java deleted file mode 100644 index 61030cd6..00000000 --- a/commons/src/main/java/org/archive/util/OneLineSimpleLayout.java +++ /dev/null @@ -1,54 +0,0 @@ -package org.archive.util; - -import java.util.logging.LogRecord; - -import org.apache.log4j.Layout; -import org.apache.log4j.spi.LoggingEvent; - -public class OneLineSimpleLayout extends Layout { - - private OneLineSimpleLogger logger = new OneLineSimpleLogger(); - - @Override - public void activateOptions() { - } - - @Override - public String format(LoggingEvent event) { - java.util.logging.Level level = convertLevel(event.getLevel()); - - LogRecord logRecord = new LogRecord(level, event.getMessage().toString()); - logRecord.setLoggerName(event.getLoggerName()); - logRecord.setMillis(event.getTimeStamp()); - logRecord.setSourceClassName(event.getLoggerName()); - logRecord.setSourceMethodName(event.getLocationInformation().getMethodName()); - logRecord.setThreadID((int) Thread.currentThread().getId()); - - return logger.format(logRecord); - } - - protected java.util.logging.Level convertLevel(org.apache.log4j.Level log4jLevel) { - switch (log4jLevel.toInt()) { - case org.apache.log4j.Level.TRACE_INT: - return java.util.logging.Level.FINER; - case org.apache.log4j.Level.DEBUG_INT: - return java.util.logging.Level.FINE; - case org.apache.log4j.Level.INFO_INT: - return java.util.logging.Level.INFO; - case org.apache.log4j.Level.WARN_INT: - return java.util.logging.Level.WARNING; - case org.apache.log4j.Level.ERROR_INT: - return java.util.logging.Level.SEVERE; - case org.apache.log4j.Level.FATAL_INT: - return java.util.logging.Level.SEVERE; - default: - return java.util.logging.Level.ALL; - } - } - - @Override - public boolean ignoresThrowable() { - return true; - } - -} diff --git a/commons/src/test/resources/log4j.xml b/commons/src/test/resources/log4j.xml deleted file mode 100644 index e04f34ab..00000000 --- a/commons/src/test/resources/log4j.xml +++ /dev/null @@ -1,23 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - diff --git a/contrib/src/main/resources/log4j.xml b/contrib/src/main/resources/log4j.xml deleted file mode 100644 index 8a4dae8e..00000000 --- a/contrib/src/main/resources/log4j.xml +++ /dev/null @@ -1,27 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/contrib/src/test/resources/log4j.xml b/contrib/src/test/resources/log4j.xml deleted file mode 100644 index 8a4dae8e..00000000 --- a/contrib/src/test/resources/log4j.xml +++ /dev/null @@ -1,27 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/dist/src/test/resources/log4j.xml b/dist/src/test/resources/log4j.xml deleted file mode 100644 index e04f34ab..00000000 --- a/dist/src/test/resources/log4j.xml +++ /dev/null @@ -1,23 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - diff --git a/docs/operating.rst b/docs/operating.rst index 041c0357..f034cdbe 100644 --- a/docs/operating.rst +++ b/docs/operating.rst @@ -198,8 +198,7 @@ Log Files Each crawl job has its own set of log files found in the ``logs`` subdirectory of a job launch directory. Logging can be configured by modifying the ``logging.properties`` file -that is located under the ``$HERITRIX_HOME/conf`` directory. For information on using -logging properties, visit http://logging.apache.org/log4j/. +that is located under the ``$HERITRIX_HOME/conf`` directory. alerts.log ~~~~~~~~~~ diff --git a/engine/src/test/resources/log4j.xml b/engine/src/test/resources/log4j.xml deleted file mode 100644 index e04f34ab..00000000 --- a/engine/src/test/resources/log4j.xml +++ /dev/null @@ -1,23 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - diff --git a/modules/src/test/resources/log4j.xml b/modules/src/test/resources/log4j.xml deleted file mode 100644 index e04f34ab..00000000 --- a/modules/src/test/resources/log4j.xml +++ /dev/null @@ -1,23 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - From cbd7f37ed9f2319a9fc593627b9baa8f070ba200 Mon Sep 17 00:00:00 2001 From: Alex Osborne Date: Fri, 18 Oct 2024 17:24:02 +0900 Subject: [PATCH 3/3] Replace littleproxy with jetty-proxy (to remove dependency on log4j) littleproxy has not been updated since 2017 and has various out of date dependencies. Since we're using jetty for HTTP server testing anyway, we may as well use it as the proxy too. --- engine/pom.xml | 6 -- modules/pom.xml | 18 ++-- .../modules/fetcher/FetchHTTPTest.java | 102 +++++++++--------- 3 files changed, 61 insertions(+), 65 deletions(-) diff --git a/engine/pom.xml b/engine/pom.xml index fb345c71..dd160615 100644 --- a/engine/pom.xml +++ b/engine/pom.xml @@ -82,12 +82,6 @@ 2.3.1 runtime - - org.littleshoot - littleproxy - 0.4 - test - org.slf4j slf4j-api diff --git a/modules/pom.xml b/modules/pom.xml index ed7ab631..6ebbbc6e 100644 --- a/modules/pom.xml +++ b/modules/pom.xml @@ -50,16 +50,16 @@ ${jetty.version} - org.littleshoot - littleproxy - 0.4 + org.eclipse.jetty + jetty-servlet + ${jetty.version} + test + + + org.eclipse.jetty + jetty-proxy + ${jetty.version} test - - - org.slf4j - slf4j-log4j12 - - org.slf4j diff --git a/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTest.java b/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTest.java index ee02c3a6..98ad98b0 100644 --- a/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTest.java +++ b/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTest.java @@ -32,27 +32,18 @@ import static org.junit.Assert.*; import java.io.IOException; import java.io.InterruptedIOException; import java.io.UnsupportedEncodingException; -import java.net.Inet4Address; -import java.net.InetAddress; -import java.net.NetworkInterface; -import java.net.ServerSocket; -import java.net.Socket; -import java.net.SocketTimeoutException; +import java.net.*; import java.nio.charset.Charset; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedList; -import java.util.List; -import java.util.Set; +import java.util.*; import java.util.concurrent.*; import java.util.logging.Level; import java.util.logging.Logger; import javax.net.ServerSocketFactory; import javax.net.ssl.SSLException; +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; import org.apache.commons.httpclient.URIException; import org.apache.commons.io.IOUtils; @@ -72,15 +63,16 @@ import org.archive.net.UURIFactory; import org.archive.util.Recorder; import org.archive.util.TmpDirTestCase; import org.bbottema.javasocksproxyserver.SocksServer; -import org.jboss.netty.handler.codec.http.HttpRequest; +import org.eclipse.jetty.client.api.Response; +import org.eclipse.jetty.proxy.ConnectHandler; +import org.eclipse.jetty.proxy.ProxyServlet; +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.servlet.ServletContextHandler; +import org.eclipse.jetty.servlet.ServletHolder; import org.junit.After; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; -import org.littleshoot.proxy.DefaultHttpProxyServer; -import org.littleshoot.proxy.HttpFilter; -import org.littleshoot.proxy.HttpRequestFilter; -import org.littleshoot.proxy.ProxyAuthorizationHandler; public class FetchHTTPTest { @@ -440,27 +432,51 @@ public class FetchHTTPTest { runDefaultChecks(curi, "httpBindAddress"); } - protected static class ProxiedRequestRememberer implements HttpRequestFilter { - protected HttpRequest lastProxiedRequest = null; - public HttpRequest getLastProxiedRequest() { - return lastProxiedRequest; + public static class TestProxyServlet extends ProxyServlet { + @Override + protected void onServerResponseHeaders(HttpServletRequest clientRequest, HttpServletResponse proxyResponse, Response serverResponse) { + super.onServerResponseHeaders(clientRequest, proxyResponse, serverResponse); + proxyResponse.addHeader("Via", "test-proxy-servlet"); } @Override - public void filter(HttpRequest httpRequest) { - lastProxiedRequest = httpRequest; + protected void service(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { + String expectedUser = (String) getServletContext().getAttribute("proxy-user"); + if (expectedUser != null) { + String expectedPassword = (String) getServletContext().getAttribute("proxy-password"); + String expectedHeader = "Basic " + Base64.getEncoder().encodeToString((expectedUser + ":" + expectedPassword).getBytes()); + String authHeader = request.getHeader("Proxy-Authorization"); + if (!expectedHeader.equals(authHeader)) { + response.setHeader("Proxy-Authenticate", "Basic realm=\"test\""); + response.sendError(407); + return; + } + } + super.service(request, response); } + } - public void clear() { - lastProxiedRequest = null; - } + private static Server newHttpProxy() { + return newHttpProxy(null, null); + } + + private static Server newHttpProxy(String user, String password) { + Server httpProxyServer = new Server(new InetSocketAddress("localhost", 7877)); + ConnectHandler connectHandler = new ConnectHandler(); + httpProxyServer.setHandler(connectHandler); + ServletContextHandler context = new ServletContextHandler(connectHandler, "/", + ServletContextHandler.SESSIONS); + ServletHolder proxyServlet = new ServletHolder(TestProxyServlet.class); + context.addServlet(proxyServlet, "/*"); + context.setAttribute("proxy-user", user); + context.setAttribute("proxy-password", password); + return httpProxyServer; } @Test public void testHttpProxy() throws Exception { - ProxiedRequestRememberer proxiedRequestRememberer = new ProxiedRequestRememberer(); - DefaultHttpProxyServer httpProxyServer = new DefaultHttpProxyServer(7877, proxiedRequestRememberer, new HashMap()); - httpProxyServer.start(true, false); + Server httpProxyServer = newHttpProxy(null, null); + httpProxyServer.start(); try { fetcher().setHttpProxyHost("localhost"); @@ -471,13 +487,10 @@ public class FetchHTTPTest { String requestString = httpRequestString(curi); assertTrue(requestString.startsWith("GET http://localhost:7777/ HTTP/1.0\r\n")); - assertNotNull(curi.getHttpResponseHeader("Via")); + assertEquals("test-proxy-servlet", curi.getHttpResponseHeader("Via")); assertTrue(requestString.contains("Proxy-Connection: close\r\n")); - // check that our little proxy server really handled a request - assertNotNull(proxiedRequestRememberer.getLastProxiedRequest()); - runDefaultChecks(curi, "requestLine"); } finally { httpProxyServer.stop(); @@ -486,16 +499,8 @@ public class FetchHTTPTest { @Test public void testHttpProxyAuth() throws Exception { - ProxiedRequestRememberer proxiedRequestRememberer = new ProxiedRequestRememberer(); - DefaultHttpProxyServer httpProxyServer = new DefaultHttpProxyServer(7877, proxiedRequestRememberer, new HashMap()); - httpProxyServer.addProxyAuthenticationHandler(new ProxyAuthorizationHandler() { - @Override - public boolean authenticate(String userName, String password) { - // logger.info("username=" + userName + " password=" + password); - return "http-proxy-user".equals(userName) && "http-proxy-password".equals(password); - } - }); - httpProxyServer.start(true, false); + Server httpProxyServer = newHttpProxy("http-proxy-user", "http-proxy-password"); + httpProxyServer.start(); try { fetcher().setHttpProxyHost("localhost"); @@ -511,12 +516,10 @@ public class FetchHTTPTest { assertTrue(requestString.startsWith("GET http://localhost:7777/ HTTP/1.1\r\n")); assertTrue(requestString.contains("Proxy-Connection: close\r\n")); - assertNull(proxiedRequestRememberer.getLastProxiedRequest()); // request didn't make it this far assertNotNull(curi.getHttpResponseHeader("Proxy-Authenticate")); assertEquals(407, curi.getFetchStatus()); // fetch original again now that credentials should be populated - proxiedRequestRememberer.clear(); curi = makeCrawlURI("http://localhost:7777/"); fetcher().process(curi); @@ -524,7 +527,6 @@ public class FetchHTTPTest { assertTrue(requestString.startsWith("GET http://localhost:7777/ HTTP/1.1\r\n")); assertTrue(requestString.contains("Proxy-Connection: close\r\n")); assertNotNull(curi.getHttpResponseHeader("Via")); - assertNotNull(proxiedRequestRememberer.getLastProxiedRequest()); runDefaultChecks(curi, "requestLine"); } finally { httpProxyServer.stop(); @@ -533,8 +535,8 @@ public class FetchHTTPTest { @Test public void testHttpsProxy() throws Exception { - DefaultHttpProxyServer httpProxyServer = new DefaultHttpProxyServer(7877); - httpProxyServer.start(true, false); + Server httpProxyServer = newHttpProxy(); + httpProxyServer.start(); try { fetcher().setHttpProxyHost("localhost");