diff --git a/commons/pom.xml b/commons/pom.xml index 3f47bcb0..a5e76309 100644 --- a/commons/pom.xml +++ b/commons/pom.xml @@ -151,14 +151,9 @@ org.netpreserve.commons webarchive-commons - 1.1.8 - - - org.apache.hadoop - hadoop-core - - + 1.1.10 + com.jcraft jsch diff --git a/commons/src/main/java/org/archive/util/OneLineSimpleLayout.java b/commons/src/main/java/org/archive/util/OneLineSimpleLayout.java deleted file mode 100644 index 61030cd6..00000000 --- a/commons/src/main/java/org/archive/util/OneLineSimpleLayout.java +++ /dev/null @@ -1,54 +0,0 @@ -package org.archive.util; - -import java.util.logging.LogRecord; - -import org.apache.log4j.Layout; -import org.apache.log4j.spi.LoggingEvent; - -public class OneLineSimpleLayout extends Layout { - - private OneLineSimpleLogger logger = new OneLineSimpleLogger(); - - @Override - public void activateOptions() { - } - - @Override - public String format(LoggingEvent event) { - java.util.logging.Level level = convertLevel(event.getLevel()); - - LogRecord logRecord = new LogRecord(level, event.getMessage().toString()); - logRecord.setLoggerName(event.getLoggerName()); - logRecord.setMillis(event.getTimeStamp()); - logRecord.setSourceClassName(event.getLoggerName()); - logRecord.setSourceMethodName(event.getLocationInformation().getMethodName()); - logRecord.setThreadID((int) Thread.currentThread().getId()); - - return logger.format(logRecord); - } - - protected java.util.logging.Level convertLevel(org.apache.log4j.Level log4jLevel) { - switch (log4jLevel.toInt()) { - case org.apache.log4j.Level.TRACE_INT: - return java.util.logging.Level.FINER; - case org.apache.log4j.Level.DEBUG_INT: - return java.util.logging.Level.FINE; - case org.apache.log4j.Level.INFO_INT: - return java.util.logging.Level.INFO; - case org.apache.log4j.Level.WARN_INT: - return java.util.logging.Level.WARNING; - case org.apache.log4j.Level.ERROR_INT: - return java.util.logging.Level.SEVERE; - case org.apache.log4j.Level.FATAL_INT: - return java.util.logging.Level.SEVERE; - default: - return java.util.logging.Level.ALL; - } - } - - @Override - public boolean ignoresThrowable() { - return true; - } - -} diff --git a/commons/src/test/resources/log4j.xml b/commons/src/test/resources/log4j.xml deleted file mode 100644 index e04f34ab..00000000 --- a/commons/src/test/resources/log4j.xml +++ /dev/null @@ -1,23 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - diff --git a/contrib/src/main/resources/log4j.xml b/contrib/src/main/resources/log4j.xml deleted file mode 100644 index 8a4dae8e..00000000 --- a/contrib/src/main/resources/log4j.xml +++ /dev/null @@ -1,27 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/contrib/src/test/resources/log4j.xml b/contrib/src/test/resources/log4j.xml deleted file mode 100644 index 8a4dae8e..00000000 --- a/contrib/src/test/resources/log4j.xml +++ /dev/null @@ -1,27 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/dist/src/test/resources/log4j.xml b/dist/src/test/resources/log4j.xml deleted file mode 100644 index e04f34ab..00000000 --- a/dist/src/test/resources/log4j.xml +++ /dev/null @@ -1,23 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - diff --git a/docs/operating.rst b/docs/operating.rst index 041c0357..f034cdbe 100644 --- a/docs/operating.rst +++ b/docs/operating.rst @@ -198,8 +198,7 @@ Log Files Each crawl job has its own set of log files found in the ``logs`` subdirectory of a job launch directory. Logging can be configured by modifying the ``logging.properties`` file -that is located under the ``$HERITRIX_HOME/conf`` directory. For information on using -logging properties, visit http://logging.apache.org/log4j/. +that is located under the ``$HERITRIX_HOME/conf`` directory. alerts.log ~~~~~~~~~~ diff --git a/engine/pom.xml b/engine/pom.xml index fb345c71..dd160615 100644 --- a/engine/pom.xml +++ b/engine/pom.xml @@ -82,12 +82,6 @@ 2.3.1 runtime - - org.littleshoot - littleproxy - 0.4 - test - org.slf4j slf4j-api diff --git a/engine/src/test/resources/log4j.xml b/engine/src/test/resources/log4j.xml deleted file mode 100644 index e04f34ab..00000000 --- a/engine/src/test/resources/log4j.xml +++ /dev/null @@ -1,23 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - diff --git a/modules/pom.xml b/modules/pom.xml index ed7ab631..6ebbbc6e 100644 --- a/modules/pom.xml +++ b/modules/pom.xml @@ -50,16 +50,16 @@ ${jetty.version} - org.littleshoot - littleproxy - 0.4 + org.eclipse.jetty + jetty-servlet + ${jetty.version} + test + + + org.eclipse.jetty + jetty-proxy + ${jetty.version} test - - - org.slf4j - slf4j-log4j12 - - org.slf4j diff --git a/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTest.java b/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTest.java index ee02c3a6..98ad98b0 100644 --- a/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTest.java +++ b/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTest.java @@ -32,27 +32,18 @@ import static org.junit.Assert.*; import java.io.IOException; import java.io.InterruptedIOException; import java.io.UnsupportedEncodingException; -import java.net.Inet4Address; -import java.net.InetAddress; -import java.net.NetworkInterface; -import java.net.ServerSocket; -import java.net.Socket; -import java.net.SocketTimeoutException; +import java.net.*; import java.nio.charset.Charset; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedList; -import java.util.List; -import java.util.Set; +import java.util.*; import java.util.concurrent.*; import java.util.logging.Level; import java.util.logging.Logger; import javax.net.ServerSocketFactory; import javax.net.ssl.SSLException; +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; import org.apache.commons.httpclient.URIException; import org.apache.commons.io.IOUtils; @@ -72,15 +63,16 @@ import org.archive.net.UURIFactory; import org.archive.util.Recorder; import org.archive.util.TmpDirTestCase; import org.bbottema.javasocksproxyserver.SocksServer; -import org.jboss.netty.handler.codec.http.HttpRequest; +import org.eclipse.jetty.client.api.Response; +import org.eclipse.jetty.proxy.ConnectHandler; +import org.eclipse.jetty.proxy.ProxyServlet; +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.servlet.ServletContextHandler; +import org.eclipse.jetty.servlet.ServletHolder; import org.junit.After; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; -import org.littleshoot.proxy.DefaultHttpProxyServer; -import org.littleshoot.proxy.HttpFilter; -import org.littleshoot.proxy.HttpRequestFilter; -import org.littleshoot.proxy.ProxyAuthorizationHandler; public class FetchHTTPTest { @@ -440,27 +432,51 @@ public class FetchHTTPTest { runDefaultChecks(curi, "httpBindAddress"); } - protected static class ProxiedRequestRememberer implements HttpRequestFilter { - protected HttpRequest lastProxiedRequest = null; - public HttpRequest getLastProxiedRequest() { - return lastProxiedRequest; + public static class TestProxyServlet extends ProxyServlet { + @Override + protected void onServerResponseHeaders(HttpServletRequest clientRequest, HttpServletResponse proxyResponse, Response serverResponse) { + super.onServerResponseHeaders(clientRequest, proxyResponse, serverResponse); + proxyResponse.addHeader("Via", "test-proxy-servlet"); } @Override - public void filter(HttpRequest httpRequest) { - lastProxiedRequest = httpRequest; + protected void service(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { + String expectedUser = (String) getServletContext().getAttribute("proxy-user"); + if (expectedUser != null) { + String expectedPassword = (String) getServletContext().getAttribute("proxy-password"); + String expectedHeader = "Basic " + Base64.getEncoder().encodeToString((expectedUser + ":" + expectedPassword).getBytes()); + String authHeader = request.getHeader("Proxy-Authorization"); + if (!expectedHeader.equals(authHeader)) { + response.setHeader("Proxy-Authenticate", "Basic realm=\"test\""); + response.sendError(407); + return; + } + } + super.service(request, response); } + } - public void clear() { - lastProxiedRequest = null; - } + private static Server newHttpProxy() { + return newHttpProxy(null, null); + } + + private static Server newHttpProxy(String user, String password) { + Server httpProxyServer = new Server(new InetSocketAddress("localhost", 7877)); + ConnectHandler connectHandler = new ConnectHandler(); + httpProxyServer.setHandler(connectHandler); + ServletContextHandler context = new ServletContextHandler(connectHandler, "/", + ServletContextHandler.SESSIONS); + ServletHolder proxyServlet = new ServletHolder(TestProxyServlet.class); + context.addServlet(proxyServlet, "/*"); + context.setAttribute("proxy-user", user); + context.setAttribute("proxy-password", password); + return httpProxyServer; } @Test public void testHttpProxy() throws Exception { - ProxiedRequestRememberer proxiedRequestRememberer = new ProxiedRequestRememberer(); - DefaultHttpProxyServer httpProxyServer = new DefaultHttpProxyServer(7877, proxiedRequestRememberer, new HashMap()); - httpProxyServer.start(true, false); + Server httpProxyServer = newHttpProxy(null, null); + httpProxyServer.start(); try { fetcher().setHttpProxyHost("localhost"); @@ -471,13 +487,10 @@ public class FetchHTTPTest { String requestString = httpRequestString(curi); assertTrue(requestString.startsWith("GET http://localhost:7777/ HTTP/1.0\r\n")); - assertNotNull(curi.getHttpResponseHeader("Via")); + assertEquals("test-proxy-servlet", curi.getHttpResponseHeader("Via")); assertTrue(requestString.contains("Proxy-Connection: close\r\n")); - // check that our little proxy server really handled a request - assertNotNull(proxiedRequestRememberer.getLastProxiedRequest()); - runDefaultChecks(curi, "requestLine"); } finally { httpProxyServer.stop(); @@ -486,16 +499,8 @@ public class FetchHTTPTest { @Test public void testHttpProxyAuth() throws Exception { - ProxiedRequestRememberer proxiedRequestRememberer = new ProxiedRequestRememberer(); - DefaultHttpProxyServer httpProxyServer = new DefaultHttpProxyServer(7877, proxiedRequestRememberer, new HashMap()); - httpProxyServer.addProxyAuthenticationHandler(new ProxyAuthorizationHandler() { - @Override - public boolean authenticate(String userName, String password) { - // logger.info("username=" + userName + " password=" + password); - return "http-proxy-user".equals(userName) && "http-proxy-password".equals(password); - } - }); - httpProxyServer.start(true, false); + Server httpProxyServer = newHttpProxy("http-proxy-user", "http-proxy-password"); + httpProxyServer.start(); try { fetcher().setHttpProxyHost("localhost"); @@ -511,12 +516,10 @@ public class FetchHTTPTest { assertTrue(requestString.startsWith("GET http://localhost:7777/ HTTP/1.1\r\n")); assertTrue(requestString.contains("Proxy-Connection: close\r\n")); - assertNull(proxiedRequestRememberer.getLastProxiedRequest()); // request didn't make it this far assertNotNull(curi.getHttpResponseHeader("Proxy-Authenticate")); assertEquals(407, curi.getFetchStatus()); // fetch original again now that credentials should be populated - proxiedRequestRememberer.clear(); curi = makeCrawlURI("http://localhost:7777/"); fetcher().process(curi); @@ -524,7 +527,6 @@ public class FetchHTTPTest { assertTrue(requestString.startsWith("GET http://localhost:7777/ HTTP/1.1\r\n")); assertTrue(requestString.contains("Proxy-Connection: close\r\n")); assertNotNull(curi.getHttpResponseHeader("Via")); - assertNotNull(proxiedRequestRememberer.getLastProxiedRequest()); runDefaultChecks(curi, "requestLine"); } finally { httpProxyServer.stop(); @@ -533,8 +535,8 @@ public class FetchHTTPTest { @Test public void testHttpsProxy() throws Exception { - DefaultHttpProxyServer httpProxyServer = new DefaultHttpProxyServer(7877); - httpProxyServer.start(true, false); + Server httpProxyServer = newHttpProxy(); + httpProxyServer.start(); try { fetcher().setHttpProxyHost("localhost"); diff --git a/modules/src/test/resources/log4j.xml b/modules/src/test/resources/log4j.xml deleted file mode 100644 index e04f34ab..00000000 --- a/modules/src/test/resources/log4j.xml +++ /dev/null @@ -1,23 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - -