Merge pull request #616 from internetarchive/webarchive-commons-1.1.10

Remove log4j from main Heritrix distribution

commons/pom.xml

@@ -151,14 +151,9 @@
 		<dependency>
 			<groupId>org.netpreserve.commons</groupId>
 			<artifactId>webarchive-commons</artifactId>
-			<version>1.1.8</version>
-			<exclusions>
-				<exclusion>
-					<groupId>org.apache.hadoop</groupId>
-					<artifactId>hadoop-core</artifactId>
-				</exclusion>
-			</exclusions>
+			<version>1.1.10</version>
 		</dependency>
+
 		<dependency>
 			<groupId>com.jcraft</groupId>
 			<artifactId>jsch</artifactId>

commons/src/main/java/org/archive/util/OneLineSimpleLayout.java

@@ -1,54 +0,0 @@
-package org.archive.util;
-
-import java.util.logging.LogRecord;
-
-import org.apache.log4j.Layout;
-import org.apache.log4j.spi.LoggingEvent;
-
-public class OneLineSimpleLayout extends Layout {
-
-    private OneLineSimpleLogger logger = new OneLineSimpleLogger();
-
-    @Override
-    public void activateOptions() {
-    }
-
-    @Override
-    public String format(LoggingEvent event) {
-        java.util.logging.Level level = convertLevel(event.getLevel());
-
-        LogRecord logRecord = new LogRecord(level, event.getMessage().toString());
-        logRecord.setLoggerName(event.getLoggerName());
-        logRecord.setMillis(event.getTimeStamp());
-        logRecord.setSourceClassName(event.getLoggerName());
-        logRecord.setSourceMethodName(event.getLocationInformation().getMethodName());
-        logRecord.setThreadID((int) Thread.currentThread().getId());
-
-        return logger.format(logRecord);
-    }
-
-    protected java.util.logging.Level convertLevel(org.apache.log4j.Level log4jLevel) {
-        switch (log4jLevel.toInt()) {
-        case org.apache.log4j.Level.TRACE_INT:
-            return java.util.logging.Level.FINER;
-        case org.apache.log4j.Level.DEBUG_INT:
-            return java.util.logging.Level.FINE;
-        case org.apache.log4j.Level.INFO_INT:
-            return java.util.logging.Level.INFO;
-        case org.apache.log4j.Level.WARN_INT:
-            return java.util.logging.Level.WARNING;
-        case org.apache.log4j.Level.ERROR_INT:
-            return java.util.logging.Level.SEVERE;
-        case org.apache.log4j.Level.FATAL_INT:
-            return java.util.logging.Level.SEVERE;
-        default:
-            return java.util.logging.Level.ALL;
-        }
-    }
-
-    @Override
-    public boolean ignoresThrowable() {
-        return true;
-    }
-
-}

commons/src/test/resources/log4j.xml

@@ -1,23 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
-
-<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/">
-
-	<appender name="console" class="org.apache.log4j.ConsoleAppender">
-		<layout class="org.archive.util.OneLineSimpleLayout" />
-	</appender>
-
-	<logger name="org.apache.commons.httpclient">
-		<level value="ERROR" />
-	</logger>
-
-	<logger name="org.eclipse.jetty">
-		<level value="ERROR" />
-	</logger>
-
-	<root>
-		<level value="WARN" /> 
-		<appender-ref ref="console" />
-	</root>
-
-</log4j:configuration>

contrib/src/main/resources/log4j.xml

@@ -1,27 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
-
-<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/">
-
-	<appender name="console" class="org.apache.log4j.ConsoleAppender">
-		<layout class="org.archive.util.OneLineSimpleLayout" />
-	</appender>
-
-	<logger name="org.apache.commons.httpclient">
-		<level value="ERROR" />
-	</logger>
-
-	<logger name="org.apache.pdfbox">
-		<level value="ERROR" />
-	</logger>
-
-	<logger name="org.eclipse.jetty">
-		<level value="ERROR" />
-	</logger>
-
-	<root>
-		<level value="WARN" /> 
-		<appender-ref ref="console" />
-	</root>
-
-</log4j:configuration>

contrib/src/test/resources/log4j.xml

@@ -1,27 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
-
-<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/">
-
-	<appender name="console" class="org.apache.log4j.ConsoleAppender">
-		<layout class="org.archive.util.OneLineSimpleLayout" />
-	</appender>
-
-	<logger name="org.apache.commons.httpclient">
-		<level value="ERROR" />
-	</logger>
-
-	<logger name="org.apache.pdfbox">
-		<level value="ERROR" />
-	</logger>
-
-	<logger name="org.eclipse.jetty">
-		<level value="ERROR" />
-	</logger>
-
-	<root>
-		<level value="WARN" /> 
-		<appender-ref ref="console" />
-	</root>
-
-</log4j:configuration>

dist/src/test/resources/log4j.xml

@@ -1,23 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
-
-<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/">
-
-	<appender name="console" class="org.apache.log4j.ConsoleAppender">
-		<layout class="org.archive.util.OneLineSimpleLayout" />
-	</appender>
-
-	<logger name="org.apache.commons.httpclient">
-		<level value="ERROR" />
-	</logger>
-
-	<logger name="org.eclipse.jetty">
-		<level value="ERROR" />
-	</logger>
-
-	<root>
-		<level value="WARN" /> 
-		<appender-ref ref="console" />
-	</root>
-
-</log4j:configuration>

docs/operating.rst

@@ -198,8 +198,7 @@ Log Files
 Each crawl job has its own set of log files found in the ``logs`` subdirectory of a job launch directory.
 
 Logging can be configured by modifying the ``logging.properties`` file
-that is located under the ``$HERITRIX_HOME/conf`` directory. For information on using
-logging properties, visit http://logging.apache.org/log4j/.
+that is located under the ``$HERITRIX_HOME/conf`` directory.
 
 alerts.log
 ~~~~~~~~~~

engine/pom.xml

@@ -82,12 +82,6 @@
 			<version>2.3.1</version>
 			<scope>runtime</scope>
 		</dependency>
-		<dependency>
-			<groupId>org.littleshoot</groupId>
-			<artifactId>littleproxy</artifactId>
-			<version>0.4</version>
-			<scope>test</scope>
-		</dependency>
 		<dependency>
 			<groupId>org.slf4j</groupId>
 			<artifactId>slf4j-api</artifactId>

engine/src/test/resources/log4j.xml

@@ -1,23 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
-
-<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/">
-
-	<appender name="console" class="org.apache.log4j.ConsoleAppender">
-		<layout class="org.archive.util.OneLineSimpleLayout" />
-	</appender>
-
-	<logger name="org.apache.commons.httpclient">
-		<level value="ERROR" />
-	</logger>
-
-	<logger name="org.eclipse.jetty">
-		<level value="ERROR" />
-	</logger>
-
-	<root>
-		<level value="WARN" /> 
-		<appender-ref ref="console" />
-	</root>
-
-</log4j:configuration>

modules/pom.xml

@@ -50,16 +50,16 @@
 			<version>${jetty.version}</version>
 		</dependency>
 		<dependency>
-			<groupId>org.littleshoot</groupId>
-			<artifactId>littleproxy</artifactId>
-			<version>0.4</version>
+			<groupId>org.eclipse.jetty</groupId>
+			<artifactId>jetty-servlet</artifactId>
+			<version>${jetty.version}</version>
+			<scope>test</scope>
+		</dependency>
+		<dependency>
+			<groupId>org.eclipse.jetty</groupId>
+			<artifactId>jetty-proxy</artifactId>
+			<version>${jetty.version}</version>
 			<scope>test</scope>
-			<exclusions>
-			   <exclusion>
-			      <groupId>org.slf4j</groupId>
-			      <artifactId>slf4j-log4j12</artifactId>
-			   </exclusion>
-			</exclusions>
 		</dependency>
 		<dependency>
 			<groupId>org.slf4j</groupId>

modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTest.java

@@ -32,27 +32,18 @@ import static org.junit.Assert.*;
 import java.io.IOException;
 import java.io.InterruptedIOException;
 import java.io.UnsupportedEncodingException;
-import java.net.Inet4Address;
-import java.net.InetAddress;
-import java.net.NetworkInterface;
-import java.net.ServerSocket;
-import java.net.Socket;
-import java.net.SocketTimeoutException;
+import java.net.*;
 import java.nio.charset.Charset;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Set;
+import java.util.*;
 import java.util.concurrent.*;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
 import javax.net.ServerSocketFactory;
 import javax.net.ssl.SSLException;
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
 
 import org.apache.commons.httpclient.URIException;
 import org.apache.commons.io.IOUtils;
@@ -72,15 +63,16 @@ import org.archive.net.UURIFactory;
 import org.archive.util.Recorder;
 import org.archive.util.TmpDirTestCase;
 import org.bbottema.javasocksproxyserver.SocksServer;
-import org.jboss.netty.handler.codec.http.HttpRequest;
+import org.eclipse.jetty.client.api.Response;
+import org.eclipse.jetty.proxy.ConnectHandler;
+import org.eclipse.jetty.proxy.ProxyServlet;
+import org.eclipse.jetty.server.Server;
+import org.eclipse.jetty.servlet.ServletContextHandler;
+import org.eclipse.jetty.servlet.ServletHolder;
 import org.junit.After;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.Test;
-import org.littleshoot.proxy.DefaultHttpProxyServer;
-import org.littleshoot.proxy.HttpFilter;
-import org.littleshoot.proxy.HttpRequestFilter;
-import org.littleshoot.proxy.ProxyAuthorizationHandler;
 
 public class FetchHTTPTest {
     
@@ -440,27 +432,51 @@ public class FetchHTTPTest {
         runDefaultChecks(curi, "httpBindAddress");
     }
 
-    protected static class ProxiedRequestRememberer implements HttpRequestFilter {
-        protected HttpRequest lastProxiedRequest = null;
-        public HttpRequest getLastProxiedRequest() {
-            return lastProxiedRequest;
+    public static class TestProxyServlet extends ProxyServlet {
+        @Override
+        protected void onServerResponseHeaders(HttpServletRequest clientRequest, HttpServletResponse proxyResponse, Response serverResponse) {
+            super.onServerResponseHeaders(clientRequest, proxyResponse, serverResponse);
+            proxyResponse.addHeader("Via", "test-proxy-servlet");
         }
 
         @Override
-        public void filter(HttpRequest httpRequest) {
-            lastProxiedRequest = httpRequest;
+        protected void service(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
+            String expectedUser = (String) getServletContext().getAttribute("proxy-user");
+            if (expectedUser != null) {
+                String expectedPassword = (String) getServletContext().getAttribute("proxy-password");
+                String expectedHeader = "Basic " + Base64.getEncoder().encodeToString((expectedUser + ":" + expectedPassword).getBytes());
+                String authHeader = request.getHeader("Proxy-Authorization");
+                if (!expectedHeader.equals(authHeader)) {
+                    response.setHeader("Proxy-Authenticate", "Basic realm=\"test\"");
+                    response.sendError(407);
+                    return;
+                }
+            }
+            super.service(request, response);
         }
+    }
 
-        public void clear() {
-            lastProxiedRequest = null;
-        }
+    private static Server newHttpProxy() {
+        return newHttpProxy(null, null);
+    }
+
+    private static Server newHttpProxy(String user, String password) {
+        Server httpProxyServer = new Server(new InetSocketAddress("localhost", 7877));
+        ConnectHandler connectHandler = new ConnectHandler();
+        httpProxyServer.setHandler(connectHandler);
+        ServletContextHandler context = new ServletContextHandler(connectHandler, "/",
+                ServletContextHandler.SESSIONS);
+        ServletHolder proxyServlet = new ServletHolder(TestProxyServlet.class);
+        context.addServlet(proxyServlet, "/*");
+        context.setAttribute("proxy-user", user);
+        context.setAttribute("proxy-password", password);
+        return httpProxyServer;
     }
 
     @Test
     public void testHttpProxy() throws Exception {
-        ProxiedRequestRememberer proxiedRequestRememberer = new ProxiedRequestRememberer();
-        DefaultHttpProxyServer httpProxyServer = new DefaultHttpProxyServer(7877, proxiedRequestRememberer, new HashMap<String, HttpFilter>());
-        httpProxyServer.start(true, false);
+        Server httpProxyServer = newHttpProxy(null, null);
+        httpProxyServer.start();
 
         try {
             fetcher().setHttpProxyHost("localhost");
@@ -471,13 +487,10 @@ public class FetchHTTPTest {
 
             String requestString = httpRequestString(curi);
             assertTrue(requestString.startsWith("GET http://localhost:7777/ HTTP/1.0\r\n"));
-            assertNotNull(curi.getHttpResponseHeader("Via"));
+            assertEquals("test-proxy-servlet", curi.getHttpResponseHeader("Via"));
             
             assertTrue(requestString.contains("Proxy-Connection: close\r\n"));
             
-            // check that our little proxy server really handled a request
-            assertNotNull(proxiedRequestRememberer.getLastProxiedRequest());
-            
             runDefaultChecks(curi, "requestLine");
         } finally {
             httpProxyServer.stop();
@@ -486,16 +499,8 @@ public class FetchHTTPTest {
 
     @Test
     public void testHttpProxyAuth() throws Exception {
-        ProxiedRequestRememberer proxiedRequestRememberer = new ProxiedRequestRememberer();
-        DefaultHttpProxyServer httpProxyServer = new DefaultHttpProxyServer(7877, proxiedRequestRememberer, new HashMap<String, HttpFilter>());
-        httpProxyServer.addProxyAuthenticationHandler(new ProxyAuthorizationHandler() {
-            @Override
-            public boolean authenticate(String userName, String password) {
-                // logger.info("username=" + userName + " password=" + password);
-                return "http-proxy-user".equals(userName) && "http-proxy-password".equals(password);
-            }
-        });
-        httpProxyServer.start(true, false);
+        Server httpProxyServer = newHttpProxy("http-proxy-user", "http-proxy-password");
+        httpProxyServer.start();
 
         try {
             fetcher().setHttpProxyHost("localhost");
@@ -511,12 +516,10 @@ public class FetchHTTPTest {
             assertTrue(requestString.startsWith("GET http://localhost:7777/ HTTP/1.1\r\n"));
             assertTrue(requestString.contains("Proxy-Connection: close\r\n"));
 
-            assertNull(proxiedRequestRememberer.getLastProxiedRequest()); // request didn't make it this far
             assertNotNull(curi.getHttpResponseHeader("Proxy-Authenticate"));
             assertEquals(407, curi.getFetchStatus());
 
             // fetch original again now that credentials should be populated
-            proxiedRequestRememberer.clear();
             curi = makeCrawlURI("http://localhost:7777/");
             fetcher().process(curi);
 
@@ -524,7 +527,6 @@ public class FetchHTTPTest {
             assertTrue(requestString.startsWith("GET http://localhost:7777/ HTTP/1.1\r\n"));
             assertTrue(requestString.contains("Proxy-Connection: close\r\n"));
             assertNotNull(curi.getHttpResponseHeader("Via"));
-            assertNotNull(proxiedRequestRememberer.getLastProxiedRequest());
             runDefaultChecks(curi, "requestLine");
         } finally {
             httpProxyServer.stop();
@@ -533,8 +535,8 @@ public class FetchHTTPTest {
 
     @Test
     public void testHttpsProxy() throws Exception {
-        DefaultHttpProxyServer httpProxyServer = new DefaultHttpProxyServer(7877);
-        httpProxyServer.start(true, false);
+        Server httpProxyServer = newHttpProxy();
+        httpProxyServer.start();
 
         try {
             fetcher().setHttpProxyHost("localhost");

modules/src/test/resources/log4j.xml

@@ -1,23 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
-
-<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/">
-
-	<appender name="console" class="org.apache.log4j.ConsoleAppender">
-		<layout class="org.archive.util.OneLineSimpleLayout" />
-	</appender>
-
-	<logger name="org.apache.commons.httpclient">
-		<level value="ERROR" />
-	</logger>
-
-	<logger name="org.eclipse.jetty">
-		<level value="ERROR" />
-	</logger>
-
-	<root>
-		<level value="WARN" /> 
-		<appender-ref ref="console" />
-	</root>
-
-</log4j:configuration>