From cb5ec19b6cd3106fc1ec18a35a02ebfce82bc7f8 Mon Sep 17 00:00:00 2001 From: gojomo Date: Fri, 23 Apr 2010 00:44:25 +0000 Subject: [PATCH] HER-1765 quiet excessive INFO level logging * GenericReplayCharSequence.java * Scoper.java * FetchFTP.java * FetchHTTP.java demote a bunch of INFO logging to FINE so that be default, console isn't filled with output for every URI handled by a busy crawl --- .../archive/io/GenericReplayCharSequence.java | 12 +++++----- .../org/archive/crawler/framework/Scoper.java | 5 ++-- .../org/archive/modules/fetcher/FetchFTP.java | 8 +++---- .../archive/modules/fetcher/FetchHTTP.java | 24 +++++++++---------- 4 files changed, 24 insertions(+), 25 deletions(-) diff --git a/commons/src/main/java/org/archive/io/GenericReplayCharSequence.java b/commons/src/main/java/org/archive/io/GenericReplayCharSequence.java index 11a4c380..11161010 100644 --- a/commons/src/main/java/org/archive/io/GenericReplayCharSequence.java +++ b/commons/src/main/java/org/archive/io/GenericReplayCharSequence.java @@ -146,7 +146,7 @@ public class GenericReplayCharSequence implements ReplayCharSequence { ReplayInputStream contentReplayInputStream, String backingFilename, String charsetName) throws IOException { super(); - logger.info("new GenericReplayCharSequence() characterEncoding=" + logger.fine("new GenericReplayCharSequence() characterEncoding=" + charsetName + " backingFilename=" + backingFilename); Charset charset; try { @@ -157,7 +157,7 @@ public class GenericReplayCharSequence implements ReplayCharSequence { charset = Charset.forName(FALLBACK_CHARSET_NAME); } if (charset.newEncoder().maxBytesPerChar() == 1.0) { - logger.info("charset=" + charsetName + logger.fine("charset=" + charsetName + ": supports random access, using backing file directly"); this.bytesPerChar = 1; this.backingFileIn = new FileInputStream(backingFilename); @@ -165,7 +165,7 @@ public class GenericReplayCharSequence implements ReplayCharSequence { this.prefixBuffer = this.decoder.decode( ByteBuffer.wrap(contentReplayInputStream.getBuffer())); } else { - logger.info("charset=" + charsetName + logger.fine("charset=" + charsetName + ": may not support random access, decoding to separate file"); // decodes only up to Integer.MAX_VALUE characters @@ -239,7 +239,7 @@ public class GenericReplayCharSequence implements ReplayCharSequence { this.decodedFile = new File(backingFilename + "." + WRITE_ENCODING); - logger.info("decodeToFile: backingFilename=" + backingFilename + logger.fine("decodeToFile: backingFilename=" + backingFilename + " encoding=" + encoding + " decodedFile=" + decodedFile); FileOutputStream fos; @@ -340,13 +340,13 @@ public class GenericReplayCharSequence implements ReplayCharSequence { + e.toString()); } if (fileToDelete != null && fileToDelete.exists()) { - logger.info("deleting file: " + fileToDelete); + logger.fine("deleting file: " + fileToDelete); fileToDelete.delete(); } } public void close() throws IOException { - logger.info("closing"); + logger.fine("closing"); if (this.backingFileChannel != null && this.backingFileChannel.isOpen()) { this.backingFileChannel.close(); diff --git a/engine/src/main/java/org/archive/crawler/framework/Scoper.java b/engine/src/main/java/org/archive/crawler/framework/Scoper.java index 1eb9e61e..b93f3ca5 100644 --- a/engine/src/main/java/org/archive/crawler/framework/Scoper.java +++ b/engine/src/main/java/org/archive/crawler/framework/Scoper.java @@ -128,7 +128,6 @@ public abstract class Scoper extends Processor implements Lifecycle { */ protected boolean isInScope(CrawlURI caUri) { boolean result = false; -// FIXME!: getController().setStateProvider(caUri); DecideResult dr = scope.decisionFor(caUri); if (dr == DecideResult.ACCEPT) { result = true; @@ -147,10 +146,10 @@ public abstract class Scoper extends Processor implements Lifecycle { * @param caUri CrawlURI that is out of scope. */ protected void outOfScope(CrawlURI caUri) { - if (!LOGGER.isLoggable(Level.INFO)) { + if (!LOGGER.isLoggable(Level.FINE)) { return; } - LOGGER.info(caUri.getUURI().toString()); + LOGGER.fine(caUri.getUURI().toString()); } diff --git a/modules/src/main/java/org/archive/modules/fetcher/FetchFTP.java b/modules/src/main/java/org/archive/modules/fetcher/FetchFTP.java index 1a2b7a16..9847274d 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/FetchFTP.java +++ b/modules/src/main/java/org/archive/modules/fetcher/FetchFTP.java @@ -257,8 +257,8 @@ public class FetchFTP extends Processor { Recorder recorder = curi.getRecorder(); try { - if (logger.isLoggable(Level.INFO)) { - logger.info("attempting to fetch ftp uri: " + curi); + if (logger.isLoggable(Level.FINE)) { + logger.fine("attempting to fetch ftp uri: " + curi); } fetch(curi, client, recorder); } catch (IOException e) { @@ -366,8 +366,8 @@ public class FetchFTP extends Processor { curi.setContentType("application/octet-stream"); } - if (logger.isLoggable(Level.INFO)) { - logger.info("read " + recorder.getRecordedInput().getSize() + if (logger.isLoggable(Level.FINE)) { + logger.fine("read " + recorder.getRecordedInput().getSize() + " bytes from ftp data socket"); } diff --git a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java index 103483d3..14aa97cb 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java +++ b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java @@ -629,8 +629,8 @@ public class FetchHTTP extends Processor implements Lifecycle { curi.setContentDigest(algorithm, rec.getRecordedInput().getDigestValue()); } - if (logger.isLoggable(Level.INFO)) { - logger.info(((curi.getFetchType() == HTTP_POST) ? "POST" : "GET") + if (logger.isLoggable(Level.FINE)) { + logger.fine(((curi.getFetchType() == HTTP_POST) ? "POST" : "GET") + " " + curi.getUURI().toString() + " " + method.getStatusCode() + " " + rec.getRecordedInput().getSize() + " " @@ -1155,16 +1155,16 @@ public class FetchHTTP extends Processor implements Lifecycle { HttpAuthenticationCredential.class, server.getName()); if (storeRfc2617Credentials == null || storeRfc2617Credentials.size() <= 0) { - logger.info("No rfc2617 credentials for " + curi); + logger.fine("No rfc2617 credentials for " + curi); } else { HttpAuthenticationCredential found = HttpAuthenticationCredential.getByRealm( storeRfc2617Credentials, realm, curi); if (found == null) { - logger.info("No rfc2617 credentials for realm " + realm + logger.fine("No rfc2617 credentials for realm " + realm + " in " + curi); } else { found.attach(curi, authscheme.getRealm()); - logger.info("Found credential for realm " + realm + logger.fine("Found credential for realm " + realm + " in store for " + curi.toString()); } } @@ -1183,7 +1183,7 @@ public class FetchHTTP extends Processor implements Lifecycle { final CrawlURI curi) { Header[] headers = method.getResponseHeaders("WWW-Authenticate"); if (headers == null || headers.length <= 0) { - logger.info("We got a 401 but no WWW-Authenticate challenge: " + logger.fine("We got a 401 but no WWW-Authenticate challenge: " + curi.toString()); return null; } @@ -1192,10 +1192,10 @@ public class FetchHTTP extends Processor implements Lifecycle { try { authschemes = AuthChallengeParser.parseChallenges(headers); } catch (MalformedChallengeException e) { - logger.info("Failed challenge parse: " + e.getMessage()); + logger.fine("Failed challenge parse: " + e.getMessage()); } if (authschemes == null || authschemes.size() <= 0) { - logger.info("We got a 401 and WWW-Authenticate challenge" + logger.fine("We got a 401 and WWW-Authenticate challenge" + " but failed parse of the header " + curi.toString()); return null; } @@ -1217,24 +1217,24 @@ public class FetchHTTP extends Processor implements Lifecycle { } else if (key.equals("digest")) { authscheme = new DigestScheme(); } else { - logger.info("Unsupported scheme: " + key); + logger.fine("Unsupported scheme: " + key); continue; } try { authscheme.processChallenge(challenge); } catch (MalformedChallengeException e) { - logger.info(e.getMessage() + " " + curi + " " + headers); + logger.fine(e.getMessage() + " " + curi + " " + headers); continue; } if (authscheme.isConnectionBased()) { - logger.info("Connection based " + authscheme); + logger.fine("Connection based " + authscheme); continue; } if (authscheme.getRealm() == null || authscheme.getRealm().length() <= 0) { - logger.info("Empty realm " + authscheme + " for " + curi); + logger.fine("Empty realm " + authscheme + " for " + curi); continue; } result = authscheme;