lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-09-23 22:26:01 +00:00
Code Issues Packages Projects Releases Wiki Activity

HER-1765 quiet excessive INFO level logging

Browse Source 
* GenericReplayCharSequence.java
* Scoper.java
* FetchFTP.java
* FetchHTTP.java
    demote a bunch of INFO logging to FINE so that be default, console isn't filled with output for every URI handled by a busy crawl
This commit is contained in:
gojomo
2010-04-23 00:44:25 +00:00
parent 3f0059c8a8
commit cb5ec19b6c
4 changed files with 24 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files
commons/src/main/java/org/archive/io/GenericReplayCharSequence.java
+6 -6
View File
@@ -146,7 +146,7 @@ public class GenericReplayCharSequence implements ReplayCharSequence {
ReplayInputStream contentReplayInputStream, String backingFilename,
String charsetName) throws IOException {
super();
logger.info("new GenericReplayCharSequence() characterEncoding="
logger.fine("new GenericReplayCharSequence() characterEncoding="
+ charsetName + " backingFilename=" + backingFilename);
Charset charset;
try {
@@ -157,7 +157,7 @@ public class GenericReplayCharSequence implements ReplayCharSequence {
charset = Charset.forName(FALLBACK_CHARSET_NAME);
}
if (charset.newEncoder().maxBytesPerChar() == 1.0) {
logger.info("charset=" + charsetName
logger.fine("charset=" + charsetName
+ ": supports random access, using backing file directly");
this.bytesPerChar = 1;
this.backingFileIn = new FileInputStream(backingFilename);
@@ -165,7 +165,7 @@ public class GenericReplayCharSequence implements ReplayCharSequence {
this.prefixBuffer = this.decoder.decode(
ByteBuffer.wrap(contentReplayInputStream.getBuffer()));
} else {
logger.info("charset=" + charsetName
logger.fine("charset=" + charsetName
+ ": may not support random access, decoding to separate file");
// decodes only up to Integer.MAX_VALUE characters
@@ -239,7 +239,7 @@ public class GenericReplayCharSequence implements ReplayCharSequence {
this.decodedFile = new File(backingFilename + "." + WRITE_ENCODING);
logger.info("decodeToFile: backingFilename=" + backingFilename
logger.fine("decodeToFile: backingFilename=" + backingFilename
+ " encoding=" + encoding + " decodedFile=" + decodedFile);
FileOutputStream fos;
@@ -340,13 +340,13 @@ public class GenericReplayCharSequence implements ReplayCharSequence {
+ e.toString());
}
if (fileToDelete != null && fileToDelete.exists()) {
logger.info("deleting file: " + fileToDelete);
logger.fine("deleting file: " + fileToDelete);
fileToDelete.delete();
}
}
public void close() throws IOException {
logger.info("closing");
logger.fine("closing");
if (this.backingFileChannel != null && this.backingFileChannel.isOpen()) {
this.backingFileChannel.close();
engine/src/main/java/org/archive/crawler/framework/Scoper.java
+2 -3
View File
@@ -128,7 +128,6 @@ public abstract class Scoper extends Processor implements Lifecycle {
*/
protected boolean isInScope(CrawlURI caUri) {
boolean result = false;
// FIXME!: getController().setStateProvider(caUri);
DecideResult dr = scope.decisionFor(caUri);
if (dr == DecideResult.ACCEPT) {
result = true;
@@ -147,10 +146,10 @@ public abstract class Scoper extends Processor implements Lifecycle {
* @param caUri CrawlURI that is out of scope.
*/
protected void outOfScope(CrawlURI caUri) {
if (!LOGGER.isLoggable(Level.INFO)) {
if (!LOGGER.isLoggable(Level.FINE)) {
return;
}
LOGGER.info(caUri.getUURI().toString());
LOGGER.fine(caUri.getUURI().toString());
}
modules/src/main/java/org/archive/modules/fetcher/FetchFTP.java
+4 -4
View File
@@ -257,8 +257,8 @@ public class FetchFTP extends Processor {
Recorder recorder = curi.getRecorder();
try {
if (logger.isLoggable(Level.INFO)) {
logger.info("attempting to fetch ftp uri: " + curi);
if (logger.isLoggable(Level.FINE)) {
logger.fine("attempting to fetch ftp uri: " + curi);
}
fetch(curi, client, recorder);
} catch (IOException e) {
@@ -366,8 +366,8 @@ public class FetchFTP extends Processor {
curi.setContentType("application/octet-stream");
}
if (logger.isLoggable(Level.INFO)) {
logger.info("read " + recorder.getRecordedInput().getSize()
if (logger.isLoggable(Level.FINE)) {
logger.fine("read " + recorder.getRecordedInput().getSize()
+ " bytes from ftp data socket");
}
modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java
+12 -12
View File
@@ -629,8 +629,8 @@ public class FetchHTTP extends Processor implements Lifecycle {
curi.setContentDigest(algorithm,
rec.getRecordedInput().getDigestValue());
}
if (logger.isLoggable(Level.INFO)) {
logger.info(((curi.getFetchType() == HTTP_POST) ? "POST" : "GET")
if (logger.isLoggable(Level.FINE)) {
logger.fine(((curi.getFetchType() == HTTP_POST) ? "POST" : "GET")
+ " " + curi.getUURI().toString() + " "
+ method.getStatusCode() + " "
+ rec.getRecordedInput().getSize() + " "
@@ -1155,16 +1155,16 @@ public class FetchHTTP extends Processor implements Lifecycle {
HttpAuthenticationCredential.class, server.getName());
if (storeRfc2617Credentials == null
|| storeRfc2617Credentials.size() <= 0) {
logger.info("No rfc2617 credentials for " + curi);
logger.fine("No rfc2617 credentials for " + curi);
} else {
HttpAuthenticationCredential found = HttpAuthenticationCredential.getByRealm(
storeRfc2617Credentials, realm, curi);
if (found == null) {
logger.info("No rfc2617 credentials for realm " + realm
logger.fine("No rfc2617 credentials for realm " + realm
+ " in " + curi);
} else {
found.attach(curi, authscheme.getRealm());
logger.info("Found credential for realm " + realm
logger.fine("Found credential for realm " + realm
+ " in store for " + curi.toString());
}
}
@@ -1183,7 +1183,7 @@ public class FetchHTTP extends Processor implements Lifecycle {
final CrawlURI curi) {
Header[] headers = method.getResponseHeaders("WWW-Authenticate");
if (headers == null || headers.length <= 0) {
logger.info("We got a 401 but no WWW-Authenticate challenge: "
logger.fine("We got a 401 but no WWW-Authenticate challenge: "
+ curi.toString());
return null;
}
@@ -1192,10 +1192,10 @@ public class FetchHTTP extends Processor implements Lifecycle {
try {
authschemes = AuthChallengeParser.parseChallenges(headers);
} catch (MalformedChallengeException e) {
logger.info("Failed challenge parse: " + e.getMessage());
logger.fine("Failed challenge parse: " + e.getMessage());
}
if (authschemes == null || authschemes.size() <= 0) {
logger.info("We got a 401 and WWW-Authenticate challenge"
logger.fine("We got a 401 and WWW-Authenticate challenge"
+ " but failed parse of the header " + curi.toString());
return null;
}
@@ -1217,24 +1217,24 @@ public class FetchHTTP extends Processor implements Lifecycle {
} else if (key.equals("digest")) {
authscheme = new DigestScheme();
} else {
logger.info("Unsupported scheme: " + key);
logger.fine("Unsupported scheme: " + key);
continue;
}
try {
authscheme.processChallenge(challenge);
} catch (MalformedChallengeException e) {
logger.info(e.getMessage() + " " + curi + " " + headers);
logger.fine(e.getMessage() + " " + curi + " " + headers);
continue;
}
if (authscheme.isConnectionBased()) {
logger.info("Connection based " + authscheme);
logger.fine("Connection based " + authscheme);
continue;
}
if (authscheme.getRealm() == null
|| authscheme.getRealm().length() <= 0) {
logger.info("Empty realm " + authscheme + " for " + curi);
logger.fine("Empty realm " + authscheme + " for " + curi);
continue;
}
result = authscheme;