diff --git a/commons/src/main/java/org/archive/io/GenericReplayCharSequence.java b/commons/src/main/java/org/archive/io/GenericReplayCharSequence.java index 8763cdfe..c5219c3c 100644 --- a/commons/src/main/java/org/archive/io/GenericReplayCharSequence.java +++ b/commons/src/main/java/org/archive/io/GenericReplayCharSequence.java @@ -273,7 +273,7 @@ public class GenericReplayCharSequence implements ReplayCharSequence { } writer.close(); - logger.info("decodeToFile: wrote " + count + " characters to " + logger.fine("decodeToFile: wrote " + count + " characters to " + decodedFile); } @@ -378,7 +378,7 @@ public class GenericReplayCharSequence implements ReplayCharSequence { */ protected void finalize() throws Throwable { super.finalize(); - logger.info("finalizing"); + logger.fine("finalizing"); close(); } diff --git a/dist/src/main/conf/logging.properties b/dist/src/main/conf/logging.properties index e4ca2773..0bce5936 100644 --- a/dist/src/main/conf/logging.properties +++ b/dist/src/main/conf/logging.properties @@ -1,120 +1,13 @@ +# Recommended logging configuration for Heritrix -# Basic logging setup; to console, all levels. -# Note, after startup Heritrix adds a special logging handler called SinkHandler -# that keeps around all instances of severe and warning messages for display by -# the Heritrix alerting system. We used to initialize it here with the -# ConsoleHandler but it wasn't being found when Heritrix was packaged as a webapp -# deployed in Tomcat. +# Default global logging level: only WARNINGs or higher from most code.. +.level= WARNING +# ...and even less from the too-chatty-with-WARNINGs HttpClient library... +org.apache.commons.httpclient.level = SEVERE +# ...but INFO for our classes, which reserve FINE/FINER/FINEST for bulk/trivia... +org.archive.level = INFO + +# Basic logging setup; to console (and thus usually heritrix_out.log), all levels handlers = java.util.logging.ConsoleHandler java.util.logging.ConsoleHandler.level = ALL java.util.logging.ConsoleHandler.formatter= org.archive.util.OneLineSimpleLogger - -# Default global logging level: only warnings or higher -.level= WARNING - -# Enable frontier INFO logging -# org.archive.crawler.frontier.BdbFrontier.level = FINER -# org.archive.crawler.frontier.BdbMultipleWorkQueues.level = INFO -# org.archive.crawler.frontier.AbstractFrontier.level = INFO -# org.archive.crawler.frontier.WorkQueueFrontier.level = INFO - -# Currently necessary (?) for standard logs to work -crawl.level= INFO -runtime-errors.level= INFO -local-errors.level= INFO -uri-errors.level= INFO -progress-statistics.level= INFO -recover.level= INFO - -# Set the selftest console logger level. -org.archive.crawler.admin.SelftestCrawlJobHandler.level = INFO - -# Enable info level on CrawlJob so can see it registered jobs w/ mbeanserver -org.archive.crawler.admin.CrawlJob.level = INFO - -# Enable the following lines to watch authentications running. -# org.archive.crawler.fetcher.FetchHTTP.level = FINE -# org.archive.crawler.prefetch.PreconditionEnforcer.level = FINE - -# HttpClient is too chatty... only want to hear about severe problems -# For more on httpclient logging, -# see http://jakarta.apache.org/commons/httpclient/logging.html -org.apache.commons.httpclient.level = SEVERE - -# If you need verbose console logging of HttpClient traffic, uncomment the -# following line: -# httpclient.wire.level = FINE - -# Enable ARCWriter.level line below to see logging of the opening and closing -# of arc files in console output (Console goes into heritrix_out.log). If you'd -# rather have console output, including the opening and closing of ARCs, go to -# a file, enable the FileHandler logger -- see the commented out 'handlers' -# line above -- and enable the lines below related to FileHandler (Configure to -# your preference -- the '%h' in the below means value of java.home system -# property, usually your home directory). There does not seem to be a way -# other than in code to configure only ARCWriter writing to the FileHandler. -# Even if you do do code changes, its awkward making the log show in each -# individual jobs log directory. Note, the closing of arc files on shutdown is -# not logged. -# org.archive.io.arc.ARCWriter.level = INFO - -# Enable to watch authentication progress amongst other details on http fetch. -# org.archive.crawler.fetcher.FetchHTTP.level = FINE - -# Enable logging of already seen memory grows; this growing is probably -# main culprit for OOMEs. Log goes into heritrix_out.log. -org.archive.util.MemLongFPSet.level = INFO -# org.archive.crawler.url.Canonicalizer.level = INFO - -# Enable logging of uris rejected by scope by setting level to INFO. -# org.archive.crawler.postprocessor.LinksScoper.level = INFO - -# Below values are used by classes that set their own FileHandler as default -# values. The pattern value is used as a suffix. Note, the below configuration -# limits log file size (to 500000 bytes) and because count is 1, old log data -# will be lost. -java.util.logging.FileHandler.level = ALL -java.util.logging.FileHandler.pattern = %u.log -java.util.logging.FileHandler.formatter = org.archive.util.OneLineSimpleLogger -java.util.logging.FileHandler.limit = 500000 -java.util.logging.FileHandler.count = 1 - -# org.archive.crawler.admin.StatisticsTracker.level = INFO - -# Enable info logging so can see note on checkpoint completion in -# heritrix_out.log. -org.archive.crawler.framework.CrawlController.level = INFO - -# AdaptiveRevist module -# org.archive.crawler.frontier.AdaptiveRevisitFrontier.level = FINE -# org.archive.crawler.frontier.AdaptiveRevisitHostQueue.level = FINE -# org.archive.crawler.frontier.AdaptiveRevisitQueueList.level = FINE -# org.archive.crawler.processor.ChangeEvaluator.level = FINE -# org.archive.crawler.processor.WaitEvaluator.level = FINE -# org.archive.crawler.processor.HTTPContentDigest.level = FINE - -# ServerCache creation of crawl host (Make it FINER if you want to see -# crawl server creation also). -# org.archive.crawler.datamodel.ServerCache.level = FINE - -# Enable to see average alreadyseen lookup times. -# org.archive.crawler.util.BdbUriUniqFilter.level = INFO - -# Enable to see duplicated-detection stats on each merge, -# if using an FPMergeUriUniqFilter -# org.archive.crawler.util.FPMergeUriUniqFilter.level = INFO - -# Enable to see report of duplicates encountered every 50K unique adds -# org.archive.crawler.util.SetBasedUriUniqFilter.level = FINE -# Used to set a log of all URIs presented to filter, for -# later comparison profiling -# org.archive.crawler.util.SetBasedUriUniqFilter.profileLogFile = uriUniq.log - -# Leave the below enabled so can see launch info messages on the console. -org.archive.crawler.Heritrix.level = INFO - -# Leave RecoveryJournal level at INFO to see recovery progress -org.archive.crawler.frontier.RecoveryJournal.level = INFO - -# Leave PersistLoadProcessor level at INFO to see if dedup info is loaded -org.archive.modules.recrawl.PersistLoadProcessor.level = INFO diff --git a/engine/src/main/java/org/archive/crawler/util/BdbUriUniqFilter.java b/engine/src/main/java/org/archive/crawler/util/BdbUriUniqFilter.java index ef27d6ce..7a48bccd 100644 --- a/engine/src/main/java/org/archive/crawler/util/BdbUriUniqFilter.java +++ b/engine/src/main/java/org/archive/crawler/util/BdbUriUniqFilter.java @@ -228,9 +228,7 @@ implements Lifecycle, Checkpointable, BeanNameAware { } public synchronized void close() { - if (logger.isLoggable(Level.INFO)) { - logger.info("Count of alreadyseen on close " + count.get()); - } + logger.fine("Count of alreadyseen on close " + count.get()); Environment env = null; if (this.alreadySeen != null) { try { @@ -331,10 +329,10 @@ implements Lifecycle, Checkpointable, BeanNameAware { } if (status == OperationStatus.SUCCESS) { count.incrementAndGet(); - if (logger.isLoggable(Level.INFO)) { + if (logger.isLoggable(Level.FINE)) { final int logAt = 10000; if (count.get() > 0 && ((count.get() % logAt) == 0)) { - logger.info("Average lookup " + + logger.fine("Average lookup " + (aggregatedLookupTime / logAt) + "ms."); aggregatedLookupTime = 0; }