From 93cf655bed7850ae5344dc96ca6fc2d06e308cc4 Mon Sep 17 00:00:00 2001 From: nlevitt Date: Thu, 6 Jan 2011 18:39:19 +0000 Subject: [PATCH] Add logToFile support to DecideRuleSequence. File is logs/{spring-bean-id}.log, format: [timestamp] [decisive-rule-num] [decisive-rule-class] [decision] [uri] * Scoper.java start(),stop() - call scope.start()/scope.stop() * CrawlerLoggerModule.java setupSimpleLog() - new logToFile paradigm * DecideRule.java start(),stop() - noop * DecideRuleSequence.java logToFile support * profile-crawler-beans.cxml --- .../profile-crawler-beans.cxml | 1 + .../org/archive/crawler/framework/Scoper.java | 2 + .../reporting/CrawlerLoggerModule.java | 26 ++++++++- .../modules/deciderules/DecideRule.java | 5 ++ .../deciderules/DecideRuleSequence.java | 58 +++++++++++++++++-- 5 files changed, 87 insertions(+), 5 deletions(-) diff --git a/dist/src/main/conf/jobs/profile-defaults/profile-crawler-beans.cxml b/dist/src/main/conf/jobs/profile-defaults/profile-crawler-beans.cxml index 71c0872a..4c817863 100644 --- a/dist/src/main/conf/jobs/profile-defaults/profile-crawler-beans.cxml +++ b/dist/src/main/conf/jobs/profile-defaults/profile-crawler-beans.cxml @@ -110,6 +110,7 @@ http://example.example/example + diff --git a/engine/src/main/java/org/archive/crawler/framework/Scoper.java b/engine/src/main/java/org/archive/crawler/framework/Scoper.java index b93f3ca5..e2d8d805 100644 --- a/engine/src/main/java/org/archive/crawler/framework/Scoper.java +++ b/engine/src/main/java/org/archive/crawler/framework/Scoper.java @@ -107,6 +107,7 @@ public abstract class Scoper extends Processor implements Lifecycle { this.getClass().getName(), Logger.getLogger(this.getClass().getName())); } + scope.start(); isRunning = true; } @@ -117,6 +118,7 @@ public abstract class Scoper extends Processor implements Lifecycle { if(fileLogger!=null) { fileLogger.close(); } + scope.stop(); isRunning = false; } diff --git a/engine/src/main/java/org/archive/crawler/reporting/CrawlerLoggerModule.java b/engine/src/main/java/org/archive/crawler/reporting/CrawlerLoggerModule.java index 29336746..64ecb403 100644 --- a/engine/src/main/java/org/archive/crawler/reporting/CrawlerLoggerModule.java +++ b/engine/src/main/java/org/archive/crawler/reporting/CrawlerLoggerModule.java @@ -22,6 +22,8 @@ package org.archive.crawler.reporting; import java.io.File; import java.io.IOException; import java.io.ObjectInputStream; +import java.text.SimpleDateFormat; +import java.util.Date; import java.util.HashMap; import java.util.Map; import java.util.logging.FileHandler; @@ -271,7 +273,29 @@ public class CrawlerLoggerModule this.fileHandlers.put(logger, fh); } - + public Logger setupSimpleLog(String logName) { + Logger logger = Logger.getLogger(logName + ".log"); + + Formatter f = new Formatter() { + private SimpleDateFormat dateFmt = new SimpleDateFormat("yyyy-MM-dd+HH:mm:ss.SSS"); + + public String format(java.util.logging.LogRecord record) { + String timestamp = dateFmt.format(new Date(record.getMillis())); + return timestamp + " " + record.getMessage() + '\n'; + } + }; + + ConfigPath logPath = new ConfigPath(logName + ".log", logName + ".log"); + logPath.setBase(getPath()); + try { + setupLogFile(logger, logPath.getFile().getAbsolutePath(), f, true); + } catch (IOException e) { + throw new IllegalStateException(e); + } + + return logger; + } + private void setupAlertLog(String logsPath) throws IOException { Logger logger = Logger.getLogger(LOGNAME_ALERTS + "." + logsPath); String filename = getAlertsLogPath().getFile().getAbsolutePath(); diff --git a/modules/src/main/java/org/archive/modules/deciderules/DecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/DecideRule.java index e6086cff..389ee6eb 100644 --- a/modules/src/main/java/org/archive/modules/deciderules/DecideRule.java +++ b/modules/src/main/java/org/archive/modules/deciderules/DecideRule.java @@ -77,4 +77,9 @@ public abstract class DecideRule implements Serializable, HasKeyedProperties { return DecideResult.ACCEPT == decisionFor(uri); } + public void start() { + } + public void stop() { + } + } diff --git a/modules/src/main/java/org/archive/modules/deciderules/DecideRuleSequence.java b/modules/src/main/java/org/archive/modules/deciderules/DecideRuleSequence.java index f69838e6..cf9dfd43 100644 --- a/modules/src/main/java/org/archive/modules/deciderules/DecideRuleSequence.java +++ b/modules/src/main/java/org/archive/modules/deciderules/DecideRuleSequence.java @@ -23,13 +23,41 @@ import java.util.List; import java.util.logging.Level; import java.util.logging.Logger; +import org.archive.crawler.reporting.CrawlerLoggerModule; import org.archive.modules.CrawlURI; +import org.springframework.beans.factory.BeanNameAware; +import org.springframework.beans.factory.annotation.Autowired; -public class DecideRuleSequence extends DecideRule { +public class DecideRuleSequence extends DecideRule implements BeanNameAware{ final private static Logger LOGGER = Logger.getLogger(DecideRuleSequence.class.getName()); private static final long serialVersionUID = 3L; + protected Logger fileLogger = null; + + /** + * If enabled, log decisions to file named logs/{spring-bean-id}.log. Format is: + * [timestamp] [decisive-rule-num] [decisive-rule-class] [decision] [uri] + */ + { + setLogToFile(false); + } + public boolean getLogToFile() { + return (Boolean) kp.get("logToFile"); + } + public void setLogToFile(boolean enabled) { + kp.put("logToFile",enabled); + } + + protected CrawlerLoggerModule loggerModule; + public CrawlerLoggerModule getLoggerModule() { + return this.loggerModule; + } + @Autowired + public void setLoggerModule(CrawlerLoggerModule loggerModule) { + this.loggerModule = loggerModule; + } + @SuppressWarnings("unchecked") public List getRules() { return (List) kp.get("rules"); @@ -39,9 +67,12 @@ public class DecideRuleSequence extends DecideRule { } public DecideResult innerDecide(CrawlURI uri) { + DecideRule decisiveRule = null; + int decisiveRuleNumber = -1; DecideResult result = DecideResult.NONE; List rules = getRules(); int max = rules.size(); + for (int i = 0; i < max; i++) { DecideRule rule = rules.get(i); if (rule.onlyDecision(uri) != result) { @@ -52,13 +83,32 @@ public class DecideRuleSequence extends DecideRule { } if (r != DecideResult.NONE) { result = r; + decisiveRule = rule; + decisiveRuleNumber = i; } } } - - if (LOGGER.isLoggable(Level.FINEST)) { - LOGGER.finest("DecideRuleSequence returned " + result + " for url: " + uri); + + if (fileLogger != null) { + fileLogger.info(decisiveRuleNumber + " " + decisiveRule.getClass().getSimpleName() + " " + result + " " + uri); } + return result; } + + public void start() { + if (getLogToFile() && fileLogger == null) { + fileLogger = loggerModule.setupSimpleLog(getBeanName()); + } + } + + protected String beanName; + public String getBeanName() { + return this.beanName; + } + @Override + public void setBeanName(String name) { + this.beanName = name; + } + }