From e15f49dd608617fa9020315f5cd394388d25cff4 Mon Sep 17 00:00:00 2001 From: gojomo Date: Wed, 2 Dec 2009 06:25:23 +0000 Subject: [PATCH] [HER-1708] h3 - crawl summary report byte totals off somehow * CrawlController.java send event of crawl-end before appCtx.stop() * StatisticsTracker.java force final progressStatisticsEvent and dumpReports at crawlEnded() before stop() add force-dump of reports (so dumpReports never reuses prior reports) * CrawledBytesHistotable.java add getTotalBytes(), a total of just the byte-counts (getTotal() includes URI counts) * CrawlStatSnapshot.java use getTotalBytes() --- .../crawler/framework/CrawlController.java | 5 +++-- .../crawler/reporting/CrawlStatSnapshot.java | 2 +- .../crawler/reporting/StatisticsTracker.java | 17 +++++++++-------- .../crawler/util/CrawledBytesHistotable.java | 6 +++++- 4 files changed, 18 insertions(+), 12 deletions(-) diff --git a/engine/src/main/java/org/archive/crawler/framework/CrawlController.java b/engine/src/main/java/org/archive/crawler/framework/CrawlController.java index ebc50260..814dc54b 100644 --- a/engine/src/main/java/org/archive/crawler/framework/CrawlController.java +++ b/engine/src/main/java/org/archive/crawler/framework/CrawlController.java @@ -371,13 +371,14 @@ implements Serializable, LOGGER.fine("Finished crawl."); + // notify before lifecycle beans stopped (closing files/DBs/etc) + sendCrawlStateChangeEvent(State.FINISHED, this.sExit); + try { appCtx.stop(); } catch (RuntimeException re) { LOGGER.log(Level.SEVERE,re.getMessage(),re); } - // Ok, now we are ready to exit. - sendCrawlStateChangeEvent(State.FINISHED, this.sExit); } protected synchronized void completePause() { diff --git a/engine/src/main/java/org/archive/crawler/reporting/CrawlStatSnapshot.java b/engine/src/main/java/org/archive/crawler/reporting/CrawlStatSnapshot.java index cb1ec95b..2d393847 100644 --- a/engine/src/main/java/org/archive/crawler/reporting/CrawlStatSnapshot.java +++ b/engine/src/main/java/org/archive/crawler/reporting/CrawlStatSnapshot.java @@ -68,7 +68,7 @@ public class CrawlStatSnapshot { public void collect(CrawlController controller, StatisticsTracker stats) { // TODO: reconsider names of these methods, inline? downloadedUriCount = controller.getFrontier().succeededFetchCount(); - bytesProcessed = stats.crawledBytes.getTotal(); + bytesProcessed = stats.crawledBytes.getTotalBytes(); timestamp = System.currentTimeMillis(); elapsedMilliseconds = stats.getCrawlElapsedTime(); diff --git a/engine/src/main/java/org/archive/crawler/reporting/StatisticsTracker.java b/engine/src/main/java/org/archive/crawler/reporting/StatisticsTracker.java index 2231e07d..f968b1f0 100644 --- a/engine/src/main/java/org/archive/crawler/reporting/StatisticsTracker.java +++ b/engine/src/main/java/org/archive/crawler/reporting/StatisticsTracker.java @@ -358,7 +358,6 @@ public class StatisticsTracker public void stop() { isRunning = false; executor.shutdownNow(); - dumpReports(); } @SuppressWarnings("unchecked") @@ -567,10 +566,10 @@ public class StatisticsTracker } public void crawlEnded(String sExitMessage) { - // Note the time when the crawl stops. + logNote("CRAWL ENDED - " + sExitMessage); crawlEndTime = System.currentTimeMillis(); progressStatisticsEvent(); - logNote("CRAWL ENDED - " + sExitMessage); + dumpReports(); } /** @@ -943,13 +942,13 @@ public class StatisticsTracker public File writeReportFile(String reportName) { for(Class reportClass : ALL_REPORTS) { if(reportClass.getSimpleName().equals(reportName)) { - return writeReportFile(reportClass); + return writeReportFile(reportClass, false); } } return null; } - protected File writeReportFile(Class reportClass) { + protected File writeReportFile(Class reportClass, boolean force) { Report r; try { r = reportClass.newInstance(); @@ -959,8 +958,10 @@ public class StatisticsTracker r.setStats(this); File f = new File(getReportsDir().getFile(), r.getFilename()); - if(f.exists() && !controller.isRunning() && controller.hasStarted()) { - // controller already started and stopped and file exists + if(f.exists() && !controller.isRunning() && controller.hasStarted() && !force) { + // controller already started and stopped + // and file exists + // and force not requested // so, don't overwrite logger.warning("reusing report: " + f.getAbsolutePath()); return f; @@ -996,7 +997,7 @@ public class StatisticsTracker for(Class reportClass : END_REPORTS) { try { - writeReportFile(reportClass); + writeReportFile(reportClass, true); } catch (RuntimeException re) { logger.log(Level.SEVERE, re.getMessage(), re); } diff --git a/engine/src/main/java/org/archive/crawler/util/CrawledBytesHistotable.java b/engine/src/main/java/org/archive/crawler/util/CrawledBytesHistotable.java index 40766cd0..0bb0fddb 100644 --- a/engine/src/main/java/org/archive/crawler/util/CrawledBytesHistotable.java +++ b/engine/src/main/java/org/archive/crawler/util/CrawledBytesHistotable.java @@ -56,7 +56,7 @@ implements CoreAttributeConstants { public String summary() { StringBuilder sb = new StringBuilder(); - sb.append(ArchiveUtils.formatBytesForDisplay(getTotal())); + sb.append(ArchiveUtils.formatBytesForDisplay(getTotalBytes())); sb.append(" crawled ("); sb.append(ArchiveUtils.formatBytesForDisplay(get(NOVEL))); sb.append(" novel"); @@ -75,4 +75,8 @@ implements CoreAttributeConstants { sb.append(")"); return sb.toString(); } + + public long getTotalBytes() { + return get(NOVEL) + get(DUPLICATE) + get(NOTMODIFIED); + } }