[HER-1708] h3 - crawl summary report byte totals off somehow

* CrawlController.java
    send event of crawl-end before appCtx.stop()
* StatisticsTracker.java
    force final progressStatisticsEvent and dumpReports at crawlEnded() before stop()
    add force-dump of reports (so dumpReports never reuses prior reports)
* CrawledBytesHistotable.java
    add getTotalBytes(), a total of just the byte-counts (getTotal() includes URI counts)
* CrawlStatSnapshot.java
    use getTotalBytes()

engine/src/main/java/org/archive/crawler/framework/CrawlController.java

@@ -371,13 +371,14 @@ implements Serializable,
 
         LOGGER.fine("Finished crawl.");
 
+        // notify before lifecycle beans stopped (closing files/DBs/etc)
+        sendCrawlStateChangeEvent(State.FINISHED, this.sExit);
+
         try {
             appCtx.stop(); 
         } catch (RuntimeException re) {
             LOGGER.log(Level.SEVERE,re.getMessage(),re);
         }
-        // Ok, now we are ready to exit.
-        sendCrawlStateChangeEvent(State.FINISHED, this.sExit);
     }
     
     protected synchronized void completePause() {

engine/src/main/java/org/archive/crawler/reporting/CrawlStatSnapshot.java

@@ -68,7 +68,7 @@ public class CrawlStatSnapshot {
     public void collect(CrawlController controller, StatisticsTracker stats) {
         // TODO: reconsider names of these methods, inline?    
         downloadedUriCount = controller.getFrontier().succeededFetchCount();
-        bytesProcessed = stats.crawledBytes.getTotal();
+        bytesProcessed = stats.crawledBytes.getTotalBytes();
         timestamp = System.currentTimeMillis();
         
         elapsedMilliseconds = stats.getCrawlElapsedTime();

engine/src/main/java/org/archive/crawler/reporting/StatisticsTracker.java

@@ -358,7 +358,6 @@ public class StatisticsTracker
     public void stop() {
         isRunning = false;
         executor.shutdownNow();
-        dumpReports();
     }
     
     @SuppressWarnings("unchecked")
@@ -567,10 +566,10 @@ public class StatisticsTracker
     }
 
     public void crawlEnded(String sExitMessage) {
-        // Note the time when the crawl stops.
+        logNote("CRAWL ENDED - " + sExitMessage);
         crawlEndTime = System.currentTimeMillis();
         progressStatisticsEvent();
-        logNote("CRAWL ENDED - " + sExitMessage);
+        dumpReports();
     }
 
     /**
@@ -943,13 +942,13 @@ public class StatisticsTracker
     public File writeReportFile(String reportName) {
         for(Class<Report> reportClass : ALL_REPORTS) {
             if(reportClass.getSimpleName().equals(reportName)) {
-             return writeReportFile(reportClass);
+             return writeReportFile(reportClass, false);
             }
         }
         return null; 
     }
     
-    protected File writeReportFile(Class<Report> reportClass) {
+    protected File writeReportFile(Class<Report> reportClass, boolean force) {
         Report r;
         try {
             r = reportClass.newInstance();
@@ -959,8 +958,10 @@ public class StatisticsTracker
         r.setStats(this);
         File f = new File(getReportsDir().getFile(), r.getFilename());
         
-        if(f.exists() && !controller.isRunning() && controller.hasStarted()) {
-            // controller already started and stopped and file exists
+        if(f.exists() && !controller.isRunning() && controller.hasStarted() && !force) {
+            // controller already started and stopped 
+            // and file exists
+            // and force not requested
             // so, don't overwrite
             logger.warning("reusing report: " + f.getAbsolutePath());
             return f;
@@ -996,7 +997,7 @@ public class StatisticsTracker
         
         for(Class<Report> reportClass : END_REPORTS) {
             try {
-                writeReportFile(reportClass);
+                writeReportFile(reportClass, true);
             } catch (RuntimeException re) {
                 logger.log(Level.SEVERE, re.getMessage(), re);
             }

engine/src/main/java/org/archive/crawler/util/CrawledBytesHistotable.java

@@ -56,7 +56,7 @@ implements CoreAttributeConstants {
     
     public String summary() {
         StringBuilder sb = new StringBuilder();
-        sb.append(ArchiveUtils.formatBytesForDisplay(getTotal()));
+        sb.append(ArchiveUtils.formatBytesForDisplay(getTotalBytes()));
         sb.append(" crawled (");
         sb.append(ArchiveUtils.formatBytesForDisplay(get(NOVEL)));
         sb.append(" novel");
@@ -75,4 +75,8 @@ implements CoreAttributeConstants {
         sb.append(")");
         return sb.toString();
     }
+    
+    public long getTotalBytes() {
+        return get(NOVEL) + get(DUPLICATE) + get(NOTMODIFIED);
+    }
 }