diff --git a/contrib/pom.xml b/contrib/pom.xml
index 405421b7..c6c7ec38 100644
--- a/contrib/pom.xml
+++ b/contrib/pom.xml
@@ -42,6 +42,12 @@
+
+ org.archive.heritrix
+ heritrix-engine
+ 3.1.2-SNAPSHOT
+
+
org.archive.heritrix
heritrix-modules
diff --git a/contrib/src/main/java/org/archive/crawler/reporting/XmlCrawlSummaryReport.java b/contrib/src/main/java/org/archive/crawler/reporting/XmlCrawlSummaryReport.java
new file mode 100644
index 00000000..4167fe3c
--- /dev/null
+++ b/contrib/src/main/java/org/archive/crawler/reporting/XmlCrawlSummaryReport.java
@@ -0,0 +1,96 @@
+package org.archive.crawler.reporting;
+
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+import org.archive.crawler.restlet.XmlMarshaller;
+import org.archive.modules.writer.WARCWriterProcessor;
+import org.archive.util.ArchiveUtils;
+
+public class XmlCrawlSummaryReport extends Report {
+
+ private String scheduledDate;
+
+ public void setScheduledDate(String scheduledDate) {
+ this.scheduledDate = scheduledDate;
+ }
+
+ public String getScheduledDate() {
+ return this.scheduledDate;
+ }
+
+ @Override
+ public void write(PrintWriter writer, StatisticsTracker stats) {
+ Map info = new LinkedHashMap();
+
+ CrawlStatSnapshot snapshot = stats.getLastSnapshot();
+
+ info.put("crawlName",
+ ((WARCWriterProcessor) stats.appCtx.getBean("warcWriter")).getPrefix());
+ info.put("crawlJobShortName",
+ stats.getCrawlController().getMetadata().getJobName());
+ info.put("scheduledDate", this.scheduledDate);
+ info.put("crawlStatus",
+ stats.getCrawlController().getCrawlExitStatus().desc);
+ info.put("duration",
+ ArchiveUtils.formatMillisecondsToConventional(stats.getCrawlElapsedTime()));
+
+ stats.tallySeeds();
+ info.put("seedsCrawled", stats.seedsCrawled);
+ info.put("seedsUncrawled",stats.seedsTotal - stats.seedsCrawled);
+
+ info.put("hostsVisited",stats.serverCache.hostKeys().size() - 1);
+
+ info.put("urisProcessed", snapshot.finishedUriCount);
+ info.put("uriSuccesses", snapshot.downloadedUriCount);
+ info.put("uriFailures", snapshot.downloadFailures);
+ info.put("uriDisregards", snapshot.downloadDisregards);
+
+ info.put("novelUris", stats.crawledBytes.get("novelCount"));
+
+ long duplicateCount = stats.crawledBytes.containsKey("dupByHashCount") ? stats.crawledBytes
+ .get("dupByHashCount").longValue() : 0L;
+
+ info.put("duplicateByHashUris", duplicateCount);
+ long notModifiedCount = stats.crawledBytes
+ .containsKey("notModifiedCount") ? stats.crawledBytes.get(
+ "notModifiedCount").longValue() : 0L;
+
+ info.put("notModifiedUris", notModifiedCount);
+
+ info.put("totalCrawledBytes", snapshot.bytesProcessed);
+
+ info.put("novelCrawledBytes", stats.crawledBytes.get("novel"));
+
+ long duplicateByHashCrawledBytes = stats.crawledBytes
+ .containsKey("dupByHash") ? stats.crawledBytes.get("dupByHash")
+ .longValue() : 0L;
+
+ info.put("duplicateByHashCrawledBytes",duplicateByHashCrawledBytes);
+ long notModifiedCrawledBytes = stats.crawledBytes
+ .containsKey("notModified") ? stats.crawledBytes.get(
+ "notModified").longValue() : 0L;
+
+ info.put("notModifiedCrawledBytes",notModifiedCrawledBytes);
+
+ info.put("urisPerSec",
+ ArchiveUtils.doubleToString(snapshot.docsPerSecond, 2));
+ info.put("kbPerSec", snapshot.totalKiBPerSec);
+
+ try {
+ XmlMarshaller.marshalDocument(writer,
+ XmlCrawlSummaryReport.class.getCanonicalName(), info);
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+
+ }
+
+ @Override
+ public String getFilename() {
+ return "crawl-report.xml";
+ }
+
+}
\ No newline at end of file
diff --git a/contrib/src/main/java/org/archive/modules/extractor/ExtractorYoutubeFormatStream.java b/contrib/src/main/java/org/archive/modules/extractor/ExtractorYoutubeFormatStream.java
index 9bdec485..2356bedb 100644
--- a/contrib/src/main/java/org/archive/modules/extractor/ExtractorYoutubeFormatStream.java
+++ b/contrib/src/main/java/org/archive/modules/extractor/ExtractorYoutubeFormatStream.java
@@ -24,7 +24,23 @@ import org.json.JSONObject;
* This will check the content of the youtube watch page looking for the url_encoded_fmt_stream_map json value.
* The json object is decoded and the stream URIs are constructed and queued.
*
+ *
+ *
* @contributor adam
+ * @contributor nlevitt
*
*/
public class ExtractorYoutubeFormatStream extends Extractor {