pom.xml

Adding heritrix-engine dependency

XmlCrawlSummaryReport.java
	XML version of the CrawlSummaryReport

ExtractorYoutubeFormatStream.java
	Adding sample bean to docs

contrib/pom.xml

@@ -42,6 +42,12 @@
 				</exclusion>
 			</exclusions>
 		</dependency>
+		<dependency>
+			<groupId>org.archive.heritrix</groupId>
+			<artifactId>heritrix-engine</artifactId>
+			<version>3.1.2-SNAPSHOT</version>
+			<!-- scope>compile</scope -->
+		</dependency>
 		<dependency>
 			<groupId>org.archive.heritrix</groupId>
 			<artifactId>heritrix-modules</artifactId>

contrib/src/main/java/org/archive/crawler/reporting/XmlCrawlSummaryReport.java

@@ -0,0 +1,96 @@
+package org.archive.crawler.reporting;
+
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+import org.archive.crawler.restlet.XmlMarshaller;
+import org.archive.modules.writer.WARCWriterProcessor;
+import org.archive.util.ArchiveUtils;
+
+public class XmlCrawlSummaryReport extends Report {
+
+    private String scheduledDate;
+
+    public void setScheduledDate(String scheduledDate) {
+        this.scheduledDate = scheduledDate;
+    }
+
+    public String getScheduledDate() {
+        return this.scheduledDate;
+    }
+
+    @Override
+    public void write(PrintWriter writer, StatisticsTracker stats) {
+        Map<String,Object> info = new LinkedHashMap<String,Object>();
+
+        CrawlStatSnapshot snapshot = stats.getLastSnapshot();
+
+        info.put("crawlName", 
+                ((WARCWriterProcessor) stats.appCtx.getBean("warcWriter")).getPrefix());
+        info.put("crawlJobShortName", 
+                stats.getCrawlController().getMetadata().getJobName());
+        info.put("scheduledDate", this.scheduledDate);
+        info.put("crawlStatus",
+                stats.getCrawlController().getCrawlExitStatus().desc);
+        info.put("duration", 
+                ArchiveUtils.formatMillisecondsToConventional(stats.getCrawlElapsedTime()));
+
+        stats.tallySeeds();
+        info.put("seedsCrawled", stats.seedsCrawled);
+        info.put("seedsUncrawled",stats.seedsTotal - stats.seedsCrawled);
+
+        info.put("hostsVisited",stats.serverCache.hostKeys().size() - 1);
+
+        info.put("urisProcessed", snapshot.finishedUriCount);
+        info.put("uriSuccesses", snapshot.downloadedUriCount);
+        info.put("uriFailures", snapshot.downloadFailures);
+        info.put("uriDisregards", snapshot.downloadDisregards);
+
+        info.put("novelUris", stats.crawledBytes.get("novelCount"));
+
+        long duplicateCount = stats.crawledBytes.containsKey("dupByHashCount") ? stats.crawledBytes
+                .get("dupByHashCount").longValue() : 0L;
+
+        info.put("duplicateByHashUris", duplicateCount);
+        long notModifiedCount = stats.crawledBytes
+                .containsKey("notModifiedCount") ? stats.crawledBytes.get(
+                "notModifiedCount").longValue() : 0L;
+
+        info.put("notModifiedUris", notModifiedCount);
+
+        info.put("totalCrawledBytes", snapshot.bytesProcessed);
+
+        info.put("novelCrawledBytes", stats.crawledBytes.get("novel"));
+
+        long duplicateByHashCrawledBytes = stats.crawledBytes
+                .containsKey("dupByHash") ? stats.crawledBytes.get("dupByHash")
+                .longValue() : 0L;
+
+        info.put("duplicateByHashCrawledBytes",duplicateByHashCrawledBytes);
+        long notModifiedCrawledBytes = stats.crawledBytes
+                .containsKey("notModified") ? stats.crawledBytes.get(
+                "notModified").longValue() : 0L;
+
+        info.put("notModifiedCrawledBytes",notModifiedCrawledBytes);
+
+        info.put("urisPerSec",
+                ArchiveUtils.doubleToString(snapshot.docsPerSecond, 2));
+        info.put("kbPerSec", snapshot.totalKiBPerSec);
+        
+        try {
+            XmlMarshaller.marshalDocument(writer,
+                    XmlCrawlSummaryReport.class.getCanonicalName(), info);
+        } catch (IOException e) {
+            e.printStackTrace();
+        }
+
+    }
+
+    @Override
+    public String getFilename() {
+        return "crawl-report.xml";
+    }
+
+}

contrib/src/main/java/org/archive/modules/extractor/ExtractorYoutubeFormatStream.java

@@ -24,7 +24,23 @@ import org.json.JSONObject;
  * This will check the content of the youtube watch page looking for the url_encoded_fmt_stream_map json value.
  * The json object is decoded and the stream URIs are constructed and queued.
  * 
+ * <bean id="extractorYoutubeFormatStream" class="org.archive.modules.extractor.ExtractorYoutubeFormatStream">
+ *  <property name="enabled" value="false" /> <!-- enable via sheet for http://(com,youtube, -->
+ *  <property name="extractLimit" value="1" />
+ *  <property name="itagPriority" >
+ *   <list>
+ *     <value>38</value> <!-- MP4 3072p -->
+ *     <value>37</value> <!-- MP4 1080p -->
+ *     <value>22</value> <!-- MP4 720p -->
+ *     <value>18</value> <!-- MP4 270p/360p -->
+ *     <value>35</value> <!-- FLV 480p -->
+ *     <value>34</value> <!-- FLV 360p -->
+ *   </list>
+ *  </property>
+ * </bean>
+ * 
  * @contributor adam
+ * @contributor nlevitt
  *
  */
 public class ExtractorYoutubeFormatStream extends Extractor {