diff --git a/contrib/pom.xml b/contrib/pom.xml index 405421b7..c6c7ec38 100644 --- a/contrib/pom.xml +++ b/contrib/pom.xml @@ -42,6 +42,12 @@ + + org.archive.heritrix + heritrix-engine + 3.1.2-SNAPSHOT + + org.archive.heritrix heritrix-modules diff --git a/contrib/src/main/java/org/archive/crawler/reporting/XmlCrawlSummaryReport.java b/contrib/src/main/java/org/archive/crawler/reporting/XmlCrawlSummaryReport.java new file mode 100644 index 00000000..4167fe3c --- /dev/null +++ b/contrib/src/main/java/org/archive/crawler/reporting/XmlCrawlSummaryReport.java @@ -0,0 +1,96 @@ +package org.archive.crawler.reporting; + +import java.io.IOException; +import java.io.PrintWriter; +import java.util.LinkedHashMap; +import java.util.Map; + +import org.archive.crawler.restlet.XmlMarshaller; +import org.archive.modules.writer.WARCWriterProcessor; +import org.archive.util.ArchiveUtils; + +public class XmlCrawlSummaryReport extends Report { + + private String scheduledDate; + + public void setScheduledDate(String scheduledDate) { + this.scheduledDate = scheduledDate; + } + + public String getScheduledDate() { + return this.scheduledDate; + } + + @Override + public void write(PrintWriter writer, StatisticsTracker stats) { + Map info = new LinkedHashMap(); + + CrawlStatSnapshot snapshot = stats.getLastSnapshot(); + + info.put("crawlName", + ((WARCWriterProcessor) stats.appCtx.getBean("warcWriter")).getPrefix()); + info.put("crawlJobShortName", + stats.getCrawlController().getMetadata().getJobName()); + info.put("scheduledDate", this.scheduledDate); + info.put("crawlStatus", + stats.getCrawlController().getCrawlExitStatus().desc); + info.put("duration", + ArchiveUtils.formatMillisecondsToConventional(stats.getCrawlElapsedTime())); + + stats.tallySeeds(); + info.put("seedsCrawled", stats.seedsCrawled); + info.put("seedsUncrawled",stats.seedsTotal - stats.seedsCrawled); + + info.put("hostsVisited",stats.serverCache.hostKeys().size() - 1); + + info.put("urisProcessed", snapshot.finishedUriCount); + info.put("uriSuccesses", snapshot.downloadedUriCount); + info.put("uriFailures", snapshot.downloadFailures); + info.put("uriDisregards", snapshot.downloadDisregards); + + info.put("novelUris", stats.crawledBytes.get("novelCount")); + + long duplicateCount = stats.crawledBytes.containsKey("dupByHashCount") ? stats.crawledBytes + .get("dupByHashCount").longValue() : 0L; + + info.put("duplicateByHashUris", duplicateCount); + long notModifiedCount = stats.crawledBytes + .containsKey("notModifiedCount") ? stats.crawledBytes.get( + "notModifiedCount").longValue() : 0L; + + info.put("notModifiedUris", notModifiedCount); + + info.put("totalCrawledBytes", snapshot.bytesProcessed); + + info.put("novelCrawledBytes", stats.crawledBytes.get("novel")); + + long duplicateByHashCrawledBytes = stats.crawledBytes + .containsKey("dupByHash") ? stats.crawledBytes.get("dupByHash") + .longValue() : 0L; + + info.put("duplicateByHashCrawledBytes",duplicateByHashCrawledBytes); + long notModifiedCrawledBytes = stats.crawledBytes + .containsKey("notModified") ? stats.crawledBytes.get( + "notModified").longValue() : 0L; + + info.put("notModifiedCrawledBytes",notModifiedCrawledBytes); + + info.put("urisPerSec", + ArchiveUtils.doubleToString(snapshot.docsPerSecond, 2)); + info.put("kbPerSec", snapshot.totalKiBPerSec); + + try { + XmlMarshaller.marshalDocument(writer, + XmlCrawlSummaryReport.class.getCanonicalName(), info); + } catch (IOException e) { + e.printStackTrace(); + } + + } + + @Override + public String getFilename() { + return "crawl-report.xml"; + } + +} \ No newline at end of file diff --git a/contrib/src/main/java/org/archive/modules/extractor/ExtractorYoutubeFormatStream.java b/contrib/src/main/java/org/archive/modules/extractor/ExtractorYoutubeFormatStream.java index 9bdec485..2356bedb 100644 --- a/contrib/src/main/java/org/archive/modules/extractor/ExtractorYoutubeFormatStream.java +++ b/contrib/src/main/java/org/archive/modules/extractor/ExtractorYoutubeFormatStream.java @@ -24,7 +24,23 @@ import org.json.JSONObject; * This will check the content of the youtube watch page looking for the url_encoded_fmt_stream_map json value. * The json object is decoded and the stream URIs are constructed and queued. * + * + * + * + * + * + * 38 + * 37 + * 22 + * 18 + * 35 + * 34 + * + * + * + * * @contributor adam + * @contributor nlevitt * */ public class ExtractorYoutubeFormatStream extends Extractor {