From 8ba641ed4bf30ea69433311aeb6af4e86a09efaf Mon Sep 17 00:00:00 2001 From: nlevitt Date: Wed, 25 Nov 2009 01:43:57 +0000 Subject: [PATCH] Part of [HER-1603] Restletify: xml representations of all resources that also offer html * XmlMarshaller.java new class to write xml based on nested map structure * BeanBrowseResource.java ScriptResource.java EngineResource.java JobResource.java JobRelatedResource.java support xml representation * WorkQueueFrontier.java PrecedenceProvider.java WorkQueue.java ToeThread.java ToePool.java CrawlJob.java Engine.java Frontier.java CrawlController.java StatisticsTracker.java CrawledBytesHistotable.java FetchStats.java ProcessorChain.java CrawlURI.java Reporter.java offer same data that is returned in the single line reports in a map suitable for use with XmlMarshaller * FileUtils.java new method tryToCanonicalize(File) --- .../java/org/archive/util/ArchiveUtils.java | 2 +- .../main/java/org/archive/util/FileUtils.java | 8 + .../main/java/org/archive/util/Reporter.java | 8 + .../crawler/framework/CrawlController.java | 6 +- .../archive/crawler/framework/CrawlJob.java | 107 ++++++++-- .../org/archive/crawler/framework/Engine.java | 10 +- .../archive/crawler/framework/Frontier.java | 3 +- .../archive/crawler/framework/ToePool.java | 30 +++ .../archive/crawler/framework/ToeThread.java | 29 +++ .../archive/crawler/frontier/WorkQueue.java | 32 +++ .../crawler/frontier/WorkQueueFrontier.java | 35 ++++ .../precedence/PrecedenceProvider.java | 10 + .../crawler/reporting/StatisticsTracker.java | 4 + .../crawler/restlet/BeanBrowseResource.java | 88 +++++++-- .../crawler/restlet/EditRepresentation.java | 2 +- .../crawler/restlet/EngineResource.java | 93 +++++++-- .../crawler/restlet/JobRelatedResource.java | 182 +++++++++++++++++- .../archive/crawler/restlet/JobResource.java | 141 +++++++++++++- .../crawler/restlet/ScriptResource.java | 112 ++++++++++- .../crawler/restlet/XmlMarshaller.java | 86 +++++++++ .../crawler/util/CrawledBytesHistotable.java | 6 +- .../java/org/archive/modules/CrawlURI.java | 10 + .../org/archive/modules/ProcessorChain.java | 9 + .../archive/modules/fetcher/FetchStats.java | 16 ++ 24 files changed, 971 insertions(+), 58 deletions(-) create mode 100644 engine/src/main/java/org/archive/crawler/restlet/XmlMarshaller.java diff --git a/commons/src/main/java/org/archive/util/ArchiveUtils.java b/commons/src/main/java/org/archive/util/ArchiveUtils.java index 91410cef..e4ab9b55 100644 --- a/commons/src/main/java/org/archive/util/ArchiveUtils.java +++ b/commons/src/main/java/org/archive/util/ArchiveUtils.java @@ -551,7 +551,7 @@ public class ArchiveUtils { /** * Convert milliseconds value to a human-readable duration * @param time - * @param toMs whether to print to the ms + * @param toMs ignored * @return Human readable string version of passed time */ public static String formatMillisecondsToConventional(long time, boolean toMs) { diff --git a/commons/src/main/java/org/archive/util/FileUtils.java b/commons/src/main/java/org/archive/util/FileUtils.java index ba1a89f7..9fbc062d 100644 --- a/commons/src/main/java/org/archive/util/FileUtils.java +++ b/commons/src/main/java/org/archive/util/FileUtils.java @@ -698,4 +698,12 @@ public class FileUtils { return dir; } + + public static File tryToCanonicalize(File file) { + try { + return file.getCanonicalFile(); + } catch (IOException e) { + return file; + } + } } \ No newline at end of file diff --git a/commons/src/main/java/org/archive/util/Reporter.java b/commons/src/main/java/org/archive/util/Reporter.java index 27ea4aad..e5759162 100644 --- a/commons/src/main/java/org/archive/util/Reporter.java +++ b/commons/src/main/java/org/archive/util/Reporter.java @@ -21,6 +21,7 @@ package org.archive.util; import java.io.IOException; import java.io.PrintWriter; +import java.util.Map; public interface Reporter { /** @@ -38,6 +39,13 @@ public interface Reporter { */ public void singleLineReportTo(PrintWriter pw) throws IOException; + + /** + * @return Same data that's in the single line report, as key-value pairs + */ + public Map singleLineReportData(); + + /** * Return a legend for the single-line summary report as a String. * diff --git a/engine/src/main/java/org/archive/crawler/framework/CrawlController.java b/engine/src/main/java/org/archive/crawler/framework/CrawlController.java index e2709437..ebc50260 100644 --- a/engine/src/main/java/org/archive/crawler/framework/CrawlController.java +++ b/engine/src/main/java/org/archive/crawler/framework/CrawlController.java @@ -25,6 +25,7 @@ import java.io.PrintWriter; import java.io.Serializable; import java.io.StringWriter; import java.util.LinkedList; +import java.util.Map; import java.util.logging.Level; import java.util.logging.Logger; @@ -311,7 +312,6 @@ implements Serializable, * @param newState State change we're to tell listeners' about. * @param message Message on state change. */ - @SuppressWarnings("unchecked") protected void sendCrawlStateChangeEvent(State newState, CrawlStatus status) { if(this.state == newState) { @@ -596,6 +596,10 @@ implements Serializable, return (toePool == null) ? "" : ArchiveUtils.singleLineReport(toePool); } + public Map getToeThreadReportShortData() { + return toePool == null ? null : toePool.singleLineReportData(); + + } public String getFrontierReportShort() { return ArchiveUtils.singleLineReport(getFrontier()); diff --git a/engine/src/main/java/org/archive/crawler/framework/CrawlJob.java b/engine/src/main/java/org/archive/crawler/framework/CrawlJob.java index 11751013..1e95d974 100644 --- a/engine/src/main/java/org/archive/crawler/framework/CrawlJob.java +++ b/engine/src/main/java/org/archive/crawler/framework/CrawlJob.java @@ -24,6 +24,7 @@ import java.io.IOException; import java.io.PrintWriter; import java.util.Collections; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -653,6 +654,21 @@ public class CrawlJob implements Comparable, ApplicationListener { return cc!=null ? cc.getStatisticsTracker() : null; } + public Map rateReportData() { + StatisticsTracker stats = getStats(); + if (stats == null) { + return null; + } + + CrawlStatSnapshot snapshot = stats.getSnapshot(); + Map map = new LinkedHashMap(); + map.put("currentDocsPerSecond", snapshot.currentDocsPerSecond); + map.put("averageDocsPerSecond", snapshot.docsPerSecond); + map.put("currentKiBPerSec", snapshot.currentKiBPerSec); + map.put("averageKiBPerSec", snapshot.totalKiBPerSec); + return map; + } + public Object rateReport() { StatisticsTracker stats = getStats(); if(stats==null) { @@ -672,6 +688,23 @@ public class CrawlJob implements Comparable, ApplicationListener { return sb.toString(); } + public Map loadReportData() { + StatisticsTracker stats = getStats(); + if (stats == null) { + return null; + } + + CrawlStatSnapshot snapshot = stats.getSnapshot(); + Map map = new LinkedHashMap(); + + map.put("busyThreads", snapshot.busyThreads); + map.put("totalThreads", stats.threadCount()); + map.put("congestionRatio", snapshot.congestionRatio); + map.put("averageQueueDepth", snapshot.averageDepth); + map.put("deepestQueueDepth", snapshot.deepestUri); + return map; + } + public Object loadReport() { StatisticsTracker stats = getStats(); if(stats==null) { @@ -693,32 +726,54 @@ public class CrawlJob implements Comparable, ApplicationListener { return sb.toString(); } - public String uriTotalsReport() { + public Map uriTotalsReportData() { StatisticsTracker stats = getStats(); - if(stats==null) { + if (stats == null) { + return null; + } + + CrawlStatSnapshot snapshot = stats.getSnapshot(); + + Map totals = new LinkedHashMap(); + totals.put("downloadedUriCount", snapshot.downloadedUriCount); + totals.put("queuedUriCount", snapshot.queuedUriCount); + totals.put("totalUriCount", snapshot.totalCount()); + totals.put("futureUriCount", snapshot.futureUriCount); + + return totals; + } + + public String uriTotalsReport() { + Map uriTotals = uriTotalsReportData(); + if (uriTotals == null) { return "n/a"; } - CrawlStatSnapshot snapshot = stats.getSnapshot(); - long downloaded = snapshot.downloadedUriCount; - long total = snapshot.totalCount(); - long queued = snapshot.queuedUriCount; + StringBuilder sb = new StringBuilder(64); sb - .append(downloaded) + .append(uriTotals.get("downloadedUriCount")) .append(" downloaded + ") - .append(queued) + .append(uriTotals.get("queuedUriCount")) .append(" queued = ") - .append(total) + .append(uriTotals.get("totalUriCount")) .append(" total"); - if(snapshot.futureUriCount>0) { + if(uriTotals.get("futureUriCount") >0) { sb .append(" (") - .append(snapshot.futureUriCount) + .append(uriTotals.get("futureUriCount")) .append(" future)"); } return sb.toString(); } + public Map sizeTotalsReportData() { + StatisticsTracker stats = getStats(); + if(stats==null) { + return null; + } + return stats.getCrawledBytes(); + } + public String sizeTotalsReport() { StatisticsTracker stats = getStats(); if(stats==null) { @@ -727,6 +782,20 @@ public class CrawlJob implements Comparable, ApplicationListener { return stats.crawledBytesSummary(); } + public Map elapsedReportData() { + StatisticsTracker stats = getStats(); + if(stats==null) { + return null; + } + + Map map = new LinkedHashMap(); + long timeElapsed = stats.getCrawlElapsedTime(); + map.put("elapsedMilliseconds", timeElapsed); + map.put("elapsedPretty", ArchiveUtils.formatMillisecondsToConventional(timeElapsed)); + + return map; + } + public String elapsedReport() { StatisticsTracker stats = getStats(); if(stats==null) { @@ -736,6 +805,14 @@ public class CrawlJob implements Comparable, ApplicationListener { return ArchiveUtils.formatMillisecondsToConventional(timeElapsed,false); } + public Map threadReportData() { + CrawlController cc = getCrawlController(); + if (cc == null) { + return null; + } + return cc.getToeThreadReportShortData(); + } + public String threadReport() { CrawlController cc = getCrawlController(); if(cc==null) { @@ -744,6 +821,14 @@ public class CrawlJob implements Comparable, ApplicationListener { return cc.getToeThreadReportShort(); } + public Map frontierReportData() { + CrawlController cc = getCrawlController(); + if (cc == null) { + return null; + } + return cc.getFrontier().singleLineReportData(); + } + public String frontierReport() { CrawlController cc = getCrawlController(); if(cc==null) { diff --git a/engine/src/main/java/org/archive/crawler/framework/Engine.java b/engine/src/main/java/org/archive/crawler/framework/Engine.java index 93de1a41..48949a63 100644 --- a/engine/src/main/java/org/archive/crawler/framework/Engine.java +++ b/engine/src/main/java/org/archive/crawler/framework/Engine.java @@ -24,6 +24,7 @@ import java.io.FilenameFilter; import java.io.IOException; import java.io.InputStream; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.Map; import java.util.logging.Level; import java.util.logging.Logger; @@ -249,6 +250,14 @@ public class Engine { return jobsDir; } + public Map heapReportData() { + Map map = new LinkedHashMap(); + map.put("usedBytes", Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()); + map.put("totalBytes", Runtime.getRuntime().totalMemory()); + map.put("maxBytes", Runtime.getRuntime().maxMemory()); + return map; + } + public String heapReport() { long totalMemory = Runtime.getRuntime().totalMemory(); long freeMemory = Runtime.getRuntime().freeMemory(); @@ -348,5 +357,4 @@ public class Engine { LOGGER.log(Level.SEVERE,"job: "+jobName+" not found in jobConfig!"); } } - } \ No newline at end of file diff --git a/engine/src/main/java/org/archive/crawler/framework/Frontier.java b/engine/src/main/java/org/archive/crawler/framework/Frontier.java index 5280ce75..dfeea33e 100644 --- a/engine/src/main/java/org/archive/crawler/framework/Frontier.java +++ b/engine/src/main/java/org/archive/crawler/framework/Frontier.java @@ -517,6 +517,7 @@ public interface Frontier extends Lifecycle, MultiReporter { HOLD, // enter a consistent, stable, checkpointable state ASAP PAUSE, // enter a stable state where no URIs are in-progress; unlike // HOLD requires all in-process URIs to complete - FINISH }; // end and cleanup; may not return to any other state after + FINISH // end and cleanup; may not return to any other state after // this state is requested/reached + } } diff --git a/engine/src/main/java/org/archive/crawler/framework/ToePool.java b/engine/src/main/java/org/archive/crawler/framework/ToePool.java index d871fbba..9d222a58 100644 --- a/engine/src/main/java/org/archive/crawler/framework/ToePool.java +++ b/engine/src/main/java/org/archive/crawler/framework/ToePool.java @@ -21,9 +21,11 @@ package org.archive.crawler.framework; import java.io.PrintWriter; import java.util.Iterator; +import java.util.LinkedHashMap; import java.util.Map; import java.util.TreeSet; +import org.apache.commons.lang.StringUtils; import org.archive.crawler.reporting.AlertThreadGroup; import org.archive.util.ArchiveUtils; import org.archive.util.Histotable; @@ -259,6 +261,34 @@ public class ToePool extends ThreadGroup implements MultiReporter { } } + public Map singleLineReportData() { + Histotable steps = new Histotable(); + Histotable processors = new Histotable(); + Thread[] toes = getToes(); + for (int i = 0; i < toes.length; i++) { + if(!(toes[i] instanceof ToeThread)) { + continue; + } + ToeThread tt = (ToeThread)toes[i]; + if(tt!=null) { + steps.tally(tt.getStep().toString()); + String currentProcessorName = tt.getCurrentProcessorName(); + if (StringUtils.isEmpty(currentProcessorName)) { + currentProcessorName = "noActiveProcessor"; + } + processors.tally(currentProcessorName); + } + } + + Map data = new LinkedHashMap(); + + data.put("toeCount", getToeCount()); + data.put("steps", steps.getSortedByCounts()); + data.put("processors", processors.getSortedByCounts()); + + return data; + } + public void singleLineReportTo(PrintWriter w) { Histotable steps = new Histotable(); Histotable processors = new Histotable(); diff --git a/engine/src/main/java/org/archive/crawler/framework/ToeThread.java b/engine/src/main/java/org/archive/crawler/framework/ToeThread.java index e89ac1ad..de4a5209 100644 --- a/engine/src/main/java/org/archive/crawler/framework/ToeThread.java +++ b/engine/src/main/java/org/archive/crawler/framework/ToeThread.java @@ -29,6 +29,8 @@ import java.lang.management.ManagementFactory; import java.lang.management.ThreadInfo; import java.lang.management.ThreadMXBean; import java.net.InetAddress; +import java.util.LinkedHashMap; +import java.util.Map; import java.util.logging.Level; import java.util.logging.Logger; @@ -479,6 +481,33 @@ implements RecorderMarker, MultiReporter, ProgressStatisticsReporter, } } + public Map singleLineReportData() { + Map data = new LinkedHashMap(); + data.put("serialNumber", serialNumber); + CrawlURI c = currentCuri; + if (c != null) { + data.put("currentURI", c.toString()); + data.put("currentProcessor", currentProcessorName); + data.put("fetchAttempts", c.getFetchAttempts()); + } else { + data.put("currentURI", null); + } + + long now = System.currentTimeMillis(); + long time = 0; + if (lastFinishTime > lastStartTime) { + data.put("status", "WAITING"); + time = now - lastFinishTime; + } else if (lastStartTime > 0) { + data.put("status", "ACTIVE"); + time = now - lastStartTime; + } + data.put("currentStatusElapsedMilliseconds", time); + data.put("currentStatusElapsedPretty", ArchiveUtils.formatMillisecondsToConventional(time)); + data.put("step", step); + return data; + } + /** * @param w PrintWriter to write to. */ diff --git a/engine/src/main/java/org/archive/crawler/frontier/WorkQueue.java b/engine/src/main/java/org/archive/crawler/frontier/WorkQueue.java index f9fa6b54..cae3e341 100644 --- a/engine/src/main/java/org/archive/crawler/frontier/WorkQueue.java +++ b/engine/src/main/java/org/archive/crawler/frontier/WorkQueue.java @@ -22,7 +22,10 @@ package org.archive.crawler.frontier; import java.io.IOException; import java.io.PrintWriter; import java.io.Serializable; +import java.util.Date; import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.Map; import java.util.Set; import java.util.concurrent.Delayed; import java.util.concurrent.TimeUnit; @@ -526,6 +529,35 @@ public abstract class WorkQueue implements Frontier.FrontierGroup, reportTo(null,writer); } + public Map singleLineReportData() { + Map map = new LinkedHashMap(); + + map.put("queueName", classKey); + map.put("precedence", getPrecedence()); + map.put("itemCount", count); + map.put("enqueueCount", enqueueCount); + map.put("sessionBalance", sessionBalance); + map.put("lastCost", lastCost); + map.put("averageCost", (double) totalExpenditure / costCount); + if (lastDequeueTime != 0) { + map.put("lastDequeueTime", new Date(lastDequeueTime)); + } else { + map.put("lastDequeueTime", null); + } + if (wakeTime != 0) { + map.put("lastDequeueTime", new Date(wakeTime)); + } else { + map.put("lastDequeueTime", null); + } + map.put("totalExpenditure", totalExpenditure); + map.put("totalBudget", totalBudget); + map.put("errorCount", errorCount); + map.put("lastPeeked", lastPeeked); + map.put("lastQueued", lastQueued); + + return map; + } + /* (non-Javadoc) * @see org.archive.util.Reporter#singleLineReportTo(java.io.Writer) */ diff --git a/engine/src/main/java/org/archive/crawler/frontier/WorkQueueFrontier.java b/engine/src/main/java/org/archive/crawler/frontier/WorkQueueFrontier.java index e942f518..feb3c2a5 100644 --- a/engine/src/main/java/org/archive/crawler/frontier/WorkQueueFrontier.java +++ b/engine/src/main/java/org/archive/crawler/frontier/WorkQueueFrontier.java @@ -35,6 +35,7 @@ import java.lang.ref.SoftReference; import java.util.ArrayList; import java.util.Collection; import java.util.Iterator; +import java.util.LinkedHashMap; import java.util.Map; import java.util.Queue; import java.util.SortedMap; @@ -1087,6 +1088,40 @@ implements Closeable, return REPORTS; } + public Map singleLineReportData() { + if (this.allQueues == null) { + return null; + } + + int allCount = allQueues.size(); + int inProcessCount = inProcessQueues.uniqueSet().size(); + int readyCount = readyClassQueues.size(); + int snoozedCount = snoozedClassQueues.size(); + int activeCount = inProcessCount + readyCount + snoozedCount; + int inactiveCount = getTotalEligibleInactiveQueues(); + int ineligibleCount = getTotalIneligibleInactiveQueues(); + int retiredCount = getRetiredQueues().size(); + int exhaustedCount = allCount - activeCount - inactiveCount - retiredCount; + int inCount = inbound.size(); + int outCount = outbound.size(); + + Map map = new LinkedHashMap(); + map.put("totalQueues", allCount); + map.put("inProcessQueues", inProcessCount); + map.put("readyQueues", readyCount); + map.put("snoozedQueues", snoozedCount); + map.put("activeQueues", activeCount); + map.put("inactiveQueues", inactiveCount); + map.put("ineligibleQueues", ineligibleCount); + map.put("retiredQueues", retiredCount); + map.put("exhaustedQueues", exhaustedCount); + map.put("lastReachedState", lastReachedState); + map.put("inboundCount", inCount); + map.put("outboundCount", outCount); + + return map; + } + /** * @param w Where to write to. */ diff --git a/engine/src/main/java/org/archive/crawler/frontier/precedence/PrecedenceProvider.java b/engine/src/main/java/org/archive/crawler/frontier/precedence/PrecedenceProvider.java index 506e6693..3414d0bc 100644 --- a/engine/src/main/java/org/archive/crawler/frontier/precedence/PrecedenceProvider.java +++ b/engine/src/main/java/org/archive/crawler/frontier/precedence/PrecedenceProvider.java @@ -20,6 +20,8 @@ package org.archive.crawler.frontier.precedence; import java.io.PrintWriter; import java.io.Serializable; +import java.util.LinkedHashMap; +import java.util.Map; import org.archive.modules.CrawlURI; import org.archive.modules.fetcher.FetchStats; @@ -34,6 +36,8 @@ import org.archive.util.MultiReporter; abstract public class PrecedenceProvider implements MultiReporter, FetchStats.CollectsFetchStats, Serializable { + private static final long serialVersionUID = 1L; + abstract public int getPrecedence(); /* (non-Javadoc) @@ -72,6 +76,12 @@ FetchStats.CollectsFetchStats, Serializable { return ArchiveUtils.singleLineReport(this); } + public Map singleLineReportData() { + Map data = new LinkedHashMap(); + data.put("precedence", getPrecedence()); + return data; + } + public void singleLineReportTo(PrintWriter writer) { writer.print(getPrecedence()); } diff --git a/engine/src/main/java/org/archive/crawler/reporting/StatisticsTracker.java b/engine/src/main/java/org/archive/crawler/reporting/StatisticsTracker.java index 285ac372..2231e07d 100644 --- a/engine/src/main/java/org/archive/crawler/reporting/StatisticsTracker.java +++ b/engine/src/main/java/org/archive/crawler/reporting/StatisticsTracker.java @@ -306,6 +306,10 @@ public class StatisticsTracker /** tally sizes novel, verified (same hash), vouched (not-modified) */ CrawledBytesHistotable crawledBytes = new CrawledBytesHistotable(); + public CrawledBytesHistotable getCrawledBytes() { + return crawledBytes; + } + // TODO: fortify these against key explosion with bigmaps like other tallies /** Keep track of the file types we see (mime type -> count) */ protected ObjectIdentityCache mimeTypeDistribution diff --git a/engine/src/main/java/org/archive/crawler/restlet/BeanBrowseResource.java b/engine/src/main/java/org/archive/crawler/restlet/BeanBrowseResource.java index 62a84a57..825edb35 100644 --- a/engine/src/main/java/org/archive/crawler/restlet/BeanBrowseResource.java +++ b/engine/src/main/java/org/archive/crawler/restlet/BeanBrowseResource.java @@ -26,6 +26,8 @@ import java.io.Writer; import java.net.URLDecoder; import java.util.Collection; import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.LinkedList; import java.util.Set; import org.apache.commons.lang.StringUtils; @@ -45,6 +47,7 @@ import org.restlet.resource.Variant; import org.restlet.resource.WriterRepresentation; import org.springframework.beans.BeanWrapperImpl; import org.springframework.beans.BeansException; +import org.xml.sax.SAXException; /** * Restlet Resource which allows browsing the constructed beans in @@ -59,6 +62,7 @@ public class BeanBrowseResource extends JobRelatedResource { public BeanBrowseResource(Context ctx, Request req, Response res) throws ResourceException { super(ctx, req, res); getVariants().add(new Variant(MediaType.TEXT_HTML)); + getVariants().add(new Variant(MediaType.APPLICATION_XML)); setModifiable(true); // accept POSTs appCtx = cj.getJobContext(); beanPath = (String)req.getAttributes().get("beanPath"); @@ -71,9 +75,11 @@ public class BeanBrowseResource extends JobRelatedResource { } else { beanPath = ""; } - if(appCtx==null) { + if (appCtx == null) { getResponse().setStatus(Status.CLIENT_ERROR_NOT_FOUND); - getResponse().setEntity("No beans; crawl not yet built.",MediaType.TEXT_HTML); + + // XXX xml? + getResponse().setEntity("No beans; crawl not yet built.", MediaType.TEXT_HTML); //throw new ResourceException(404); } } @@ -85,7 +91,6 @@ public class BeanBrowseResource extends JobRelatedResource { String newVal = form.getFirstValue("newVal"); if(newVal!=null) { - int i = beanPath.indexOf("."); String beanName = i<0?beanPath:beanPath.substring(0,i); Object namedBean = appCtx.getBean(beanName); @@ -93,13 +98,11 @@ public class BeanBrowseResource extends JobRelatedResource { String propPath = beanPath.substring(i+1); Object coercedVal = bwrap.convertIfNecessary(newVal, bwrap.getPropertyValue(propPath).getClass()); bwrap.setPropertyValue(propPath, coercedVal); - } Reference ref = getRequest().getResourceRef(); ref.setPath(getBeansRefPath()); ref.addSegment(beanPath); getResponse().redirectSeeOther(ref); - } public String getBeansRefPath() { @@ -114,20 +117,79 @@ public class BeanBrowseResource extends JobRelatedResource { } return path; } - public Representation represent(Variant variant) throws ResourceException { - Representation representation = new WriterRepresentation( - MediaType.TEXT_HTML) { - public void write(Writer writer) throws IOException { - BeanBrowseResource.this.writeHtml(writer); - } - }; + Representation representation; + if (variant.getMediaType() == MediaType.APPLICATION_XML) { + representation = new WriterRepresentation(MediaType.APPLICATION_XML) { + public void write(Writer writer) throws IOException { + try { + new XmlMarshaller(writer).marshalDocument("script", presentablify()); + } catch (SAXException e) { + throw new IOException(e); + } + } + }; + } else { + representation = new WriterRepresentation( + MediaType.TEXT_HTML) { + public void write(Writer writer) throws IOException { + BeanBrowseResource.this.writeHtml(writer); + } + }; + } // TODO: remove if not necessary in future? representation.setCharacterSet(CharacterSet.UTF_8); return representation; } + protected LinkedHashMap presentablify() { + LinkedHashMap info = new LinkedHashMap(); + + info.put("crawlJobShortName", cj.getShortName()); + info.put("crawlJobUrl", new Reference(getRequest().getResourceRef().getBaseRef(), "..").getTargetRef()); + + if (StringUtils.isNotBlank(beanPath)) { + info.put("beanPath", beanPath); + try { + int firstDot = beanPath.indexOf("."); + String beanName = firstDot<0?beanPath:beanPath.substring(0,firstDot); + Object namedBean = appCtx.getBean(beanName); + Object target; + if (firstDot < 0) { + target = namedBean; + info.put("bean", presentablifyObject(null, target, beanPath)); + } else { + BeanWrapperImpl bwrap = new BeanWrapperImpl(namedBean); + String propPath = beanPath.substring(firstDot+1); + target = bwrap.getPropertyValue(propPath); + + Class type = bwrap.getPropertyType(propPath); + if(bwrap.isWritableProperty(propPath) + && (bwrap.getDefaultEditor(type)!=null|| type == String.class) + && !Collection.class.isAssignableFrom(type)) { + info.put("editable", true); + info.put("bean", presentablifyObject(null, target)); + } else { + info.put("bean", presentablifyObject(null, target, beanPath)); + } + } + } catch (BeansException e) { + info.put("problem", e.toString()); + } + } + + Collection nestedNames = new LinkedList(); + Set alreadyWritten = new HashSet(); + presentablifyNestedNames(appCtx.getBean("crawlController"), getBeansRefPath(), alreadyWritten, nestedNames); + for(String name: appCtx.getBeanDefinitionNames()) { + presentablifyNestedNames(appCtx.getBean(name), getBeansRefPath(), alreadyWritten, nestedNames); + } + info.put("allNamedCrawlBeans", nestedNames); + + return info; + } + protected void writeHtml(Writer writer) { PrintWriter pw = new PrintWriter(writer); pw.println("Crawl beans in "+cj.getShortName()+""); @@ -153,7 +215,7 @@ public class BeanBrowseResource extends JobRelatedResource { String propPath = beanPath.substring(i+1); target = bwrap.getPropertyValue(propPath); - Class type = bwrap.getPropertyType(propPath); + Class type = bwrap.getPropertyType(propPath); if(bwrap.isWritableProperty(propPath) && (bwrap.getDefaultEditor(type)!=null|| type == String.class) && !Collection.class.isAssignableFrom(type)) { diff --git a/engine/src/main/java/org/archive/crawler/restlet/EditRepresentation.java b/engine/src/main/java/org/archive/crawler/restlet/EditRepresentation.java index a596ea88..fccd76dc 100644 --- a/engine/src/main/java/org/archive/crawler/restlet/EditRepresentation.java +++ b/engine/src/main/java/org/archive/crawler/restlet/EditRepresentation.java @@ -72,7 +72,7 @@ public class EditRepresentation extends CharacterRepresentation { viewRef.setQuery(null); pw.println("view"); pw.println("
"); - pw.println(""); pw.close(); diff --git a/engine/src/main/java/org/archive/crawler/restlet/EngineResource.java b/engine/src/main/java/org/archive/crawler/restlet/EngineResource.java index 0cf89c28..1dd34a49 100644 --- a/engine/src/main/java/org/archive/crawler/restlet/EngineResource.java +++ b/engine/src/main/java/org/archive/crawler/restlet/EngineResource.java @@ -24,12 +24,19 @@ import java.io.IOException; import java.io.PrintWriter; import java.io.Writer; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; import org.apache.commons.lang.StringUtils; import org.archive.crawler.framework.CrawlJob; import org.archive.crawler.framework.Engine; +import org.archive.crawler.framework.CrawlController.State; +import org.archive.util.FileUtils; import org.restlet.Context; import org.restlet.data.CharacterSet; import org.restlet.data.Form; @@ -41,6 +48,7 @@ import org.restlet.resource.Resource; import org.restlet.resource.ResourceException; import org.restlet.resource.Variant; import org.restlet.resource.WriterRepresentation; +import org.xml.sax.SAXException; /** * Restlet Resource representing an Engine that may be used @@ -54,15 +62,28 @@ public class EngineResource extends Resource { super(ctx, req, res); setModifiable(true); getVariants().add(new Variant(MediaType.TEXT_HTML)); + getVariants().add(new Variant(MediaType.APPLICATION_XML)); } public Representation represent(Variant variant) throws ResourceException { - Representation representation = new WriterRepresentation( - MediaType.TEXT_HTML) { - public void write(Writer writer) throws IOException { - EngineResource.this.writeHtml(writer); - } - }; + Representation representation; + if (variant.getMediaType() == MediaType.APPLICATION_XML) { + representation = new WriterRepresentation(MediaType.APPLICATION_XML) { + public void write(Writer writer) throws IOException { + try { + new XmlMarshaller(writer).marshalDocument("engine", presentablify()); + } catch (SAXException e) { + throw new IOException(e); + } + } + }; + } else { + representation = new WriterRepresentation(MediaType.TEXT_HTML) { + public void write(Writer writer) throws IOException { + EngineResource.this.writeHtml(writer); + } + }; + } // TODO: remove if not necessary in future? representation.setCharacterSet(CharacterSet.UTF_8); return representation; @@ -126,6 +147,59 @@ public class EngineResource extends Resource { getResponse().redirectSeeOther(getRequest().getOriginalRef()); } + protected List getAvailableActions() { + List actions = new LinkedList(); + actions.add("rescan"); + actions.add("add"); + actions.add("create"); + return actions; + } + + protected LinkedHashMap presentablify() { + String baseRef = getRequest().getResourceRef().getBaseRef().toString(); + if(!baseRef.endsWith("/")) { + baseRef += "/"; + } + + LinkedHashMap info = new LinkedHashMap(); + Engine engine = getEngine(); + info.put("heritrixVersion", engine.getHeritrixVersion()); + File jobsDir = FileUtils.tryToCanonicalize(engine.getJobsDir()); + info.put("jobsDir", jobsDir.getAbsolutePath()); + info.put("jobsDirUrl", baseRef + "jobsdir/"); + info.put("availableActions", getAvailableActions()); + info.put("heapReport", getEngine().heapReportData()); + + // re-scan job configs on each page load + ArrayList jobs = new ArrayList(); + engine.findJobConfigs(); + jobs.addAll(engine.getJobConfigs().values()); + Collections.sort(jobs); + Collection> jobsInfo = new LinkedList>(); + for (CrawlJob cj: jobs) { + // cj.writeHtmlTo(pw,"job/"); + Map jobInfo = new LinkedHashMap(); + jobInfo.put("shortName", cj.getShortName()); + jobInfo.put("url", baseRef + "job/" + cj.getShortName()); + jobInfo.put("isProfile", cj.isProfile()); + jobInfo.put("launchCount", cj.getLaunchCount()); + jobInfo.put("lastLaunch", cj.getLastLaunch()); + File primaryConfig = FileUtils.tryToCanonicalize(cj.getPrimaryConfig()); + jobInfo.put("primaryConfig", primaryConfig.getAbsolutePath()); + jobInfo.put("primaryConfigUrl", baseRef + "job/" + cj.getShortName() + "/jobdir/" + primaryConfig.getName()); + if (cj.getCrawlController() != null) { + jobInfo.put("crawlControllerState", cj.getCrawlController().getState()); + if (cj.getCrawlController().getState() == State.FINISHED) { + jobInfo.put("crawlExitStatus", cj.getCrawlController().getCrawlExitStatus()); + } + } + jobsInfo.add(jobInfo); + } + info.put("jobs", jobsInfo); + + return info; + } + /** * wraps a message in a styled div given messsage type * @param msg message to be displayed @@ -152,12 +226,7 @@ public class EngineResource extends Resource { protected void writeHtml(Writer writer) { Engine engine = getEngine(); String engineTitle = "Heritrix Engine "+engine.getHeritrixVersion(); - File jobsDir = engine.getJobsDir(); - try { - jobsDir = jobsDir.getCanonicalFile(); - } catch (IOException ioe) { - // live with the noncanonical file - } + File jobsDir = FileUtils.tryToCanonicalize(engine.getJobsDir()); String baseRef = getRequest().getResourceRef().getBaseRef().toString(); if(!baseRef.endsWith("/")) { baseRef += "/"; diff --git a/engine/src/main/java/org/archive/crawler/restlet/JobRelatedResource.java b/engine/src/main/java/org/archive/crawler/restlet/JobRelatedResource.java index 0dc94e03..371d5ccb 100644 --- a/engine/src/main/java/org/archive/crawler/restlet/JobRelatedResource.java +++ b/engine/src/main/java/org/archive/crawler/restlet/JobRelatedResource.java @@ -24,8 +24,11 @@ import java.beans.PropertyDescriptor; import java.io.File; import java.io.PrintWriter; import java.util.Arrays; +import java.util.Collection; import java.util.HashSet; import java.util.IdentityHashMap; +import java.util.LinkedHashMap; +import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Set; @@ -34,6 +37,7 @@ import org.apache.commons.lang.StringUtils; import org.archive.crawler.framework.CrawlJob; import org.archive.crawler.framework.Engine; import org.restlet.Context; +import org.restlet.data.Reference; import org.restlet.data.Request; import org.restlet.data.Response; import org.restlet.resource.Resource; @@ -64,6 +68,55 @@ public abstract class JobRelatedResource extends Resource { return ((EngineApplication)getApplication()).getEngine(); } + protected void presentablifyNestedNames(Object obj, String prefix, + Set alreadyWritten, Collection namedBeans) { + if (obj == null || alreadyWritten.contains(obj) + || obj.getClass().getName().startsWith("org.springframework.")) { + return; + } + + Reference baseRef = getRequest().getResourceRef().getBaseRef(); + + if (getBeanToNameMap().containsKey(obj)) { + // this object is itself a named bean + Map bean = new LinkedHashMap(); + bean.put("name", getBeanToNameMap().get(obj)); + bean.put("url", new Reference(baseRef, "../beans/" + getBeanToNameMap().get(obj)).getTargetRef()); + + bean.put("class", obj.getClass().getName()); + namedBeans.add(bean); + + // nest children + namedBeans = new LinkedList(); + bean.put("children", namedBeans); + } + + if (!alreadyWritten.contains(obj)) { + alreadyWritten.add(obj); + + BeanWrapperImpl bwrap = new BeanWrapperImpl(obj); + for (PropertyDescriptor pd : bwrap.getPropertyDescriptors()) { + if (pd.getReadMethod() != null) { + String propName = pd.getName(); + Object propValue = bwrap.getPropertyValue(propName); + presentablifyNestedNames(propValue, prefix, alreadyWritten, namedBeans); + } + } + if (obj.getClass().isArray()) { + List list = Arrays.asList(obj); + for (int i = 0; i < list.size(); i++) { + presentablifyNestedNames(list.get(i), prefix, + alreadyWritten, namedBeans); + } + } + if (obj instanceof Iterable) { + for (Object next : (Iterable) obj) { + presentablifyNestedNames(next, prefix, alreadyWritten, namedBeans); + } + } + } + } + /** * Starting at (and including) the given object, write a hierarchical * list of named beans in HTML to the PrintWriter. @@ -99,7 +152,7 @@ public abstract class JobRelatedResource extends Resource { } } if(obj.getClass().isArray()) { - List list = Arrays.asList(obj); + List list = Arrays.asList(obj); for(int i = 0; i < list.size(); i++) { writeNestedNames(pw, list.get(i), prefix, alreadyWritten); } @@ -137,6 +190,127 @@ public abstract class JobRelatedResource extends Resource { writeObject(pw, field, object, new HashSet(), beanPath); } + protected Map presentablifyObject(String field, Object object) { + return presentablifyObject(field, object, new HashSet(), null); + } + + protected Map presentablifyObject(String field, Object object, String beanPath) { + return presentablifyObject(field, object, new HashSet(), beanPath); + } + + protected Map presentablifyObject(String field, Object object, HashSet alreadyWritten, String beanPathPrefix) { + Map info = new LinkedHashMap(); + Reference baseRef = getRequest().getResourceRef().getBaseRef(); + + String beanPath = beanPathPrefix; + + if(StringUtils.isNotBlank(field)) { + info.put("field", field); + + if(StringUtils.isNotBlank(beanPathPrefix)) { + if(beanPathPrefix.endsWith(".")) { + beanPath += field; + } else if (beanPathPrefix.endsWith("[")) { + beanPath += field + "]"; + } + info.put("url", new Reference(baseRef, "../beans/" + beanPath).getTargetRef()); + } + } + String key = getBeanToNameMap().get(object); + + if (object == null) { + info.put("propValue", null); + return info; + } + if (object instanceof String || BeanUtils.isSimpleValueType(object.getClass()) || object instanceof File) { + info.put("class", object.getClass().getName()); + info.put("propValue", object); + return info; + } + if (alreadyWritten.contains(object)) { + info.put("propValuePreviouslyDescribed", null); + return info; + } + + alreadyWritten.add(object); // guard against repeats and cycles + + if (StringUtils.isNotBlank(key) && StringUtils.isNotBlank(field)) { + info.put("key", key); + info.put("url", new Reference(baseRef, "../beans/" + key).getTargetRef()); + return info; + } + + info.put("class", object.getClass().getName()); + + Collection properties = new LinkedList(); + BeanWrapperImpl bwrap = new BeanWrapperImpl(object); + for (PropertyDescriptor pd : bwrap.getPropertyDescriptors()) { + if (object instanceof DescriptorUpdater) { + ((DescriptorUpdater) object).updateDescriptor(pd); + } else { + defaultUpdateDescriptor(pd); + } + if (pd.getReadMethod() != null && !pd.isHidden()) { + String propName = pd.getName(); + if (beanPath != null) { + beanPathPrefix = beanPath + "."; + } + Object propValue = presentablifyObject(propName, + bwrap.getPropertyValue(propName), + alreadyWritten, beanPathPrefix); + properties.add(propValue); + } + } + if (properties.size() > 0) { + info.put("properties", properties); + } + + Collection propValues = new LinkedList(); + if(object.getClass().isArray()) { + Object[] array = (Object[])object; + for(int i = 0; i < array.length; i++) { + if(beanPath!=null) { + beanPathPrefix = beanPath+"["; + } + // TODO: protect against overlong content? + propValues.add(presentablifyObject(i + "", array[i], + alreadyWritten, beanPathPrefix)); + } + } + if (object instanceof List) { + List list = (List) object; + for (int i = 0; i < list.size(); i++) { + if (beanPath != null) { + beanPathPrefix = beanPath + "["; + } + // TODO: protect against overlong content? + propValues.add(presentablifyObject(i + "", list.get(i), + alreadyWritten, beanPathPrefix)); + } + } else if (object instanceof Iterable) { + for (Object next : (Iterable) object) { + propValues.add(presentablifyObject("#", next, alreadyWritten, + beanPathPrefix)); + } + } + if (object instanceof Map) { + for (Object next : ((Map) object).entrySet()) { + // TODO: protect against giant maps? + Map.Entry entry = (Map.Entry) next; + if (beanPath != null) { + beanPathPrefix = beanPath + "["; + } + propValues.add(presentablifyObject(entry.getKey().toString(), + entry.getValue(), alreadyWritten, beanPathPrefix)); + } + } + if (propValues.size() > 0) { + info.put("propValue", propValues); + } + + return info; + } + /** * Get a map giving object beanNames. * @@ -146,7 +320,7 @@ public abstract class JobRelatedResource extends Resource { if(beanToNameMap == null) { beanToNameMap = new IdentityHashMap(); for(Object entryObj : cj.getJobContext().getBeansOfType(Object.class).entrySet()) { - Map.Entry entry = (Map.Entry)entryObj; + Map.Entry entry = (Map.Entry)entryObj; beanToNameMap.put(entry.getValue(),(String)entry.getKey()); } } @@ -242,7 +416,7 @@ public abstract class JobRelatedResource extends Resource { } } if(object instanceof List) { - List list = (List)object; + List list = (List)object; for(int i = 0; i < list.size(); i++) { if(beanPath!=null) { beanPathPrefix = beanPath+"["; @@ -258,7 +432,7 @@ public abstract class JobRelatedResource extends Resource { if(object instanceof Map) { for (Object next : ((Map)object).entrySet()) { // TODO: protect against giant maps? - Map.Entry entry = (Map.Entry)next; + Map.Entry entry = (Map.Entry)next; if(beanPath!=null) { beanPathPrefix = beanPath+"["; } diff --git a/engine/src/main/java/org/archive/crawler/restlet/JobResource.java b/engine/src/main/java/org/archive/crawler/restlet/JobResource.java index 5215f82f..8e6e88d7 100644 --- a/engine/src/main/java/org/archive/crawler/restlet/JobResource.java +++ b/engine/src/main/java/org/archive/crawler/restlet/JobResource.java @@ -24,8 +24,14 @@ import java.io.IOException; import java.io.PrintWriter; import java.io.Writer; import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; import java.util.LinkedList; import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.logging.Level; +import java.util.logging.Logger; import org.apache.commons.io.filefilter.IOFileFilter; import org.apache.commons.lang.StringEscapeUtils; @@ -33,6 +39,7 @@ import org.apache.commons.lang.StringUtils; import org.archive.checkpointing.Checkpoint; import org.archive.crawler.framework.CrawlJob; import org.archive.crawler.framework.Engine; +import org.archive.crawler.framework.CrawlController.State; import org.archive.crawler.reporting.AlertHandler; import org.archive.crawler.reporting.AlertThreadGroup; import org.archive.crawler.reporting.Report; @@ -45,6 +52,7 @@ import org.restlet.Context; import org.restlet.data.CharacterSet; import org.restlet.data.Form; import org.restlet.data.MediaType; +import org.restlet.data.Reference; import org.restlet.data.Request; import org.restlet.data.Response; import org.restlet.data.Status; @@ -53,6 +61,7 @@ import org.restlet.resource.Resource; import org.restlet.resource.ResourceException; import org.restlet.resource.Variant; import org.restlet.resource.WriterRepresentation; +import org.xml.sax.SAXException; /** * Restlet Resource representing a single local CrawlJob inside an @@ -64,12 +73,15 @@ public class JobResource extends Resource { public static final IOFileFilter EDIT_FILTER = FileUtils.getRegexFileFilter(".*\\.((c?xml)|(txt))$"); - CrawlJob cj; + private static final Logger logger = Logger.getLogger(JobResource.class.getName()); + + protected CrawlJob cj; public JobResource(Context ctx, Request req, Response res) throws ResourceException { super(ctx, req, res); setModifiable(true); getVariants().add(new Variant(MediaType.TEXT_HTML)); + getVariants().add(new Variant(MediaType.APPLICATION_XML)); cj = getEngine().getJob(TextUtils.urlUnescape((String)req.getAttributes().get("job"))); if(cj==null) { throw new ResourceException(404); @@ -77,17 +89,131 @@ public class JobResource extends Resource { } public Representation represent(Variant variant) throws ResourceException { - Representation representation = new WriterRepresentation( - MediaType.TEXT_HTML) { + Representation representation = null; + if (variant.getMediaType() == MediaType.APPLICATION_XML) { + representation = new WriterRepresentation(MediaType.APPLICATION_XML) { + public void write(Writer writer) throws IOException { + try { + new XmlMarshaller(writer).marshalDocument("job", presentablify()); + } catch (SAXException e) { + throw new IOException(e); + } + } + }; + } else { + representation = new WriterRepresentation(MediaType.TEXT_HTML) { public void write(Writer writer) throws IOException { JobResource.this.writeHtml(writer); } }; + } + // TODO: remove if not necessary in future? + // honor requested charset? representation.setCharacterSet(CharacterSet.UTF_8); return representation; } + protected LinkedHashMap presentablify() { + LinkedHashMap info = new LinkedHashMap(); + + String baseRef = getRequest().getResourceRef().getBaseRef().toString(); + if (!baseRef.endsWith("/")) { + baseRef += "/"; + } + Reference baseRefRef = new Reference(baseRef); + + info.put("shortName", cj.getShortName()); + if (cj.getCrawlController() != null) { + info.put("crawlControllerState", cj.getCrawlController().getState()); + if (cj.getCrawlController().getState() == State.FINISHED) { + info.put("crawlExitStatus", cj.getCrawlController().getCrawlExitStatus()); + } + } + info.put("statusDescription", getJobStatusDescription()); + info.put("availableActions", getAvailableActions()); + + info.put("launchCount", cj.getLaunchCount()); + info.put("lastLaunch", cj.getLastLaunch()); + info.put("isProfile", cj.isProfile()); + File primaryConfig = FileUtils.tryToCanonicalize(cj.getPrimaryConfig()); + info.put("primaryConfig", primaryConfig.getAbsolutePath()); + info.put("primaryConfigUrl", baseRef + "jobdir/" + primaryConfig.getName()); + + if (cj.getJobLog().exists()) try { + List logLines = new LinkedList(); + FileUtils.pagedLines(cj.getJobLog(), -1, -5, logLines); + info.put("jobLogTail", logLines); + } catch (IOException ioe) { + throw new RuntimeException(ioe); + } + + if (cj.hasApplicationContext()) { + info.put("uriTotalsReport", cj.uriTotalsReportData()); + info.put("sizeTotalsReport", cj.sizeTotalsReportData()); + info.put("rateReport", cj.rateReportData()); + info.put("loadReport", cj.loadReportData()); + info.put("elapsedReport", cj.elapsedReportData()); + info.put("threadReport", cj.threadReportData()); + info.put("frontierReport", cj.frontierReportData()); + info.put("heapReport", getEngine().heapReportData()); + + if(cj.isRunning() || (cj.hasApplicationContext() && !cj.isLaunchable())) { + try { + List logLines = new LinkedList(); + FileUtils.pagedLines( + cj.getCrawlController().getLoggerModule().getCrawlLogPath().getFile(), + -1, -10, logLines); + info.put("crawlLogTail", logLines); + } catch (IOException ioe) { + throw new RuntimeException(ioe); + } + } + } + + List> configFiles = new LinkedList>(); + for(String cppp: cj.getConfigPaths().keySet()) { + Map configFileInfo = new LinkedHashMap(); + configFileInfo.put("key", cppp); + File path = FileUtils.tryToCanonicalize(cj.getConfigPaths().get(cppp).getFile()); + configFileInfo.put("path", path.getAbsolutePath()); + Reference urlRef = new Reference(baseRefRef, getHrefPath(path, cj)).getTargetRef(); + configFileInfo.put("url", urlRef.toString()); + configFiles.add(configFileInfo); + } + info.put("configFiles", configFiles); + + return info; + } + + protected Set getAvailableActions() { + Set actions = new LinkedHashSet(); + + if (!cj.hasApplicationContext()) { + actions.add("build"); + } + if (!cj.isProfile() && cj.isLaunchable()) { + actions.add("launch"); + } + if (cj.isPausable()) { + actions.add("pause"); + } + if (cj.isUnpausable()) { + actions.add("unpause"); + } + if (cj.getCheckpointService() != null && cj.isRunning()) { + actions.add("checkpoint"); + } + if (cj.isRunning()) { + actions.add("terminate"); + } + if (cj.hasApplicationContext()) { + actions.add("teardown"); + } + + return actions; + } + protected void writeHtml(Writer writer) { PrintWriter pw = new PrintWriter(writer); String jobTitle = cj.getShortName() + " - " @@ -420,7 +546,9 @@ public class JobResource extends Resource { @Override public void acceptRepresentation(Representation entity) throws ResourceException { // copy op? - Form form = getRequest().getEntityAsForm(); + Form form = null; + try { + form = getRequest().getEntityAsForm(); String copyTo = form.getFirstValue("copyTo"); if(copyTo!=null) { copyJob(copyTo,"on".equals(form.getFirstValue("asProfile"))); @@ -460,8 +588,13 @@ public class JobResource extends Resource { cj.terminate(); } AlertThreadGroup.setThreadLogger(null); + // default: redirect to GET self getResponse().redirectSeeOther(getRequest().getOriginalRef()); + } catch (IllegalStateException e) { + logger.log(Level.WARNING, "problem accepting input (redirecting to GET self anyway): " + e, e); + getResponse().setStatus(Status.CLIENT_ERROR_BAD_REQUEST, e, "problem accepting input"); + } } protected void copyJob(String copyTo, boolean asProfile) throws ResourceException { diff --git a/engine/src/main/java/org/archive/crawler/restlet/ScriptResource.java b/engine/src/main/java/org/archive/crawler/restlet/ScriptResource.java index e104bdd7..a3007a52 100644 --- a/engine/src/main/java/org/archive/crawler/restlet/ScriptResource.java +++ b/engine/src/main/java/org/archive/crawler/restlet/ScriptResource.java @@ -23,7 +23,11 @@ import java.io.IOException; import java.io.PrintWriter; import java.io.StringWriter; import java.io.Writer; +import java.util.Collection; +import java.util.LinkedHashMap; import java.util.LinkedList; +import java.util.List; +import java.util.Map; import javax.script.ScriptEngine; import javax.script.ScriptEngineFactory; @@ -37,12 +41,14 @@ import org.restlet.Context; import org.restlet.data.CharacterSet; import org.restlet.data.Form; import org.restlet.data.MediaType; +import org.restlet.data.Reference; import org.restlet.data.Request; import org.restlet.data.Response; import org.restlet.resource.Representation; import org.restlet.resource.ResourceException; import org.restlet.resource.Variant; import org.restlet.resource.WriterRepresentation; +import org.xml.sax.SAXException; /** * Restlet Resource which runs an arbitrary script, which is supplied @@ -72,6 +78,7 @@ public class ScriptResource extends JobRelatedResource { super(ctx, req, res); setModifiable(true); getVariants().add(new Variant(MediaType.TEXT_HTML)); + getVariants().add(new Variant(MediaType.APPLICATION_XML)); } @Override @@ -113,17 +120,110 @@ public class ScriptResource extends JobRelatedResource { } public Representation represent(Variant variant) throws ResourceException { - Representation representation = new WriterRepresentation( - MediaType.TEXT_HTML) { - public void write(Writer writer) throws IOException { - ScriptResource.this.writeHtml(writer); - } - }; + Representation representation; + if (variant.getMediaType() == MediaType.APPLICATION_XML) { + representation = new WriterRepresentation(MediaType.APPLICATION_XML) { + public void write(Writer writer) throws IOException { + try { + new XmlMarshaller(writer).marshalDocument("script", presentablify()); + } catch (SAXException e) { + throw new IOException(e); + } + } + }; + } else { + representation = new WriterRepresentation(MediaType.TEXT_HTML) { + public void write(Writer writer) throws IOException { + ScriptResource.this.writeHtml(writer); + } + }; + } // TODO: remove if not necessary in future? representation.setCharacterSet(CharacterSet.UTF_8); return representation; } + protected List getAvailableActions() { + List actions = new LinkedList(); + actions.add("rescan"); + actions.add("add"); + actions.add("create"); + return actions; + } + + protected Collection> getAvailableScriptEngines() { + List> engines = new LinkedList>(); + for (ScriptEngineFactory f: FACTORIES) { + Map engine = new LinkedHashMap(); + engine.put("engine", f.getNames().get(0)); + engine.put("language", f.getLanguageName()); + engines.add(engine); + } + return engines; + } + + protected Collection> getAvailableGlobalVariables() { + List> vars = new LinkedList>(); + Map var; + + var = new LinkedHashMap(); + var.put("variable", "rawOut"); + var.put("description", "a PrintWriter for arbitrary text output to this page"); + vars.add(var); + + var = new LinkedHashMap(); + var.put("variable", "htmlOut"); + var.put("description", "a PrintWriter for HTML output to this page"); + vars.add(var); + + var = new LinkedHashMap(); + var.put("variable", "job"); + var.put("description", "the current CrawlJob instance"); + vars.add(var); + + var = new LinkedHashMap(); + var.put("variable", "appCtx"); + var.put("description", "current job ApplicationContext, if any"); + vars.add(var); + + var = new LinkedHashMap(); + var.put("variable", "scriptResource"); + var.put("description", "the ScriptResource implementing this page, which offers utility methods"); + vars.add(var); + + return vars; + } + + protected LinkedHashMap presentablify() { + LinkedHashMap info = new LinkedHashMap(); + + String baseRef = getRequest().getResourceRef().getBaseRef().toString(); + if(!baseRef.endsWith("/")) { + baseRef += "/"; + } + Reference baseRefRef = new Reference(baseRef); + + info.put("crawlJobShortName", cj.getShortName()); + info.put("crawlJobUrl", new Reference(baseRefRef, "..").getTargetRef()); + info.put("availableScriptEngines", getAvailableScriptEngines()); + info.put("availableGlobalVariables", getAvailableGlobalVariables()); + + if(linesExecuted>0) { + info.put("linesExecuted", linesExecuted); + } + if(ex!=null) { + info.put("exception", ex); + } + if(StringUtils.isNotBlank(rawOutput)) { + info.put("rawOutput", rawOutput); + } + if(StringUtils.isNotBlank(htmlOutput)) { + info.put("htmlOutput", htmlOutput); + } + + return info; + } + protected void writeHtml(Writer writer) { PrintWriter pw = new PrintWriter(writer); pw.println("Script in "+cj.getShortName()+""); diff --git a/engine/src/main/java/org/archive/crawler/restlet/XmlMarshaller.java b/engine/src/main/java/org/archive/crawler/restlet/XmlMarshaller.java new file mode 100644 index 00000000..951c22e2 --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/restlet/XmlMarshaller.java @@ -0,0 +1,86 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.crawler.restlet; + +import java.io.Writer; +import java.util.Map; + +import org.restlet.util.XmlWriter; +import org.xml.sax.SAXException; + +/** + * @contributor nlevitt + */ +public class XmlMarshaller { + protected XmlWriter xmlWriter; + + public XmlMarshaller(Writer writer) throws SAXException { + xmlWriter = new XmlWriter(writer); + // this looks ok, but makes the response much bigger - pipe to + // "xmllint --format -" instead + // xmlWriter.setDataFormat(true); + // xmlWriter.setIndentStep(2); + } + + public void marshalDocument(String rootTag, Object content) throws SAXException { + xmlWriter.startDocument(); + marshal(rootTag, content); + xmlWriter.endDocument(); // calls flush() + } + + protected void marshal(String key, Object value) throws SAXException { + if (value == null) { + xmlWriter.emptyElement(key); + } else if (value instanceof Map) { + marshal(key, (Map) value); + } else if (value instanceof Iterable) { + marshal(key, (Iterable) value); + } else { + xmlWriter.dataElement(key, value.toString()); + } + } + + protected void marshal(String key, Map map) throws SAXException { + xmlWriter.startElement(key); + for (Map.Entry entry: map.entrySet()) { + marshal(entry.getKey().toString(), entry.getValue()); + } + xmlWriter.endElement(key); + } + + protected void marshal(String key, Iterable iterable) throws SAXException { + xmlWriter.startElement(key); + for (Object item: iterable) { + marshal(item); + } + xmlWriter.endElement(key); + } + + // something we have no name for goes in a tag + protected void marshal(Object item) throws SAXException { + if (item instanceof Map.Entry) { + // if it happens to have a name, use it + Map.Entry entry = (Map.Entry) item; + marshal(entry.getKey().toString(), entry.getValue()); + } else { + marshal("value", item); + } + } +} diff --git a/engine/src/main/java/org/archive/crawler/util/CrawledBytesHistotable.java b/engine/src/main/java/org/archive/crawler/util/CrawledBytesHistotable.java index d95de882..40766cd0 100644 --- a/engine/src/main/java/org/archive/crawler/util/CrawledBytesHistotable.java +++ b/engine/src/main/java/org/archive/crawler/util/CrawledBytesHistotable.java @@ -33,9 +33,9 @@ implements CoreAttributeConstants { public static final String NOTMODIFIED = "not-modified"; public static final String DUPLICATE = "dup-by-hash"; public static final String NOVEL = "novel"; - public static final String NOTMODIFIEDCOUNT="not-modified-count"; - public static final String DUPLICATECOUNT="dup-by-hash-count"; - public static final String NOVELCOUNT="novel-count"; + public static final String NOTMODIFIEDCOUNT = "not-modified-count"; + public static final String DUPLICATECOUNT = "dup-by-hash-count"; + public static final String NOVELCOUNT = "novel-count"; public CrawledBytesHistotable() { super(); diff --git a/modules/src/main/java/org/archive/modules/CrawlURI.java b/modules/src/main/java/org/archive/modules/CrawlURI.java index 708c10da..fb258a95 100644 --- a/modules/src/main/java/org/archive/modules/CrawlURI.java +++ b/modules/src/main/java/org/archive/modules/CrawlURI.java @@ -65,6 +65,7 @@ import java.util.Collection; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; +import java.util.LinkedHashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -1489,6 +1490,15 @@ implements MultiReporter, Serializable, OverlayContext { return ArchiveUtils.singleLineReport(this); } + public Map singleLineReportData() { + Map map = new LinkedHashMap(); + map.put("class", getClass().getName()); + map.put("uri", getUURI().toString()); + map.put("pathFromSeed", pathFromSeed); + map.put("flattenVia", flattenVia()); + return map; + } + public void singleLineReportTo(PrintWriter w) { String className = this.getClass().getName(); className = className.substring(className.lastIndexOf(".")+1); diff --git a/modules/src/main/java/org/archive/modules/ProcessorChain.java b/modules/src/main/java/org/archive/modules/ProcessorChain.java index 13839e55..04715f69 100644 --- a/modules/src/main/java/org/archive/modules/ProcessorChain.java +++ b/modules/src/main/java/org/archive/modules/ProcessorChain.java @@ -2,7 +2,9 @@ package org.archive.modules; import java.io.PrintWriter; import java.util.Iterator; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; import org.archive.spring.HasKeyedProperties; import org.archive.spring.KeyedProperties; @@ -96,6 +98,13 @@ implements Iterable, return ""; } + public Map singleLineReportData() { + Map data = new LinkedHashMap(); + data.put("processorCount", size()); + data.put("processors", getProcessors()); + return data; + } + public void singleLineReportTo(PrintWriter pw) { pw.print(size()); pw.print(" processors: "); diff --git a/modules/src/main/java/org/archive/modules/fetcher/FetchStats.java b/modules/src/main/java/org/archive/modules/fetcher/FetchStats.java index 81e21562..cd19ab96 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/FetchStats.java +++ b/modules/src/main/java/org/archive/modules/fetcher/FetchStats.java @@ -20,6 +20,8 @@ package org.archive.modules.fetcher; import java.io.PrintWriter; import java.io.Serializable; +import java.util.LinkedHashMap; +import java.util.Map; import org.archive.modules.CrawlURI; import org.archive.util.ArchiveUtils; @@ -232,4 +234,18 @@ public class FetchStats implements Serializable, FetchStatusCodes, MultiReporter writer.print(" "); writer.print(fetchNonResponses); } + + public Map singleLineReportData() { + Map map = new LinkedHashMap(); + map.put("totalScheduled", totalScheduled); + map.put("fetchSuccesses", fetchSuccesses); + map.put("fetchFailures", fetchFailures); + map.put("fetchDisregards", fetchDisregards); + map.put("fetchResponses", fetchResponses); + map.put("robotsDenials", robotsDenials); + map.put("successBytes", successBytes); + map.put("totalBytes", totalBytes); + map.put("fetchNonResponses", fetchNonResponses); + return map; + } }