lee/heritrix3
1
0 Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-11-09 07:24:41 +00:00
Code Issues Packages Projects Releases Wiki Activity

Part of [HER-1603] Restletify: xml representations of all resources that also

Browse Source 
offer html
* XmlMarshaller.java
    new class to write xml based on nested map structure
* BeanBrowseResource.java ScriptResource.java EngineResource.java
  JobResource.java JobRelatedResource.java
    support xml representation
* WorkQueueFrontier.java PrecedenceProvider.java WorkQueue.java ToeThread.java
  ToePool.java CrawlJob.java Engine.java Frontier.java CrawlController.java
  StatisticsTracker.java CrawledBytesHistotable.java FetchStats.java
  ProcessorChain.java CrawlURI.java Reporter.java
    offer same data that is returned in the single line reports in a map
    suitable for use with XmlMarshaller
* FileUtils.java
    new method tryToCanonicalize(File)
This commit is contained in:
nlevitt committed 2009-11-25 01:43:57 +00:00
1 parent a1022c8246
commit 8ba641ed4b
24 files changed
+971 -58
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

engine/src/main/java/org/archive/crawler/framework/CrawlController.java
+5 -1
View File
@@ -25,6 +25,7 @@ import java.io.PrintWriter;
import java.io.Serializable;
import java.io.StringWriter;
import java.util.LinkedList;
import java.util.Map;
import java.util.logging.Level;
import java.util.logging.Logger;
@@ -311,7 +312,6 @@ implements Serializable,
* @param newState State change we're to tell listeners' about.
* @param message Message on state change.
*/
@SuppressWarnings("unchecked")
protected void sendCrawlStateChangeEvent(State newState,
CrawlStatus status) {
if(this.state == newState) {
@@ -596,6 +596,10 @@ implements Serializable,
return (toePool == null) ? "" : ArchiveUtils.singleLineReport(toePool);
}
public Map<String,Object> getToeThreadReportShortData() {
return toePool == null ? null : toePool.singleLineReportData();
}
public String getFrontierReportShort() {
return ArchiveUtils.singleLineReport(getFrontier());
engine/src/main/java/org/archive/crawler/framework/CrawlJob.java
+96 -11
View File
@@ -24,6 +24,7 @@ import java.io.IOException;
import java.io.PrintWriter;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
@@ -653,6 +654,21 @@ public class CrawlJob implements Comparable<CrawlJob>, ApplicationListener {
return cc!=null ? cc.getStatisticsTracker() : null;
}
public Map<String,Number> rateReportData() {
StatisticsTracker stats = getStats();
if (stats == null) {
return null;
}
CrawlStatSnapshot snapshot = stats.getSnapshot();
Map<String,Number> map = new LinkedHashMap<String,Number>();
map.put("currentDocsPerSecond", snapshot.currentDocsPerSecond);
map.put("averageDocsPerSecond", snapshot.docsPerSecond);
map.put("currentKiBPerSec", snapshot.currentKiBPerSec);
map.put("averageKiBPerSec", snapshot.totalKiBPerSec);
return map;
}
public Object rateReport() {
StatisticsTracker stats = getStats();
if(stats==null) {
@@ -672,6 +688,23 @@ public class CrawlJob implements Comparable<CrawlJob>, ApplicationListener {
return sb.toString();
}
public Map<String,Number> loadReportData() {
StatisticsTracker stats = getStats();
if (stats == null) {
return null;
}
CrawlStatSnapshot snapshot = stats.getSnapshot();
Map<String,Number> map = new LinkedHashMap<String,Number>();
map.put("busyThreads", snapshot.busyThreads);
map.put("totalThreads", stats.threadCount());
map.put("congestionRatio", snapshot.congestionRatio);
map.put("averageQueueDepth", snapshot.averageDepth);
map.put("deepestQueueDepth", snapshot.deepestUri);
return map;
}
public Object loadReport() {
StatisticsTracker stats = getStats();
if(stats==null) {
@@ -693,32 +726,54 @@ public class CrawlJob implements Comparable<CrawlJob>, ApplicationListener {
return sb.toString();
}
public String uriTotalsReport() {
public Map<String,Long> uriTotalsReportData() {
StatisticsTracker stats = getStats();
if(stats==null) {
if (stats == null) {
return null;
}
CrawlStatSnapshot snapshot = stats.getSnapshot();
Map<String,Long> totals = new LinkedHashMap<String,Long>();
totals.put("downloadedUriCount", snapshot.downloadedUriCount);
totals.put("queuedUriCount", snapshot.queuedUriCount);
totals.put("totalUriCount", snapshot.totalCount());
totals.put("futureUriCount", snapshot.futureUriCount);
return totals;
}
public String uriTotalsReport() {
Map<String,Long> uriTotals = uriTotalsReportData();
if (uriTotals == null) {
return "<i>n/a</i>";
}
CrawlStatSnapshot snapshot = stats.getSnapshot();
long downloaded = snapshot.downloadedUriCount;
long total = snapshot.totalCount();
long queued = snapshot.queuedUriCount;
StringBuilder sb = new StringBuilder(64);
sb
.append(downloaded)
.append(uriTotals.get("downloadedUriCount"))
.append(" downloaded + ")
.append(queued)
.append(uriTotals.get("queuedUriCount"))
.append(" queued = ")
.append(total)
.append(uriTotals.get("totalUriCount"))
.append(" total");
if(snapshot.futureUriCount>0) {
if(uriTotals.get("futureUriCount") >0) {
sb
.append(" (")
.append(snapshot.futureUriCount)
.append(uriTotals.get("futureUriCount"))
.append(" future)");
}
return sb.toString();
}
public Map<String,Long> sizeTotalsReportData() {
StatisticsTracker stats = getStats();
if(stats==null) {
return null;
}
return stats.getCrawledBytes();
}
public String sizeTotalsReport() {
StatisticsTracker stats = getStats();
if(stats==null) {
@@ -727,6 +782,20 @@ public class CrawlJob implements Comparable<CrawlJob>, ApplicationListener {
return stats.crawledBytesSummary();
}
public Map<String,Object> elapsedReportData() {
StatisticsTracker stats = getStats();
if(stats==null) {
return null;
}
Map<String,Object> map = new LinkedHashMap<String,Object>();
long timeElapsed = stats.getCrawlElapsedTime();
map.put("elapsedMilliseconds", timeElapsed);
map.put("elapsedPretty", ArchiveUtils.formatMillisecondsToConventional(timeElapsed));
return map;
}
public String elapsedReport() {
StatisticsTracker stats = getStats();
if(stats==null) {
@@ -736,6 +805,14 @@ public class CrawlJob implements Comparable<CrawlJob>, ApplicationListener {
return ArchiveUtils.formatMillisecondsToConventional(timeElapsed,false);
}
public Map<String,Object> threadReportData() {
CrawlController cc = getCrawlController();
if (cc == null) {
return null;
}
return cc.getToeThreadReportShortData();
}
public String threadReport() {
CrawlController cc = getCrawlController();
if(cc==null) {
@@ -744,6 +821,14 @@ public class CrawlJob implements Comparable<CrawlJob>, ApplicationListener {
return cc.getToeThreadReportShort();
}
public Map<String,Object> frontierReportData() {
CrawlController cc = getCrawlController();
if (cc == null) {
return null;
}
return cc.getFrontier().singleLineReportData();
}
public String frontierReport() {
CrawlController cc = getCrawlController();
if(cc==null) {
engine/src/main/java/org/archive/crawler/framework/Engine.java
+9 -1
View File
@@ -24,6 +24,7 @@ import java.io.FilenameFilter;
import java.io.IOException;
import java.io.InputStream;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.logging.Level;
import java.util.logging.Logger;
@@ -249,6 +250,14 @@ public class Engine {
return jobsDir;
}
public Map<String,Object> heapReportData() {
Map<String,Object> map = new LinkedHashMap<String, Object>();
map.put("usedBytes", Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory());
map.put("totalBytes", Runtime.getRuntime().totalMemory());
map.put("maxBytes", Runtime.getRuntime().maxMemory());
return map;
}
public String heapReport() {
long totalMemory = Runtime.getRuntime().totalMemory();
long freeMemory = Runtime.getRuntime().freeMemory();
@@ -348,5 +357,4 @@ public class Engine {
LOGGER.log(Level.SEVERE,"job: "+jobName+" not found in jobConfig!");
}
}
}
engine/src/main/java/org/archive/crawler/framework/Frontier.java
+2 -1
View File
@@ -517,6 +517,7 @@ public interface Frontier extends Lifecycle, MultiReporter {
HOLD, // enter a consistent, stable, checkpointable state ASAP
PAUSE, // enter a stable state where no URIs are in-progress; unlike
// HOLD requires all in-process URIs to complete
FINISH }; // end and cleanup; may not return to any other state after
FINISH // end and cleanup; may not return to any other state after
// this state is requested/reached
}
}
engine/src/main/java/org/archive/crawler/framework/ToePool.java
+30
View File
@@ -21,9 +21,11 @@ package org.archive.crawler.framework;
import java.io.PrintWriter;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.TreeSet;
import org.apache.commons.lang.StringUtils;
import org.archive.crawler.reporting.AlertThreadGroup;
import org.archive.util.ArchiveUtils;
import org.archive.util.Histotable;
@@ -259,6 +261,34 @@ public class ToePool extends ThreadGroup implements MultiReporter {
}
}
public Map<String, Object> singleLineReportData() {
Histotable<Object> steps = new Histotable<Object>();
Histotable<Object> processors = new Histotable<Object>();
Thread[] toes = getToes();
for (int i = 0; i < toes.length; i++) {
if(!(toes[i] instanceof ToeThread)) {
continue;
}
ToeThread tt = (ToeThread)toes[i];
if(tt!=null) {
steps.tally(tt.getStep().toString());
String currentProcessorName = tt.getCurrentProcessorName();
if (StringUtils.isEmpty(currentProcessorName)) {
currentProcessorName = "noActiveProcessor";
}
processors.tally(currentProcessorName);
}
}
Map<String,Object> data = new LinkedHashMap<String, Object>();
data.put("toeCount", getToeCount());
data.put("steps", steps.getSortedByCounts());
data.put("processors", processors.getSortedByCounts());
return data;
}
public void singleLineReportTo(PrintWriter w) {
Histotable<Object> steps = new Histotable<Object>();
Histotable<Object> processors = new Histotable<Object>();
engine/src/main/java/org/archive/crawler/framework/ToeThread.java
+29
View File
@@ -29,6 +29,8 @@ import java.lang.management.ManagementFactory;
import java.lang.management.ThreadInfo;
import java.lang.management.ThreadMXBean;
import java.net.InetAddress;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.logging.Level;
import java.util.logging.Logger;
@@ -479,6 +481,33 @@ implements RecorderMarker, MultiReporter, ProgressStatisticsReporter,
}
}
public Map<String, Object> singleLineReportData() {
Map<String,Object> data = new LinkedHashMap<String, Object>();
data.put("serialNumber", serialNumber);
CrawlURI c = currentCuri;
if (c != null) {
data.put("currentURI", c.toString());
data.put("currentProcessor", currentProcessorName);
data.put("fetchAttempts", c.getFetchAttempts());
} else {
data.put("currentURI", null);
}
long now = System.currentTimeMillis();
long time = 0;
if (lastFinishTime > lastStartTime) {
data.put("status", "WAITING");
time = now - lastFinishTime;
} else if (lastStartTime > 0) {
data.put("status", "ACTIVE");
time = now - lastStartTime;
}
data.put("currentStatusElapsedMilliseconds", time);
data.put("currentStatusElapsedPretty", ArchiveUtils.formatMillisecondsToConventional(time));
data.put("step", step);
return data;
}
/**
* @param w PrintWriter to write to.
*/
engine/src/main/java/org/archive/crawler/frontier/WorkQueue.java
+32
View File
@@ -22,7 +22,10 @@ package org.archive.crawler.frontier;
import java.io.IOException;
import java.io.PrintWriter;
import java.io.Serializable;
import java.util.Date;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.Delayed;
import java.util.concurrent.TimeUnit;
@@ -526,6 +529,35 @@ public abstract class WorkQueue implements Frontier.FrontierGroup,
reportTo(null,writer);
}
public Map<String, Object> singleLineReportData() {
Map<String,Object> map = new LinkedHashMap<String, Object>();
map.put("queueName", classKey);
map.put("precedence", getPrecedence());
map.put("itemCount", count);
map.put("enqueueCount", enqueueCount);
map.put("sessionBalance", sessionBalance);
map.put("lastCost", lastCost);
map.put("averageCost", (double) totalExpenditure / costCount);
if (lastDequeueTime != 0) {
map.put("lastDequeueTime", new Date(lastDequeueTime));
} else {
map.put("lastDequeueTime", null);
}
if (wakeTime != 0) {
map.put("lastDequeueTime", new Date(wakeTime));
} else {
map.put("lastDequeueTime", null);
}
map.put("totalExpenditure", totalExpenditure);
map.put("totalBudget", totalBudget);
map.put("errorCount", errorCount);
map.put("lastPeeked", lastPeeked);
map.put("lastQueued", lastQueued);
return map;
}
/* (non-Javadoc)
* @see org.archive.util.Reporter#singleLineReportTo(java.io.Writer)
*/
engine/src/main/java/org/archive/crawler/frontier/WorkQueueFrontier.java
+35
View File
@@ -35,6 +35,7 @@ import java.lang.ref.SoftReference;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Queue;
import java.util.SortedMap;
@@ -1087,6 +1088,40 @@ implements Closeable,
return REPORTS;
}
public Map<String, Object> singleLineReportData() {
if (this.allQueues == null) {
return null;
}
int allCount = allQueues.size();
int inProcessCount = inProcessQueues.uniqueSet().size();
int readyCount = readyClassQueues.size();
int snoozedCount = snoozedClassQueues.size();
int activeCount = inProcessCount + readyCount + snoozedCount;
int inactiveCount = getTotalEligibleInactiveQueues();
int ineligibleCount = getTotalIneligibleInactiveQueues();
int retiredCount = getRetiredQueues().size();
int exhaustedCount = allCount - activeCount - inactiveCount - retiredCount;
int inCount = inbound.size();
int outCount = outbound.size();
Map<String,Object> map = new LinkedHashMap<String, Object>();
map.put("totalQueues", allCount);
map.put("inProcessQueues", inProcessCount);
map.put("readyQueues", readyCount);
map.put("snoozedQueues", snoozedCount);
map.put("activeQueues", activeCount);
map.put("inactiveQueues", inactiveCount);
map.put("ineligibleQueues", ineligibleCount);
map.put("retiredQueues", retiredCount);
map.put("exhaustedQueues", exhaustedCount);
map.put("lastReachedState", lastReachedState);
map.put("inboundCount", inCount);
map.put("outboundCount", outCount);
return map;
}
/**
* @param w Where to write to.
*/
engine/src/main/java/org/archive/crawler/frontier/precedence/PrecedenceProvider.java
+10
View File
@@ -20,6 +20,8 @@ package org.archive.crawler.frontier.precedence;
import java.io.PrintWriter;
import java.io.Serializable;
import java.util.LinkedHashMap;
import java.util.Map;
import org.archive.modules.CrawlURI;
import org.archive.modules.fetcher.FetchStats;
@@ -34,6 +36,8 @@ import org.archive.util.MultiReporter;
abstract public class PrecedenceProvider implements MultiReporter,
FetchStats.CollectsFetchStats, Serializable {
private static final long serialVersionUID = 1L;
abstract public int getPrecedence();
/* (non-Javadoc)
@@ -72,6 +76,12 @@ FetchStats.CollectsFetchStats, Serializable {
return ArchiveUtils.singleLineReport(this);
}
public Map<String, Object> singleLineReportData() {
Map<String,Object> data = new LinkedHashMap<String, Object>();
data.put("precedence", getPrecedence());
return data;
}
public void singleLineReportTo(PrintWriter writer) {
writer.print(getPrecedence());
}
engine/src/main/java/org/archive/crawler/reporting/StatisticsTracker.java
+4
View File
@@ -306,6 +306,10 @@ public class StatisticsTracker
/** tally sizes novel, verified (same hash), vouched (not-modified) */
CrawledBytesHistotable crawledBytes = new CrawledBytesHistotable();
public CrawledBytesHistotable getCrawledBytes() {
return crawledBytes;
}
// TODO: fortify these against key explosion with bigmaps like other tallies
/** Keep track of the file types we see (mime type -> count) */
protected ObjectIdentityCache<String,AtomicLong> mimeTypeDistribution
engine/src/main/java/org/archive/crawler/restlet/BeanBrowseResource.java
+75 -13
View File
@@ -26,6 +26,8 @@ import java.io.Writer;
import java.net.URLDecoder;
import java.util.Collection;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.LinkedList;
import java.util.Set;
import org.apache.commons.lang.StringUtils;
@@ -45,6 +47,7 @@ import org.restlet.resource.Variant;
import org.restlet.resource.WriterRepresentation;
import org.springframework.beans.BeanWrapperImpl;
import org.springframework.beans.BeansException;
import org.xml.sax.SAXException;
/**
* Restlet Resource which allows browsing the constructed beans in
@@ -59,6 +62,7 @@ public class BeanBrowseResource extends JobRelatedResource {
public BeanBrowseResource(Context ctx, Request req, Response res) throws ResourceException {
super(ctx, req, res);
getVariants().add(new Variant(MediaType.TEXT_HTML));
getVariants().add(new Variant(MediaType.APPLICATION_XML));
setModifiable(true); // accept POSTs
appCtx = cj.getJobContext();
beanPath = (String)req.getAttributes().get("beanPath");
@@ -71,9 +75,11 @@ public class BeanBrowseResource extends JobRelatedResource {
} else {
beanPath = "";
}
if(appCtx==null) {
if (appCtx == null) {
getResponse().setStatus(Status.CLIENT_ERROR_NOT_FOUND);
getResponse().setEntity("No beans; crawl not yet built.",MediaType.TEXT_HTML);
// XXX xml?
getResponse().setEntity("No beans; crawl not yet built.", MediaType.TEXT_HTML);
//throw new ResourceException(404);
}
}
@@ -85,7 +91,6 @@ public class BeanBrowseResource extends JobRelatedResource {
String newVal = form.getFirstValue("newVal");
if(newVal!=null) {
int i = beanPath.indexOf(".");
String beanName = i<0?beanPath:beanPath.substring(0,i);
Object namedBean = appCtx.getBean(beanName);
@@ -93,13 +98,11 @@ public class BeanBrowseResource extends JobRelatedResource {
String propPath = beanPath.substring(i+1);
Object coercedVal = bwrap.convertIfNecessary(newVal, bwrap.getPropertyValue(propPath).getClass());
bwrap.setPropertyValue(propPath, coercedVal);
}
Reference ref = getRequest().getResourceRef();
ref.setPath(getBeansRefPath());
ref.addSegment(beanPath);
getResponse().redirectSeeOther(ref);
}
public String getBeansRefPath() {
@@ -114,20 +117,79 @@ public class BeanBrowseResource extends JobRelatedResource {
}
return path;
}
public Representation represent(Variant variant) throws ResourceException {
Representation representation = new WriterRepresentation(
MediaType.TEXT_HTML) {
public void write(Writer writer) throws IOException {
BeanBrowseResource.this.writeHtml(writer);
}
};
Representation representation;
if (variant.getMediaType() == MediaType.APPLICATION_XML) {
representation = new WriterRepresentation(MediaType.APPLICATION_XML) {
public void write(Writer writer) throws IOException {
try {
new XmlMarshaller(writer).marshalDocument("script", presentablify());
} catch (SAXException e) {
throw new IOException(e);
}
}
};
} else {
representation = new WriterRepresentation(
MediaType.TEXT_HTML) {
public void write(Writer writer) throws IOException {
BeanBrowseResource.this.writeHtml(writer);
}
};
}
// TODO: remove if not necessary in future?
representation.setCharacterSet(CharacterSet.UTF_8);
return representation;
}
protected LinkedHashMap<String,Object> presentablify() {
LinkedHashMap<String,Object> info = new LinkedHashMap<String,Object>();
info.put("crawlJobShortName", cj.getShortName());
info.put("crawlJobUrl", new Reference(getRequest().getResourceRef().getBaseRef(), "..").getTargetRef());
if (StringUtils.isNotBlank(beanPath)) {
info.put("beanPath", beanPath);
try {
int firstDot = beanPath.indexOf(".");
String beanName = firstDot<0?beanPath:beanPath.substring(0,firstDot);
Object namedBean = appCtx.getBean(beanName);
Object target;
if (firstDot < 0) {
target = namedBean;
info.put("bean", presentablifyObject(null, target, beanPath));
} else {
BeanWrapperImpl bwrap = new BeanWrapperImpl(namedBean);
String propPath = beanPath.substring(firstDot+1);
target = bwrap.getPropertyValue(propPath);
Class<?> type = bwrap.getPropertyType(propPath);
if(bwrap.isWritableProperty(propPath)
&& (bwrap.getDefaultEditor(type)!=null|| type == String.class)
&& !Collection.class.isAssignableFrom(type)) {
info.put("editable", true);
info.put("bean", presentablifyObject(null, target));
} else {
info.put("bean", presentablifyObject(null, target, beanPath));
}
}
} catch (BeansException e) {
info.put("problem", e.toString());
}
}
Collection<Object> nestedNames = new LinkedList<Object>();
Set<Object> alreadyWritten = new HashSet<Object>();
presentablifyNestedNames(appCtx.getBean("crawlController"), getBeansRefPath(), alreadyWritten, nestedNames);
for(String name: appCtx.getBeanDefinitionNames()) {
presentablifyNestedNames(appCtx.getBean(name), getBeansRefPath(), alreadyWritten, nestedNames);
}
info.put("allNamedCrawlBeans", nestedNames);
return info;
}
protected void writeHtml(Writer writer) {
PrintWriter pw = new PrintWriter(writer);
pw.println("<head><title>Crawl beans in "+cj.getShortName()+"</title></head>");
@@ -153,7 +215,7 @@ public class BeanBrowseResource extends JobRelatedResource {
String propPath = beanPath.substring(i+1);
target = bwrap.getPropertyValue(propPath);
Class type = bwrap.getPropertyType(propPath);
Class<?> type = bwrap.getPropertyType(propPath);
if(bwrap.isWritableProperty(propPath)
&& (bwrap.getDefaultEditor(type)!=null|| type == String.class)
&& !Collection.class.isAssignableFrom(type)) {
engine/src/main/java/org/archive/crawler/restlet/EditRepresentation.java
+1 -1
View File
@@ -72,7 +72,7 @@ public class EditRepresentation extends CharacterRepresentation {
viewRef.setQuery(null);
pw.println("<a href='"+viewRef+"'>view</a>");
pw.println("<br/>");
pw.println("<textarea style='width:100%; height:100%' name='contents'>");
pw.println("<textarea style='width:100%;height:94%;background-color:#ffc;font-size:small;' name='contents'>");
StringEscapeUtils.escapeHtml(pw,fileRepresentation.getText());
pw.println("</textarea></form>");
pw.close();
engine/src/main/java/org/archive/crawler/restlet/EngineResource.java
+81 -12
View File
@@ -24,12 +24,19 @@ import java.io.IOException;
import java.io.PrintWriter;
import java.io.Writer;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import org.apache.commons.lang.StringUtils;
import org.archive.crawler.framework.CrawlJob;
import org.archive.crawler.framework.Engine;
import org.archive.crawler.framework.CrawlController.State;
import org.archive.util.FileUtils;
import org.restlet.Context;
import org.restlet.data.CharacterSet;
import org.restlet.data.Form;
@@ -41,6 +48,7 @@ import org.restlet.resource.Resource;
import org.restlet.resource.ResourceException;
import org.restlet.resource.Variant;
import org.restlet.resource.WriterRepresentation;
import org.xml.sax.SAXException;
/**
* Restlet Resource representing an Engine that may be used
@@ -54,15 +62,28 @@ public class EngineResource extends Resource {
super(ctx, req, res);
setModifiable(true);
getVariants().add(new Variant(MediaType.TEXT_HTML));
getVariants().add(new Variant(MediaType.APPLICATION_XML));
}
public Representation represent(Variant variant) throws ResourceException {
Representation representation = new WriterRepresentation(
MediaType.TEXT_HTML) {
public void write(Writer writer) throws IOException {
EngineResource.this.writeHtml(writer);
}
};
Representation representation;
if (variant.getMediaType() == MediaType.APPLICATION_XML) {
representation = new WriterRepresentation(MediaType.APPLICATION_XML) {
public void write(Writer writer) throws IOException {
try {
new XmlMarshaller(writer).marshalDocument("engine", presentablify());
} catch (SAXException e) {
throw new IOException(e);
}
}
};
} else {
representation = new WriterRepresentation(MediaType.TEXT_HTML) {
public void write(Writer writer) throws IOException {
EngineResource.this.writeHtml(writer);
}
};
}
// TODO: remove if not necessary in future?
representation.setCharacterSet(CharacterSet.UTF_8);
return representation;
@@ -126,6 +147,59 @@ public class EngineResource extends Resource {
getResponse().redirectSeeOther(getRequest().getOriginalRef());
}
protected List<String> getAvailableActions() {
List<String> actions = new LinkedList<String>();
actions.add("rescan");
actions.add("add");
actions.add("create");
return actions;
}
protected LinkedHashMap<String,Object> presentablify() {
String baseRef = getRequest().getResourceRef().getBaseRef().toString();
if(!baseRef.endsWith("/")) {
baseRef += "/";
}
LinkedHashMap<String,Object> info = new LinkedHashMap<String,Object>();
Engine engine = getEngine();
info.put("heritrixVersion", engine.getHeritrixVersion());
File jobsDir = FileUtils.tryToCanonicalize(engine.getJobsDir());
info.put("jobsDir", jobsDir.getAbsolutePath());
info.put("jobsDirUrl", baseRef + "jobsdir/");
info.put("availableActions", getAvailableActions());
info.put("heapReport", getEngine().heapReportData());
// re-scan job configs on each page load
ArrayList<CrawlJob> jobs = new ArrayList<CrawlJob>();
engine.findJobConfigs();
jobs.addAll(engine.getJobConfigs().values());
Collections.sort(jobs);
Collection<Map<String,Object>> jobsInfo = new LinkedList<Map<String,Object>>();
for (CrawlJob cj: jobs) {
// cj.writeHtmlTo(pw,"job/");
Map<String,Object> jobInfo = new LinkedHashMap<String, Object>();
jobInfo.put("shortName", cj.getShortName());
jobInfo.put("url", baseRef + "job/" + cj.getShortName());
jobInfo.put("isProfile", cj.isProfile());
jobInfo.put("launchCount", cj.getLaunchCount());
jobInfo.put("lastLaunch", cj.getLastLaunch());
File primaryConfig = FileUtils.tryToCanonicalize(cj.getPrimaryConfig());
jobInfo.put("primaryConfig", primaryConfig.getAbsolutePath());
jobInfo.put("primaryConfigUrl", baseRef + "job/" + cj.getShortName() + "/jobdir/" + primaryConfig.getName());
if (cj.getCrawlController() != null) {
jobInfo.put("crawlControllerState", cj.getCrawlController().getState());
if (cj.getCrawlController().getState() == State.FINISHED) {
jobInfo.put("crawlExitStatus", cj.getCrawlController().getCrawlExitStatus());
}
}
jobsInfo.add(jobInfo);
}
info.put("jobs", jobsInfo);
return info;
}
/**
* wraps a message in a styled div given messsage type
* @param msg message to be displayed
@@ -152,12 +226,7 @@ public class EngineResource extends Resource {
protected void writeHtml(Writer writer) {
Engine engine = getEngine();
String engineTitle = "Heritrix Engine "+engine.getHeritrixVersion();
File jobsDir = engine.getJobsDir();
try {
jobsDir = jobsDir.getCanonicalFile();
} catch (IOException ioe) {
// live with the noncanonical file
}
File jobsDir = FileUtils.tryToCanonicalize(engine.getJobsDir());
String baseRef = getRequest().getResourceRef().getBaseRef().toString();
if(!baseRef.endsWith("/")) {
baseRef += "/";
engine/src/main/java/org/archive/crawler/restlet/JobRelatedResource.java
+178 -4
View File
@@ -24,8 +24,11 @@ import java.beans.PropertyDescriptor;
import java.io.File;
import java.io.PrintWriter;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashSet;
import java.util.IdentityHashMap;
import java.util.LinkedHashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -34,6 +37,7 @@ import org.apache.commons.lang.StringUtils;
import org.archive.crawler.framework.CrawlJob;
import org.archive.crawler.framework.Engine;
import org.restlet.Context;
import org.restlet.data.Reference;
import org.restlet.data.Request;
import org.restlet.data.Response;
import org.restlet.resource.Resource;
@@ -64,6 +68,55 @@ public abstract class JobRelatedResource extends Resource {
return ((EngineApplication)getApplication()).getEngine();
}
protected void presentablifyNestedNames(Object obj, String prefix,
Set<Object> alreadyWritten, Collection<Object> namedBeans) {
if (obj == null || alreadyWritten.contains(obj)
|| obj.getClass().getName().startsWith("org.springframework.")) {
return;
}
Reference baseRef = getRequest().getResourceRef().getBaseRef();
if (getBeanToNameMap().containsKey(obj)) {
// this object is itself a named bean
Map<String, Object> bean = new LinkedHashMap<String, Object>();
bean.put("name", getBeanToNameMap().get(obj));
bean.put("url", new Reference(baseRef, "../beans/" + getBeanToNameMap().get(obj)).getTargetRef());
bean.put("class", obj.getClass().getName());
namedBeans.add(bean);
// nest children
namedBeans = new LinkedList<Object>();
bean.put("children", namedBeans);
}
if (!alreadyWritten.contains(obj)) {
alreadyWritten.add(obj);
BeanWrapperImpl bwrap = new BeanWrapperImpl(obj);
for (PropertyDescriptor pd : bwrap.getPropertyDescriptors()) {
if (pd.getReadMethod() != null) {
String propName = pd.getName();
Object propValue = bwrap.getPropertyValue(propName);
presentablifyNestedNames(propValue, prefix, alreadyWritten, namedBeans);
}
}
if (obj.getClass().isArray()) {
List<?> list = Arrays.asList(obj);
for (int i = 0; i < list.size(); i++) {
presentablifyNestedNames(list.get(i), prefix,
alreadyWritten, namedBeans);
}
}
if (obj instanceof Iterable<?>) {
for (Object next : (Iterable<?>) obj) {
presentablifyNestedNames(next, prefix, alreadyWritten, namedBeans);
}
}
}
}
/**
* Starting at (and including) the given object, write a hierarchical
* list of named beans in HTML to the PrintWriter.
@@ -99,7 +152,7 @@ public abstract class JobRelatedResource extends Resource {
}
}
if(obj.getClass().isArray()) {
List list = Arrays.asList(obj);
List<?> list = Arrays.asList(obj);
for(int i = 0; i < list.size(); i++) {
writeNestedNames(pw, list.get(i), prefix, alreadyWritten);
}
@@ -137,6 +190,127 @@ public abstract class JobRelatedResource extends Resource {
writeObject(pw, field, object, new HashSet<Object>(), beanPath);
}
protected Map<String, Object> presentablifyObject(String field, Object object) {
return presentablifyObject(field, object, new HashSet<Object>(), null);
}
protected Map<String, Object> presentablifyObject(String field, Object object, String beanPath) {
return presentablifyObject(field, object, new HashSet<Object>(), beanPath);
}
protected Map<String, Object> presentablifyObject(String field, Object object, HashSet<Object> alreadyWritten, String beanPathPrefix) {
Map<String,Object> info = new LinkedHashMap<String, Object>();
Reference baseRef = getRequest().getResourceRef().getBaseRef();
String beanPath = beanPathPrefix;
if(StringUtils.isNotBlank(field)) {
info.put("field", field);
if(StringUtils.isNotBlank(beanPathPrefix)) {
if(beanPathPrefix.endsWith(".")) {
beanPath += field;
} else if (beanPathPrefix.endsWith("[")) {
beanPath += field + "]";
}
info.put("url", new Reference(baseRef, "../beans/" + beanPath).getTargetRef());
}
}
String key = getBeanToNameMap().get(object);
if (object == null) {
info.put("propValue", null);
return info;
}
if (object instanceof String || BeanUtils.isSimpleValueType(object.getClass()) || object instanceof File) {
info.put("class", object.getClass().getName());
info.put("propValue", object);
return info;
}
if (alreadyWritten.contains(object)) {
info.put("propValuePreviouslyDescribed", null);
return info;
}
alreadyWritten.add(object); // guard against repeats and cycles
if (StringUtils.isNotBlank(key) && StringUtils.isNotBlank(field)) {
info.put("key", key);
info.put("url", new Reference(baseRef, "../beans/" + key).getTargetRef());
return info;
}
info.put("class", object.getClass().getName());
Collection<Object> properties = new LinkedList<Object>();
BeanWrapperImpl bwrap = new BeanWrapperImpl(object);
for (PropertyDescriptor pd : bwrap.getPropertyDescriptors()) {
if (object instanceof DescriptorUpdater) {
((DescriptorUpdater) object).updateDescriptor(pd);
} else {
defaultUpdateDescriptor(pd);
}
if (pd.getReadMethod() != null && !pd.isHidden()) {
String propName = pd.getName();
if (beanPath != null) {
beanPathPrefix = beanPath + ".";
}
Object propValue = presentablifyObject(propName,
bwrap.getPropertyValue(propName),
alreadyWritten, beanPathPrefix);
properties.add(propValue);
}
}
if (properties.size() > 0) {
info.put("properties", properties);
}
Collection<Object> propValues = new LinkedList<Object>();
if(object.getClass().isArray()) {
Object[] array = (Object[])object;
for(int i = 0; i < array.length; i++) {
if(beanPath!=null) {
beanPathPrefix = beanPath+"[";
}
// TODO: protect against overlong content?
propValues.add(presentablifyObject(i + "", array[i],
alreadyWritten, beanPathPrefix));
}
}
if (object instanceof List<?>) {
List<?> list = (List<?>) object;
for (int i = 0; i < list.size(); i++) {
if (beanPath != null) {
beanPathPrefix = beanPath + "[";
}
// TODO: protect against overlong content?
propValues.add(presentablifyObject(i + "", list.get(i),
alreadyWritten, beanPathPrefix));
}
} else if (object instanceof Iterable<?>) {
for (Object next : (Iterable<?>) object) {
propValues.add(presentablifyObject("#", next, alreadyWritten,
beanPathPrefix));
}
}
if (object instanceof Map<?,?>) {
for (Object next : ((Map<?,?>) object).entrySet()) {
// TODO: protect against giant maps?
Map.Entry<?, ?> entry = (Map.Entry<?, ?>) next;
if (beanPath != null) {
beanPathPrefix = beanPath + "[";
}
propValues.add(presentablifyObject(entry.getKey().toString(),
entry.getValue(), alreadyWritten, beanPathPrefix));
}
}
if (propValues.size() > 0) {
info.put("propValue", propValues);
}
return info;
}
/**
* Get a map giving object beanNames.
*
@@ -146,7 +320,7 @@ public abstract class JobRelatedResource extends Resource {
if(beanToNameMap == null) {
beanToNameMap = new IdentityHashMap<Object, String>();
for(Object entryObj : cj.getJobContext().getBeansOfType(Object.class).entrySet()) {
Map.Entry entry = (Map.Entry)entryObj;
Map.Entry<?, ?> entry = (Map.Entry<?, ?>)entryObj;
beanToNameMap.put(entry.getValue(),(String)entry.getKey());
}
}
@@ -242,7 +416,7 @@ public abstract class JobRelatedResource extends Resource {
}
}
if(object instanceof List) {
List list = (List)object;
List<?> list = (List<?>)object;
for(int i = 0; i < list.size(); i++) {
if(beanPath!=null) {
beanPathPrefix = beanPath+"[";
@@ -258,7 +432,7 @@ public abstract class JobRelatedResource extends Resource {
if(object instanceof Map) {
for (Object next : ((Map)object).entrySet()) {
// TODO: protect against giant maps?
Map.Entry entry = (Map.Entry)next;
Map.Entry<?, ?> entry = (Map.Entry<?, ?>)next;
if(beanPath!=null) {
beanPathPrefix = beanPath+"[";
}
engine/src/main/java/org/archive/crawler/restlet/JobResource.java
+137 -4
View File
@@ -24,8 +24,14 @@ import java.io.IOException;
import java.io.PrintWriter;
import java.io.Writer;
import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.apache.commons.io.filefilter.IOFileFilter;
import org.apache.commons.lang.StringEscapeUtils;
@@ -33,6 +39,7 @@ import org.apache.commons.lang.StringUtils;
import org.archive.checkpointing.Checkpoint;
import org.archive.crawler.framework.CrawlJob;
import org.archive.crawler.framework.Engine;
import org.archive.crawler.framework.CrawlController.State;
import org.archive.crawler.reporting.AlertHandler;
import org.archive.crawler.reporting.AlertThreadGroup;
import org.archive.crawler.reporting.Report;
@@ -45,6 +52,7 @@ import org.restlet.Context;
import org.restlet.data.CharacterSet;
import org.restlet.data.Form;
import org.restlet.data.MediaType;
import org.restlet.data.Reference;
import org.restlet.data.Request;
import org.restlet.data.Response;
import org.restlet.data.Status;
@@ -53,6 +61,7 @@ import org.restlet.resource.Resource;
import org.restlet.resource.ResourceException;
import org.restlet.resource.Variant;
import org.restlet.resource.WriterRepresentation;
import org.xml.sax.SAXException;
/**
* Restlet Resource representing a single local CrawlJob inside an
@@ -64,12 +73,15 @@ public class JobResource extends Resource {
public static final IOFileFilter EDIT_FILTER =
FileUtils.getRegexFileFilter(".*\\.((c?xml)|(txt))$");
CrawlJob cj;
private static final Logger logger = Logger.getLogger(JobResource.class.getName());
protected CrawlJob cj;
public JobResource(Context ctx, Request req, Response res) throws ResourceException {
super(ctx, req, res);
setModifiable(true);
getVariants().add(new Variant(MediaType.TEXT_HTML));
getVariants().add(new Variant(MediaType.APPLICATION_XML));
cj = getEngine().getJob(TextUtils.urlUnescape((String)req.getAttributes().get("job")));
if(cj==null) {
throw new ResourceException(404);
@@ -77,17 +89,131 @@ public class JobResource extends Resource {
}
public Representation represent(Variant variant) throws ResourceException {
Representation representation = new WriterRepresentation(
MediaType.TEXT_HTML) {
Representation representation = null;
if (variant.getMediaType() == MediaType.APPLICATION_XML) {
representation = new WriterRepresentation(MediaType.APPLICATION_XML) {
public void write(Writer writer) throws IOException {
try {
new XmlMarshaller(writer).marshalDocument("job", presentablify());
} catch (SAXException e) {
throw new IOException(e);
}
}
};
} else {
representation = new WriterRepresentation(MediaType.TEXT_HTML) {
public void write(Writer writer) throws IOException {
JobResource.this.writeHtml(writer);
}
};
}
// TODO: remove if not necessary in future?
// honor requested charset?
representation.setCharacterSet(CharacterSet.UTF_8);
return representation;
}
protected LinkedHashMap<String,Object> presentablify() {
LinkedHashMap<String,Object> info = new LinkedHashMap<String,Object>();
String baseRef = getRequest().getResourceRef().getBaseRef().toString();
if (!baseRef.endsWith("/")) {
baseRef += "/";
}
Reference baseRefRef = new Reference(baseRef);
info.put("shortName", cj.getShortName());
if (cj.getCrawlController() != null) {
info.put("crawlControllerState", cj.getCrawlController().getState());
if (cj.getCrawlController().getState() == State.FINISHED) {
info.put("crawlExitStatus", cj.getCrawlController().getCrawlExitStatus());
}
}
info.put("statusDescription", getJobStatusDescription());
info.put("availableActions", getAvailableActions());
info.put("launchCount", cj.getLaunchCount());
info.put("lastLaunch", cj.getLastLaunch());
info.put("isProfile", cj.isProfile());
File primaryConfig = FileUtils.tryToCanonicalize(cj.getPrimaryConfig());
info.put("primaryConfig", primaryConfig.getAbsolutePath());
info.put("primaryConfigUrl", baseRef + "jobdir/" + primaryConfig.getName());
if (cj.getJobLog().exists()) try {
List<String> logLines = new LinkedList<String>();
FileUtils.pagedLines(cj.getJobLog(), -1, -5, logLines);
info.put("jobLogTail", logLines);
} catch (IOException ioe) {
throw new RuntimeException(ioe);
}
if (cj.hasApplicationContext()) {
info.put("uriTotalsReport", cj.uriTotalsReportData());
info.put("sizeTotalsReport", cj.sizeTotalsReportData());
info.put("rateReport", cj.rateReportData());
info.put("loadReport", cj.loadReportData());
info.put("elapsedReport", cj.elapsedReportData());
info.put("threadReport", cj.threadReportData());
info.put("frontierReport", cj.frontierReportData());
info.put("heapReport", getEngine().heapReportData());
if(cj.isRunning() || (cj.hasApplicationContext() && !cj.isLaunchable())) {
try {
List<String> logLines = new LinkedList<String>();
FileUtils.pagedLines(
cj.getCrawlController().getLoggerModule().getCrawlLogPath().getFile(),
-1, -10, logLines);
info.put("crawlLogTail", logLines);
} catch (IOException ioe) {
throw new RuntimeException(ioe);
}
}
}
List<Map<String,String>> configFiles = new LinkedList<Map<String,String>>();
for(String cppp: cj.getConfigPaths().keySet()) {
Map<String,String> configFileInfo = new LinkedHashMap<String,String>();
configFileInfo.put("key", cppp);
File path = FileUtils.tryToCanonicalize(cj.getConfigPaths().get(cppp).getFile());
configFileInfo.put("path", path.getAbsolutePath());
Reference urlRef = new Reference(baseRefRef, getHrefPath(path, cj)).getTargetRef();
configFileInfo.put("url", urlRef.toString());
configFiles.add(configFileInfo);
}
info.put("configFiles", configFiles);
return info;
}
protected Set<String> getAvailableActions() {
Set<String> actions = new LinkedHashSet<String>();
if (!cj.hasApplicationContext()) {
actions.add("build");
}
if (!cj.isProfile() && cj.isLaunchable()) {
actions.add("launch");
}
if (cj.isPausable()) {
actions.add("pause");
}
if (cj.isUnpausable()) {
actions.add("unpause");
}
if (cj.getCheckpointService() != null && cj.isRunning()) {
actions.add("checkpoint");
}
if (cj.isRunning()) {
actions.add("terminate");
}
if (cj.hasApplicationContext()) {
actions.add("teardown");
}
return actions;
}
protected void writeHtml(Writer writer) {
PrintWriter pw = new PrintWriter(writer);
String jobTitle = cj.getShortName() + " - "
@@ -420,7 +546,9 @@ public class JobResource extends Resource {
@Override
public void acceptRepresentation(Representation entity) throws ResourceException {
// copy op?
Form form = getRequest().getEntityAsForm();
Form form = null;
try {
form = getRequest().getEntityAsForm();
String copyTo = form.getFirstValue("copyTo");
if(copyTo!=null) {
copyJob(copyTo,"on".equals(form.getFirstValue("asProfile")));
@@ -460,8 +588,13 @@ public class JobResource extends Resource {
cj.terminate();
}
AlertThreadGroup.setThreadLogger(null);
// default: redirect to GET self
getResponse().redirectSeeOther(getRequest().getOriginalRef());
} catch (IllegalStateException e) {
logger.log(Level.WARNING, "problem accepting input (redirecting to GET self anyway): " + e, e);
getResponse().setStatus(Status.CLIENT_ERROR_BAD_REQUEST, e, "problem accepting input");
}
}
protected void copyJob(String copyTo, boolean asProfile) throws ResourceException {
engine/src/main/java/org/archive/crawler/restlet/ScriptResource.java
+106 -6
View File
@@ -23,7 +23,11 @@ import java.io.IOException;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.io.Writer;
import java.util.Collection;
import java.util.LinkedHashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import javax.script.ScriptEngine;
import javax.script.ScriptEngineFactory;
@@ -37,12 +41,14 @@ import org.restlet.Context;
import org.restlet.data.CharacterSet;
import org.restlet.data.Form;
import org.restlet.data.MediaType;
import org.restlet.data.Reference;
import org.restlet.data.Request;
import org.restlet.data.Response;
import org.restlet.resource.Representation;
import org.restlet.resource.ResourceException;
import org.restlet.resource.Variant;
import org.restlet.resource.WriterRepresentation;
import org.xml.sax.SAXException;
/**
* Restlet Resource which runs an arbitrary script, which is supplied
@@ -72,6 +78,7 @@ public class ScriptResource extends JobRelatedResource {
super(ctx, req, res);
setModifiable(true);
getVariants().add(new Variant(MediaType.TEXT_HTML));
getVariants().add(new Variant(MediaType.APPLICATION_XML));
}
@Override
@@ -113,17 +120,110 @@ public class ScriptResource extends JobRelatedResource {
}
public Representation represent(Variant variant) throws ResourceException {
Representation representation = new WriterRepresentation(
MediaType.TEXT_HTML) {
public void write(Writer writer) throws IOException {
ScriptResource.this.writeHtml(writer);
}
};
Representation representation;
if (variant.getMediaType() == MediaType.APPLICATION_XML) {
representation = new WriterRepresentation(MediaType.APPLICATION_XML) {
public void write(Writer writer) throws IOException {
try {
new XmlMarshaller(writer).marshalDocument("script", presentablify());
} catch (SAXException e) {
throw new IOException(e);
}
}
};
} else {
representation = new WriterRepresentation(MediaType.TEXT_HTML) {
public void write(Writer writer) throws IOException {
ScriptResource.this.writeHtml(writer);
}
};
}
// TODO: remove if not necessary in future?
representation.setCharacterSet(CharacterSet.UTF_8);
return representation;
}
protected List<String> getAvailableActions() {
List<String> actions = new LinkedList<String>();
actions.add("rescan");
actions.add("add");
actions.add("create");
return actions;
}
protected Collection<Map<String,String>> getAvailableScriptEngines() {
List<Map<String,String>> engines = new LinkedList<Map<String,String>>();
for (ScriptEngineFactory f: FACTORIES) {
Map<String,String> engine = new LinkedHashMap<String, String>();
engine.put("engine", f.getNames().get(0));
engine.put("language", f.getLanguageName());
engines.add(engine);
}
return engines;
}
protected Collection<Map<String,String>> getAvailableGlobalVariables() {
List<Map<String,String>> vars = new LinkedList<Map<String,String>>();
Map<String,String> var;
var = new LinkedHashMap<String,String>();
var.put("variable", "rawOut");
var.put("description", "a PrintWriter for arbitrary text output to this page");
vars.add(var);
var = new LinkedHashMap<String,String>();
var.put("variable", "htmlOut");
var.put("description", "a PrintWriter for HTML output to this page");
vars.add(var);
var = new LinkedHashMap<String,String>();
var.put("variable", "job");
var.put("description", "the current CrawlJob instance");
vars.add(var);
var = new LinkedHashMap<String,String>();
var.put("variable", "appCtx");
var.put("description", "current job ApplicationContext, if any");
vars.add(var);
var = new LinkedHashMap<String,String>();
var.put("variable", "scriptResource");
var.put("description", "the ScriptResource implementing this page, which offers utility methods");
vars.add(var);
return vars;
}
protected LinkedHashMap<String,Object> presentablify() {
LinkedHashMap<String,Object> info = new LinkedHashMap<String,Object>();
String baseRef = getRequest().getResourceRef().getBaseRef().toString();
if(!baseRef.endsWith("/")) {
baseRef += "/";
}
Reference baseRefRef = new Reference(baseRef);
info.put("crawlJobShortName", cj.getShortName());
info.put("crawlJobUrl", new Reference(baseRefRef, "..").getTargetRef());
info.put("availableScriptEngines", getAvailableScriptEngines());
info.put("availableGlobalVariables", getAvailableGlobalVariables());
if(linesExecuted>0) {
info.put("linesExecuted", linesExecuted);
}
if(ex!=null) {
info.put("exception", ex);
}
if(StringUtils.isNotBlank(rawOutput)) {
info.put("rawOutput", rawOutput);
}
if(StringUtils.isNotBlank(htmlOutput)) {
info.put("htmlOutput", htmlOutput);
}
return info;
}
protected void writeHtml(Writer writer) {
PrintWriter pw = new PrintWriter(writer);
pw.println("<head><title>Script in "+cj.getShortName()+"</title></head>");
engine/src/main/java/org/archive/crawler/restlet/XmlMarshaller.java
+86
View File
@@ -0,0 +1,86 @@
/*
* This file is part of the Heritrix web crawler (crawler.archive.org).
*
* Licensed to the Internet Archive (IA) by one or more individual
* contributors.
*
* The IA licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.archive.crawler.restlet;
import java.io.Writer;
import java.util.Map;
import org.restlet.util.XmlWriter;
import org.xml.sax.SAXException;
/**
* @contributor nlevitt
*/
public class XmlMarshaller {
protected XmlWriter xmlWriter;
public XmlMarshaller(Writer writer) throws SAXException {
xmlWriter = new XmlWriter(writer);
// this looks ok, but makes the response much bigger - pipe to
// "xmllint --format -" instead
// xmlWriter.setDataFormat(true);
// xmlWriter.setIndentStep(2);
}
public void marshalDocument(String rootTag, Object content) throws SAXException {
xmlWriter.startDocument();
marshal(rootTag, content);
xmlWriter.endDocument(); // calls flush()
}
protected void marshal(String key, Object value) throws SAXException {
if (value == null) {
xmlWriter.emptyElement(key);
} else if (value instanceof Map<?,?>) {
marshal(key, (Map<?,?>) value);
} else if (value instanceof Iterable<?>) {
marshal(key, (Iterable<?>) value);
} else {
xmlWriter.dataElement(key, value.toString());
}
}
protected void marshal(String key, Map<?,?> map) throws SAXException {
xmlWriter.startElement(key);
for (Map.Entry<?,?> entry: map.entrySet()) {
marshal(entry.getKey().toString(), entry.getValue());
}
xmlWriter.endElement(key);
}
protected void marshal(String key, Iterable<?> iterable) throws SAXException {
xmlWriter.startElement(key);
for (Object item: iterable) {
marshal(item);
}
xmlWriter.endElement(key);
}
// something we have no name for goes in a <value/> tag
protected void marshal(Object item) throws SAXException {
if (item instanceof Map.Entry<?, ?>) {
// if it happens to have a name, use it
Map.Entry<?, ?> entry = (Map.Entry<?, ?>) item;
marshal(entry.getKey().toString(), entry.getValue());
} else {
marshal("value", item);
}
}
}
engine/src/main/java/org/archive/crawler/util/CrawledBytesHistotable.java
+3 -3
View File
@@ -33,9 +33,9 @@ implements CoreAttributeConstants {
public static final String NOTMODIFIED = "not-modified";
public static final String DUPLICATE = "dup-by-hash";
public static final String NOVEL = "novel";
public static final String NOTMODIFIEDCOUNT="not-modified-count";
public static final String DUPLICATECOUNT="dup-by-hash-count";
public static final String NOVELCOUNT="novel-count";
public static final String NOTMODIFIEDCOUNT = "not-modified-count";
public static final String DUPLICATECOUNT = "dup-by-hash-count";
public static final String NOVELCOUNT = "novel-count";
public CrawledBytesHistotable() {
super();