lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-08-22 06:26:32 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'topic/scripting-xml-fix'

Browse Source
This commit is contained in:
Kenji Nagahashi
2013-06-10 15:36:13 -07:00
parent f2984881f6 358afd4486
commit 9ef2fb0f7f
7 changed files with 480 additions and 163 deletions
Download Patch File Download Diff File Expand all files Collapse all files
engine/src/main/java/org/archive/crawler/restlet/ScriptResource.java
+10 -103
View File
@@ -20,8 +20,6 @@
package org.archive.crawler.restlet;
import java.io.IOException;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.io.Writer;
import java.util.Collection;
import java.util.Collections;
@@ -31,14 +29,11 @@ import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import javax.script.Bindings;
import javax.script.ScriptEngine;
import javax.script.ScriptEngineFactory;
import javax.script.ScriptEngineManager;
import javax.script.ScriptException;
import org.apache.commons.lang.StringUtils;
import org.archive.crawler.framework.BeanLookupBindings;
import org.archive.crawler.restlet.models.ScriptModel;
import org.archive.crawler.restlet.models.ViewModel;
import org.restlet.Context;
@@ -52,7 +47,6 @@ import org.restlet.resource.Representation;
import org.restlet.resource.ResourceException;
import org.restlet.resource.Variant;
import org.restlet.resource.WriterRepresentation;
import org.springframework.context.ApplicationContext;
import freemarker.template.Configuration;
import freemarker.template.ObjectWrapper;
@@ -84,7 +78,6 @@ public class ScriptResource extends JobRelatedResource {
}
});
}
ScriptExecution scriptExec = null;
protected String chosenEngine = FACTORIES.isEmpty() ? "" : FACTORIES.getFirst().getNames().get(0);
private Configuration _templateConfiguration;
@@ -99,6 +92,8 @@ public class ScriptResource extends JobRelatedResource {
tmpltCfg.setClassForTemplateLoading(this.getClass(),"");
tmpltCfg.setObjectWrapper(ObjectWrapper.BEANS_WRAPPER);
setTemplateConfiguration(tmpltCfg);
scriptingConsole = new ScriptingConsole(cj);
}
public void setTemplateConfiguration(Configuration tmpltCfg) {
_templateConfiguration=tmpltCfg;
@@ -106,92 +101,8 @@ public class ScriptResource extends JobRelatedResource {
public Configuration getTemplateConfiguration(){
return _templateConfiguration;
}
/**
* JavaBean that packages script, its execution and the result.
*
* TODO: the script could create an object that persists and retains a
* reference to the Bindings. by encapsulating this way, I've made it
* a bit more difficult? we could {@code eng} and {@code script} arguments
* to execute() method.
*/
public static class ScriptExecution {
private ScriptEngine eng;
private ApplicationContext appCtx;
private Bindings bindings;
private String script;
private StringWriter rawString;
private StringWriter htmlString;
private Throwable exception;
private int linesExecuted;
public ScriptExecution(ScriptEngine eng, ApplicationContext appCtx, String script) {
this.eng = eng;
this.appCtx = appCtx;
this.bindings = new BeanLookupBindings(appCtx);
this.script = script;
if (StringUtils.isBlank(this.script)) {
this.script = "";
}
this.rawString = new StringWriter();
this.htmlString = new StringWriter();
}
public void bind(String name, Object obj) {
bindings.put(name, obj);
}
public Object unbind(String name) {
return bindings.remove(name);
}
public void execute() {
PrintWriter rawOut = new PrintWriter(rawString);
PrintWriter htmlOut = new PrintWriter(htmlString);
bind("rawOut", rawOut);
bind("htmlOut", htmlOut);
bind("appCtx", appCtx);
try {
eng.eval(script, bindings);
// TODO: should count with RE rather than creating String[]?
linesExecuted = script.split("\r?\n").length;
} catch (ScriptException ex) {
Throwable cause = ex.getCause();
exception = cause != null ? cause : ex;
} catch (RuntimeException ex) {
exception = ex;
} finally {
rawOut.flush();
htmlOut.flush();
// TODO: are these really necessary?
unbind("rawOut");
unbind("htmlOut");
unbind("appCtx");
}
}
public boolean isFailure() {
return exception != null;
}
public String getStackTrace() {
if (exception == null) return "";
StringWriter s = new StringWriter();
exception.printStackTrace(new PrintWriter(s));
return s.toString();
}
public Throwable getException() {
return exception;
}
public int getLinesExecuted() {
return linesExecuted;
}
public String getRawOutput() {
return rawString.toString();
}
public String getHtmlOutput() {
return htmlString.toString();
}
public String getScript() {
return script;
}
}
private ScriptingConsole scriptingConsole;
@Override
public void acceptRepresentation(Representation entity) throws ResourceException {
@@ -204,12 +115,9 @@ public class ScriptResource extends JobRelatedResource {
ScriptEngine eng = MANAGER.getEngineByName(chosenEngine);
scriptExec = new ScriptExecution(eng, cj.getJobContext(), script);
scriptExec.bind("job", cj);
scriptExec.bind("scriptResource", this);
scriptExec.execute();
scriptExec.unbind("job");
scriptExec.unbind("scriptResource");
scriptingConsole.bind("scriptResource", this);
scriptingConsole.execute(eng, script);
scriptingConsole.unbind("scriptResource");
//TODO: log script, results somewhere; job log INFO?
@@ -262,10 +170,9 @@ public class ScriptResource extends JobRelatedResource {
}
Reference baseRefRef = new Reference(baseRef);
ScriptModel model = new ScriptModel(cj.getShortName(),
ScriptModel model = new ScriptModel(scriptingConsole,
new Reference(baseRefRef, "..").getTargetRef().toString(),
getAvailableScriptEngines(),
scriptExec);
getAvailableScriptEngines());
return model;
}
@@ -284,7 +191,7 @@ public class ScriptResource extends JobRelatedResource {
viewModel.put("cssRef", getStylesheetRef());
viewModel.put("staticRef", getStaticRef(""));
viewModel.put("baseResourceRef",getRequest().getRootRef().toString()+"/engine/static/");
viewModel.put("model",makeDataModel());
viewModel.put("model", makeDataModel());
viewModel.put("selectedEngine", chosenEngine);
try {
engine/src/main/java/org/archive/crawler/restlet/ScriptingConsole.java
+129
View File
@@ -0,0 +1,129 @@
/**
*
*/
package org.archive.crawler.restlet;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.util.LinkedHashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import javax.script.Bindings;
import javax.script.ScriptEngine;
import javax.script.ScriptException;
import org.archive.crawler.framework.BeanLookupBindings;
import org.archive.crawler.framework.CrawlJob;
/**
* ScriptingConsole implements view-independent logic of scripting console.
*
* Currently it is short-lived; it is created by ScriptResource for each request and
* destroyed after rendering the view.
*
* @contributor kenji
*
*/
public class ScriptingConsole {
private final CrawlJob cj;
private ScriptEngine eng;
private String script;
private Bindings bindings;
private StringWriter rawString;
private StringWriter htmlString;
private Throwable exception;
private int linesExecuted;
private List<Map<String, String>> availableGlobalVariables;
public ScriptingConsole(CrawlJob job) {
this.cj = job;
this.bindings = new BeanLookupBindings(this.cj.getJobContext());
this.script = "";
setupAvailableGlobalVariables();
}
protected void addGlobalVariable(String name, String desc) {
Map<String, String> var = new LinkedHashMap<String, String>();
var.put("variable", name);
var.put("description", desc);
availableGlobalVariables.add(var);
}
private void setupAvailableGlobalVariables() {
availableGlobalVariables = new LinkedList<Map<String,String>>();
addGlobalVariable("rawOut", "a PrintWriter for arbitrary text output to this page");
addGlobalVariable("htmlOut", "a PrintWriter for HTML output to this page");
addGlobalVariable("job", "the current CrawlJob instance");
addGlobalVariable("appCtx", "current job ApplicationContext, if any");
// TODO: a bit awkward to have this here, because ScriptingConsole has no ref to
// ScriptResource. better to have ScriptResource call #addGlobalVariable(String, String)?
addGlobalVariable("scriptResource",
"the ScriptResource implementing this page, which offers utility methods");
}
public void bind(String name, Object obj) {
bindings.put(name, obj);
}
public Object unbind(String name) {
return bindings.remove(name);
}
public void execute(ScriptEngine eng, String script) {
// TODO: update through setter rather than passing as method arguments?
this.eng = eng;
this.script = script;
bind("job", cj);
rawString = new StringWriter();
htmlString = new StringWriter();
PrintWriter rawOut = new PrintWriter(rawString);
PrintWriter htmlOut = new PrintWriter(htmlString);
bind("rawOut", rawOut);
bind("htmlOut", htmlOut);
bind("appCtx", cj.getJobContext());
exception = null;
try {
this.eng.eval(this.script, bindings);
// TODO: should count with RE rather than creating String[]?
linesExecuted = script.split("\r?\n").length;
} catch (ScriptException ex) {
Throwable cause = ex.getCause();
exception = cause != null ? cause : ex;
} catch (RuntimeException ex) {
exception = ex;
} finally {
rawOut.flush();
htmlOut.flush();
// TODO: are these really necessary?
unbind("rawOut");
unbind("htmlOut");
unbind("appCtx");
unbind("job");
}
}
public CrawlJob getCrawlJob( ) {
return cj;
}
public Throwable getException() {
return exception;
}
public int getLinesExecuted() {
return linesExecuted;
}
public String getRawOutput() {
return rawString != null ? rawString.toString() : "";
}
public String getHtmlOutput() {
return htmlString != null ? htmlString.toString() : "";
}
public String getScript() {
return script;
}
public List<Map<String, String>> getAvailableGlobalVariables() {
return availableGlobalVariables;
}
}
engine/src/main/java/org/archive/crawler/restlet/XmlMarshaller.java
+97
View File
@@ -19,10 +19,23 @@
package org.archive.crawler.restlet;
import java.beans.BeanInfo;
import java.beans.IntrospectionException;
import java.beans.Introspector;
import java.beans.PropertyDescriptor;
import java.io.IOException;
import java.io.Writer;
import java.lang.reflect.Method;
import java.util.Arrays;
import java.util.Comparator;
import java.util.HashMap;
import java.util.Map;
import javax.xml.bind.annotation.XmlRootElement;
import javax.xml.bind.annotation.XmlTransient;
import javax.xml.bind.annotation.XmlType;
import org.apache.commons.lang.StringUtils;
import org.restlet.util.XmlWriter;
import org.xml.sax.SAXException;
@@ -90,6 +103,88 @@ public class XmlMarshaller {
}
}
}
/**
* sort PropertyDescriptors according to propOrder.
* properties listed in propOrder come first, in the order they are listed, and
* then come remaining unlisted properties, in arbitrary order (likely alphabetical).
* this semantics is rather relaxed compared to original JAXB semantics, where props
* and propOrder must be the same set (except for those marked XmlTransient).
* @param props PropertyDescriptor array to be sorted in-place.
* @param propOrder list of property names in order of desired appearance.
*/
protected static void orderProperties(PropertyDescriptor[] props, final String[] propOrder) {
if (propOrder == null || propOrder.length == 0) return;
final Map<String, Integer> order = new HashMap<String, Integer>();
for (int i = 0; i < propOrder.length; i++) {
order.put(propOrder[i], i);
}
final Integer LAST = Integer.valueOf(propOrder.length);
Arrays.sort(props, new Comparator<PropertyDescriptor>() {
@Override
public int compare(PropertyDescriptor o1, PropertyDescriptor o2) {
Integer c1 = order.get(o1.getName());
Integer c2 = order.get(o2.getName());
return (c1 != null ? c1 : LAST).compareTo(c2 != null ? c2 : LAST);
}
});
}
/**
* test if {@code obj} has {@link XmlRootElement} annotation.
* to avoid unexpected side effects, objects are mapped to nested XML structure
* only when its class has {@link XmlRootElement} annotation.
* note semantics is slightly different from JAXB - just borrowing XmlRootElement
* as substitute of XmlElement, because Map entry cannot be annotated.
* @param obj object to test
* @return true if obj's class has XmlRootElement annotation.
*/
protected static boolean marshalAsElement(Object obj) {
XmlRootElement ann = obj.getClass().getAnnotation(XmlRootElement.class);
return ann != null;
}
/**
* generate nested XML structure for a bean {@code obj}. enclosing element
* will not be generated if {@code key} is empty. each readable JavaBeans property
* is mapped to an nested element named after its name. Those properties
* annotated with {@link XmlTransient} are ignored.
* @param xmlWriter XmlWriter
* @param key name of enclosing element
* @param obj bean
* @throws SAXException
*/
protected static void marshalBean(XmlWriter xmlWriter, String key, Object obj) throws SAXException {
if (!StringUtils.isEmpty(key))
xmlWriter.startElement(key);
try {
BeanInfo beanInfo = Introspector.getBeanInfo(obj.getClass(), Object.class);
PropertyDescriptor[] props = beanInfo.getPropertyDescriptors();
XmlType xmlType = obj.getClass().getAnnotation(XmlType.class);
if (xmlType != null) {
String[] propOrder = xmlType.propOrder();
if (propOrder != null) {
// TODO: should cache this sorted version?
orderProperties(props, propOrder);
}
}
for (PropertyDescriptor prop : props) {
Method m = prop.getReadMethod();
if (m == null || m.getAnnotation(XmlTransient.class) != null)
continue;
try {
Object propValue = m.invoke(obj);
if (propValue != null && !"".equals(propValue)) {
marshal(xmlWriter, prop.getName(), propValue);
}
} catch (Exception ex) {
// generate empty element, for now. generate comment?
xmlWriter.emptyElement(prop.getName());
}
}
} catch (IntrospectionException ex) {
// ignored, for now.
}
if (!StringUtils.isEmpty(key))
xmlWriter.endElement(key);
}
protected static void marshal(XmlWriter xmlWriter, String key, Object value) throws SAXException {
if (value == null) {
@@ -98,6 +193,8 @@ public class XmlMarshaller {
marshal(xmlWriter, key, (Map<?,?>) value);
} else if (value instanceof Iterable<?>) {
marshal(xmlWriter, key, (Iterable<?>) value);
} else if (marshalAsElement(value)) {
marshalBean(xmlWriter, key, value);
} else {
xmlWriter.dataElement(key, value.toString());
}
engine/src/main/java/org/archive/crawler/restlet/models/ScriptModel.java
+61 -46
View File
@@ -1,56 +1,71 @@
package org.archive.crawler.restlet.models;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.util.Collection;
import java.util.LinkedHashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import org.archive.crawler.restlet.ScriptResource.ScriptExecution;
import javax.xml.bind.annotation.XmlRootElement;
import javax.xml.bind.annotation.XmlType;
@SuppressWarnings("serial")
public class ScriptModel extends LinkedHashMap<String, Object> {
import org.archive.crawler.restlet.ScriptingConsole;
public ScriptModel(String crawlJobShortName, String crawlJobUrl,
Collection<Map<String, String>> scriptEngines,
ScriptExecution scriptExec) {// int linesExecuted, Exception
// exception, String rawOutput,
// String htmlOutput){
super();
this.put("crawlJobUrl",crawlJobUrl);
this.put("crawlJobShortName", crawlJobShortName);
this.put("availableScriptEngines", scriptEngines);
if (scriptExec != null)
this.put("scriptExec", scriptExec);
List<Map<String,String>> vars = new LinkedList<Map<String,String>>();
Map<String,String> var;
var = new LinkedHashMap<String,String>();
var.put("variable", "rawOut");
var.put("description", "a PrintWriter for arbitrary text output to this page");
vars.add(var);
var = new LinkedHashMap<String,String>();
var.put("variable", "htmlOut");
var.put("description", "a PrintWriter for HTML output to this page");
vars.add(var);
var = new LinkedHashMap<String,String>();
var.put("variable", "job");
var.put("description", "the current CrawlJob instance");
vars.add(var);
var = new LinkedHashMap<String,String>();
var.put("variable", "appCtx");
var.put("description", "current job ApplicationContext, if any");
vars.add(var);
var = new LinkedHashMap<String,String>();
var.put("variable", "scriptResource");
var.put("description", "the ScriptResource implementing this page, which offers utility methods");
vars.add(var);
this.put("availableGlobalVariables", vars);
@XmlRootElement(name="script")
@XmlType(propOrder={
"crawlJobUrl", "crawlJobShortName", "availableScriptEngines",
"script", "linesExecuted", "exception", "rawOutput", "htmlOutput",
"availableGlobalVariables"
})
public class ScriptModel {
private String crawlJobUrl;
private Collection<Map<String, String>> availableScriptEngines;
private ScriptingConsole scriptingConsole;
public ScriptModel(ScriptingConsole cc,
String crawlJobUrl,
Collection<Map<String, String>> scriptEngines) {
scriptingConsole = cc;
this.crawlJobUrl = crawlJobUrl;
this.availableScriptEngines = scriptEngines;
}
public boolean isFailure() {
return scriptingConsole.getException() != null;
}
public String getStackTrace() {
Throwable exception = scriptingConsole.getException();
if (exception == null) return "";
StringWriter s = new StringWriter();
exception.printStackTrace(new PrintWriter(s));
return s.toString();
}
public Throwable getException() {
return scriptingConsole.getException();
}
public int getLinesExecuted() {
return scriptingConsole.getLinesExecuted();
}
public String getRawOutput() {
return scriptingConsole.getRawOutput();
}
public String getHtmlOutput() {
return scriptingConsole.getHtmlOutput();
}
public String getScript() {
return scriptingConsole.getScript();
}
public String getCrawlJobShortName() {
return scriptingConsole.getCrawlJob().getShortName();
}
public Collection<Map<String, String>> getAvailableScriptEngines() {
return availableScriptEngines;
}
public List<Map<String, String>> getAvailableGlobalVariables() {
return scriptingConsole.getAvailableGlobalVariables();
}
public String getCrawlJobUrl() {
return crawlJobUrl;
}
}