mirror of
https://github.com/internetarchive/heritrix3.git
synced 2026-09-24 06:36:12 +00:00
[HER-1505] Expand BeanshellProcessor/BeanshellDecideRule for every JVM-friendly scripting language (Javascript/Jython/Groovy/JRuby)
* BeanShellProcessor.java -> ScriptedProcessor.java; BeanShellDecideRule.java -> ScriptedDecideRule.java
update to use any present JSR-223 scripting engine (requested by free-text name)
update to allow script to come from inline string or file (just as with seeds)
* modules/pom.xml
include groovy and jsr-223-capable JARs
* .classpath
add new JARs to classpath
* BeanShellProcessorTest.java -> ScriptedProcessorTest.java
rename (empty) test class
This commit is contained in:
gojomo
+3
-2
@@ -12,13 +12,13 @@
|
||||
<classpathentry kind="src" path="commons/src/test/resources"/>
|
||||
<classpathentry kind="src" path="dist/src/test/java"/>
|
||||
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
|
||||
<classpathentry kind="var" path="M2_REPO/org/beanshell/bsh/2.0b4/bsh-2.0b4.jar"/>
|
||||
<classpathentry kind="var" path="M2_REPO/org/beanshell/bsh/2.0b5/bsh-2.0b5.jar"/>
|
||||
<classpathentry kind="var" path="M2_REPO/org/mortbay/jetty/jetty/6.1.14/jetty-6.1.14.jar"/>
|
||||
<classpathentry kind="var" path="M2_REPO/commons-collections/commons-collections/3.1/commons-collections-3.1.jar" sourcepath="M2_REPO/commons-collections/commons-collections/3.1/commons-collections-3.1-sources.jar"/>
|
||||
<classpathentry kind="var" path="M2_REPO/commons-logging/commons-logging/1.0.4/commons-logging-1.0.4.jar" sourcepath="M2_REPO/commons-logging/commons-logging/1.0.4/commons-logging-1.0.4-sources.jar"/>
|
||||
<classpathentry kind="var" path="M2_REPO/poi/poi/2.5.1-final-20040804/poi-2.5.1-final-20040804.jar" sourcepath="M2_REPO/poi/poi/2.5.1-final-20040804/poi-2.5.1-final-20040804-sources.jar"/>
|
||||
<classpathentry kind="var" path="M2_REPO/commons-cli/commons-cli/1.1/commons-cli-1.1.jar"/>
|
||||
<classpathentry kind="var" path="M2_REPO/net/htmlparser/jericho/jericho-html/2.5/jericho-html-2.5.jar"/>
|
||||
<classpathentry kind="var" path="M2_REPO/net/htmlparser/jericho/jericho-html/2.6.1/jericho-html-2.6.1.jar"/>
|
||||
<classpathentry kind="var" path="M2_REPO/org/gnu/inet/libidn/0.6.5/libidn-0.6.5.jar"/>
|
||||
<classpathentry kind="var" path="M2_REPO/commons-lang/commons-lang/2.3/commons-lang-2.3.jar" sourcepath="/M2SRC/commons-lang-2.4-src/src/java"/>
|
||||
<classpathentry kind="var" path="M2_REPO/com/lowagie/itext/1.3/itext-1.3.jar" sourcepath="M2_REPO/com/lowagie/itext/1.3/itext-1.3-sources.jar"/>
|
||||
@@ -59,5 +59,6 @@
|
||||
<classpathentry kind="var" path="M2_REPO/org/restlet/org.restlet/1.1.2/org.restlet-1.1.2.jar" sourcepath="/M2SRC/restlet-1.1.1/src/org.restlet"/>
|
||||
<classpathentry kind="var" path="M2_REPO/com/noelios/restlet/com.noelios.restlet/1.1.2/com.noelios.restlet-1.1.2.jar" sourcepath="/M2SRC/restlet-1.1.1/src/com.noelios.restlet"/>
|
||||
<classpathentry kind="var" path="M2_REPO/com/noelios/restlet/com.noelios.restlet.ext.jetty/1.1.2/com.noelios.restlet.ext.jetty-1.1.2.jar"/>
|
||||
<classpathentry kind="var" path="M2_REPO/org/codehaus/groovy/groovy-all/1.6.3/groovy-all-1.6.3.jar"/>
|
||||
<classpathentry kind="output" path="bin"/>
|
||||
</classpath>
|
||||
|
||||
+6
-1
@@ -48,9 +48,14 @@ be used in applications other than Heritrix, however.
|
||||
<dependency>
|
||||
<groupId>org.beanshell</groupId>
|
||||
<artifactId>bsh</artifactId>
|
||||
<version>2.0b4</version>
|
||||
<version>2.0b5</version>
|
||||
<scope>compile</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.codehaus.groovy</groupId>
|
||||
<artifactId>groovy-all</artifactId>
|
||||
<version>1.6.3</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
<build>
|
||||
<plugins>
|
||||
|
||||
@@ -1,176 +0,0 @@
|
||||
/* BeanShellProcessor
|
||||
*
|
||||
* Created on Aug 4, 2006
|
||||
*
|
||||
* Copyright (C) 2006 Internet Archive.
|
||||
*
|
||||
* This file is part of the Heritrix web crawler (crawler.archive.org).
|
||||
*
|
||||
* Heritrix is free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Lesser Public License as published by
|
||||
* the Free Software Foundation; either version 2.1 of the License, or
|
||||
* any later version.
|
||||
*
|
||||
* Heritrix is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Lesser Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser Public License
|
||||
* along with Heritrix; if not, write to the Free Software
|
||||
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
*/
|
||||
package org.archive.modules;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import java.util.logging.Level;
|
||||
import java.util.logging.Logger;
|
||||
|
||||
import org.archive.spring.ConfigPath;
|
||||
import org.springframework.beans.BeansException;
|
||||
import org.springframework.beans.factory.annotation.Required;
|
||||
import org.springframework.context.ApplicationContext;
|
||||
|
||||
import bsh.EvalError;
|
||||
import bsh.Interpreter;
|
||||
|
||||
/**
|
||||
* A processor which runs a BeanShell script on the CrawlURI.
|
||||
*
|
||||
* Script source may be provided via a file
|
||||
* local to the crawler.
|
||||
* Script source should define
|
||||
* a method with one argument, 'run(curi)'. Each processed CrawlURI is
|
||||
* passed to this script method.
|
||||
*
|
||||
* Other variables available to the script include 'self' (this
|
||||
* BeanShellProcessor instance) and 'controller' (the crawl's
|
||||
* CrawlController instance).
|
||||
*
|
||||
* @author gojomo
|
||||
* @version $Date$, $Revision$
|
||||
*/
|
||||
public class BeanShellProcessor extends Processor {
|
||||
|
||||
private static final long serialVersionUID = 3L;
|
||||
|
||||
private static final Logger logger =
|
||||
Logger.getLogger(BeanShellProcessor.class.getName());
|
||||
|
||||
|
||||
/**
|
||||
* BeanShell script file.
|
||||
*/
|
||||
ConfigPath scriptFile = null;
|
||||
public ConfigPath getScriptFile() {
|
||||
return this.scriptFile;
|
||||
}
|
||||
@Required
|
||||
public void setScriptFile(ConfigPath file) {
|
||||
this.scriptFile = file;
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether each ToeThread should get its own independent script context, or
|
||||
* they should share synchronized access to one context. Default is true,
|
||||
* meaning each threads gets its own isolated context.
|
||||
*
|
||||
*/
|
||||
protected boolean isolateThreads = true;
|
||||
public boolean getIsolateThreads() {
|
||||
return isolateThreads;
|
||||
}
|
||||
public void setIsolateThreads(boolean isolateThreads) {
|
||||
this.isolateThreads = isolateThreads;
|
||||
}
|
||||
|
||||
ApplicationContext appCtx;
|
||||
public void setApplicationContext(ApplicationContext applicationContext) throws BeansException {
|
||||
this.appCtx = applicationContext;
|
||||
}
|
||||
|
||||
protected ThreadLocal<Interpreter> threadInterpreter;
|
||||
protected Interpreter sharedInterpreter;
|
||||
public Map<Object,Object> sharedMap = Collections.synchronizedMap(
|
||||
new HashMap<Object,Object>());
|
||||
|
||||
/**
|
||||
* Constructor.
|
||||
*/
|
||||
public BeanShellProcessor() {
|
||||
super();
|
||||
}
|
||||
|
||||
protected boolean shouldProcess(ProcessorURI curi) {
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void innerProcess(ProcessorURI curi) {
|
||||
// depending on previous configuration, interpreter may
|
||||
// be local to this thread or shared
|
||||
Interpreter interpreter = getInterpreter();
|
||||
synchronized(interpreter) {
|
||||
// synchronization is harmless for local thread interpreter,
|
||||
// necessary for shared interpreter
|
||||
try {
|
||||
interpreter.set("curi",curi);
|
||||
interpreter.eval("process(curi)");
|
||||
} catch (EvalError e) {
|
||||
logger.log(Level.WARNING,"BeanShell error", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the proper Interpreter instance -- either shared or local
|
||||
* to this thread.
|
||||
* @return Interpreter to use
|
||||
*/
|
||||
protected synchronized Interpreter getInterpreter() {
|
||||
if(sharedInterpreter==null
|
||||
&& getIsolateThreads()) {
|
||||
// initialize
|
||||
sharedInterpreter = newInterpreter();
|
||||
}
|
||||
if(sharedInterpreter!=null) {
|
||||
return sharedInterpreter;
|
||||
}
|
||||
Interpreter interpreter = threadInterpreter.get();
|
||||
if(interpreter==null) {
|
||||
interpreter = newInterpreter();
|
||||
threadInterpreter.set(interpreter);
|
||||
}
|
||||
return interpreter;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a new Interpreter instance, preloaded with any supplied
|
||||
* source code or source file and the variables 'self' (this
|
||||
* BeanShellProcessor) and 'controller' (the CrawlController).
|
||||
*
|
||||
* @return the new Interpreter instance
|
||||
*/
|
||||
protected Interpreter newInterpreter() {
|
||||
Interpreter interpreter = new Interpreter();
|
||||
try {
|
||||
interpreter.set("self", this);
|
||||
interpreter.set("context", appCtx);
|
||||
|
||||
File file = getScriptFile().getFile();
|
||||
try {
|
||||
interpreter.source(file.getPath());
|
||||
} catch (IOException e) {
|
||||
logger.log(Level.SEVERE,"unable to read script file",e);
|
||||
}
|
||||
} catch (EvalError e) {
|
||||
logger.log(Level.SEVERE,"error in source file",e);
|
||||
}
|
||||
|
||||
return interpreter;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,193 @@
|
||||
/*
|
||||
* This file is part of the Heritrix web crawler (crawler.archive.org).
|
||||
*
|
||||
* Licensed to the Internet Archive (IA) by one or more individual
|
||||
* contributors.
|
||||
*
|
||||
* The IA licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.archive.modules;
|
||||
|
||||
import java.io.Reader;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import java.util.logging.Level;
|
||||
import java.util.logging.Logger;
|
||||
|
||||
import javax.script.ScriptEngine;
|
||||
import javax.script.ScriptEngineManager;
|
||||
import javax.script.ScriptException;
|
||||
|
||||
import org.apache.commons.io.IOUtils;
|
||||
import org.archive.io.ReadSource;
|
||||
import org.springframework.beans.BeansException;
|
||||
import org.springframework.beans.factory.BeanInitializationException;
|
||||
import org.springframework.beans.factory.InitializingBean;
|
||||
import org.springframework.beans.factory.annotation.Required;
|
||||
import org.springframework.context.ApplicationContext;
|
||||
|
||||
/**
|
||||
* A processor which runs a JSR-223 script on the CrawlURI.
|
||||
*
|
||||
* Script source may be provided via a file local to the crawler or
|
||||
* an inline configuration string.
|
||||
*
|
||||
* The source must include a function "run()" taking one argument.
|
||||
* Each processed CrawlURI is passed to this script function.
|
||||
*
|
||||
* Other variables available to the script include 'self' (this
|
||||
* ScriptedProcessor instance) and 'context' (the crawl's
|
||||
* ApplicationContext instance, from which all named beans are
|
||||
* reachable).
|
||||
*
|
||||
* TODO: provide way to trigger reload of script mid-crawl; perhaps
|
||||
* by watching for a certain applicationEvent?
|
||||
*
|
||||
* @author gojomo
|
||||
* @version $Date$, $Revision$
|
||||
*/
|
||||
public class ScriptedProcessor extends Processor implements InitializingBean {
|
||||
|
||||
private static final long serialVersionUID = 3L;
|
||||
|
||||
private static final Logger logger =
|
||||
Logger.getLogger(ScriptedProcessor.class.getName());
|
||||
|
||||
/** engine name; default "beanshell" */
|
||||
protected String engineName = "beanshell";
|
||||
public String getEngineName() {
|
||||
return this.engineName;
|
||||
}
|
||||
public void setEngineName(String name) {
|
||||
this.engineName = name;
|
||||
}
|
||||
|
||||
ReadSource scriptSource = null;
|
||||
public ReadSource getScriptSource() {
|
||||
return this.scriptSource;
|
||||
}
|
||||
@Required
|
||||
public void setScriptSource(ReadSource source) {
|
||||
this.scriptSource = source;
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether each ToeThread should get its own independent script
|
||||
* engine, or they should share synchronized access to one
|
||||
* engine. Default is true, meaning each thread gets its own
|
||||
* isolated engine.
|
||||
*/
|
||||
protected boolean isolateThreads = true;
|
||||
public boolean getIsolateThreads() {
|
||||
return isolateThreads;
|
||||
}
|
||||
public void setIsolateThreads(boolean isolateThreads) {
|
||||
this.isolateThreads = isolateThreads;
|
||||
}
|
||||
|
||||
ApplicationContext appCtx;
|
||||
public void setApplicationContext(ApplicationContext applicationContext) throws BeansException {
|
||||
this.appCtx = applicationContext;
|
||||
}
|
||||
|
||||
transient protected ThreadLocal<ScriptEngine> threadEngine =
|
||||
new ThreadLocal<ScriptEngine>();
|
||||
protected ScriptEngine sharedEngine;
|
||||
/** map for optional use by scripts */
|
||||
public Map<Object,Object> sharedMap =
|
||||
Collections.synchronizedMap(new HashMap<Object,Object>());
|
||||
|
||||
/**
|
||||
* Constructor.
|
||||
*/
|
||||
public ScriptedProcessor() {
|
||||
super();
|
||||
}
|
||||
|
||||
public void afterPropertiesSet() throws Exception {
|
||||
// fail at build-time if script engine not available
|
||||
if(null == new ScriptEngineManager().getEngineByName(engineName)) {
|
||||
throw new BeanInitializationException("named ScriptEngine not available");
|
||||
}
|
||||
}
|
||||
protected boolean shouldProcess(ProcessorURI curi) {
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void innerProcess(ProcessorURI curi) {
|
||||
// depending on previous configuration, engine may
|
||||
// be local to this thread or shared
|
||||
ScriptEngine engine = getEngine();
|
||||
synchronized(engine) {
|
||||
// synchronization is harmless for local thread engine,
|
||||
// necessary for shared engine
|
||||
try {
|
||||
engine.put("curi",curi);
|
||||
engine.eval("process(curi)");
|
||||
} catch (ScriptException e) {
|
||||
logger.log(Level.WARNING,e.getMessage(),e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the proper ScriptEngine instance -- either shared or local
|
||||
* to this thread.
|
||||
* @return ScriptEngine to use
|
||||
*/
|
||||
protected synchronized ScriptEngine getEngine() {
|
||||
if(sharedEngine==null
|
||||
&& getIsolateThreads()) {
|
||||
// initialize
|
||||
sharedEngine = newEngine();
|
||||
}
|
||||
if(sharedEngine!=null) {
|
||||
return sharedEngine;
|
||||
}
|
||||
ScriptEngine engine = threadEngine.get();
|
||||
if(engine==null) {
|
||||
engine = newEngine();
|
||||
threadEngine.set(engine);
|
||||
}
|
||||
return engine;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a new ScriptEngine instance, preloaded with any supplied
|
||||
* source file and the variables 'self' (this ScriptedDecideRule)
|
||||
* and 'context' (the ApplicationContext).
|
||||
*
|
||||
* @return the new Interpreter instance
|
||||
*/
|
||||
protected ScriptEngine newEngine() {
|
||||
ScriptEngine interpreter = new ScriptEngineManager().getEngineByName(engineName);
|
||||
|
||||
interpreter.put("self", this);
|
||||
interpreter.put("context", appCtx);
|
||||
|
||||
Reader reader = null;
|
||||
try {
|
||||
reader = getScriptSource().getReader();
|
||||
interpreter.eval(reader);
|
||||
} catch (ScriptException e) {
|
||||
logger.log(Level.SEVERE,"script problem",e);
|
||||
} finally {
|
||||
IOUtils.closeQuietly(reader);
|
||||
}
|
||||
|
||||
return interpreter;
|
||||
}
|
||||
}
|
||||
@@ -1,173 +0,0 @@
|
||||
/* BeanShellDecideRule
|
||||
*
|
||||
* $Id$
|
||||
*
|
||||
* Created on Aug 7, 2006
|
||||
*
|
||||
* Copyright (C) 2006 Internet Archive.
|
||||
*
|
||||
* This file is part of the Heritrix web crawler (crawler.archive.org).
|
||||
*
|
||||
* Heritrix is free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Lesser Public License as published by
|
||||
* the Free Software Foundation; either version 2.1 of the License, or
|
||||
* any later version.
|
||||
*
|
||||
* Heritrix is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Lesser Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser Public License
|
||||
* along with Heritrix; if not, write to the Free Software
|
||||
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
*/
|
||||
package org.archive.modules.deciderules;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import java.util.logging.Level;
|
||||
import java.util.logging.Logger;
|
||||
|
||||
import org.archive.modules.ProcessorURI;
|
||||
import org.archive.spring.ConfigPath;
|
||||
import org.springframework.beans.BeansException;
|
||||
import org.springframework.beans.factory.annotation.Required;
|
||||
import org.springframework.context.ApplicationContext;
|
||||
import org.springframework.context.ApplicationContextAware;
|
||||
|
||||
import bsh.EvalError;
|
||||
import bsh.Interpreter;
|
||||
|
||||
|
||||
/**
|
||||
* Rule which runs a groovy script to make its decision.
|
||||
*
|
||||
* Script source may be provided via a file local to the crawler.
|
||||
*
|
||||
* Variables available to the script include 'object' (the object to be
|
||||
* evaluated, typically a CandidateURI or CrawlURI), 'self'
|
||||
* (this GroovyDecideRule instance), and 'controller' (the crawl's
|
||||
* CrawlController instance).
|
||||
*
|
||||
* TODO: reduce copy & paste with GroovyProcessor
|
||||
*
|
||||
* @author gojomo
|
||||
*/
|
||||
public class BeanShellDecideRule extends DecideRule
|
||||
implements ApplicationContextAware {
|
||||
|
||||
private static final long serialVersionUID = 3L;
|
||||
|
||||
private static final Logger logger =
|
||||
Logger.getLogger(BeanShellDecideRule.class.getName());
|
||||
|
||||
/** BeanShell script file. */
|
||||
protected ConfigPath scriptFile = null;
|
||||
public ConfigPath getScriptFile() {
|
||||
return scriptFile;
|
||||
}
|
||||
@Required
|
||||
public void setScriptFile(ConfigPath scriptFile) {
|
||||
this.scriptFile = scriptFile;
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether each ToeThread should get its own independent script context, or
|
||||
* they should share synchronized access to one context. Default is true,
|
||||
* meaning each threads gets its own isolated context.
|
||||
*/
|
||||
protected boolean isolateThreads = true;
|
||||
public boolean getIsolateThreads() {
|
||||
return isolateThreads;
|
||||
}
|
||||
public void setIsolateThreads(boolean isolateThreads) {
|
||||
this.isolateThreads = isolateThreads;
|
||||
}
|
||||
|
||||
ApplicationContext appCtx;
|
||||
public void setApplicationContext(ApplicationContext applicationContext) throws BeansException {
|
||||
this.appCtx = applicationContext;
|
||||
}
|
||||
|
||||
protected ThreadLocal<Interpreter> threadInterpreter =
|
||||
new ThreadLocal<Interpreter>();;
|
||||
protected Interpreter sharedInterpreter;
|
||||
public Map<Object,Object> sharedMap =
|
||||
Collections.synchronizedMap(new HashMap<Object,Object>());
|
||||
protected boolean initialized = false;
|
||||
|
||||
public BeanShellDecideRule() {
|
||||
}
|
||||
|
||||
@Override
|
||||
public DecideResult innerDecide(ProcessorURI uri) {
|
||||
// depending on previous configuration, interpreter may
|
||||
// be local to this thread or shared
|
||||
Interpreter interpreter = getInterpreter();
|
||||
synchronized(interpreter) {
|
||||
// synchronization is harmless for local thread interpreter,
|
||||
// necessary for shared interpreter
|
||||
try {
|
||||
interpreter.set("object",uri);
|
||||
return (DecideResult)interpreter.eval("decisionFor(object)");
|
||||
} catch (EvalError e) {
|
||||
// TODO Auto-generated catch block
|
||||
e.printStackTrace();
|
||||
return DecideResult.PASS;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the proper Interpreter instance -- either shared or local
|
||||
* to this thread.
|
||||
* @return Interpreter to use
|
||||
*/
|
||||
protected synchronized Interpreter getInterpreter() {
|
||||
if(sharedInterpreter==null
|
||||
&& getIsolateThreads()) {
|
||||
// initialize
|
||||
sharedInterpreter = newInterpreter();
|
||||
}
|
||||
if(sharedInterpreter!=null) {
|
||||
return sharedInterpreter;
|
||||
}
|
||||
Interpreter interpreter = threadInterpreter.get();
|
||||
if(interpreter==null) {
|
||||
interpreter = newInterpreter();
|
||||
threadInterpreter.set(interpreter);
|
||||
}
|
||||
return interpreter;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a new Interpreter instance, preloaded with any supplied
|
||||
* source file and the variables 'self' (this
|
||||
* BeanShellProcessor) and 'controller' (the CrawlController).
|
||||
*
|
||||
* @return the new Interpreter instance
|
||||
*/
|
||||
protected Interpreter newInterpreter() {
|
||||
Interpreter interpreter = new Interpreter();
|
||||
try {
|
||||
interpreter.set("self", this);
|
||||
interpreter.set("context", appCtx);
|
||||
|
||||
File file = getScriptFile().getFile();
|
||||
try {
|
||||
interpreter.source(file.getPath());
|
||||
} catch (IOException e) {
|
||||
logger.log(Level.SEVERE,"unable to read script file",e);
|
||||
}
|
||||
} catch (EvalError e) {
|
||||
// TODO Auto-generated catch block
|
||||
e.printStackTrace();
|
||||
}
|
||||
|
||||
return interpreter;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,190 @@
|
||||
/*
|
||||
* This file is part of the Heritrix web crawler (crawler.archive.org).
|
||||
*
|
||||
* Licensed to the Internet Archive (IA) by one or more individual
|
||||
* contributors.
|
||||
*
|
||||
* The IA licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.archive.modules.deciderules;
|
||||
|
||||
import java.io.Reader;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import java.util.logging.Level;
|
||||
import java.util.logging.Logger;
|
||||
|
||||
import javax.script.ScriptEngine;
|
||||
import javax.script.ScriptEngineManager;
|
||||
import javax.script.ScriptException;
|
||||
|
||||
import org.apache.commons.io.IOUtils;
|
||||
import org.archive.io.ReadSource;
|
||||
import org.archive.modules.ProcessorURI;
|
||||
import org.springframework.beans.BeansException;
|
||||
import org.springframework.beans.factory.BeanInitializationException;
|
||||
import org.springframework.beans.factory.InitializingBean;
|
||||
import org.springframework.beans.factory.annotation.Required;
|
||||
import org.springframework.context.ApplicationContext;
|
||||
import org.springframework.context.ApplicationContextAware;
|
||||
|
||||
|
||||
/**
|
||||
* Rule which runs a JSR-223 script to make its decision.
|
||||
*
|
||||
* Script source may be provided via a file local to the crawler or
|
||||
* an inline configuration string.
|
||||
*
|
||||
* The source must include a one-argument function "decisionFor" which
|
||||
* returns the appropriate DecideResult.
|
||||
*
|
||||
* Variables available to the script include 'object' (the object to be
|
||||
* evaluated, typically a CrawlURI), 'self' (this ScriptedDecideRule
|
||||
* instance), and 'context' (the crawl's ApplicationContext, from
|
||||
* which all named crawl beans are easily reachable).
|
||||
*
|
||||
* TODO: provide way to trigger reload of script mid-crawl; perhaps
|
||||
* by watching for a certain applicationEvent?
|
||||
*
|
||||
* @author gojomo
|
||||
*/
|
||||
public class ScriptedDecideRule extends DecideRule
|
||||
implements ApplicationContextAware, InitializingBean {
|
||||
private static final long serialVersionUID = 3L;
|
||||
|
||||
private static final Logger logger =
|
||||
Logger.getLogger(ScriptedDecideRule.class.getName());
|
||||
|
||||
/** engine name; default "beanshell" */
|
||||
protected String engineName = "beanshell";
|
||||
public String getEngineName() {
|
||||
return this.engineName;
|
||||
}
|
||||
public void setEngineName(String name) {
|
||||
this.engineName = name;
|
||||
}
|
||||
|
||||
protected ReadSource scriptSource = null;
|
||||
public ReadSource getScriptSource() {
|
||||
return scriptSource;
|
||||
}
|
||||
@Required
|
||||
public void setScriptSource(ReadSource scriptSource) {
|
||||
this.scriptSource = scriptSource;
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether each ToeThread should get its own independent script
|
||||
* engine, or they should share synchronized access to one
|
||||
* engine. Default is true, meaning each thread gets its own
|
||||
* isolated engine.
|
||||
*/
|
||||
protected boolean isolateThreads = true;
|
||||
public boolean getIsolateThreads() {
|
||||
return isolateThreads;
|
||||
}
|
||||
public void setIsolateThreads(boolean isolateThreads) {
|
||||
this.isolateThreads = isolateThreads;
|
||||
}
|
||||
|
||||
ApplicationContext appCtx;
|
||||
public void setApplicationContext(ApplicationContext applicationContext) throws BeansException {
|
||||
this.appCtx = applicationContext;
|
||||
}
|
||||
|
||||
transient protected ThreadLocal<ScriptEngine> threadEngine =
|
||||
new ThreadLocal<ScriptEngine>();
|
||||
protected ScriptEngine sharedEngine;
|
||||
/** map for optional use by scripts */
|
||||
public Map<Object,Object> sharedMap =
|
||||
Collections.synchronizedMap(new HashMap<Object,Object>());
|
||||
|
||||
public ScriptedDecideRule() {
|
||||
}
|
||||
|
||||
public void afterPropertiesSet() throws Exception {
|
||||
// fail at build-time if script engine not available
|
||||
if(null == new ScriptEngineManager().getEngineByName(engineName)) {
|
||||
throw new BeanInitializationException("named ScriptEngine not available");
|
||||
}
|
||||
}
|
||||
@Override
|
||||
public DecideResult innerDecide(ProcessorURI uri) {
|
||||
// depending on previous configuration, engine may
|
||||
// be local to this thread or shared
|
||||
ScriptEngine engine = getEngine();
|
||||
synchronized(engine) {
|
||||
// synchronization is harmless for local thread engine,
|
||||
// necessary for shared engine
|
||||
try {
|
||||
engine.put("object",uri);
|
||||
return (DecideResult)engine.eval("decisionFor(object)");
|
||||
} catch (ScriptException e) {
|
||||
logger.log(Level.WARNING,e.getMessage(),e);
|
||||
return DecideResult.PASS;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the proper ScriptEngine instance -- either shared or local
|
||||
* to this thread.
|
||||
* @return ScriptEngine to use
|
||||
*/
|
||||
protected synchronized ScriptEngine getEngine() {
|
||||
if(sharedEngine==null
|
||||
&& getIsolateThreads()) {
|
||||
// initialize
|
||||
sharedEngine = newEngine();
|
||||
}
|
||||
if(sharedEngine!=null) {
|
||||
return sharedEngine;
|
||||
}
|
||||
ScriptEngine engine = threadEngine.get();
|
||||
if(engine==null) {
|
||||
engine = newEngine();
|
||||
threadEngine.set(engine);
|
||||
}
|
||||
return engine;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a new ScriptEngine instance, preloaded with any supplied
|
||||
* source file and the variables 'self' (this ScriptedDecideRule)
|
||||
* and 'context' (the ApplicationContext).
|
||||
*
|
||||
* @return the new Interpreter instance
|
||||
*/
|
||||
protected ScriptEngine newEngine() {
|
||||
ScriptEngine interpreter = new ScriptEngineManager().getEngineByName(engineName);
|
||||
|
||||
interpreter.put("self", this);
|
||||
interpreter.put("context", appCtx);
|
||||
|
||||
Reader reader = null;
|
||||
try {
|
||||
reader = getScriptSource().getReader();
|
||||
interpreter.eval(reader);
|
||||
} catch (ScriptException e) {
|
||||
logger.log(Level.SEVERE,"script problem",e);
|
||||
} finally {
|
||||
IOUtils.closeQuietly(reader);
|
||||
}
|
||||
|
||||
return interpreter;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
@@ -1,37 +0,0 @@
|
||||
/*
|
||||
* Copyright (C) 2007 Internet Archive.
|
||||
*
|
||||
* This file is part of the Heritrix web crawler (crawler.archive.org).
|
||||
*
|
||||
* Heritrix is free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Lesser Public License as published by
|
||||
* the Free Software Foundation; either version 2.1 of the License, or
|
||||
* any later version.
|
||||
*
|
||||
* Heritrix is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Lesser Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser Public License
|
||||
* along with Heritrix; if not, write to the Free Software
|
||||
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
*
|
||||
* BeanShellProcessor.java
|
||||
*
|
||||
* Created on Jan 31, 2007
|
||||
*
|
||||
* $Id:$
|
||||
*/
|
||||
package org.archive.modules;
|
||||
|
||||
/**
|
||||
* Unit test for {@link BeanShellProcessor}.
|
||||
*
|
||||
* @author pjack
|
||||
*/
|
||||
public class BeanShellProcessorTest extends ProcessorTestBase {
|
||||
|
||||
// TODO TESTME!
|
||||
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
/*
|
||||
* This file is part of the Heritrix web crawler (crawler.archive.org).
|
||||
*
|
||||
* Licensed to the Internet Archive (IA) by one or more individual
|
||||
* contributors.
|
||||
*
|
||||
* The IA licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.archive.modules;
|
||||
|
||||
/**
|
||||
* Unit test for {@link ScriptedProcessor}.
|
||||
*
|
||||
* @author pjack
|
||||
*/
|
||||
public class ScriptedProcessorTest extends ProcessorTestBase {
|
||||
|
||||
// TODO TESTME!
|
||||
|
||||
}
|
||||
Reference in New Issue
Block a user