diff --git a/.classpath b/.classpath index 7bfef98e..8fb5c72f 100644 --- a/.classpath +++ b/.classpath @@ -12,13 +12,13 @@ - + - + @@ -59,5 +59,6 @@ + diff --git a/modules/pom.xml b/modules/pom.xml index 3784c793..598dec87 100644 --- a/modules/pom.xml +++ b/modules/pom.xml @@ -48,9 +48,14 @@ be used in applications other than Heritrix, however. org.beanshell bsh - 2.0b4 + 2.0b5 compile + + org.codehaus.groovy + groovy-all + 1.6.3 + diff --git a/modules/src/main/java/org/archive/modules/BeanShellProcessor.java b/modules/src/main/java/org/archive/modules/BeanShellProcessor.java deleted file mode 100644 index 55500a3c..00000000 --- a/modules/src/main/java/org/archive/modules/BeanShellProcessor.java +++ /dev/null @@ -1,176 +0,0 @@ -/* BeanShellProcessor - * - * Created on Aug 4, 2006 - * - * Copyright (C) 2006 Internet Archive. - * - * This file is part of the Heritrix web crawler (crawler.archive.org). - * - * Heritrix is free software; you can redistribute it and/or modify - * it under the terms of the GNU Lesser Public License as published by - * the Free Software Foundation; either version 2.1 of the License, or - * any later version. - * - * Heritrix is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser Public License for more details. - * - * You should have received a copy of the GNU Lesser Public License - * along with Heritrix; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - */ -package org.archive.modules; - -import java.io.File; -import java.io.IOException; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; -import java.util.logging.Level; -import java.util.logging.Logger; - -import org.archive.spring.ConfigPath; -import org.springframework.beans.BeansException; -import org.springframework.beans.factory.annotation.Required; -import org.springframework.context.ApplicationContext; - -import bsh.EvalError; -import bsh.Interpreter; - -/** - * A processor which runs a BeanShell script on the CrawlURI. - * - * Script source may be provided via a file - * local to the crawler. - * Script source should define - * a method with one argument, 'run(curi)'. Each processed CrawlURI is - * passed to this script method. - * - * Other variables available to the script include 'self' (this - * BeanShellProcessor instance) and 'controller' (the crawl's - * CrawlController instance). - * - * @author gojomo - * @version $Date$, $Revision$ - */ -public class BeanShellProcessor extends Processor { - - private static final long serialVersionUID = 3L; - - private static final Logger logger = - Logger.getLogger(BeanShellProcessor.class.getName()); - - - /** - * BeanShell script file. - */ - ConfigPath scriptFile = null; - public ConfigPath getScriptFile() { - return this.scriptFile; - } - @Required - public void setScriptFile(ConfigPath file) { - this.scriptFile = file; - } - - /** - * Whether each ToeThread should get its own independent script context, or - * they should share synchronized access to one context. Default is true, - * meaning each threads gets its own isolated context. - * - */ - protected boolean isolateThreads = true; - public boolean getIsolateThreads() { - return isolateThreads; - } - public void setIsolateThreads(boolean isolateThreads) { - this.isolateThreads = isolateThreads; - } - - ApplicationContext appCtx; - public void setApplicationContext(ApplicationContext applicationContext) throws BeansException { - this.appCtx = applicationContext; - } - - protected ThreadLocal threadInterpreter; - protected Interpreter sharedInterpreter; - public Map sharedMap = Collections.synchronizedMap( - new HashMap()); - - /** - * Constructor. - */ - public BeanShellProcessor() { - super(); - } - - protected boolean shouldProcess(ProcessorURI curi) { - return true; - } - - @Override - protected void innerProcess(ProcessorURI curi) { - // depending on previous configuration, interpreter may - // be local to this thread or shared - Interpreter interpreter = getInterpreter(); - synchronized(interpreter) { - // synchronization is harmless for local thread interpreter, - // necessary for shared interpreter - try { - interpreter.set("curi",curi); - interpreter.eval("process(curi)"); - } catch (EvalError e) { - logger.log(Level.WARNING,"BeanShell error", e); - } - } - } - - /** - * Get the proper Interpreter instance -- either shared or local - * to this thread. - * @return Interpreter to use - */ - protected synchronized Interpreter getInterpreter() { - if(sharedInterpreter==null - && getIsolateThreads()) { - // initialize - sharedInterpreter = newInterpreter(); - } - if(sharedInterpreter!=null) { - return sharedInterpreter; - } - Interpreter interpreter = threadInterpreter.get(); - if(interpreter==null) { - interpreter = newInterpreter(); - threadInterpreter.set(interpreter); - } - return interpreter; - } - - /** - * Create a new Interpreter instance, preloaded with any supplied - * source code or source file and the variables 'self' (this - * BeanShellProcessor) and 'controller' (the CrawlController). - * - * @return the new Interpreter instance - */ - protected Interpreter newInterpreter() { - Interpreter interpreter = new Interpreter(); - try { - interpreter.set("self", this); - interpreter.set("context", appCtx); - - File file = getScriptFile().getFile(); - try { - interpreter.source(file.getPath()); - } catch (IOException e) { - logger.log(Level.SEVERE,"unable to read script file",e); - } - } catch (EvalError e) { - logger.log(Level.SEVERE,"error in source file",e); - } - - return interpreter; - } -} diff --git a/modules/src/main/java/org/archive/modules/ScriptedProcessor.java b/modules/src/main/java/org/archive/modules/ScriptedProcessor.java new file mode 100644 index 00000000..5184e96f --- /dev/null +++ b/modules/src/main/java/org/archive/modules/ScriptedProcessor.java @@ -0,0 +1,193 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.modules; + +import java.io.Reader; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.logging.Level; +import java.util.logging.Logger; + +import javax.script.ScriptEngine; +import javax.script.ScriptEngineManager; +import javax.script.ScriptException; + +import org.apache.commons.io.IOUtils; +import org.archive.io.ReadSource; +import org.springframework.beans.BeansException; +import org.springframework.beans.factory.BeanInitializationException; +import org.springframework.beans.factory.InitializingBean; +import org.springframework.beans.factory.annotation.Required; +import org.springframework.context.ApplicationContext; + +/** + * A processor which runs a JSR-223 script on the CrawlURI. + * + * Script source may be provided via a file local to the crawler or + * an inline configuration string. + * + * The source must include a function "run()" taking one argument. + * Each processed CrawlURI is passed to this script function. + * + * Other variables available to the script include 'self' (this + * ScriptedProcessor instance) and 'context' (the crawl's + * ApplicationContext instance, from which all named beans are + * reachable). + * + * TODO: provide way to trigger reload of script mid-crawl; perhaps + * by watching for a certain applicationEvent? + * + * @author gojomo + * @version $Date$, $Revision$ + */ +public class ScriptedProcessor extends Processor implements InitializingBean { + + private static final long serialVersionUID = 3L; + + private static final Logger logger = + Logger.getLogger(ScriptedProcessor.class.getName()); + + /** engine name; default "beanshell" */ + protected String engineName = "beanshell"; + public String getEngineName() { + return this.engineName; + } + public void setEngineName(String name) { + this.engineName = name; + } + + ReadSource scriptSource = null; + public ReadSource getScriptSource() { + return this.scriptSource; + } + @Required + public void setScriptSource(ReadSource source) { + this.scriptSource = source; + } + + /** + * Whether each ToeThread should get its own independent script + * engine, or they should share synchronized access to one + * engine. Default is true, meaning each thread gets its own + * isolated engine. + */ + protected boolean isolateThreads = true; + public boolean getIsolateThreads() { + return isolateThreads; + } + public void setIsolateThreads(boolean isolateThreads) { + this.isolateThreads = isolateThreads; + } + + ApplicationContext appCtx; + public void setApplicationContext(ApplicationContext applicationContext) throws BeansException { + this.appCtx = applicationContext; + } + + transient protected ThreadLocal threadEngine = + new ThreadLocal(); + protected ScriptEngine sharedEngine; + /** map for optional use by scripts */ + public Map sharedMap = + Collections.synchronizedMap(new HashMap()); + + /** + * Constructor. + */ + public ScriptedProcessor() { + super(); + } + + public void afterPropertiesSet() throws Exception { + // fail at build-time if script engine not available + if(null == new ScriptEngineManager().getEngineByName(engineName)) { + throw new BeanInitializationException("named ScriptEngine not available"); + } + } + protected boolean shouldProcess(ProcessorURI curi) { + return true; + } + + @Override + protected void innerProcess(ProcessorURI curi) { + // depending on previous configuration, engine may + // be local to this thread or shared + ScriptEngine engine = getEngine(); + synchronized(engine) { + // synchronization is harmless for local thread engine, + // necessary for shared engine + try { + engine.put("curi",curi); + engine.eval("process(curi)"); + } catch (ScriptException e) { + logger.log(Level.WARNING,e.getMessage(),e); + } + } + } + + /** + * Get the proper ScriptEngine instance -- either shared or local + * to this thread. + * @return ScriptEngine to use + */ + protected synchronized ScriptEngine getEngine() { + if(sharedEngine==null + && getIsolateThreads()) { + // initialize + sharedEngine = newEngine(); + } + if(sharedEngine!=null) { + return sharedEngine; + } + ScriptEngine engine = threadEngine.get(); + if(engine==null) { + engine = newEngine(); + threadEngine.set(engine); + } + return engine; + } + + /** + * Create a new ScriptEngine instance, preloaded with any supplied + * source file and the variables 'self' (this ScriptedDecideRule) + * and 'context' (the ApplicationContext). + * + * @return the new Interpreter instance + */ + protected ScriptEngine newEngine() { + ScriptEngine interpreter = new ScriptEngineManager().getEngineByName(engineName); + + interpreter.put("self", this); + interpreter.put("context", appCtx); + + Reader reader = null; + try { + reader = getScriptSource().getReader(); + interpreter.eval(reader); + } catch (ScriptException e) { + logger.log(Level.SEVERE,"script problem",e); + } finally { + IOUtils.closeQuietly(reader); + } + + return interpreter; + } +} diff --git a/modules/src/main/java/org/archive/modules/deciderules/BeanShellDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/BeanShellDecideRule.java deleted file mode 100644 index ce50fe74..00000000 --- a/modules/src/main/java/org/archive/modules/deciderules/BeanShellDecideRule.java +++ /dev/null @@ -1,173 +0,0 @@ -/* BeanShellDecideRule -* -* $Id$ -* -* Created on Aug 7, 2006 -* -* Copyright (C) 2006 Internet Archive. -* -* This file is part of the Heritrix web crawler (crawler.archive.org). -* -* Heritrix is free software; you can redistribute it and/or modify -* it under the terms of the GNU Lesser Public License as published by -* the Free Software Foundation; either version 2.1 of the License, or -* any later version. -* -* Heritrix is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Lesser Public License for more details. -* -* You should have received a copy of the GNU Lesser Public License -* along with Heritrix; if not, write to the Free Software -* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -*/ -package org.archive.modules.deciderules; - -import java.io.File; -import java.io.IOException; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; -import java.util.logging.Level; -import java.util.logging.Logger; - -import org.archive.modules.ProcessorURI; -import org.archive.spring.ConfigPath; -import org.springframework.beans.BeansException; -import org.springframework.beans.factory.annotation.Required; -import org.springframework.context.ApplicationContext; -import org.springframework.context.ApplicationContextAware; - -import bsh.EvalError; -import bsh.Interpreter; - - -/** - * Rule which runs a groovy script to make its decision. - * - * Script source may be provided via a file local to the crawler. - * - * Variables available to the script include 'object' (the object to be - * evaluated, typically a CandidateURI or CrawlURI), 'self' - * (this GroovyDecideRule instance), and 'controller' (the crawl's - * CrawlController instance). - * - * TODO: reduce copy & paste with GroovyProcessor - * - * @author gojomo - */ -public class BeanShellDecideRule extends DecideRule -implements ApplicationContextAware { - - private static final long serialVersionUID = 3L; - - private static final Logger logger = - Logger.getLogger(BeanShellDecideRule.class.getName()); - - /** BeanShell script file. */ - protected ConfigPath scriptFile = null; - public ConfigPath getScriptFile() { - return scriptFile; - } - @Required - public void setScriptFile(ConfigPath scriptFile) { - this.scriptFile = scriptFile; - } - - /** - * Whether each ToeThread should get its own independent script context, or - * they should share synchronized access to one context. Default is true, - * meaning each threads gets its own isolated context. - */ - protected boolean isolateThreads = true; - public boolean getIsolateThreads() { - return isolateThreads; - } - public void setIsolateThreads(boolean isolateThreads) { - this.isolateThreads = isolateThreads; - } - - ApplicationContext appCtx; - public void setApplicationContext(ApplicationContext applicationContext) throws BeansException { - this.appCtx = applicationContext; - } - - protected ThreadLocal threadInterpreter = - new ThreadLocal();; - protected Interpreter sharedInterpreter; - public Map sharedMap = - Collections.synchronizedMap(new HashMap()); - protected boolean initialized = false; - - public BeanShellDecideRule() { - } - - @Override - public DecideResult innerDecide(ProcessorURI uri) { - // depending on previous configuration, interpreter may - // be local to this thread or shared - Interpreter interpreter = getInterpreter(); - synchronized(interpreter) { - // synchronization is harmless for local thread interpreter, - // necessary for shared interpreter - try { - interpreter.set("object",uri); - return (DecideResult)interpreter.eval("decisionFor(object)"); - } catch (EvalError e) { - // TODO Auto-generated catch block - e.printStackTrace(); - return DecideResult.PASS; - } - } - } - - /** - * Get the proper Interpreter instance -- either shared or local - * to this thread. - * @return Interpreter to use - */ - protected synchronized Interpreter getInterpreter() { - if(sharedInterpreter==null - && getIsolateThreads()) { - // initialize - sharedInterpreter = newInterpreter(); - } - if(sharedInterpreter!=null) { - return sharedInterpreter; - } - Interpreter interpreter = threadInterpreter.get(); - if(interpreter==null) { - interpreter = newInterpreter(); - threadInterpreter.set(interpreter); - } - return interpreter; - } - - /** - * Create a new Interpreter instance, preloaded with any supplied - * source file and the variables 'self' (this - * BeanShellProcessor) and 'controller' (the CrawlController). - * - * @return the new Interpreter instance - */ - protected Interpreter newInterpreter() { - Interpreter interpreter = new Interpreter(); - try { - interpreter.set("self", this); - interpreter.set("context", appCtx); - - File file = getScriptFile().getFile(); - try { - interpreter.source(file.getPath()); - } catch (IOException e) { - logger.log(Level.SEVERE,"unable to read script file",e); - } - } catch (EvalError e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - - return interpreter; - } -} diff --git a/modules/src/main/java/org/archive/modules/deciderules/ScriptedDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/ScriptedDecideRule.java new file mode 100644 index 00000000..a1710445 --- /dev/null +++ b/modules/src/main/java/org/archive/modules/deciderules/ScriptedDecideRule.java @@ -0,0 +1,190 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.modules.deciderules; + +import java.io.Reader; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.logging.Level; +import java.util.logging.Logger; + +import javax.script.ScriptEngine; +import javax.script.ScriptEngineManager; +import javax.script.ScriptException; + +import org.apache.commons.io.IOUtils; +import org.archive.io.ReadSource; +import org.archive.modules.ProcessorURI; +import org.springframework.beans.BeansException; +import org.springframework.beans.factory.BeanInitializationException; +import org.springframework.beans.factory.InitializingBean; +import org.springframework.beans.factory.annotation.Required; +import org.springframework.context.ApplicationContext; +import org.springframework.context.ApplicationContextAware; + + +/** + * Rule which runs a JSR-223 script to make its decision. + * + * Script source may be provided via a file local to the crawler or + * an inline configuration string. + * + * The source must include a one-argument function "decisionFor" which + * returns the appropriate DecideResult. + * + * Variables available to the script include 'object' (the object to be + * evaluated, typically a CrawlURI), 'self' (this ScriptedDecideRule + * instance), and 'context' (the crawl's ApplicationContext, from + * which all named crawl beans are easily reachable). + * + * TODO: provide way to trigger reload of script mid-crawl; perhaps + * by watching for a certain applicationEvent? + * + * @author gojomo + */ +public class ScriptedDecideRule extends DecideRule +implements ApplicationContextAware, InitializingBean { + private static final long serialVersionUID = 3L; + + private static final Logger logger = + Logger.getLogger(ScriptedDecideRule.class.getName()); + + /** engine name; default "beanshell" */ + protected String engineName = "beanshell"; + public String getEngineName() { + return this.engineName; + } + public void setEngineName(String name) { + this.engineName = name; + } + + protected ReadSource scriptSource = null; + public ReadSource getScriptSource() { + return scriptSource; + } + @Required + public void setScriptSource(ReadSource scriptSource) { + this.scriptSource = scriptSource; + } + + /** + * Whether each ToeThread should get its own independent script + * engine, or they should share synchronized access to one + * engine. Default is true, meaning each thread gets its own + * isolated engine. + */ + protected boolean isolateThreads = true; + public boolean getIsolateThreads() { + return isolateThreads; + } + public void setIsolateThreads(boolean isolateThreads) { + this.isolateThreads = isolateThreads; + } + + ApplicationContext appCtx; + public void setApplicationContext(ApplicationContext applicationContext) throws BeansException { + this.appCtx = applicationContext; + } + + transient protected ThreadLocal threadEngine = + new ThreadLocal(); + protected ScriptEngine sharedEngine; + /** map for optional use by scripts */ + public Map sharedMap = + Collections.synchronizedMap(new HashMap()); + + public ScriptedDecideRule() { + } + + public void afterPropertiesSet() throws Exception { + // fail at build-time if script engine not available + if(null == new ScriptEngineManager().getEngineByName(engineName)) { + throw new BeanInitializationException("named ScriptEngine not available"); + } + } + @Override + public DecideResult innerDecide(ProcessorURI uri) { + // depending on previous configuration, engine may + // be local to this thread or shared + ScriptEngine engine = getEngine(); + synchronized(engine) { + // synchronization is harmless for local thread engine, + // necessary for shared engine + try { + engine.put("object",uri); + return (DecideResult)engine.eval("decisionFor(object)"); + } catch (ScriptException e) { + logger.log(Level.WARNING,e.getMessage(),e); + return DecideResult.PASS; + } + } + } + + /** + * Get the proper ScriptEngine instance -- either shared or local + * to this thread. + * @return ScriptEngine to use + */ + protected synchronized ScriptEngine getEngine() { + if(sharedEngine==null + && getIsolateThreads()) { + // initialize + sharedEngine = newEngine(); + } + if(sharedEngine!=null) { + return sharedEngine; + } + ScriptEngine engine = threadEngine.get(); + if(engine==null) { + engine = newEngine(); + threadEngine.set(engine); + } + return engine; + } + + /** + * Create a new ScriptEngine instance, preloaded with any supplied + * source file and the variables 'self' (this ScriptedDecideRule) + * and 'context' (the ApplicationContext). + * + * @return the new Interpreter instance + */ + protected ScriptEngine newEngine() { + ScriptEngine interpreter = new ScriptEngineManager().getEngineByName(engineName); + + interpreter.put("self", this); + interpreter.put("context", appCtx); + + Reader reader = null; + try { + reader = getScriptSource().getReader(); + interpreter.eval(reader); + } catch (ScriptException e) { + logger.log(Level.SEVERE,"script problem",e); + } finally { + IOUtils.closeQuietly(reader); + } + + return interpreter; + } + + +} diff --git a/modules/src/test/java/org/archive/modules/BeanShellProcessorTest.java b/modules/src/test/java/org/archive/modules/BeanShellProcessorTest.java deleted file mode 100644 index 3d4e0320..00000000 --- a/modules/src/test/java/org/archive/modules/BeanShellProcessorTest.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (C) 2007 Internet Archive. - * - * This file is part of the Heritrix web crawler (crawler.archive.org). - * - * Heritrix is free software; you can redistribute it and/or modify - * it under the terms of the GNU Lesser Public License as published by - * the Free Software Foundation; either version 2.1 of the License, or - * any later version. - * - * Heritrix is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser Public License for more details. - * - * You should have received a copy of the GNU Lesser Public License - * along with Heritrix; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - * - * BeanShellProcessor.java - * - * Created on Jan 31, 2007 - * - * $Id:$ - */ -package org.archive.modules; - -/** - * Unit test for {@link BeanShellProcessor}. - * - * @author pjack - */ -public class BeanShellProcessorTest extends ProcessorTestBase { - - // TODO TESTME! - -} diff --git a/modules/src/test/java/org/archive/modules/ScriptedProcessorTest.java b/modules/src/test/java/org/archive/modules/ScriptedProcessorTest.java new file mode 100644 index 00000000..bbcf0fad --- /dev/null +++ b/modules/src/test/java/org/archive/modules/ScriptedProcessorTest.java @@ -0,0 +1,30 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.archive.modules; + +/** + * Unit test for {@link ScriptedProcessor}. + * + * @author pjack + */ +public class ScriptedProcessorTest extends ProcessorTestBase { + + // TODO TESTME! + +}