diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorMultipleRegex.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorMultipleRegex.java index f0160d17..6e36a671 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorMultipleRegex.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorMultipleRegex.java @@ -22,7 +22,6 @@ import groovy.text.SimpleTemplateEngine; import groovy.text.Template; import java.io.IOException; -import java.util.HashMap; import java.util.LinkedHashMap; import java.util.LinkedList; import java.util.Map; @@ -37,6 +36,54 @@ import org.archive.modules.CrawlURI; import org.archive.modules.fetcher.FetchStatusCodes; import org.archive.util.TextUtils; +/** + * An extractor that uses regular expressions to find strings in the fetched + * content of a URI, and constructs outlink URIs from those strings. + * + *

+ * The crawl operator configures these parameters: + * + *

+ * + *

+ * The URI is checked against uriRegex. The match is done using + * {@link Matcher#matches()}, so the full URI string must match, not just a + * substring. If it does match, then the matching groups are available to the + * URI-building template as ${uriRegex[n]}. If it does not match, + * processing of the URI is finished and no outlinks are extracted. + * + *

+ * Then the extractor looks for matches for each of the + * contentRegexes in the fetched content. If any of the regular + * expressions produce no matches, processing of the URI is finished and no + * outlinks are extracted. If at least one match is found for each regular + * expression, then an outlink is constructed, using the URI-building template, + * for every combination of matches. The matching groups are available to the + * template as ${name[n]}. + * + *

+ * Outlinks are constructed using the URI-building template. + * Variable interpolation using the familiar ${...} syntax is supported. The + * template is evaluated for each combination of regular expression matches + * found, and the matching groups are available to the template as + * ${regexName[n]}. An example template might look like: + * http://example.org/${uriRegex[1]}/foo?bar=${myContentRegex[0]}. + * + *

+ * The template is evaluated as a Groovy Template, so further capabilities + * beyond simple variable interpolation are available. + * + * @see http://groovy.codehaus.org/Groovy+Templates + * + * @contributor nlevitt + * @contributor travis + */ public class ExtractorMultipleRegex extends Extractor { private static final Logger LOGGER = @@ -45,18 +92,34 @@ public class ExtractorMultipleRegex extends Extractor { { setUriRegex(""); } - public void setUriRegex(String reg) { - kp.put("uriRegex", reg); + /** + * Regular expression against which to match the URI. If the URI matches, + * then the matching groups are available to the URI-building template as + * ${uriRegex[n]}. If it does not match, processing of this URI + * is finished and no outlinks are extracted. + */ + public void setUriRegex(String uriRegex) { + kp.put("uriRegex", uriRegex); } public String getUriRegex() { return (String) kp.get("uriRegex"); } { - setContentRegexes(new HashMap()); + setContentRegexes(new LinkedHashMap()); } - public void setContentRegexes(Map regexes) { - kp.put("contentRegexes", regexes); + + /** + * A map of { name => regex }. The extractor looks for matches for each + * regular expression in the content of the URI being processed. If any of + * the regular expressions produce no matches, processing of the URI is + * finished and no outlinks are extracted. If at least one match is found + * for each regular expression, then an outlink is constructed for every + * combination of matches. The matching groups are available to the + * URI-building template as ${name[n]}. + */ + public void setContentRegexes(Map contentRegexes) { + kp.put("contentRegexes", contentRegexes); } @SuppressWarnings("unchecked") public Map getContentRegexes() { @@ -66,12 +129,28 @@ public class ExtractorMultipleRegex extends Extractor { { setTemplate(""); } + + /** + * URI-building template. Provides variable interpolation using the familiar + * ${...} syntax. The template is evaluated for each combination of regular + * expression matches found, and the matching groups are available to the + * template as ${regexName[n]}. An example template might look + * like: + * http://example.org/${uriRegex[1]}/foo?bar=${myContentRegex[0]}. + * + *

+ * The template is evaluated as a Groovy Template, so further capabilities + * beyond simple variable interpolation are available. + * + * @see http://groovy.codehaus.org/Groovy+Templates + */ + public void setTemplate(String template) { + kp.put("template", template); + } public String getTemplate() { return (String) kp.get("template"); } - public void setTemplate(String templ) { - kp.put("template", templ); - } @Override protected boolean shouldProcess(CrawlURI uri) { @@ -142,6 +221,9 @@ public class ExtractorMultipleRegex extends Extractor { matchLists.put(regexName, matchList); } + // XXX how expensive is this? should we cache it? would we need a + // separate one per thread? what about making sure we have the right one + // in case of a sheet overlay? Template groovyTemplate; try { groovyTemplate = new SimpleTemplateEngine().createTemplate(getTemplate());