diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorMultipleRegex.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorMultipleRegex.java index f0160d17..6e36a671 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorMultipleRegex.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorMultipleRegex.java @@ -22,7 +22,6 @@ import groovy.text.SimpleTemplateEngine; import groovy.text.Template; import java.io.IOException; -import java.util.HashMap; import java.util.LinkedHashMap; import java.util.LinkedList; import java.util.Map; @@ -37,6 +36,54 @@ import org.archive.modules.CrawlURI; import org.archive.modules.fetcher.FetchStatusCodes; import org.archive.util.TextUtils; +/** + * An extractor that uses regular expressions to find strings in the fetched + * content of a URI, and constructs outlink URIs from those strings. + * + *
+ * The crawl operator configures these parameters: + * + *
uriRegex: a regular expression to match against the urlcontentRegexes a map of named regular expressions { name =>
+ * regex } to run against the contenttemplate: the template for constructing the outlinks
+ * The URI is checked against uriRegex. The match is done using
+ * {@link Matcher#matches()}, so the full URI string must match, not just a
+ * substring. If it does match, then the matching groups are available to the
+ * URI-building template as ${uriRegex[n]}. If it does not match,
+ * processing of the URI is finished and no outlinks are extracted.
+ *
+ *
+ * Then the extractor looks for matches for each of the
+ * contentRegexes in the fetched content. If any of the regular
+ * expressions produce no matches, processing of the URI is finished and no
+ * outlinks are extracted. If at least one match is found for each regular
+ * expression, then an outlink is constructed, using the URI-building template,
+ * for every combination of matches. The matching groups are available to the
+ * template as ${name[n]}.
+ *
+ *
+ * Outlinks are constructed using the URI-building template.
+ * Variable interpolation using the familiar ${...} syntax is supported. The
+ * template is evaluated for each combination of regular expression matches
+ * found, and the matching groups are available to the template as
+ * ${regexName[n]}. An example template might look like:
+ * http://example.org/${uriRegex[1]}/foo?bar=${myContentRegex[0]}.
+ *
+ *
+ * The template is evaluated as a Groovy Template, so further capabilities
+ * beyond simple variable interpolation are available.
+ *
+ * @see http://groovy.codehaus.org/Groovy+Templates
+ *
+ * @contributor nlevitt
+ * @contributor travis
+ */
public class ExtractorMultipleRegex extends Extractor {
private static final Logger LOGGER =
@@ -45,18 +92,34 @@ public class ExtractorMultipleRegex extends Extractor {
{
setUriRegex("");
}
- public void setUriRegex(String reg) {
- kp.put("uriRegex", reg);
+ /**
+ * Regular expression against which to match the URI. If the URI matches,
+ * then the matching groups are available to the URI-building template as
+ *
+ * The template is evaluated as a Groovy Template, so further capabilities
+ * beyond simple variable interpolation are available.
+ *
+ * @see http://groovy.codehaus.org/Groovy+Templates
+ */
+ public void setTemplate(String template) {
+ kp.put("template", template);
+ }
public String getTemplate() {
return (String) kp.get("template");
}
- public void setTemplate(String templ) {
- kp.put("template", templ);
- }
@Override
protected boolean shouldProcess(CrawlURI uri) {
@@ -142,6 +221,9 @@ public class ExtractorMultipleRegex extends Extractor {
matchLists.put(regexName, matchList);
}
+ // XXX how expensive is this? should we cache it? would we need a
+ // separate one per thread? what about making sure we have the right one
+ // in case of a sheet overlay?
Template groovyTemplate;
try {
groovyTemplate = new SimpleTemplateEngine().createTemplate(getTemplate());
${uriRegex[n]}. If it does not match, processing of this URI
+ * is finished and no outlinks are extracted.
+ */
+ public void setUriRegex(String uriRegex) {
+ kp.put("uriRegex", uriRegex);
}
public String getUriRegex() {
return (String) kp.get("uriRegex");
}
{
- setContentRegexes(new HashMap${name[n]}.
+ */
+ public void setContentRegexes(Map${regexName[n]}. An example template might look
+ * like:
+ * http://example.org/${uriRegex[1]}/foo?bar=${myContentRegex[0]}.
+ *
+ *