diff --git a/contrib/src/main/java/org/archive/modules/extractor/KnowledgableExtractorJS.java b/contrib/src/main/java/org/archive/modules/extractor/KnowledgableExtractorJS.java new file mode 100644 index 00000000..4645693f --- /dev/null +++ b/contrib/src/main/java/org/archive/modules/extractor/KnowledgableExtractorJS.java @@ -0,0 +1,115 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.archive.modules.extractor; + +import java.util.Collection; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.regex.Matcher; + +import org.apache.commons.httpclient.URIException; +import org.apache.commons.lang.StringEscapeUtils; +import org.apache.commons.lang.exception.NestableRuntimeException; +import org.archive.modules.CrawlURI; +import org.archive.net.UURI; +import org.archive.net.UURIFactory; +import org.archive.util.TextUtils; + +/** + * A subclass of {@link ExtractorJS} that has some customized behavior for + * specific kinds of web pages. As of April 2015, the one special behavior it + * has is for drupal generated pages. See https://webarchive.jira.com/browse/ARI-4190 + */ +public class KnowledgableExtractorJS extends ExtractorJS { + + private static Logger LOGGER = + Logger.getLogger(KnowledgableExtractorJS.class.getName()); + + /** + * Wraps a {@link CrawlURI}, allowing baseURI to be overridden, without + * changing the underlying CrawlURI. The only methods implemented are the + * ones necessary for {@link ExtractorJS} to work properly. + */ + protected static class CustomizedCrawlURIFacade extends CrawlURI { + private static final long serialVersionUID = 1l; + + protected CrawlURI wrapped; + protected UURI baseURI; + + public CustomizedCrawlURIFacade(CrawlURI wrapped, UURI baseURI) { + super(wrapped.getUURI(), wrapped.getPathFromSeed(), wrapped.getVia(), wrapped.getViaContext()); + this.wrapped = wrapped; + this.baseURI = baseURI; + } + + /** + * @return value set in {@link #KnowledgableExtractorJS(CrawlURI, UURI)} + */ + @Override + public UURI getBaseURI() { + return baseURI; + } + + /** Delegates to wrapped CrawlURI */ + @Override + public CrawlURI createCrawlURI(UURI destination, LinkContext context, + Hop hop) throws URIException { + return wrapped.createCrawlURI(destination, context, hop); + } + + /** Delegates to wrapped CrawlURI */ + @Override + public Collection getOutLinks() { + return wrapped.getOutLinks(); + } + + /** Delegates to wrapped CrawlURI */ + @Override + public void incrementDiscardedOutLinks() { + wrapped.incrementDiscardedOutLinks(); + } + } + + public long considerStrings(Extractor ext, + CrawlURI curi, CharSequence cs, boolean handlingJSFile) { + + CrawlURI baseUri = curi; + + Matcher m = TextUtils.getMatcher("jQuery\\.extend\\(Drupal\\.settings,[^'\"]*['\"]basePath['\"]:[^'\"]*['\"]([^'\"]+)['\"]", cs); + if (m.find()) { + String basePath = m.group(1); + try { + basePath = StringEscapeUtils.unescapeJavaScript(basePath); + } catch (NestableRuntimeException e) { + LOGGER.log(Level.WARNING, "problem unescaping purported drupal basePath '" + basePath + "'", e); + } + + try { + UURI baseUURI = UURIFactory.getInstance(curi.getUURI(), basePath); + baseUri = new CustomizedCrawlURIFacade(curi, baseUURI); + } catch (URIException e) { + LOGGER.log(Level.WARNING, "problem creating UURI from drupal basePath '" + basePath + "'", e); + } + } + TextUtils.recycleMatcher(m); + + return super.considerStrings(ext, baseUri, cs, handlingJSFile); + } + +}