From e9fcb3450856878ce0926913f5fce804f2618e57 Mon Sep 17 00:00:00 2001 From: Noah Levitt Date: Tue, 21 Apr 2015 01:19:33 -0700 Subject: [PATCH 1/3] new contrib module KnowledgableExtractorJS, a subclass of ExtractorJS that has some customized behavior for specific kinds of web pages; initially, the one special behavior it has is for drupal generated pages --- .../extractor/KnowledgableExtractorJS.java | 96 +++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100644 contrib/src/main/java/org/archive/modules/extractor/KnowledgableExtractorJS.java diff --git a/contrib/src/main/java/org/archive/modules/extractor/KnowledgableExtractorJS.java b/contrib/src/main/java/org/archive/modules/extractor/KnowledgableExtractorJS.java new file mode 100644 index 00000000..0a00e770 --- /dev/null +++ b/contrib/src/main/java/org/archive/modules/extractor/KnowledgableExtractorJS.java @@ -0,0 +1,96 @@ +package org.archive.modules.extractor; + +import java.util.Collection; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.regex.Matcher; + +import org.apache.commons.httpclient.URIException; +import org.apache.commons.lang.StringEscapeUtils; +import org.apache.commons.lang.exception.NestableRuntimeException; +import org.archive.modules.CrawlURI; +import org.archive.net.UURI; +import org.archive.net.UURIFactory; +import org.archive.util.TextUtils; + +/** + * A subclass of {@link ExtractorJS} that has some customized behavior for + * specific kinds of web pages. As of April 2015, the one special behavior it + * has is for drupal generated pages. See https://webarchive.jira.com/browse/ARI-4190 + */ +public class KnowledgableExtractorJS extends ExtractorJS { + + private static Logger LOGGER = + Logger.getLogger(KnowledgableExtractorJS.class.getName()); + + /** + * XXX explain + * Only wraps the methods we need to work, or it would be very long. + */ + protected static class CustomizedCrawlURIFacade extends CrawlURI { + private static final long serialVersionUID = 1l; + + protected CrawlURI wrapped; + protected UURI baseURI; + + public CustomizedCrawlURIFacade(CrawlURI wrapped, UURI baseURI) { + super(wrapped.getUURI(), wrapped.getPathFromSeed(), wrapped.getVia(), wrapped.getViaContext()); + this.wrapped = wrapped; + this.baseURI = baseURI; + } + + /** + * @return value set in {@link #KnowledgableExtractorJS(CrawlURI, UURI)} + */ + @Override + public UURI getBaseURI() { + return baseURI; + } + + /** Delegates to wrapped CrawlURI */ + @Override + public CrawlURI createCrawlURI(UURI destination, LinkContext context, + Hop hop) throws URIException { + return wrapped.createCrawlURI(destination, context, hop); + } + + /** Delegates to wrapped CrawlURI */ + @Override + public Collection getOutLinks() { + return wrapped.getOutLinks(); + } + + /** Delegates to wrapped CrawlURI */ + @Override + public void incrementDiscardedOutLinks() { + wrapped.incrementDiscardedOutLinks(); + } + } + + public long considerStrings(Extractor ext, + CrawlURI curi, CharSequence cs, boolean handlingJSFile) { + + CrawlURI baseUri = curi; + + Matcher m = TextUtils.getMatcher("jQuery\\.extend\\(Drupal\\.settings,[^'\"]*['\"]basePath['\"]:[^'\"]*['\"]([^'\"]+)['\"]", cs); + if (m.find()) { + String basePath = m.group(1); + try { + basePath = StringEscapeUtils.unescapeJavaScript(basePath); + } catch (NestableRuntimeException e) { + LOGGER.log(Level.WARNING, "problem unescaping purported drupal basePath '" + basePath + "'", e); + } + + try { + UURI baseUURI = UURIFactory.getInstance(curi.getUURI(), basePath); + baseUri = new CustomizedCrawlURIFacade(curi, baseUURI); + } catch (URIException e) { + LOGGER.log(Level.WARNING, "problem creating UURI from drupal basePath '" + basePath + "'", e); + } + } + TextUtils.recycleMatcher(m); + + return super.considerStrings(ext, baseUri, cs, handlingJSFile); + } + +} From 0f43a3dae67f2c7198de4485a8cebe91065c5c36 Mon Sep 17 00:00:00 2001 From: Noah Levitt Date: Mon, 4 May 2015 15:08:52 -0700 Subject: [PATCH 2/3] add license header --- .../extractor/KnowledgableExtractorJS.java | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/contrib/src/main/java/org/archive/modules/extractor/KnowledgableExtractorJS.java b/contrib/src/main/java/org/archive/modules/extractor/KnowledgableExtractorJS.java index 0a00e770..d15891a0 100644 --- a/contrib/src/main/java/org/archive/modules/extractor/KnowledgableExtractorJS.java +++ b/contrib/src/main/java/org/archive/modules/extractor/KnowledgableExtractorJS.java @@ -1,3 +1,21 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.archive.modules.extractor; import java.util.Collection; From fea49f0e30f722a02b90452ac6c9548432a1705a Mon Sep 17 00:00:00 2001 From: Noah Levitt Date: Mon, 4 May 2015 15:16:11 -0700 Subject: [PATCH 3/3] expand stubbed javadoc comment --- .../archive/modules/extractor/KnowledgableExtractorJS.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/contrib/src/main/java/org/archive/modules/extractor/KnowledgableExtractorJS.java b/contrib/src/main/java/org/archive/modules/extractor/KnowledgableExtractorJS.java index d15891a0..4645693f 100644 --- a/contrib/src/main/java/org/archive/modules/extractor/KnowledgableExtractorJS.java +++ b/contrib/src/main/java/org/archive/modules/extractor/KnowledgableExtractorJS.java @@ -42,8 +42,9 @@ public class KnowledgableExtractorJS extends ExtractorJS { Logger.getLogger(KnowledgableExtractorJS.class.getName()); /** - * XXX explain - * Only wraps the methods we need to work, or it would be very long. + * Wraps a {@link CrawlURI}, allowing baseURI to be overridden, without + * changing the underlying CrawlURI. The only methods implemented are the + * ones necessary for {@link ExtractorJS} to work properly. */ protected static class CustomizedCrawlURIFacade extends CrawlURI { private static final long serialVersionUID = 1l;