From 54aca17757ef6e5bcdfc38f5f78d419e8463231a Mon Sep 17 00:00:00 2001 From: nlevitt Date: Wed, 14 Sep 2011 21:34:26 +0000 Subject: [PATCH] Removing very old abandoned line of development, package org.archive.extractor --- .../extractor/CharSequenceLinkExtractor.java | 189 ------- .../extractor/CharSequenceProvider.java | 34 -- .../extractor/ExtractErrorListener.java | 41 -- .../org/archive/extractor/LinkExtractor.java | 83 ---- .../extractor/RegexCSSLinkExtractor.java | 107 ---- .../extractor/RegexHTMLLinkExtractor.java | 461 ------------------ .../extractor/RegexJSLinkExtractor.java | 107 ---- .../java/org/archive/extractor/overview.html | 14 - 8 files changed, 1036 deletions(-) delete mode 100644 engine/src/main/java/org/archive/extractor/CharSequenceLinkExtractor.java delete mode 100644 engine/src/main/java/org/archive/extractor/CharSequenceProvider.java delete mode 100644 engine/src/main/java/org/archive/extractor/ExtractErrorListener.java delete mode 100644 engine/src/main/java/org/archive/extractor/LinkExtractor.java delete mode 100644 engine/src/main/java/org/archive/extractor/RegexCSSLinkExtractor.java delete mode 100644 engine/src/main/java/org/archive/extractor/RegexHTMLLinkExtractor.java delete mode 100644 engine/src/main/java/org/archive/extractor/RegexJSLinkExtractor.java delete mode 100644 engine/src/main/java/org/archive/extractor/overview.html diff --git a/engine/src/main/java/org/archive/extractor/CharSequenceLinkExtractor.java b/engine/src/main/java/org/archive/extractor/CharSequenceLinkExtractor.java deleted file mode 100644 index 8afe98a8..00000000 --- a/engine/src/main/java/org/archive/extractor/CharSequenceLinkExtractor.java +++ /dev/null @@ -1,189 +0,0 @@ -/* - * This file is part of the Heritrix web crawler (crawler.archive.org). - * - * Licensed to the Internet Archive (IA) by one or more individual - * contributors. - * - * The IA licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.archive.extractor; - -import java.io.InputStream; -import java.nio.charset.Charset; -import java.util.LinkedList; -import java.util.List; -import java.util.NoSuchElementException; - -import org.archive.modules.extractor.Link; -import org.archive.net.UURI; - -/** - * Abstract superclass providing utility methods for LinkExtractors which - * would prefer to work on a CharSequence rather than a stream. - * - * ROUGH DRAFT IN PROGRESS / incomplete... untested... - * - * @author gojomo - */ -public abstract class CharSequenceLinkExtractor implements LinkExtractor { - - protected UURI source; - protected UURI base; - protected ExtractErrorListener extractErrorListener; - - protected CharSequence sourceContent; - protected LinkedList next; - - public void setup(UURI source, UURI base, InputStream content, - Charset charset, ExtractErrorListener listener) { - setup(source, base, charSequenceFrom(content,charset), listener); - } - - /** - * @param source - * @param base - * @param content - * @param listener - */ - public void setup(UURI source, UURI base, CharSequence content, - ExtractErrorListener listener) { - this.source = source; - this.base = base; - this.extractErrorListener = listener; - this.sourceContent = content; - this.next = new LinkedList(); - } - - - /** - * Convenience method for when source and base are same. - * - * @param sourceandbase - * @param content - * @param listener - */ - public void setup(UURI sourceandbase, CharSequence content, - ExtractErrorListener listener) { - setup(sourceandbase, sourceandbase, content, listener); - } - - /* (non-Javadoc) - * @see org.archive.extractor.LinkExtractor#setup(org.archive.crawler.datamodel.UURI, java.io.InputStream, java.nio.charset.Charset) - */ - public void setup(UURI sourceandbase, InputStream content, Charset charset, - ExtractErrorListener listener) { - setup(sourceandbase,sourceandbase,content,charset,listener); - } - - /* (non-Javadoc) - * @see org.archive.extractor.LinkExtractor#nextLink() - */ - public Link nextLink() { - if(!hasNext()) { - throw new NoSuchElementException(); - } - // next will have been filled with at least one item - return (Link) next.removeFirst(); - } - - /** - * Discard all state. Another setup() is required to use again. - */ - public void reset() { - base = null; - source = null; - sourceContent = null; // TODO: discard other resources - } - - /* (non-Javadoc) - * @see java.util.Iterator#hasNext() - */ - public boolean hasNext() { - if (!next.isEmpty()) { - return true; - } - return findNextLink(); - } - - /** - * Scan to the next link(s), if any, loading it into the next buffer. - * - * @return true if any links are found/available, false otherwise - */ - abstract protected boolean findNextLink(); - - /* (non-Javadoc) - * @see java.util.Iterator#next() - */ - public Link next() { - return nextLink(); - } - - /* (non-Javadoc) - * @see java.util.Iterator#remove() - */ - public void remove() { - throw new UnsupportedOperationException(); - } - - /** - * @param content - * @param charset - * @return CharSequence obtained from stream in given charset - */ - protected CharSequence charSequenceFrom(InputStream content, Charset charset) { - // See if content InputStream can provide - if(content instanceof CharSequenceProvider) { - return ((CharSequenceProvider)content).getCharSequence(); - } - // otherwise, create one - return createCharSequenceFrom(content, charset); - } - - /** - * @param content - * @param charset - * @return CharSequence built over given stream in given charset - */ - protected CharSequence createCharSequenceFrom(InputStream content, Charset charset) { - // TODO: implement - return null; - // TODO: consider cleanup in reset() - } - - /** - * Convenience method to do default extraction. - * - * @param content - * @param source - * @param base - * @param collector - * @param extractErrorListener - */ - public static void extract(CharSequence content, UURI source, UURI base, - List collector, ExtractErrorListener extractErrorListener) { - // TODO: arrange for inheritance of prefs... eg when HTML includes JS - // includes HTML, have inner HTML follow robots, etc from outer - CharSequenceLinkExtractor extractor = newDefaultInstance(); - extractor.setup(source, base, content, extractErrorListener); - while (extractor.hasNext()) { - collector.add(extractor.nextLink()); - } - extractor.reset(); - } - - protected static CharSequenceLinkExtractor newDefaultInstance() { - // override in subclasses - return null; - } -} diff --git a/engine/src/main/java/org/archive/extractor/CharSequenceProvider.java b/engine/src/main/java/org/archive/extractor/CharSequenceProvider.java deleted file mode 100644 index 8ef4176c..00000000 --- a/engine/src/main/java/org/archive/extractor/CharSequenceProvider.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * This file is part of the Heritrix web crawler (crawler.archive.org). - * - * Licensed to the Internet Archive (IA) by one or more individual - * contributors. - * - * The IA licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.archive.extractor; - -/** - * Interface indicating an object can efficiently provide a - * (perhaps cached or simulated) CharSequence version of itself. - * - * @author gojomo - */ -public interface CharSequenceProvider { - - /** - * @return CharSequence linked/cached/implied by this object - */ - CharSequence getCharSequence(); - -} diff --git a/engine/src/main/java/org/archive/extractor/ExtractErrorListener.java b/engine/src/main/java/org/archive/extractor/ExtractErrorListener.java deleted file mode 100644 index 9856e109..00000000 --- a/engine/src/main/java/org/archive/extractor/ExtractErrorListener.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * This file is part of the Heritrix web crawler (crawler.archive.org). - * - * Licensed to the Internet Archive (IA) by one or more individual - * contributors. - * - * The IA licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.archive.extractor; - -import java.io.IOException; - -import org.archive.net.UURI; - -/** - * ExtractErrorListener receives exceptions that may need to be logged - * from inside a LinkExtractor, allowing the extraction to continue - * without raising an exception through hasNext()/next()/nextLink(). - * - * @author gojomo - */ -public interface ExtractErrorListener { - /** - * Callback to report an extraction error. - * - * @param ex - * @param source - * @param context - */ - public void noteExtractError(IOException ex, UURI source, CharSequence context); -} diff --git a/engine/src/main/java/org/archive/extractor/LinkExtractor.java b/engine/src/main/java/org/archive/extractor/LinkExtractor.java deleted file mode 100644 index ea4b90d8..00000000 --- a/engine/src/main/java/org/archive/extractor/LinkExtractor.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * This file is part of the Heritrix web crawler (crawler.archive.org). - * - * Licensed to the Internet Archive (IA) by one or more individual - * contributors. - * - * The IA licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.archive.extractor; - -import java.io.InputStream; -import java.nio.charset.Charset; -import java.util.Iterator; - -import org.archive.modules.extractor.Link; -import org.archive.net.UURI; - -/** - * LinkExtractor is a general interface for classes which, when given an - * InputStream and Charset, can scan for Links and return them via - * an Iterator interface. - * - * Implementors may in fact complete all extraction on the first - * hasNext(), then trickle Links out from an internal collection, - * depending on whether the link-extraction technique used is amenable - * to incremental scanning. - * - * ROUGH DRAFT IN PROGRESS / incomplete... untested... - * - * @author gojomo - */ -public interface LinkExtractor extends Iterator { - /** - * Setup the LinkExtractor to operate on the given stream and charset, - * considering the given contextURI as the initial 'base' URI for - * resolving relative URIs. - * - * May be called to 'reset' a LinkExtractor to start with new input. - * - * @param source source URI - * @param base base URI (usually the source URI) for URI derelativizing - * @param content input stream of content to scan for links - * @param charset Charset to consult to decode stream to characters - * @param listener ExtractErrorListener to notify, rather than raising - * exception through extraction loop - */ - public void setup(UURI source, UURI base, InputStream content, - Charset charset, ExtractErrorListener listener); - - /** - * Convenience version of above for common case where source and base are - * same. - * - * @param sourceandbase URI to use as source and base for derelativizing - * @param content input stream of content to scan for links - * @param charset Charset to consult to decode stream to characters - * @param listener ExtractErrorListener to notify, rather than raising - * exception through extraction loop - */ - public void setup(UURI sourceandbase, InputStream content, - Charset charset, ExtractErrorListener listener); - - /** - * Alternative to Iterator.next() which returns type Link. - * @return a discovered Link - */ - public Link nextLink(); - - /** - * Discard all state and release any used resources. - */ - public void reset(); -} diff --git a/engine/src/main/java/org/archive/extractor/RegexCSSLinkExtractor.java b/engine/src/main/java/org/archive/extractor/RegexCSSLinkExtractor.java deleted file mode 100644 index 52d3096e..00000000 --- a/engine/src/main/java/org/archive/extractor/RegexCSSLinkExtractor.java +++ /dev/null @@ -1,107 +0,0 @@ -/* - * This file is part of the Heritrix web crawler (crawler.archive.org). - * - * Licensed to the Internet Archive (IA) by one or more individual - * contributors. - * - * The IA licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.archive.extractor; - -import java.util.regex.Matcher; - -import org.apache.commons.httpclient.URIException; -import org.archive.modules.extractor.Hop; -import org.archive.modules.extractor.Link; -import org.archive.modules.extractor.LinkContext; -import org.archive.net.UURIFactory; -import org.archive.util.DevUtils; -import org.archive.util.TextUtils; - -/** - * This extractor is parsing URIs from CSS type files. - * The format of a CSS URL value is 'url(' followed by optional white space - * followed by an optional single quote (') or double quote (") character - * followed by the URL itself followed by an optional single quote (') or - * double quote (") character followed by optional white space followed by ')'. - * Parentheses, commas, white space characters, single quotes (') and double - * quotes (") appearing in a URL must be escaped with a backslash: - * '\(', '\)', '\,'. Partial URLs are interpreted relative to the source of - * the style sheet, not relative to the document. - * Source: www.w3.org - * - * ROUGH DRAFT IN PROGRESS / incomplete... untested... major changes likely - * - * @author igor gojomo - * - **/ - -public class RegexCSSLinkExtractor extends CharSequenceLinkExtractor { - - private static String ESCAPED_AMP = "&"; - // CSS escapes: "Parentheses, commas, whitespace characters, single - // quotes (') and double quotes (") appearing in a URL must be - // escaped with a backslash" - static final String CSS_BACKSLASH_ESCAPE = "\\\\([,'\"\\(\\)\\s])"; - - protected Matcher uris; - - /** - * CSS URL extractor pattern. - * - * This pattern extracts URIs for CSS files - **/ - static final String CSS_URI_EXTRACTOR = - "(?:@import (?:url[(]|)|url[(])\\s*([\\\"\']?)([^\\\"\'].*?)\\1\\s*[);]"; - - protected boolean findNextLink() { - if (uris == null) { - uris = TextUtils.getMatcher(CSS_URI_EXTRACTOR, sourceContent); - // NOTE: this matcher can't be recycled in this method because - // it is reused on rentry - } - String cssUri; - try { - while (uris.find()) { - cssUri = uris.group(2); - // TODO: Escape more HTML Entities. - cssUri = TextUtils.replaceAll(ESCAPED_AMP, cssUri, "&"); - // Remove backslashes when used as escape character in CSS URL - cssUri = TextUtils.replaceAll(CSS_BACKSLASH_ESCAPE, cssUri, "$1"); - // TODO: handle relative URIs? - try { - Link link = new Link(source, UURIFactory.getInstance(base, - cssUri), LinkContext.EMBED_MISC, Hop.EMBED); - next.addLast(link); - } catch (URIException e) { - extractErrorListener.noteExtractError(e, source, cssUri); - } - return true; - } - } catch (StackOverflowError e) { - DevUtils.warnHandle(e, "RegexCSSLinkExtractor StackOverflowError"); - } - return false; - } - - public void reset() { - super.reset(); - TextUtils.recycleMatcher(uris); - uris = null; - } - - protected static CharSequenceLinkExtractor newDefaultInstance() { - return new RegexCSSLinkExtractor(); - } -} diff --git a/engine/src/main/java/org/archive/extractor/RegexHTMLLinkExtractor.java b/engine/src/main/java/org/archive/extractor/RegexHTMLLinkExtractor.java deleted file mode 100644 index 07e5af46..00000000 --- a/engine/src/main/java/org/archive/extractor/RegexHTMLLinkExtractor.java +++ /dev/null @@ -1,461 +0,0 @@ -/* - * This file is part of the Heritrix web crawler (crawler.archive.org). - * - * Licensed to the Internet Archive (IA) by one or more individual - * contributors. - * - * The IA licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.archive.extractor; - -import java.util.ArrayList; -import java.util.Iterator; -import java.util.LinkedList; -import java.util.logging.Level; -import java.util.logging.Logger; -import java.util.regex.Matcher; - -import org.apache.commons.httpclient.URIException; -import org.archive.modules.extractor.HTMLLinkContext; -import org.archive.modules.extractor.Hop; -import org.archive.modules.extractor.Link; -import org.archive.modules.extractor.LinkContext; -import org.archive.net.UURI; -import org.archive.net.UURIFactory; -import org.archive.util.DevUtils; -import org.archive.util.TextUtils; - - -/** - * Basic link-extraction, from an HTML content-body, - * using regular expressions. - * - * ROUGH DRAFT IN PROGRESS / incomplete... untested... - * - * @author gojomo - */ -public class RegexHTMLLinkExtractor extends CharSequenceLinkExtractor { - private static Logger logger = - Logger.getLogger(RegexHTMLLinkExtractor.class.getName()); - - boolean honorRobots = true; - boolean extractInlineCss = true; - boolean extractInlineJs = true; - - protected LinkedList next = new LinkedList(); - protected Matcher tags; - - /* (non-Javadoc) - * @see org.archive.extractor.CharSequenceLinkExtractor#findNextLink() - */ - protected boolean findNextLink() { - if (tags == null) { - tags = TextUtils.getMatcher(RELEVANT_TAG_EXTRACTOR, sourceContent); - } - while(tags.find()) { - if(Thread.interrupted()){ - // TODO: throw an exception, perhaps, rather than just clear & break? - break; - } - if (tags.start(8) > 0) { - // comment match - // for now do nothing - } else if (tags.start(7) > 0) { - // match - int start = tags.start(5); - int end = tags.end(5); - processMeta(sourceContent.subSequence(start, end)); - } else if (tags.start(5) > 0) { - // generic match - int start5 = tags.start(5); - int end5 = tags.end(5); - int start6 = tags.start(6); - int end6 = tags.end(6); - processGeneralTag(sourceContent.subSequence(start6, end6), - sourceContent.subSequence(start5, end5)); - } else if (tags.start(1) > 0) { - //