diff --git a/engine/src/main/java/org/archive/extractor/CharSequenceLinkExtractor.java b/engine/src/main/java/org/archive/extractor/CharSequenceLinkExtractor.java
deleted file mode 100644
index 8afe98a8..00000000
--- a/engine/src/main/java/org/archive/extractor/CharSequenceLinkExtractor.java
+++ /dev/null
@@ -1,189 +0,0 @@
-/*
- * This file is part of the Heritrix web crawler (crawler.archive.org).
- *
- * Licensed to the Internet Archive (IA) by one or more individual
- * contributors.
- *
- * The IA licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.archive.extractor;
-
-import java.io.InputStream;
-import java.nio.charset.Charset;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.NoSuchElementException;
-
-import org.archive.modules.extractor.Link;
-import org.archive.net.UURI;
-
-/**
- * Abstract superclass providing utility methods for LinkExtractors which
- * would prefer to work on a CharSequence rather than a stream.
- *
- * ROUGH DRAFT IN PROGRESS / incomplete... untested...
- *
- * @author gojomo
- */
-public abstract class CharSequenceLinkExtractor implements LinkExtractor {
-
- protected UURI source;
- protected UURI base;
- protected ExtractErrorListener extractErrorListener;
-
- protected CharSequence sourceContent;
- protected LinkedList next;
-
- public void setup(UURI source, UURI base, InputStream content,
- Charset charset, ExtractErrorListener listener) {
- setup(source, base, charSequenceFrom(content,charset), listener);
- }
-
- /**
- * @param source
- * @param base
- * @param content
- * @param listener
- */
- public void setup(UURI source, UURI base, CharSequence content,
- ExtractErrorListener listener) {
- this.source = source;
- this.base = base;
- this.extractErrorListener = listener;
- this.sourceContent = content;
- this.next = new LinkedList();
- }
-
-
- /**
- * Convenience method for when source and base are same.
- *
- * @param sourceandbase
- * @param content
- * @param listener
- */
- public void setup(UURI sourceandbase, CharSequence content,
- ExtractErrorListener listener) {
- setup(sourceandbase, sourceandbase, content, listener);
- }
-
- /* (non-Javadoc)
- * @see org.archive.extractor.LinkExtractor#setup(org.archive.crawler.datamodel.UURI, java.io.InputStream, java.nio.charset.Charset)
- */
- public void setup(UURI sourceandbase, InputStream content, Charset charset,
- ExtractErrorListener listener) {
- setup(sourceandbase,sourceandbase,content,charset,listener);
- }
-
- /* (non-Javadoc)
- * @see org.archive.extractor.LinkExtractor#nextLink()
- */
- public Link nextLink() {
- if(!hasNext()) {
- throw new NoSuchElementException();
- }
- // next will have been filled with at least one item
- return (Link) next.removeFirst();
- }
-
- /**
- * Discard all state. Another setup() is required to use again.
- */
- public void reset() {
- base = null;
- source = null;
- sourceContent = null; // TODO: discard other resources
- }
-
- /* (non-Javadoc)
- * @see java.util.Iterator#hasNext()
- */
- public boolean hasNext() {
- if (!next.isEmpty()) {
- return true;
- }
- return findNextLink();
- }
-
- /**
- * Scan to the next link(s), if any, loading it into the next buffer.
- *
- * @return true if any links are found/available, false otherwise
- */
- abstract protected boolean findNextLink();
-
- /* (non-Javadoc)
- * @see java.util.Iterator#next()
- */
- public Link next() {
- return nextLink();
- }
-
- /* (non-Javadoc)
- * @see java.util.Iterator#remove()
- */
- public void remove() {
- throw new UnsupportedOperationException();
- }
-
- /**
- * @param content
- * @param charset
- * @return CharSequence obtained from stream in given charset
- */
- protected CharSequence charSequenceFrom(InputStream content, Charset charset) {
- // See if content InputStream can provide
- if(content instanceof CharSequenceProvider) {
- return ((CharSequenceProvider)content).getCharSequence();
- }
- // otherwise, create one
- return createCharSequenceFrom(content, charset);
- }
-
- /**
- * @param content
- * @param charset
- * @return CharSequence built over given stream in given charset
- */
- protected CharSequence createCharSequenceFrom(InputStream content, Charset charset) {
- // TODO: implement
- return null;
- // TODO: consider cleanup in reset()
- }
-
- /**
- * Convenience method to do default extraction.
- *
- * @param content
- * @param source
- * @param base
- * @param collector
- * @param extractErrorListener
- */
- public static void extract(CharSequence content, UURI source, UURI base,
- List collector, ExtractErrorListener extractErrorListener) {
- // TODO: arrange for inheritance of prefs... eg when HTML includes JS
- // includes HTML, have inner HTML follow robots, etc from outer
- CharSequenceLinkExtractor extractor = newDefaultInstance();
- extractor.setup(source, base, content, extractErrorListener);
- while (extractor.hasNext()) {
- collector.add(extractor.nextLink());
- }
- extractor.reset();
- }
-
- protected static CharSequenceLinkExtractor newDefaultInstance() {
- // override in subclasses
- return null;
- }
-}
diff --git a/engine/src/main/java/org/archive/extractor/CharSequenceProvider.java b/engine/src/main/java/org/archive/extractor/CharSequenceProvider.java
deleted file mode 100644
index 8ef4176c..00000000
--- a/engine/src/main/java/org/archive/extractor/CharSequenceProvider.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * This file is part of the Heritrix web crawler (crawler.archive.org).
- *
- * Licensed to the Internet Archive (IA) by one or more individual
- * contributors.
- *
- * The IA licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.archive.extractor;
-
-/**
- * Interface indicating an object can efficiently provide a
- * (perhaps cached or simulated) CharSequence version of itself.
- *
- * @author gojomo
- */
-public interface CharSequenceProvider {
-
- /**
- * @return CharSequence linked/cached/implied by this object
- */
- CharSequence getCharSequence();
-
-}
diff --git a/engine/src/main/java/org/archive/extractor/ExtractErrorListener.java b/engine/src/main/java/org/archive/extractor/ExtractErrorListener.java
deleted file mode 100644
index 9856e109..00000000
--- a/engine/src/main/java/org/archive/extractor/ExtractErrorListener.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * This file is part of the Heritrix web crawler (crawler.archive.org).
- *
- * Licensed to the Internet Archive (IA) by one or more individual
- * contributors.
- *
- * The IA licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.archive.extractor;
-
-import java.io.IOException;
-
-import org.archive.net.UURI;
-
-/**
- * ExtractErrorListener receives exceptions that may need to be logged
- * from inside a LinkExtractor, allowing the extraction to continue
- * without raising an exception through hasNext()/next()/nextLink().
- *
- * @author gojomo
- */
-public interface ExtractErrorListener {
- /**
- * Callback to report an extraction error.
- *
- * @param ex
- * @param source
- * @param context
- */
- public void noteExtractError(IOException ex, UURI source, CharSequence context);
-}
diff --git a/engine/src/main/java/org/archive/extractor/LinkExtractor.java b/engine/src/main/java/org/archive/extractor/LinkExtractor.java
deleted file mode 100644
index ea4b90d8..00000000
--- a/engine/src/main/java/org/archive/extractor/LinkExtractor.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * This file is part of the Heritrix web crawler (crawler.archive.org).
- *
- * Licensed to the Internet Archive (IA) by one or more individual
- * contributors.
- *
- * The IA licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.archive.extractor;
-
-import java.io.InputStream;
-import java.nio.charset.Charset;
-import java.util.Iterator;
-
-import org.archive.modules.extractor.Link;
-import org.archive.net.UURI;
-
-/**
- * LinkExtractor is a general interface for classes which, when given an
- * InputStream and Charset, can scan for Links and return them via
- * an Iterator interface.
- *
- * Implementors may in fact complete all extraction on the first
- * hasNext(), then trickle Links out from an internal collection,
- * depending on whether the link-extraction technique used is amenable
- * to incremental scanning.
- *
- * ROUGH DRAFT IN PROGRESS / incomplete... untested...
- *
- * @author gojomo
- */
-public interface LinkExtractor extends Iterator {
- /**
- * Setup the LinkExtractor to operate on the given stream and charset,
- * considering the given contextURI as the initial 'base' URI for
- * resolving relative URIs.
- *
- * May be called to 'reset' a LinkExtractor to start with new input.
- *
- * @param source source URI
- * @param base base URI (usually the source URI) for URI derelativizing
- * @param content input stream of content to scan for links
- * @param charset Charset to consult to decode stream to characters
- * @param listener ExtractErrorListener to notify, rather than raising
- * exception through extraction loop
- */
- public void setup(UURI source, UURI base, InputStream content,
- Charset charset, ExtractErrorListener listener);
-
- /**
- * Convenience version of above for common case where source and base are
- * same.
- *
- * @param sourceandbase URI to use as source and base for derelativizing
- * @param content input stream of content to scan for links
- * @param charset Charset to consult to decode stream to characters
- * @param listener ExtractErrorListener to notify, rather than raising
- * exception through extraction loop
- */
- public void setup(UURI sourceandbase, InputStream content,
- Charset charset, ExtractErrorListener listener);
-
- /**
- * Alternative to Iterator.next() which returns type Link.
- * @return a discovered Link
- */
- public Link nextLink();
-
- /**
- * Discard all state and release any used resources.
- */
- public void reset();
-}
diff --git a/engine/src/main/java/org/archive/extractor/RegexCSSLinkExtractor.java b/engine/src/main/java/org/archive/extractor/RegexCSSLinkExtractor.java
deleted file mode 100644
index 52d3096e..00000000
--- a/engine/src/main/java/org/archive/extractor/RegexCSSLinkExtractor.java
+++ /dev/null
@@ -1,107 +0,0 @@
-/*
- * This file is part of the Heritrix web crawler (crawler.archive.org).
- *
- * Licensed to the Internet Archive (IA) by one or more individual
- * contributors.
- *
- * The IA licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.archive.extractor;
-
-import java.util.regex.Matcher;
-
-import org.apache.commons.httpclient.URIException;
-import org.archive.modules.extractor.Hop;
-import org.archive.modules.extractor.Link;
-import org.archive.modules.extractor.LinkContext;
-import org.archive.net.UURIFactory;
-import org.archive.util.DevUtils;
-import org.archive.util.TextUtils;
-
-/**
- * This extractor is parsing URIs from CSS type files.
- * The format of a CSS URL value is 'url(' followed by optional white space
- * followed by an optional single quote (') or double quote (") character
- * followed by the URL itself followed by an optional single quote (') or
- * double quote (") character followed by optional white space followed by ')'.
- * Parentheses, commas, white space characters, single quotes (') and double
- * quotes (") appearing in a URL must be escaped with a backslash:
- * '\(', '\)', '\,'. Partial URLs are interpreted relative to the source of
- * the style sheet, not relative to the document.
- * Source: www.w3.org
- *
- * ROUGH DRAFT IN PROGRESS / incomplete... untested... major changes likely
- *
- * @author igor gojomo
- *
- **/
-
-public class RegexCSSLinkExtractor extends CharSequenceLinkExtractor {
-
- private static String ESCAPED_AMP = "&";
- // CSS escapes: "Parentheses, commas, whitespace characters, single
- // quotes (') and double quotes (") appearing in a URL must be
- // escaped with a backslash"
- static final String CSS_BACKSLASH_ESCAPE = "\\\\([,'\"\\(\\)\\s])";
-
- protected Matcher uris;
-
- /**
- * CSS URL extractor pattern.
- *
- * This pattern extracts URIs for CSS files
- **/
- static final String CSS_URI_EXTRACTOR =
- "(?:@import (?:url[(]|)|url[(])\\s*([\\\"\']?)([^\\\"\'].*?)\\1\\s*[);]";
-
- protected boolean findNextLink() {
- if (uris == null) {
- uris = TextUtils.getMatcher(CSS_URI_EXTRACTOR, sourceContent);
- // NOTE: this matcher can't be recycled in this method because
- // it is reused on rentry
- }
- String cssUri;
- try {
- while (uris.find()) {
- cssUri = uris.group(2);
- // TODO: Escape more HTML Entities.
- cssUri = TextUtils.replaceAll(ESCAPED_AMP, cssUri, "&");
- // Remove backslashes when used as escape character in CSS URL
- cssUri = TextUtils.replaceAll(CSS_BACKSLASH_ESCAPE, cssUri, "$1");
- // TODO: handle relative URIs?
- try {
- Link link = new Link(source, UURIFactory.getInstance(base,
- cssUri), LinkContext.EMBED_MISC, Hop.EMBED);
- next.addLast(link);
- } catch (URIException e) {
- extractErrorListener.noteExtractError(e, source, cssUri);
- }
- return true;
- }
- } catch (StackOverflowError e) {
- DevUtils.warnHandle(e, "RegexCSSLinkExtractor StackOverflowError");
- }
- return false;
- }
-
- public void reset() {
- super.reset();
- TextUtils.recycleMatcher(uris);
- uris = null;
- }
-
- protected static CharSequenceLinkExtractor newDefaultInstance() {
- return new RegexCSSLinkExtractor();
- }
-}
diff --git a/engine/src/main/java/org/archive/extractor/RegexHTMLLinkExtractor.java b/engine/src/main/java/org/archive/extractor/RegexHTMLLinkExtractor.java
deleted file mode 100644
index 07e5af46..00000000
--- a/engine/src/main/java/org/archive/extractor/RegexHTMLLinkExtractor.java
+++ /dev/null
@@ -1,461 +0,0 @@
-/*
- * This file is part of the Heritrix web crawler (crawler.archive.org).
- *
- * Licensed to the Internet Archive (IA) by one or more individual
- * contributors.
- *
- * The IA licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.archive.extractor;
-
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-import java.util.regex.Matcher;
-
-import org.apache.commons.httpclient.URIException;
-import org.archive.modules.extractor.HTMLLinkContext;
-import org.archive.modules.extractor.Hop;
-import org.archive.modules.extractor.Link;
-import org.archive.modules.extractor.LinkContext;
-import org.archive.net.UURI;
-import org.archive.net.UURIFactory;
-import org.archive.util.DevUtils;
-import org.archive.util.TextUtils;
-
-
-/**
- * Basic link-extraction, from an HTML content-body,
- * using regular expressions.
- *
- * ROUGH DRAFT IN PROGRESS / incomplete... untested...
- *
- * @author gojomo
- */
-public class RegexHTMLLinkExtractor extends CharSequenceLinkExtractor {
- private static Logger logger =
- Logger.getLogger(RegexHTMLLinkExtractor.class.getName());
-
- boolean honorRobots = true;
- boolean extractInlineCss = true;
- boolean extractInlineJs = true;
-
- protected LinkedList next = new LinkedList();
- protected Matcher tags;
-
- /* (non-Javadoc)
- * @see org.archive.extractor.CharSequenceLinkExtractor#findNextLink()
- */
- protected boolean findNextLink() {
- if (tags == null) {
- tags = TextUtils.getMatcher(RELEVANT_TAG_EXTRACTOR, sourceContent);
- }
- while(tags.find()) {
- if(Thread.interrupted()){
- // TODO: throw an exception, perhaps, rather than just clear & break?
- break;
- }
- if (tags.start(8) > 0) {
- // comment match
- // for now do nothing
- } else if (tags.start(7) > 0) {
- // match
- int start = tags.start(5);
- int end = tags.end(5);
- processMeta(sourceContent.subSequence(start, end));
- } else if (tags.start(5) > 0) {
- // generic match
- int start5 = tags.start(5);
- int end5 = tags.end(5);
- int start6 = tags.start(6);
- int end6 = tags.end(6);
- processGeneralTag(sourceContent.subSequence(start6, end6),
- sourceContent.subSequence(start5, end5));
- } else if (tags.start(1) > 0) {
- //