mirror of
https://github.com/internetarchive/heritrix3.git
synced 2026-09-23 14:15:47 +00:00
Removing very old abandoned line of development, package org.archive.extractor
This commit is contained in:
nlevitt
@@ -1,189 +0,0 @@
|
||||
/*
|
||||
* This file is part of the Heritrix web crawler (crawler.archive.org).
|
||||
*
|
||||
* Licensed to the Internet Archive (IA) by one or more individual
|
||||
* contributors.
|
||||
*
|
||||
* The IA licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.archive.extractor;
|
||||
|
||||
import java.io.InputStream;
|
||||
import java.nio.charset.Charset;
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
import java.util.NoSuchElementException;
|
||||
|
||||
import org.archive.modules.extractor.Link;
|
||||
import org.archive.net.UURI;
|
||||
|
||||
/**
|
||||
* Abstract superclass providing utility methods for LinkExtractors which
|
||||
* would prefer to work on a CharSequence rather than a stream.
|
||||
*
|
||||
* ROUGH DRAFT IN PROGRESS / incomplete... untested...
|
||||
*
|
||||
* @author gojomo
|
||||
*/
|
||||
public abstract class CharSequenceLinkExtractor implements LinkExtractor {
|
||||
|
||||
protected UURI source;
|
||||
protected UURI base;
|
||||
protected ExtractErrorListener extractErrorListener;
|
||||
|
||||
protected CharSequence sourceContent;
|
||||
protected LinkedList<Link> next;
|
||||
|
||||
public void setup(UURI source, UURI base, InputStream content,
|
||||
Charset charset, ExtractErrorListener listener) {
|
||||
setup(source, base, charSequenceFrom(content,charset), listener);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param source
|
||||
* @param base
|
||||
* @param content
|
||||
* @param listener
|
||||
*/
|
||||
public void setup(UURI source, UURI base, CharSequence content,
|
||||
ExtractErrorListener listener) {
|
||||
this.source = source;
|
||||
this.base = base;
|
||||
this.extractErrorListener = listener;
|
||||
this.sourceContent = content;
|
||||
this.next = new LinkedList<Link>();
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Convenience method for when source and base are same.
|
||||
*
|
||||
* @param sourceandbase
|
||||
* @param content
|
||||
* @param listener
|
||||
*/
|
||||
public void setup(UURI sourceandbase, CharSequence content,
|
||||
ExtractErrorListener listener) {
|
||||
setup(sourceandbase, sourceandbase, content, listener);
|
||||
}
|
||||
|
||||
/* (non-Javadoc)
|
||||
* @see org.archive.extractor.LinkExtractor#setup(org.archive.crawler.datamodel.UURI, java.io.InputStream, java.nio.charset.Charset)
|
||||
*/
|
||||
public void setup(UURI sourceandbase, InputStream content, Charset charset,
|
||||
ExtractErrorListener listener) {
|
||||
setup(sourceandbase,sourceandbase,content,charset,listener);
|
||||
}
|
||||
|
||||
/* (non-Javadoc)
|
||||
* @see org.archive.extractor.LinkExtractor#nextLink()
|
||||
*/
|
||||
public Link nextLink() {
|
||||
if(!hasNext()) {
|
||||
throw new NoSuchElementException();
|
||||
}
|
||||
// next will have been filled with at least one item
|
||||
return (Link) next.removeFirst();
|
||||
}
|
||||
|
||||
/**
|
||||
* Discard all state. Another setup() is required to use again.
|
||||
*/
|
||||
public void reset() {
|
||||
base = null;
|
||||
source = null;
|
||||
sourceContent = null; // TODO: discard other resources
|
||||
}
|
||||
|
||||
/* (non-Javadoc)
|
||||
* @see java.util.Iterator#hasNext()
|
||||
*/
|
||||
public boolean hasNext() {
|
||||
if (!next.isEmpty()) {
|
||||
return true;
|
||||
}
|
||||
return findNextLink();
|
||||
}
|
||||
|
||||
/**
|
||||
* Scan to the next link(s), if any, loading it into the next buffer.
|
||||
*
|
||||
* @return true if any links are found/available, false otherwise
|
||||
*/
|
||||
abstract protected boolean findNextLink();
|
||||
|
||||
/* (non-Javadoc)
|
||||
* @see java.util.Iterator#next()
|
||||
*/
|
||||
public Link next() {
|
||||
return nextLink();
|
||||
}
|
||||
|
||||
/* (non-Javadoc)
|
||||
* @see java.util.Iterator#remove()
|
||||
*/
|
||||
public void remove() {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
/**
|
||||
* @param content
|
||||
* @param charset
|
||||
* @return CharSequence obtained from stream in given charset
|
||||
*/
|
||||
protected CharSequence charSequenceFrom(InputStream content, Charset charset) {
|
||||
// See if content InputStream can provide
|
||||
if(content instanceof CharSequenceProvider) {
|
||||
return ((CharSequenceProvider)content).getCharSequence();
|
||||
}
|
||||
// otherwise, create one
|
||||
return createCharSequenceFrom(content, charset);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param content
|
||||
* @param charset
|
||||
* @return CharSequence built over given stream in given charset
|
||||
*/
|
||||
protected CharSequence createCharSequenceFrom(InputStream content, Charset charset) {
|
||||
// TODO: implement
|
||||
return null;
|
||||
// TODO: consider cleanup in reset()
|
||||
}
|
||||
|
||||
/**
|
||||
* Convenience method to do default extraction.
|
||||
*
|
||||
* @param content
|
||||
* @param source
|
||||
* @param base
|
||||
* @param collector
|
||||
* @param extractErrorListener
|
||||
*/
|
||||
public static void extract(CharSequence content, UURI source, UURI base,
|
||||
List<Link> collector, ExtractErrorListener extractErrorListener) {
|
||||
// TODO: arrange for inheritance of prefs... eg when HTML includes JS
|
||||
// includes HTML, have inner HTML follow robots, etc from outer
|
||||
CharSequenceLinkExtractor extractor = newDefaultInstance();
|
||||
extractor.setup(source, base, content, extractErrorListener);
|
||||
while (extractor.hasNext()) {
|
||||
collector.add(extractor.nextLink());
|
||||
}
|
||||
extractor.reset();
|
||||
}
|
||||
|
||||
protected static CharSequenceLinkExtractor newDefaultInstance() {
|
||||
// override in subclasses
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -1,34 +0,0 @@
|
||||
/*
|
||||
* This file is part of the Heritrix web crawler (crawler.archive.org).
|
||||
*
|
||||
* Licensed to the Internet Archive (IA) by one or more individual
|
||||
* contributors.
|
||||
*
|
||||
* The IA licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.archive.extractor;
|
||||
|
||||
/**
|
||||
* Interface indicating an object can efficiently provide a
|
||||
* (perhaps cached or simulated) CharSequence version of itself.
|
||||
*
|
||||
* @author gojomo
|
||||
*/
|
||||
public interface CharSequenceProvider {
|
||||
|
||||
/**
|
||||
* @return CharSequence linked/cached/implied by this object
|
||||
*/
|
||||
CharSequence getCharSequence();
|
||||
|
||||
}
|
||||
@@ -1,41 +0,0 @@
|
||||
/*
|
||||
* This file is part of the Heritrix web crawler (crawler.archive.org).
|
||||
*
|
||||
* Licensed to the Internet Archive (IA) by one or more individual
|
||||
* contributors.
|
||||
*
|
||||
* The IA licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.archive.extractor;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
import org.archive.net.UURI;
|
||||
|
||||
/**
|
||||
* ExtractErrorListener receives exceptions that may need to be logged
|
||||
* from inside a LinkExtractor, allowing the extraction to continue
|
||||
* without raising an exception through hasNext()/next()/nextLink().
|
||||
*
|
||||
* @author gojomo
|
||||
*/
|
||||
public interface ExtractErrorListener {
|
||||
/**
|
||||
* Callback to report an extraction error.
|
||||
*
|
||||
* @param ex
|
||||
* @param source
|
||||
* @param context
|
||||
*/
|
||||
public void noteExtractError(IOException ex, UURI source, CharSequence context);
|
||||
}
|
||||
@@ -1,83 +0,0 @@
|
||||
/*
|
||||
* This file is part of the Heritrix web crawler (crawler.archive.org).
|
||||
*
|
||||
* Licensed to the Internet Archive (IA) by one or more individual
|
||||
* contributors.
|
||||
*
|
||||
* The IA licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.archive.extractor;
|
||||
|
||||
import java.io.InputStream;
|
||||
import java.nio.charset.Charset;
|
||||
import java.util.Iterator;
|
||||
|
||||
import org.archive.modules.extractor.Link;
|
||||
import org.archive.net.UURI;
|
||||
|
||||
/**
|
||||
* LinkExtractor is a general interface for classes which, when given an
|
||||
* InputStream and Charset, can scan for Links and return them via
|
||||
* an Iterator interface.
|
||||
*
|
||||
* Implementors may in fact complete all extraction on the first
|
||||
* hasNext(), then trickle Links out from an internal collection,
|
||||
* depending on whether the link-extraction technique used is amenable
|
||||
* to incremental scanning.
|
||||
*
|
||||
* ROUGH DRAFT IN PROGRESS / incomplete... untested...
|
||||
*
|
||||
* @author gojomo
|
||||
*/
|
||||
public interface LinkExtractor extends Iterator<Link> {
|
||||
/**
|
||||
* Setup the LinkExtractor to operate on the given stream and charset,
|
||||
* considering the given contextURI as the initial 'base' URI for
|
||||
* resolving relative URIs.
|
||||
*
|
||||
* May be called to 'reset' a LinkExtractor to start with new input.
|
||||
*
|
||||
* @param source source URI
|
||||
* @param base base URI (usually the source URI) for URI derelativizing
|
||||
* @param content input stream of content to scan for links
|
||||
* @param charset Charset to consult to decode stream to characters
|
||||
* @param listener ExtractErrorListener to notify, rather than raising
|
||||
* exception through extraction loop
|
||||
*/
|
||||
public void setup(UURI source, UURI base, InputStream content,
|
||||
Charset charset, ExtractErrorListener listener);
|
||||
|
||||
/**
|
||||
* Convenience version of above for common case where source and base are
|
||||
* same.
|
||||
*
|
||||
* @param sourceandbase URI to use as source and base for derelativizing
|
||||
* @param content input stream of content to scan for links
|
||||
* @param charset Charset to consult to decode stream to characters
|
||||
* @param listener ExtractErrorListener to notify, rather than raising
|
||||
* exception through extraction loop
|
||||
*/
|
||||
public void setup(UURI sourceandbase, InputStream content,
|
||||
Charset charset, ExtractErrorListener listener);
|
||||
|
||||
/**
|
||||
* Alternative to Iterator.next() which returns type Link.
|
||||
* @return a discovered Link
|
||||
*/
|
||||
public Link nextLink();
|
||||
|
||||
/**
|
||||
* Discard all state and release any used resources.
|
||||
*/
|
||||
public void reset();
|
||||
}
|
||||
@@ -1,107 +0,0 @@
|
||||
/*
|
||||
* This file is part of the Heritrix web crawler (crawler.archive.org).
|
||||
*
|
||||
* Licensed to the Internet Archive (IA) by one or more individual
|
||||
* contributors.
|
||||
*
|
||||
* The IA licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.archive.extractor;
|
||||
|
||||
import java.util.regex.Matcher;
|
||||
|
||||
import org.apache.commons.httpclient.URIException;
|
||||
import org.archive.modules.extractor.Hop;
|
||||
import org.archive.modules.extractor.Link;
|
||||
import org.archive.modules.extractor.LinkContext;
|
||||
import org.archive.net.UURIFactory;
|
||||
import org.archive.util.DevUtils;
|
||||
import org.archive.util.TextUtils;
|
||||
|
||||
/**
|
||||
* This extractor is parsing URIs from CSS type files.
|
||||
* The format of a CSS URL value is 'url(' followed by optional white space
|
||||
* followed by an optional single quote (') or double quote (") character
|
||||
* followed by the URL itself followed by an optional single quote (') or
|
||||
* double quote (") character followed by optional white space followed by ')'.
|
||||
* Parentheses, commas, white space characters, single quotes (') and double
|
||||
* quotes (") appearing in a URL must be escaped with a backslash:
|
||||
* '\(', '\)', '\,'. Partial URLs are interpreted relative to the source of
|
||||
* the style sheet, not relative to the document. <a href="http://www.w3.org/TR/REC-CSS1#url">
|
||||
* Source: www.w3.org</a>
|
||||
*
|
||||
* ROUGH DRAFT IN PROGRESS / incomplete... untested... major changes likely
|
||||
*
|
||||
* @author igor gojomo
|
||||
*
|
||||
**/
|
||||
|
||||
public class RegexCSSLinkExtractor extends CharSequenceLinkExtractor {
|
||||
|
||||
private static String ESCAPED_AMP = "&";
|
||||
// CSS escapes: "Parentheses, commas, whitespace characters, single
|
||||
// quotes (') and double quotes (") appearing in a URL must be
|
||||
// escaped with a backslash"
|
||||
static final String CSS_BACKSLASH_ESCAPE = "\\\\([,'\"\\(\\)\\s])";
|
||||
|
||||
protected Matcher uris;
|
||||
|
||||
/**
|
||||
* CSS URL extractor pattern.
|
||||
*
|
||||
* This pattern extracts URIs for CSS files
|
||||
**/
|
||||
static final String CSS_URI_EXTRACTOR =
|
||||
"(?:@import (?:url[(]|)|url[(])\\s*([\\\"\']?)([^\\\"\'].*?)\\1\\s*[);]";
|
||||
|
||||
protected boolean findNextLink() {
|
||||
if (uris == null) {
|
||||
uris = TextUtils.getMatcher(CSS_URI_EXTRACTOR, sourceContent);
|
||||
// NOTE: this matcher can't be recycled in this method because
|
||||
// it is reused on rentry
|
||||
}
|
||||
String cssUri;
|
||||
try {
|
||||
while (uris.find()) {
|
||||
cssUri = uris.group(2);
|
||||
// TODO: Escape more HTML Entities.
|
||||
cssUri = TextUtils.replaceAll(ESCAPED_AMP, cssUri, "&");
|
||||
// Remove backslashes when used as escape character in CSS URL
|
||||
cssUri = TextUtils.replaceAll(CSS_BACKSLASH_ESCAPE, cssUri, "$1");
|
||||
// TODO: handle relative URIs?
|
||||
try {
|
||||
Link link = new Link(source, UURIFactory.getInstance(base,
|
||||
cssUri), LinkContext.EMBED_MISC, Hop.EMBED);
|
||||
next.addLast(link);
|
||||
} catch (URIException e) {
|
||||
extractErrorListener.noteExtractError(e, source, cssUri);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
} catch (StackOverflowError e) {
|
||||
DevUtils.warnHandle(e, "RegexCSSLinkExtractor StackOverflowError");
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
public void reset() {
|
||||
super.reset();
|
||||
TextUtils.recycleMatcher(uris);
|
||||
uris = null;
|
||||
}
|
||||
|
||||
protected static CharSequenceLinkExtractor newDefaultInstance() {
|
||||
return new RegexCSSLinkExtractor();
|
||||
}
|
||||
}
|
||||
@@ -1,461 +0,0 @@
|
||||
/*
|
||||
* This file is part of the Heritrix web crawler (crawler.archive.org).
|
||||
*
|
||||
* Licensed to the Internet Archive (IA) by one or more individual
|
||||
* contributors.
|
||||
*
|
||||
* The IA licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.archive.extractor;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Iterator;
|
||||
import java.util.LinkedList;
|
||||
import java.util.logging.Level;
|
||||
import java.util.logging.Logger;
|
||||
import java.util.regex.Matcher;
|
||||
|
||||
import org.apache.commons.httpclient.URIException;
|
||||
import org.archive.modules.extractor.HTMLLinkContext;
|
||||
import org.archive.modules.extractor.Hop;
|
||||
import org.archive.modules.extractor.Link;
|
||||
import org.archive.modules.extractor.LinkContext;
|
||||
import org.archive.net.UURI;
|
||||
import org.archive.net.UURIFactory;
|
||||
import org.archive.util.DevUtils;
|
||||
import org.archive.util.TextUtils;
|
||||
|
||||
|
||||
/**
|
||||
* Basic link-extraction, from an HTML content-body,
|
||||
* using regular expressions.
|
||||
*
|
||||
* ROUGH DRAFT IN PROGRESS / incomplete... untested...
|
||||
*
|
||||
* @author gojomo
|
||||
*/
|
||||
public class RegexHTMLLinkExtractor extends CharSequenceLinkExtractor {
|
||||
private static Logger logger =
|
||||
Logger.getLogger(RegexHTMLLinkExtractor.class.getName());
|
||||
|
||||
boolean honorRobots = true;
|
||||
boolean extractInlineCss = true;
|
||||
boolean extractInlineJs = true;
|
||||
|
||||
protected LinkedList<Link> next = new LinkedList<Link>();
|
||||
protected Matcher tags;
|
||||
|
||||
/* (non-Javadoc)
|
||||
* @see org.archive.extractor.CharSequenceLinkExtractor#findNextLink()
|
||||
*/
|
||||
protected boolean findNextLink() {
|
||||
if (tags == null) {
|
||||
tags = TextUtils.getMatcher(RELEVANT_TAG_EXTRACTOR, sourceContent);
|
||||
}
|
||||
while(tags.find()) {
|
||||
if(Thread.interrupted()){
|
||||
// TODO: throw an exception, perhaps, rather than just clear & break?
|
||||
break;
|
||||
}
|
||||
if (tags.start(8) > 0) {
|
||||
// comment match
|
||||
// for now do nothing
|
||||
} else if (tags.start(7) > 0) {
|
||||
// <meta> match
|
||||
int start = tags.start(5);
|
||||
int end = tags.end(5);
|
||||
processMeta(sourceContent.subSequence(start, end));
|
||||
} else if (tags.start(5) > 0) {
|
||||
// generic <whatever> match
|
||||
int start5 = tags.start(5);
|
||||
int end5 = tags.end(5);
|
||||
int start6 = tags.start(6);
|
||||
int end6 = tags.end(6);
|
||||
processGeneralTag(sourceContent.subSequence(start6, end6),
|
||||
sourceContent.subSequence(start5, end5));
|
||||
} else if (tags.start(1) > 0) {
|
||||
// <script> match
|
||||
int start = tags.start(1);
|
||||
int end = tags.end(1);
|
||||
processScript(sourceContent.subSequence(start, end),
|
||||
tags.end(2) - start);
|
||||
} else if (tags.start(3) > 0){
|
||||
// <style... match
|
||||
int start = tags.start(3);
|
||||
int end = tags.end(3);
|
||||
processStyle(sourceContent.subSequence(start, end),
|
||||
tags.end(4) - start);
|
||||
}
|
||||
if(!next.isEmpty()) {
|
||||
// at least one link found
|
||||
return true;
|
||||
}
|
||||
}
|
||||
// no relevant tags found
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Compiled relevant tag extractor.
|
||||
*
|
||||
* <p>
|
||||
* This pattern extracts either:
|
||||
* <li> (1) whole <script>...</script> or
|
||||
* <li> (2) <style>...</style> or
|
||||
* <li> (3) <meta ...> or
|
||||
* <li> (4) any other open-tag with at least one attribute
|
||||
* (eg matches "<a href='boo'>" but not "</a>" or "<br>")
|
||||
* <p>
|
||||
* groups:
|
||||
* <li> 1: SCRIPT SRC=foo>boo</SCRIPT
|
||||
* <li> 2: just script open tag
|
||||
* <li> 3: STYLE TYPE=moo>zoo</STYLE
|
||||
* <li> 4: just style open tag
|
||||
* <li> 5: entire other tag, without '<' '>'
|
||||
* <li> 6: element
|
||||
* <li> 7: META
|
||||
* <li> 8: !-- comment --
|
||||
*/
|
||||
static final String RELEVANT_TAG_EXTRACTOR =
|
||||
"(?is)<(?:((script[^>]*+)>.*?</script)|((style[^>]*+)>[^<]*+</style)|(((meta)|(?:\\w+))\\s+[^>]*+)|(!--.*?--))>";
|
||||
|
||||
// this pattern extracts attributes from any open-tag innards
|
||||
// matched by the above. attributes known to be URIs of various
|
||||
// sorts are matched specially
|
||||
static final String EACH_ATTRIBUTE_EXTRACTOR =
|
||||
"(?is)\\s((href)|(action)|(on\\w*)"
|
||||
+"|((?:src)|(?:lowsrc)|(?:background)|(?:cite)|(?:longdesc)"
|
||||
+"|(?:usemap)|(?:profile)|(?:datasrc)|(?:for))"
|
||||
+"|(codebase)|((?:classid)|(?:data))|(archive)|(code)"
|
||||
+"|(value)|([-\\w]+))"
|
||||
+"\\s*=\\s*"
|
||||
+"(?:(?:\"(.*?)(?:\"|$))"
|
||||
+"|(?:'(.*?)(?:'|$))"
|
||||
+"|(\\S+))";
|
||||
// groups:
|
||||
// 1: attribute name
|
||||
// 2: HREF - single URI relative to doc base, or occasionally javascript:
|
||||
// 3: ACTION - single URI relative to doc base, or occasionally javascript:
|
||||
// 4: ON[WHATEVER] - script handler
|
||||
// 5: SRC,LOWSRC,BACKGROUND,CITE,LONGDESC,USEMAP,PROFILE,DATASRC, or FOR
|
||||
// single URI relative to doc base
|
||||
// 6: CODEBASE - a single URI relative to doc base, affecting other
|
||||
// attributes
|
||||
// 7: CLASSID, DATA - a single URI relative to CODEBASE (if supplied)
|
||||
// 8: ARCHIVE - one or more space-delimited URIs relative to CODEBASE
|
||||
// (if supplied)
|
||||
// 9: CODE - a single URI relative to the CODEBASE (is specified).
|
||||
// 10: VALUE - often includes a uri path on forms
|
||||
// 11: any other attribute
|
||||
// 12: double-quote delimited attr value
|
||||
// 13: single-quote delimited attr value
|
||||
// 14: space-delimited attr value
|
||||
|
||||
|
||||
// much like the javascript likely-URI extractor, but
|
||||
// without requiring quotes -- this can indicate whether
|
||||
// an HTML tag attribute that isn't definitionally a
|
||||
// URI might be one anyway, as in form-tag VALUE attributes
|
||||
static final String LIKELY_URI_PATH =
|
||||
"(\\.{0,2}[^\\.\\n\\r\\s\"']*(\\.[^\\.\\n\\r\\s\"']+)+)";
|
||||
static final String ESCAPED_AMP = "&";
|
||||
static final String AMP ="&";
|
||||
static final String WHITESPACE = "\\s";
|
||||
static final String CLASSEXT =".class";
|
||||
static final String APPLET = "applet";
|
||||
static final String BASE = "base";
|
||||
static final String LINK = "link";
|
||||
|
||||
protected boolean processGeneralTag(CharSequence element, CharSequence cs) {
|
||||
|
||||
Matcher attr = TextUtils.getMatcher(EACH_ATTRIBUTE_EXTRACTOR, cs);
|
||||
|
||||
// Just in case it's an OBJECT or APPLET tag
|
||||
String codebase = null;
|
||||
ArrayList<String> resources = null;
|
||||
long tally = next.size();
|
||||
|
||||
while (attr.find()) {
|
||||
int valueGroup =
|
||||
(attr.start(12) > -1) ? 12 : (attr.start(13) > -1) ? 13 : 14;
|
||||
int start = attr.start(valueGroup);
|
||||
int end = attr.end(valueGroup);
|
||||
CharSequence value = cs.subSequence(start, end);
|
||||
if (attr.start(2) > -1) {
|
||||
// HREF
|
||||
LinkContext context = new HTMLLinkContext(element, attr.group(2));
|
||||
if(element.toString().equalsIgnoreCase(LINK)) {
|
||||
// <LINK> elements treated as embeds (css, ico, etc)
|
||||
processEmbed(value, context);
|
||||
} else {
|
||||
if (element.toString().equalsIgnoreCase(BASE)) {
|
||||
try {
|
||||
base = UURIFactory.getInstance(value.toString());
|
||||
} catch (URIException e) {
|
||||
extractErrorListener.noteExtractError(e,source,value);
|
||||
}
|
||||
}
|
||||
// other HREFs treated as links
|
||||
processLink(value, context);
|
||||
}
|
||||
} else if (attr.start(3) > -1) {
|
||||
// ACTION
|
||||
LinkContext context = new HTMLLinkContext(element, attr.group(3));
|
||||
processLink(value, context);
|
||||
} else if (attr.start(4) > -1) {
|
||||
// ON____
|
||||
processScriptCode(value); // TODO: context?
|
||||
} else if (attr.start(5) > -1) {
|
||||
// SRC etc.
|
||||
LinkContext context = new HTMLLinkContext(element, attr.group(5));
|
||||
processEmbed(value, context);
|
||||
} else if (attr.start(6) > -1) {
|
||||
// CODEBASE
|
||||
// TODO: more HTML deescaping?
|
||||
codebase = TextUtils.replaceAll(ESCAPED_AMP, value, AMP);
|
||||
LinkContext context = new HTMLLinkContext(element,attr.group(6));
|
||||
processEmbed(codebase, context);
|
||||
} else if (attr.start(7) > -1) {
|
||||
// CLASSID, DATA
|
||||
if (resources == null) {
|
||||
resources = new ArrayList<String>();
|
||||
}
|
||||
resources.add(value.toString());
|
||||
} else if (attr.start(8) > -1) {
|
||||
// ARCHIVE
|
||||
if (resources==null) {
|
||||
resources = new ArrayList<String>();
|
||||
}
|
||||
String[] multi = TextUtils.split(WHITESPACE, value);
|
||||
for(int i = 0; i < multi.length; i++ ) {
|
||||
resources.add(multi[i]);
|
||||
}
|
||||
} else if (attr.start(9) > -1) {
|
||||
// CODE
|
||||
if (resources==null) {
|
||||
resources = new ArrayList<String>();
|
||||
}
|
||||
// If element is applet and code value does not end with
|
||||
// '.class' then append '.class' to the code value.
|
||||
if (element.toString().toLowerCase().equals(APPLET) &&
|
||||
!value.toString().toLowerCase().endsWith(CLASSEXT)) {
|
||||
resources.add(value.toString() + CLASSEXT);
|
||||
} else {
|
||||
resources.add(value.toString());
|
||||
}
|
||||
|
||||
} else if (attr.start(10) > -1) {
|
||||
// VALUE
|
||||
if(TextUtils.matches(LIKELY_URI_PATH, value)) {
|
||||
LinkContext context = new HTMLLinkContext(element, attr.group(10));
|
||||
processLink(value, context);
|
||||
}
|
||||
|
||||
} else if (attr.start(11) > -1) {
|
||||
// any other attribute
|
||||
// ignore for now
|
||||
// could probe for path- or script-looking strings, but
|
||||
// those should be vanishingly rare in other attributes,
|
||||
// and/or symptomatic of page bugs
|
||||
}
|
||||
}
|
||||
TextUtils.recycleMatcher(attr);
|
||||
|
||||
// handle codebase/resources
|
||||
if (resources == null) {
|
||||
return (tally-next.size())>0;
|
||||
}
|
||||
Iterator<String> iter = resources.iterator();
|
||||
UURI codebaseURI = null;
|
||||
String res = null;
|
||||
try {
|
||||
if (codebase != null) {
|
||||
// TODO: Pass in the charset.
|
||||
codebaseURI = UURIFactory.getInstance(base, codebase);
|
||||
}
|
||||
while(iter.hasNext()) {
|
||||
res = iter.next().toString();
|
||||
// TODO: more HTML deescaping?
|
||||
res = TextUtils.replaceAll(ESCAPED_AMP, res, AMP);
|
||||
if (codebaseURI != null) {
|
||||
res = codebaseURI.resolve(res).toString();
|
||||
}
|
||||
processEmbed(res, new HTMLLinkContext(element.toString())); // TODO: include attribute too
|
||||
}
|
||||
} catch (URIException e) {
|
||||
extractErrorListener.noteExtractError(e,source,codebase);
|
||||
} catch (IllegalArgumentException e) {
|
||||
DevUtils.logger.log(Level.WARNING, "processGeneralTag()\n" +
|
||||
"codebase=" + codebase + " res=" + res + "\n" +
|
||||
DevUtils.extraInfo(), e);
|
||||
}
|
||||
return (tally-next.size())>0;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param cs
|
||||
*/
|
||||
protected void processScriptCode(CharSequence cs) {
|
||||
RegexJSLinkExtractor.extract(cs, source, base, next,
|
||||
extractErrorListener);
|
||||
}
|
||||
|
||||
static final String JAVASCRIPT = "(?i)^javascript:.*";
|
||||
|
||||
/**
|
||||
* @param value
|
||||
* @param context
|
||||
*/
|
||||
protected void processLink(CharSequence value, LinkContext context) {
|
||||
String link = TextUtils.replaceAll(ESCAPED_AMP, value, "&");
|
||||
|
||||
if(TextUtils.matches(JAVASCRIPT, link)) {
|
||||
processScriptCode(value.subSequence(11, value.length()));
|
||||
} else {
|
||||
addLinkFromString(link, context, Hop.NAVLINK);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param uri
|
||||
* @param context
|
||||
*/
|
||||
private void addLinkFromString(String uri, LinkContext context, Hop hop) {
|
||||
try {
|
||||
Link link = new Link(source, UURIFactory.getInstance(
|
||||
base, uri), context, hop);
|
||||
next.addLast(link);
|
||||
} catch (URIException e) {
|
||||
extractErrorListener.noteExtractError(e,source,uri);
|
||||
}
|
||||
}
|
||||
|
||||
protected long processEmbed(CharSequence value, LinkContext context) {
|
||||
String embed = TextUtils.replaceAll(ESCAPED_AMP, value, "&");
|
||||
addLinkFromString(embed, context, Hop.EMBED);
|
||||
return 1;
|
||||
}
|
||||
|
||||
static final String NON_HTML_PATH_EXTENSION =
|
||||
"(?i)(gif)|(jp(e)?g)|(png)|(tif(f)?)|(bmp)|(avi)|(mov)|(mp(e)?g)"+
|
||||
"|(mp3)|(mp4)|(swf)|(wav)|(au)|(aiff)|(mid)";
|
||||
|
||||
protected void processScript(CharSequence sequence, int endOfOpenTag) {
|
||||
// first, get attributes of script-open tag
|
||||
// as per any other tag
|
||||
processGeneralTag(sequence.subSequence(0,6),
|
||||
sequence.subSequence(0,endOfOpenTag));
|
||||
|
||||
// then, apply best-effort string-analysis heuristics
|
||||
// against any code present (false positives are OK)
|
||||
processScriptCode(sequence.subSequence(endOfOpenTag, sequence.length()));
|
||||
}
|
||||
|
||||
protected void processMeta(CharSequence cs) {
|
||||
Matcher attr = TextUtils.getMatcher(EACH_ATTRIBUTE_EXTRACTOR, cs);
|
||||
|
||||
String name = null;
|
||||
String httpEquiv = null;
|
||||
String content = null;
|
||||
|
||||
while (attr.find()) {
|
||||
int valueGroup =
|
||||
(attr.start(12) > -1) ? 12 : (attr.start(13) > -1) ? 13 : 14;
|
||||
CharSequence value =
|
||||
cs.subSequence(attr.start(valueGroup), attr.end(valueGroup));
|
||||
if (attr.group(1).equalsIgnoreCase("name")) {
|
||||
name = value.toString();
|
||||
} else if (attr.group(1).equalsIgnoreCase("http-equiv")) {
|
||||
httpEquiv = value.toString();
|
||||
} else if (attr.group(1).equalsIgnoreCase("content")) {
|
||||
content = value.toString();
|
||||
}
|
||||
// TODO: handle other stuff
|
||||
}
|
||||
TextUtils.recycleMatcher(attr);
|
||||
|
||||
// Look for the 'robots' meta-tag
|
||||
if("robots".equalsIgnoreCase(name) && content != null ) {
|
||||
if (getHonorRobots()) {
|
||||
String contentLower = content.toLowerCase();
|
||||
if ((contentLower.indexOf("nofollow") >= 0
|
||||
|| contentLower.indexOf("none") >= 0)) {
|
||||
// if 'nofollow' or 'none' is specified and we
|
||||
// are honoring robots, end html extraction
|
||||
logger.fine("HTML extraction skipped due to robots meta-tag for: "
|
||||
+ source);
|
||||
cancelFurtherExtraction();
|
||||
return;
|
||||
}
|
||||
}
|
||||
} else if ("refresh".equalsIgnoreCase(httpEquiv) && content != null) {
|
||||
String refreshUri = content.substring(content.indexOf("=") + 1);
|
||||
try {
|
||||
Link refreshLink = new Link(
|
||||
source,
|
||||
UURIFactory.getInstance(base,refreshUri),
|
||||
new HTMLLinkContext("meta", httpEquiv),
|
||||
Hop.REFER);
|
||||
next.addLast(refreshLink);
|
||||
} catch (URIException e) {
|
||||
extractErrorListener.noteExtractError(e,source,refreshUri);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @return whether to honor internal robots directives (eg meta robots)
|
||||
*/
|
||||
private boolean getHonorRobots() {
|
||||
return honorRobots;
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensure no further Links are extracted (by setting matcher up to fail)
|
||||
*/
|
||||
private void cancelFurtherExtraction() {
|
||||
// java 1.5 only:
|
||||
// tags.region(tags.regionEnd(),tags.regionEnd());
|
||||
tags.reset("");
|
||||
}
|
||||
|
||||
/**
|
||||
* @param sequence
|
||||
* @param endOfOpenTag
|
||||
*/
|
||||
protected void processStyle(CharSequence sequence,
|
||||
int endOfOpenTag)
|
||||
{
|
||||
// First, get attributes of script-open tag as per any other tag.
|
||||
processGeneralTag(sequence.subSequence(0,6),
|
||||
sequence.subSequence(0,endOfOpenTag));
|
||||
|
||||
// then, parse for URIs
|
||||
RegexCSSLinkExtractor.extract(sequence.subSequence(endOfOpenTag,
|
||||
sequence.length()), source, base, next, extractErrorListener);
|
||||
}
|
||||
|
||||
/**
|
||||
* Discard all state. Another setup() is required to use again.
|
||||
*/
|
||||
public void reset() {
|
||||
super.reset();
|
||||
TextUtils.recycleMatcher(tags);
|
||||
tags = null;
|
||||
}
|
||||
|
||||
protected static CharSequenceLinkExtractor newDefaultInstance() {
|
||||
return new RegexHTMLLinkExtractor();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,107 +0,0 @@
|
||||
/*
|
||||
* This file is part of the Heritrix web crawler (crawler.archive.org).
|
||||
*
|
||||
* Licensed to the Internet Archive (IA) by one or more individual
|
||||
* contributors.
|
||||
*
|
||||
* The IA licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.archive.extractor;
|
||||
|
||||
import java.util.LinkedList;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import org.apache.commons.httpclient.URIException;
|
||||
import org.archive.modules.extractor.Hop;
|
||||
import org.archive.modules.extractor.Link;
|
||||
import org.archive.modules.extractor.LinkContext;
|
||||
import org.archive.net.UURI;
|
||||
import org.archive.net.UURIFactory;
|
||||
import org.archive.util.TextUtils;
|
||||
|
||||
/**
|
||||
* Uses regular expressions to find likely URIs inside Javascript.
|
||||
*
|
||||
* ROUGH DRAFT IN PROGRESS / incomplete... untested...
|
||||
*
|
||||
* @author gojomo
|
||||
*/
|
||||
public class RegexJSLinkExtractor extends CharSequenceLinkExtractor {
|
||||
|
||||
static final String AMP = "&";
|
||||
static final String ESCAPED_AMP = "&";
|
||||
static final String WHITESPACE = "\\s";
|
||||
|
||||
// finds whitespace-free strings in Javascript
|
||||
// (areas between paired ' or " characters, possibly backslash-quoted
|
||||
// on the ends, but not in the middle)
|
||||
static final Pattern JAVASCRIPT_STRING_EXTRACTOR = Pattern.compile(
|
||||
"(\\\\{0,8}+(?:\"|\'))(.+?)(?:\\1)");
|
||||
|
||||
// determines whether a string is likely URI
|
||||
// (no whitespace or '<' '>', has an internal dot or some slash,
|
||||
// begins and ends with either '/' or a word-char)
|
||||
static final Pattern STRING_URI_DETECTOR = Pattern.compile(
|
||||
"(?:\\w|[\\.]{0,2}/)[\\S&&[^<>]]*(?:\\.|/)[\\S&&[^<>]]*(?:\\w|/)");
|
||||
|
||||
Matcher strings;
|
||||
LinkedList<Matcher> matcherStack = new LinkedList<Matcher>();
|
||||
|
||||
protected boolean findNextLink() {
|
||||
if(strings==null) {
|
||||
strings = JAVASCRIPT_STRING_EXTRACTOR.matcher(sourceContent);
|
||||
}
|
||||
while(strings!=null) {
|
||||
while(strings.find()) {
|
||||
CharSequence subsequence =
|
||||
sourceContent.subSequence(strings.start(2), strings.end(2));
|
||||
Matcher uri = STRING_URI_DETECTOR.matcher(subsequence);
|
||||
if ((subsequence.length() <= UURI.MAX_URL_LENGTH) && uri.matches()) {
|
||||
String string = uri.group();
|
||||
string = TextUtils.replaceAll(ESCAPED_AMP, string, AMP);
|
||||
try {
|
||||
Link link = new Link(source, UURIFactory.getInstance(
|
||||
source, string), LinkContext.JS_MISC, Hop.SPECULATIVE);
|
||||
next.add(link);
|
||||
return true;
|
||||
} catch (URIException e) {
|
||||
extractErrorListener.noteExtractError(e,source,string);
|
||||
}
|
||||
} else {
|
||||
// push current range
|
||||
matcherStack.addFirst(strings);
|
||||
// start looking inside string
|
||||
strings = JAVASCRIPT_STRING_EXTRACTOR.matcher(subsequence);
|
||||
}
|
||||
}
|
||||
// continue at enclosing range, if available
|
||||
strings = (Matcher) (matcherStack.isEmpty() ? null : matcherStack.removeFirst());
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
/* (non-Javadoc)
|
||||
* @see org.archive.extractor.LinkExtractor#reset()
|
||||
*/
|
||||
public void reset() {
|
||||
super.reset();
|
||||
matcherStack.clear();
|
||||
strings = null;
|
||||
}
|
||||
|
||||
protected static CharSequenceLinkExtractor newDefaultInstance() {
|
||||
return new RegexJSLinkExtractor();
|
||||
}
|
||||
}
|
||||
@@ -1,14 +0,0 @@
|
||||
<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
|
||||
<html>
|
||||
<head>
|
||||
<title>Overview</title>
|
||||
<meta http-equiv="content-type" content="text/html; charset=UTF-8" >
|
||||
<meta http-equiv="Content-Script-Type" content="text/javascript" >
|
||||
<meta http-equiv="Content-Style-Type" content="text/css" >
|
||||
</head>
|
||||
<body>
|
||||
package org.archive.extractor is an in-progress esperiment in a different
|
||||
decomposition of link-extraction functionality. classes in progress and
|
||||
subject to moving/refactoring.
|
||||
</body>
|
||||
</html>
|
||||
Reference in New Issue
Block a user