lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-09-23 06:05:46 +00:00
Code Issues Packages Projects Releases Wiki Activity

HER-2039: Initial work. Eliminated Link class and dealt with resultant

Browse Source 
errors. Not tested any further than that.
This commit is contained in:
Kristinn Sigurðsson
2013-05-15 13:36:01 +00:00
parent d4c5bd6e98
commit 858523db94
33 changed files with 216 additions and 474 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/src/main/java/org/archive/modules/CrawlURI.java
+40 -88
View File
@@ -88,7 +88,6 @@ import org.archive.modules.credential.Credential;
import org.archive.modules.credential.HttpAuthenticationCredential;
import org.archive.modules.extractor.HTMLLinkContext;
import org.archive.modules.extractor.Hop;
import org.archive.modules.extractor.Link;
import org.archive.modules.extractor.LinkContext;
import org.archive.net.UURI;
import org.archive.net.UURIFactory;
@@ -869,7 +868,6 @@ implements Reporter, Serializable, OverlayContext {
this.data = getPersistentDataMap();
extraInfo = null;
outCandidates = null;
outLinks = null;
method = null;
}
@@ -1080,38 +1078,25 @@ implements Reporter, Serializable, OverlayContext {
}
/**
* All discovered outbound Links (navlinks, embeds, etc.)
* Can either contain Link instances or CrawlURI instances, or both.
* The LinksScoper processor converts Link instances in this collection
* to CrawlURI instances.
* All discovered outbound urls as CrawlURIs (navlinks, embeds, etc.)
*/
protected transient Collection<Link> outLinks = new LinkedHashSet<Link>();
protected transient Collection<CrawlURI> outCandidates = new LinkedHashSet<CrawlURI>();
protected transient Collection<CrawlURI> outLinks;
/**
* Returns discovered links. The returned collection might be empty if
* no links were discovered, or if something like LinksScoper promoted
* the links to CrawlURIs.
*
* @return Collection of all discovered outbound Links
* @return Collection of all discovered outbound links
*/
public Collection<Link> getOutLinks() {
public Collection<CrawlURI> getOutLinks() {
if (outLinks==null) {
outLinks = new LinkedHashSet<CrawlURI>();
}
return outLinks;
// return Transform.subclasses(outLinks, Link.class);
}
/**
* Returns discovered candidate URIs. The returned collection will be
* emtpy until something like LinksScoper promotes discovered Links
* into CrawlURIs.
*
* @return Collection of candidate URIs
*/
public Collection<CrawlURI> getOutCandidates() {
return outCandidates;
}
/**
* Set the (HTML) Base URI used for derelativizing internal URIs.
*
@@ -1178,8 +1163,6 @@ implements Reporter, Serializable, OverlayContext {
@SuppressWarnings("unchecked")
Map<String,Object> temp = (Map<String,Object>)stream.readObject();
this.data = temp;
outLinks = new HashSet<Link>();
outCandidates = new HashSet<CrawlURI>();
}
/**
@@ -1379,6 +1362,11 @@ implements Reporter, Serializable, OverlayContext {
return StringUtils.isEmpty(pathFromSeed) ? "" : pathFromSeed.substring(pathFromSeed.length()-1);
}
public Hop getLastHopType() {
return Hop.getForChar(getLastHop().charAt(0));
}
/**
* @return URI via which this one was discovered
*/
@@ -1412,28 +1400,6 @@ implements Reporter, Serializable, OverlayContext {
}
// public void setStateProvider(SheetManager manager) {
// if(this.provider!=null) {
// return;
// }
// this.manager = manager;
//// this.provider = manager.findConfig(SURT.fromURI(toString()));
// }
//
//
// public StateProvider getStateProvider() {
// return provider;
// }
// public <T> T get(Object module, Key<T> key) {
// if (provider == null) {
// throw new AssertionError("ToeThread never set up CrawlURI's sheet.");
// }
// return provider.get(module, key);
// }
//
// Reporter implementation
//
@@ -1632,26 +1598,31 @@ implements Reporter, Serializable, OverlayContext {
}
/**
* Utility method for creation of CandidateURIs found extracting
* links from this CrawlURI.
* @param baseUURI BaseUURI for <code>link</code>.
* @param link Link to wrap CandidateURI in.
* @return New candidateURI wrapper around <code>link</code>.
* Utility method for creating CrawlURIs that were found as out links from the current CrawlURI
* links from this CrawlURI.
* <p>
* Any relative URIs will be treated as relative to this CrawlURI's UURI.
* @param destination The new URI, possibly a relative URI
* @param context
* @param hop
* @return New CrawlURI with the current CrawlURI set as the one it inherits from
* @throws URIException
*/
public CrawlURI createCrawlURI(UURI baseUURI, Link link)
throws URIException {
UURI u = (link.getDestination() instanceof UURI)?
(UURI)link.getDestination():
UURIFactory.getInstance(baseUURI,
link.getDestination().toString());
CrawlURI newCaURI = new CrawlURI(u,
extendHopsPath(getPathFromSeed(),link.getHopType().getHopChar()),
getUURI(), link.getContext());
public CrawlURI createCrawlURI(UURI destination, LinkContext context, Hop hop)
throws URIException {
return createCrawlURI(destination.toString(), context, hop);
}
public CrawlURI createCrawlURI(String destination, LinkContext context, Hop hop)
throws URIException {
UURI u = UURIFactory.getInstance(this.getBaseURI(), destination);
CrawlURI newCaURI = new CrawlURI(
u,
extendHopsPath(getPathFromSeed(),
hop.getHopChar()),
this.getUURI(),
context);
newCaURI.inheritFrom(this);
if (link.hasData()) {
newCaURI.data = link.getData();
}
return newCaURI;
}
@@ -1675,19 +1646,19 @@ implements Reporter, Serializable, OverlayContext {
}
/**
* Utility method for creation of CandidateURIs found extracting
* Utility method for creation of CrawlURIs found extracting
* links from this CrawlURI.
* @param baseUURI BaseUURI for <code>link</code>.
* @param link Link to wrap CandidateURI in.
* TODO: Fix JavaDoc
* @param scheduling How new CandidateURI should be scheduled.
* @param seed True if this CandidateURI is a seed.
* @return New candidateURI wrapper around <code>link</code>.
* @throws URIException
*/
public CrawlURI createCrawlURI(UURI baseUURI, Link link,
public CrawlURI createCrawlURI(UURI destination, LinkContext context, Hop hop,
int scheduling, boolean seed)
throws URIException {
final CrawlURI caURI = createCrawlURI(baseUURI, link);
final CrawlURI caURI = createCrawlURI(destination, context, hop);
caURI.setSchedulingDirective(scheduling);
caURI.setSeed(seed);
return caURI;
@@ -1877,7 +1848,7 @@ implements Reporter, Serializable, OverlayContext {
*/
public CrawlURI markPrerequisite(String preq)
throws URIException {
CrawlURI caUri = makeConsequentCandidate(preq, LinkContext.PREREQ_MISC, Hop.PREREQ);
CrawlURI caUri = createCrawlURI(preq, LinkContext.PREREQ_MISC, Hop.PREREQ);
caUri.setPrerequisite(true);
// TODO: consider moving some of this to configurable candidate-handling
int prereqPriority = getSchedulingDirective() - 1;
@@ -1894,25 +1865,6 @@ implements Reporter, Serializable, OverlayContext {
return caUri;
}
/**
* Create a consequent CrawlURI from this one, given the
* additional parameters
*
* @param destination URI string
* @param lc LinkContext
* @param hop Hop
* @return the newly created prerequisite CrawlURI
* @throws URIException
*/
public CrawlURI makeConsequentCandidate(String destination, LinkContext lc, Hop hop)
throws URIException {
UURI src = getUURI();
UURI dest = UURIFactory.getInstance(getBaseURI(),destination);
Link link = new Link(src, dest, lc, hop);
CrawlURI caUri = createCrawlURI(getBaseURI(), link);
return caUri;
}
public boolean containsContentTypeCharsetDeclaration() {
// TODO can this regex be improved? should the test consider if its legal?
return getContentType().matches("(?i).*charset=.*");
modules/src/main/java/org/archive/modules/extractor/Extractor.java
+51 -2
View File
@@ -43,7 +43,8 @@ import org.springframework.beans.factory.annotation.Autowired;
* @author pjack
*/
public abstract class Extractor extends Processor {
private static final Logger LOGGER = Logger.getLogger(Extractor.class.getName());
protected AtomicLong numberOfLinksExtracted = new AtomicLong(0);
/** Logger. */
@@ -143,13 +144,23 @@ public abstract class Extractor extends Processor {
Hop hop) {
try {
UURI dest = UURIFactory.getInstance(curi.getUURI(), uri);
Link link = new Link(curi.getUURI(), dest, context, hop);
CrawlURI link = curi.createCrawlURI(dest, context, hop);
curi.getOutLinks().add(link);
} catch (URIException e) {
logUriError(e, curi.getUURI(), uri);
}
}
protected void addOutlink(CrawlURI curi, UURI uuri, LinkContext context,
Hop hop) {
try {
CrawlURI link = curi.createCrawlURI(uuri, context, hop);
curi.getOutLinks().add(link);
} catch (URIException e) {
logUriError(e, curi.getUURI(), uuri.toString());
}
}
public void logUriError(URIException e, UURI uuri,
CharSequence l) {
loggerModule.logUriError(e, uuri, l);
@@ -174,5 +185,43 @@ public abstract class Extractor extends Processor {
ret.append(" " + numberOfLinksExtracted + " links from " + getURICount() +" CrawlURIs\n");
return ret.toString();
}
public static void addRelativeToBase(CrawlURI uri, int max,
String newUri, LinkContext context, Hop hop) throws URIException {
UURI dest = UURIFactory.getInstance(uri.getBaseURI(), newUri);
add2(uri, max, dest, context, hop);
}
public static void addRelativeToVia(CrawlURI uri, int max, String newUri,
LinkContext context, Hop hop) throws URIException {
UURI relTo = uri.getVia();
if (relTo == null) {
if (!uri.getAnnotations().contains("usedBaseForVia")) {
LOGGER.info("no via where expected; using base instead: " + uri);
uri.getAnnotations().add("usedBaseForVia");
}
relTo = uri.getBaseURI();
}
UURI dest = UURIFactory.getInstance(relTo, newUri);
add2(uri, max, dest, context, hop);
}
public static void add(CrawlURI uri, int max, String newUri,
LinkContext context, Hop hop) throws URIException {
UURI dest = UURIFactory.getInstance(newUri);
add2(uri, max, dest, context, hop);
}
private static void add2(CrawlURI curi, int max, UURI dest,
LinkContext context, Hop hop) throws URIException {
if (curi.getOutLinks().size() < max) {
CrawlURI link = curi.createCrawlURI(dest, context, hop);
curi.getOutLinks().add(link);
} else {
curi.incrementDiscardedOutLinks();
}
}
}
modules/src/main/java/org/archive/modules/extractor/ExtractorCSS.java
+1 -1
View File
@@ -136,7 +136,7 @@ public class ExtractorCSS extends ContentExtractor {
foundLinks++;
int max = ext.getExtractorParameters().getMaxOutlinks();
try {
Link.addRelativeToBase(curi, max, cssUri,
addRelativeToBase(curi, max, cssUri,
LinkContext.EMBED_MISC, Hop.EMBED);
} catch (URIException e) {
ext.logUriError(e, curi.getUURI(), cssUri);
modules/src/main/java/org/archive/modules/extractor/ExtractorDOC.java
+1 -2
View File
@@ -114,8 +114,7 @@ public class ExtractorDOC extends ContentExtractor {
try {
UURI dest = UURIFactory.getInstance(curi.getUURI(), hyperlink);
LinkContext lc = LinkContext.NAVLINK_MISC;
Link link = new Link(curi.getUURI(), dest, lc, Hop.NAVLINK);
curi.getOutLinks().add(link);
addOutlink(curi, hyperlink, lc, Hop.NAVLINK);
} catch (URIException e1) {
logUriError(e1, curi.getUURI(), hyperlink);
}
modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java
+2 -2
View File
@@ -656,7 +656,7 @@ public class ExtractorHTML extends ContentExtractor implements InitializingBean
// ReplayCharSequence.
HTMLLinkContext hc = HTMLLinkContext.get(context.toString());
int max = getExtractorParameters().getMaxOutlinks();
Link.addRelativeToBase(curi, max, uri.toString(), hc, hop);
addRelativeToBase(curi, max, uri.toString(), hc, hop);
} catch (URIException e) {
logUriError(e, curi.getUURI(), uri);
}
@@ -954,7 +954,7 @@ public class ExtractorHTML extends ContentExtractor implements InitializingBean
String refreshUri = content.substring(urlIndex);
try {
int max = getExtractorParameters().getMaxOutlinks();
Link.addRelativeToBase(curi, max, refreshUri,
addRelativeToBase(curi, max, refreshUri,
HTMLLinkContext.META, Hop.REFER);
} catch (URIException e) {
logUriError(e, curi.getUURI(), refreshUri);
modules/src/main/java/org/archive/modules/extractor/ExtractorHTTP.java
+1 -2
View File
@@ -83,8 +83,7 @@ public class ExtractorHTTP extends Extractor {
try {
UURI dest = UURIFactory.getInstance(curi.getUURI(), loc.getValue());
LinkContext lc = HTMLLinkContext.get(loc.getName()+":");
Link link = new Link(curi.getUURI(), dest, lc, Hop.REFER);
curi.getOutLinks().add(link);
addOutlink(curi, dest.toString(), lc, Hop.REFER);
numberOfLinksExtracted.incrementAndGet();
} catch (URIException e) {
logUriError(e, curi.getUURI(), loc.getValue());
modules/src/main/java/org/archive/modules/extractor/ExtractorImpliedURI.java
+8 -10
View File
@@ -121,36 +121,34 @@ public class ExtractorImpliedURI extends Extractor {
*/
@Override
public void extract(CrawlURI curi) {
List<Link> links = new ArrayList<Link>(curi.getOutLinks());
List<CrawlURI> links = new ArrayList<CrawlURI>(curi.getOutLinks());
int max = links.size();
for (int i = 0; i < max; i++) {
Link link = links.get(i);
CrawlURI link = links.get(i);
Pattern trigger = getRegex();
String build = getFormat();
CharSequence dest = link.getDestination();
CharSequence dest = link.getUURI();
String implied = extractImplied(dest, trigger, build);
if (implied != null) {
try {
UURI src = curi.getUURI();
UURI target = UURIFactory.getInstance(implied);
LinkContext lc = LinkContext.INFERRED_MISC;
Hop hop = Hop.INFERRED;
Link out = new Link(src, target, lc, hop);
curi.getOutLinks().add(out);
addOutlink(curi, target, lc, hop);
numberOfLinksExtracted.incrementAndGet();
boolean removeTriggerURI = getRemoveTriggerUris();
// remove trigger URI from the outlinks if configured so.
if (removeTriggerURI) {
if (curi.getOutLinks().remove(link)) {
LOGGER.log(Level.FINE, link.getDestination() +
LOGGER.log(Level.FINE, link.getURI() +
" has been removed from " +
link.getSource() + " outlinks list.");
curi.getURI() + " outlinks list.");
numberOfLinksExtracted.decrementAndGet();
} else {
LOGGER.log(Level.FINE, "Failed to remove " +
link.getDestination() + " from " +
link.getSource()+ " outlinks list.");
link.getURI() + " from " +
curi.getURI() + " outlinks list.");
}
}
} catch (URIException e) {
modules/src/main/java/org/archive/modules/extractor/ExtractorJS.java
+2 -2
View File
@@ -171,11 +171,11 @@ public class ExtractorJS extends ContentExtractor {
try {
int max = ext.getExtractorParameters().getMaxOutlinks();
if (handlingJSFile) {
Link.addRelativeToVia(curi, max, candidate, JS_MISC,
addRelativeToVia(curi, max, candidate, JS_MISC,
SPECULATIVE);
return true;
} else {
Link.addRelativeToBase(curi, max, candidate, JS_MISC,
addRelativeToBase(curi, max, candidate, JS_MISC,
SPECULATIVE);
return true;
}
modules/src/main/java/org/archive/modules/extractor/ExtractorMultipleRegex.java
+1 -1
View File
@@ -280,7 +280,7 @@ public class ExtractorMultipleRegex extends Extractor {
String outlinkUri = groovyTemplate().make(bindings).toString();
try {
Link.addRelativeToBase(curi,
addRelativeToBase(curi,
getExtractorParameters().getMaxOutlinks(), outlinkUri,
HTMLLinkContext.INFERRED_MISC, Hop.INFERRED);
} catch (URIException e) {
modules/src/main/java/org/archive/modules/extractor/ExtractorPDF.java
+1 -2
View File
@@ -117,8 +117,7 @@ public class ExtractorPDF extends ContentExtractor {
UURI dest = UURIFactory.getInstance(uri);
LinkContext lc = LinkContext.NAVLINK_MISC;
Hop hop = Hop.NAVLINK;
Link out = new Link(src, dest, lc, hop);
curi.getOutLinks().add(out);
addOutlink(curi, dest, lc, hop);
} catch (URIException e1) {
// There may not be a controller (e.g. If we're being run
// by the extractor tool).
modules/src/main/java/org/archive/modules/extractor/ExtractorSWF.java
+2 -2
View File
@@ -334,7 +334,7 @@ public class ExtractorSWF extends ContentExtractor {
}
} else {
int max = ext.getExtractorParameters().getMaxOutlinks();
Link.addRelativeToVia(curi, max, url, LinkContext.EMBED_MISC,
addRelativeToVia(curi, max, url, LinkContext.EMBED_MISC,
Hop.EMBED);
linkCount++;
}
@@ -343,7 +343,7 @@ public class ExtractorSWF extends ContentExtractor {
public void considerStringAsUri(String str) throws IOException {
if (UriUtils.isLikelyUri(str)) {
int max = ext.getExtractorParameters().getMaxOutlinks();
Link.addRelativeToVia(curi, max, str,
addRelativeToVia(curi, max, str,
LinkContext.SPECULATIVE_MISC, Hop.SPECULATIVE);
linkCount++;
}
modules/src/main/java/org/archive/modules/extractor/ExtractorURI.java
+5 -10
View File
@@ -77,11 +77,8 @@ public class ExtractorURI extends Extractor {
*/
@Override
public void extract(CrawlURI curi) {
List<Link> links = new ArrayList<Link>(curi.getOutLinks());
int max = links.size();
for (int i = 0; i < max; i++) {
Link wref = links.get(i);
extractLink(curi, wref);
for (CrawlURI link : curi.getOutLinks()) {
extractLink(curi, link);
}
}
@@ -91,10 +88,10 @@ public class ExtractorURI extends Extractor {
* @param curi CrawlURI to add discoveries to
* @param wref Link to examine for internal URIs
*/
protected void extractLink(CrawlURI curi, Link wref) {
protected void extractLink(CrawlURI curi, CrawlURI wref) {
UURI source = null;
try {
source = UURIFactory.getInstance(wref.getDestination().toString());
source = UURIFactory.getInstance(wref.getURI());
} catch (URIException e) {
LOGGER.log(Level.FINE,"bad URI",e);
}
@@ -105,13 +102,11 @@ public class ExtractorURI extends Extractor {
List<String> found = extractQueryStringLinks(source);
for (String uri : found) {
try {
UURI src = curi.getUURI();
UURI dest = UURIFactory.getInstance(uri);
LinkContext lc = LinkContext.SPECULATIVE_MISC;
Hop hop = Hop.SPECULATIVE;
Link link = new Link(src, dest, lc, hop);
addOutlink(curi, dest, lc, hop);
numberOfLinksExtracted.incrementAndGet();
curi.getOutLinks().add(link);
} catch (URIException e) {
LOGGER.log(Level.FINE, "bad URI", e);
}
modules/src/main/java/org/archive/modules/extractor/ExtractorUniversal.java
+1 -3
View File
@@ -405,12 +405,10 @@ public class ExtractorUniversal extends ContentExtractor {
// And add the URL to speculative embeds.
numberOfLinksExtracted.incrementAndGet();
UURI src = curi.getUURI();
UURI dest = UURIFactory.getInstance(newURL);
LinkContext lc = LinkContext.SPECULATIVE_MISC;
Hop hop = Hop.SPECULATIVE;
Link link = new Link(src, dest, lc, hop);
curi.getOutLinks().add(link);
addOutlink(curi, dest, lc, hop);
}
// Reset lookat for next string.
lookat = new StringBuffer();
modules/src/main/java/org/archive/modules/extractor/ExtractorXML.java
+1 -1
View File
@@ -159,7 +159,7 @@ public class ExtractorXML extends ContentExtractor {
// intends to create a followable/fetchable URI is
// unknown
int max = ext.getExtractorParameters().getMaxOutlinks();
Link.addRelativeToBase(curi, max, xmlUri,
addRelativeToBase(curi, max, xmlUri,
LinkContext.SPECULATIVE_MISC, Hop.SPECULATIVE);
} catch (URIException e) {
// There may not be a controller (e.g. If we're being run
modules/src/main/java/org/archive/modules/extractor/Hop.java
+14
View File
@@ -87,4 +87,18 @@ public enum Hop {
public String getHopString() {
return hopString;
}
/**
* Turns a character into the appropriate Hop enum.
* @param hopChar
* @return A Hop enum represented by the supplied char or null if char does not match any known hop type
*/
public static final Hop getForChar(char hopChar) {
for (Hop h : Hop.values()) {
if (h.hopChar==hopChar) {
return h;
}
}
return null;
}
}
modules/src/main/java/org/archive/modules/extractor/JerichoExtractorHTML.java
+1 -1
View File
@@ -319,7 +319,7 @@ public class JerichoExtractorHTML extends ExtractorHTML {
String refreshUri = content.substring(content.indexOf("=") + 1);
try {
int max = getExtractorParameters().getMaxOutlinks();
Link.addRelativeToBase(curi, max, refreshUri,
addRelativeToBase(curi, max, refreshUri,
HTMLLinkContext.META, Hop.REFER);
} catch (URIException e) {
logUriError(e, curi.getUURI(), refreshUri);
modules/src/main/java/org/archive/modules/extractor/Link.java
-222
View File
@@ -1,222 +0,0 @@
/*
* This file is part of the Heritrix web crawler (crawler.archive.org).
*
* Licensed to the Internet Archive (IA) by one or more individual
* contributors.
*
* The IA licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.archive.modules.extractor;
import java.io.Serializable;
import java.util.HashMap;
import java.util.Map;
import java.util.logging.Logger;
import org.apache.commons.httpclient.URIException;
import org.archive.modules.CoreAttributeConstants;
import org.archive.modules.CrawlURI;
import org.archive.net.UURI;
import org.archive.net.UURIFactory;
/**
* Link represents one discovered "edge" of the web graph: the source
* URI, the destination URI, and the type of reference (represented by the
* context in which it was found).
*
* As such, it is a suitably generic item to returned from generic
* link-extraction utility code.
*
* @author gojomo
*/
public class Link implements Serializable, Comparable<Link> {
private static final Logger LOGGER = Logger.getLogger(Link.class.getName());
private static final long serialVersionUID = 2L;
/** URI where this Link was discovered */
private CharSequence source;
/** URI (absolute) where this Link points */
private CharSequence destination;
/** context of discovery -- will be an XPath-like element[/@attribute]
* fragment for HTML URIs, a header name with trailing ':' for header
* values, or one of the stand-in constants when other context is
* unavailable */
private LinkContext context;
/** hop-type */
private Hop hop;
/**
* Flexible dynamic attributes list.
* <p>
* See further {@link Link#getData()}
*/
protected Map<String,Object> data;
/**
* Create a Link with the given fields.
* @param source
* @param destination
* @param context
* @param hopType
*/
public Link(CharSequence source, CharSequence destination,
LinkContext context, Hop hop) {
super();
this.source = source;
this.destination = destination;
this.context = context;
this.hop = hop;
}
/**
* @return Returns the context.
*/
public LinkContext getContext() {
return context;
}
/**
* @return Returns the destination.
*/
public CharSequence getDestination() {
return destination;
}
/**
* @return Returns the source.
*/
public CharSequence getSource() {
return source;
}
/**
* @return char hopType
*/
public Hop getHopType() {
return hop;
}
@Override
public String toString() {
return this.destination + " " + hop.getHopChar() + " " + this.context;
}
/**
* Attribute list
* <p>
* By convention the attribute list is keyed by constants found in the
* {@link CoreAttributeConstants} interface. Use this list to carry
* data or state produced by custom processors rather change the
* classes {@link CrawlURI} or this class, CrawlURI.
* <p>
* This list becomes {@link CrawlURI#getData()} when the Link is promoted to CrawlURI via
* {@link CrawlURI#createCrawlURI(UURI, Link)}
* <p>
* If the list is null when this method is invoked, a new instance will be created and returned.
*
* @returns a flexible map of key/value pairs for storing
* status of this URI for use by other processors.
*/
public Map<String, Object> getData() {
if (data == null) {
data = new HashMap<String,Object>();
}
return data;
}
/**
* @return true if data map is not null
*/
public boolean hasData() {
return data != null;
}
@Override
public boolean equals(Object o) {
if (!(o instanceof Link)) {
return false;
}
Link l = (Link)o;
return l.source.equals(source) && l.destination.equals(destination)
&& l.context.equals(context) && l.hop.equals(hop);
}
@Override
public int hashCode() {
int r = 37;
return r ^ source.hashCode() ^ destination.hashCode()
^ context.hashCode() ^ hop.hashCode();
}
public static void addRelativeToBase(CrawlURI uri, int max,
String newUri, LinkContext context, Hop hop) throws URIException {
UURI dest = UURIFactory.getInstance(uri.getBaseURI(), newUri);
add2(uri, max, dest, context, hop);
}
public static void addRelativeToVia(CrawlURI uri, int max, String newUri,
LinkContext context, Hop hop) throws URIException {
UURI relTo = uri.getVia();
if (relTo == null) {
if (!uri.getAnnotations().contains("usedBaseForVia")) {
LOGGER.info("no via where expected; using base instead: " + uri);
uri.getAnnotations().add("usedBaseForVia");
}
relTo = uri.getBaseURI();
}
UURI dest = UURIFactory.getInstance(relTo, newUri);
add2(uri, max, dest, context, hop);
}
public static void add(CrawlURI uri, int max, String newUri,
LinkContext context, Hop hop) throws URIException {
UURI dest = UURIFactory.getInstance(newUri);
add2(uri, max, dest, context, hop);
}
private static void add2(CrawlURI uri, int max, UURI dest,
LinkContext context, Hop hop) throws URIException {
if (uri.getOutLinks().size() < max) {
UURI src = uri.getUURI();
Link link = new Link(src, dest, context, hop);
uri.getOutLinks().add(link);
// return link;
} else {
uri.incrementDiscardedOutLinks();
}
}
public int compareTo(Link o) {
int cmp = source.toString().compareTo(o.source.toString());
if (cmp==0) {
cmp = destination.toString().compareTo(o.destination.toString());
}
if (cmp==0) {
cmp = context.toString().compareTo(o.context.toString());
}
if (cmp==0) {
cmp = hop.toString().compareTo(o.hop.toString());
}
return cmp;
}
}
modules/src/main/java/org/archive/modules/extractor/StringExtractorTestBase.java
+3 -3
View File
@@ -29,9 +29,9 @@ public abstract class StringExtractorTestBase extends ContentExtractorTestBase {
public static class TestData {
public CrawlURI uri;
public Link expectedResult;
public CrawlURI expectedResult;
public TestData(CrawlURI uri, Link expectedResult) {
public TestData(CrawlURI uri, CrawlURI expectedResult) {
this.uri = uri;
this.expectedResult = expectedResult;
}
@@ -80,7 +80,7 @@ public abstract class StringExtractorTestBase extends ContentExtractorTestBase {
Collection<TestData> testDataCol = makeData(text, expectedURL);
for (TestData testData: testDataCol) {
extractor.process(testData.uri);
HashSet<Link> expected = new HashSet<Link>();
HashSet<CrawlURI> expected = new HashSet<CrawlURI>();
if (testData.expectedResult != null) {
expected.add(testData.expectedResult);
}
modules/src/main/java/org/archive/modules/fetcher/FetchFTP.java
+2 -3
View File
@@ -44,7 +44,6 @@ import org.archive.io.ReplayCharSequence;
import org.archive.modules.CrawlURI;
import org.archive.modules.Processor;
import org.archive.modules.extractor.Hop;
import org.archive.modules.extractor.Link;
import org.archive.modules.extractor.LinkContext;
import org.archive.net.ClientFTP;
import org.archive.net.UURI;
@@ -573,7 +572,7 @@ public class FetchFTP extends Processor {
}
try {
UURI n = UURIFactory.getInstance(base + "/" + file);
Link link = new Link(curi.getUURI(), n, LinkContext.NAVLINK_MISC, Hop.NAVLINK);
CrawlURI link = curi.createCrawlURI(n, LinkContext.NAVLINK_MISC, Hop.NAVLINK);
curi.getOutLinks().add(link);
} catch (URIException e) {
logger.log(Level.WARNING, "URI error during extraction.", e);
@@ -610,7 +609,7 @@ public class FetchFTP extends Processor {
String path = uuri.getEscapedCurrentHierPath();
UURI parent = UURIFactory.getInstance(scheme + "://" + auth + path);
Link link = new Link(uuri, parent, LinkContext.NAVLINK_MISC,
CrawlURI link = curi.createCrawlURI(parent, LinkContext.NAVLINK_MISC,
Hop.NAVLINK);
curi.getOutLinks().add(link);
} catch (URIException e) {
modules/src/main/java/org/archive/modules/fetcher/FetchWhois.java
+2 -2
View File
@@ -37,8 +37,8 @@ import org.archive.modules.CoreAttributeConstants;
import org.archive.modules.CrawlURI;
import org.archive.modules.ProcessResult;
import org.archive.modules.Processor;
import org.archive.modules.extractor.Extractor;
import org.archive.modules.extractor.Hop;
import org.archive.modules.extractor.Link;
import org.archive.modules.extractor.LinkContext;
import org.archive.modules.net.CrawlHost;
import org.archive.modules.net.ServerCache;
@@ -423,7 +423,7 @@ public class FetchWhois extends Processor implements CoreAttributeConstants,
protected void addWhoisLink(CrawlURI curi, String query) {
String whoisUrl = "whois:" + query;
try {
Link.add(curi, Integer.MAX_VALUE, whoisUrl, LinkContext.INFERRED_MISC, Hop.INFERRED);
Extractor.add(curi, Integer.MAX_VALUE, whoisUrl, LinkContext.INFERRED_MISC, Hop.INFERRED);
} catch (URIException e) {
logger.log(Level.WARNING, "problem with url " + whoisUrl, e);
}
modules/src/main/java/org/archive/modules/forms/FormLoginProcessor.java
+2 -2
View File
@@ -254,7 +254,7 @@ public class FormLoginProcessor extends Processor implements Checkpointable {
protected void createFormSubmissionAttempt(CrawlURI curi, HTMLForm templateForm, String formProvince) {
LinkContext lc = new LinkContext.SimpleLinkContext("form/@action");
try {
CrawlURI submitCuri = curi.makeConsequentCandidate(templateForm.getAction(),lc, Hop.SUBMIT);
CrawlURI submitCuri = curi.createCrawlURI(templateForm.getAction(), lc, Hop.SUBMIT);
submitCuri.setFetchType(FetchType.HTTP_POST);
submitCuri.getData().put(
CoreAttributeConstants.A_SUBMIT_DATA,
@@ -264,7 +264,7 @@ public class FormLoginProcessor extends Processor implements Checkpointable {
//submitCuri.setSchedulingDirective(Math.max(curi.getSchedulingDirective()-1, 0));
submitCuri.setSchedulingDirective(SchedulingConstants.HIGH);
submitCuri.setForceFetch(true);
curi.getOutCandidates().add(submitCuri);
curi.getOutLinks().add(submitCuri);
curi.getAnnotations().add("submit:"+templateForm.getAction());
} catch (URIException ue) {
loggerModule.logUriError(ue,curi.getUURI(),templateForm.getAction());
modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java
+27 -28
View File
@@ -19,36 +19,36 @@
package org.archive.modules.writer;
import static org.archive.io.warc.WARCConstants.FTP_CONTROL_CONVERSATION_MIMETYPE;
import static org.archive.io.warc.WARCConstants.HEADER_KEY_CONCURRENT_TO;
import static org.archive.io.warc.WARCConstants.HEADER_KEY_ETAG;
import static org.archive.io.warc.WARCConstants.HEADER_KEY_IP;
import static org.archive.io.warc.WARCConstants.HEADER_KEY_LAST_MODIFIED;
import static org.archive.io.warc.WARCConstants.HEADER_KEY_PAYLOAD_DIGEST;
import static org.archive.io.warc.WARCConstants.HEADER_KEY_PROFILE;
import static org.archive.io.warc.WARCConstants.HEADER_KEY_REFERS_TO;
import static org.archive.io.warc.WARCConstants.HEADER_KEY_REFERS_TO_DATE;
import static org.archive.io.warc.WARCConstants.HEADER_KEY_REFERS_TO_FILENAME;
import static org.archive.io.warc.WARCConstants.HEADER_KEY_REFERS_TO_FILE_OFFSET;
import static org.archive.io.warc.WARCConstants.HEADER_KEY_REFERS_TO_TARGET_URI;
import static org.archive.io.warc.WARCConstants.HEADER_KEY_TRUNCATED;
import static org.archive.io.warc.WARCConstants.HTTP_REQUEST_MIMETYPE;
import static org.archive.io.warc.WARCConstants.HTTP_RESPONSE_MIMETYPE;
import static org.archive.io.warc.WARCConstants.NAMED_FIELD_TRUNCATED_VALUE_HEAD;
import static org.archive.io.warc.WARCConstants.NAMED_FIELD_TRUNCATED_VALUE_LENGTH;
import static org.archive.io.warc.WARCConstants.NAMED_FIELD_TRUNCATED_VALUE_TIME;
import static org.archive.io.warc.WARCConstants.PROFILE_REVISIT_IDENTICAL_DIGEST;
import static org.archive.io.warc.WARCConstants.PROFILE_REVISIT_NOT_MODIFIED;
import static org.archive.io.warc.WARCConstants.PROFILE_REVISIT_URI_AGNOSTIC_IDENTICAL_DIGEST;
import static org.archive.io.warc.WARCConstants.TYPE;
import static org.archive.format.warc.WARCConstants.FTP_CONTROL_CONVERSATION_MIMETYPE;
import static org.archive.format.warc.WARCConstants.HEADER_KEY_CONCURRENT_TO;
import static org.archive.format.warc.WARCConstants.HEADER_KEY_ETAG;
import static org.archive.format.warc.WARCConstants.HEADER_KEY_IP;
import static org.archive.format.warc.WARCConstants.HEADER_KEY_LAST_MODIFIED;
import static org.archive.format.warc.WARCConstants.HEADER_KEY_PAYLOAD_DIGEST;
import static org.archive.format.warc.WARCConstants.HEADER_KEY_PROFILE;
import static org.archive.format.warc.WARCConstants.HEADER_KEY_REFERS_TO;
import static org.archive.format.warc.WARCConstants.HEADER_KEY_REFERS_TO_DATE;
import static org.archive.format.warc.WARCConstants.HEADER_KEY_REFERS_TO_FILENAME;
import static org.archive.format.warc.WARCConstants.HEADER_KEY_REFERS_TO_FILE_OFFSET;
import static org.archive.format.warc.WARCConstants.HEADER_KEY_REFERS_TO_TARGET_URI;
import static org.archive.format.warc.WARCConstants.HEADER_KEY_TRUNCATED;
import static org.archive.format.warc.WARCConstants.HTTP_REQUEST_MIMETYPE;
import static org.archive.format.warc.WARCConstants.HTTP_RESPONSE_MIMETYPE;
import static org.archive.format.warc.WARCConstants.NAMED_FIELD_TRUNCATED_VALUE_HEAD;
import static org.archive.format.warc.WARCConstants.NAMED_FIELD_TRUNCATED_VALUE_LENGTH;
import static org.archive.format.warc.WARCConstants.NAMED_FIELD_TRUNCATED_VALUE_TIME;
import static org.archive.format.warc.WARCConstants.PROFILE_REVISIT_IDENTICAL_DIGEST;
import static org.archive.format.warc.WARCConstants.PROFILE_REVISIT_NOT_MODIFIED;
import static org.archive.format.warc.WARCConstants.PROFILE_REVISIT_URI_AGNOSTIC_IDENTICAL_DIGEST;
import static org.archive.format.warc.WARCConstants.TYPE;
import static org.archive.modules.CoreAttributeConstants.A_DNS_SERVER_IP_LABEL;
import static org.archive.modules.CoreAttributeConstants.A_FTP_CONTROL_CONVERSATION;
import static org.archive.modules.CoreAttributeConstants.A_FTP_FETCH_STATUS;
import static org.archive.modules.CoreAttributeConstants.A_SOURCE_TAG;
import static org.archive.modules.CoreAttributeConstants.A_WARC_RESPONSE_HEADERS;
import static org.archive.modules.CoreAttributeConstants.HEADER_TRUNC;
import static org.archive.modules.CoreAttributeConstants.LENGTH_TRUNC;
import static org.archive.modules.CoreAttributeConstants.TIMER_TRUNC;
import static org.archive.modules.CoreAttributeConstants.A_WARC_RESPONSE_HEADERS;
import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_CONTENT_DIGEST_COUNT;
import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_ETAG_HEADER;
import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_FETCH_HISTORY;
@@ -83,8 +83,8 @@ import org.apache.commons.httpclient.HttpMethod;
import org.apache.commons.httpclient.HttpStatus;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
import org.archive.io.ReplayInputStream;
import org.archive.format.warc.WARCConstants.WARCRecordType;
import org.archive.io.ReplayInputStream;
import org.archive.io.warc.WARCRecordInfo;
import org.archive.io.warc.WARCWriter;
import org.archive.io.warc.WARCWriterPool;
@@ -94,7 +94,6 @@ import org.archive.modules.CrawlMetadata;
import org.archive.modules.CrawlURI;
import org.archive.modules.ProcessResult;
import org.archive.modules.deciderules.recrawl.IdenticalDigestDecideRule;
import org.archive.modules.extractor.Link;
import org.archive.spring.ConfigPath;
import org.archive.uid.RecordIDGenerator;
import org.archive.uid.UUIDGenerator;
@@ -862,10 +861,10 @@ public class WARCWriterProcessor extends WriterPoolProcessor implements WARCWrit
}
// Add outlinks though they are effectively useless without anchor text.
Collection<Link> links = curi.getOutLinks();
Collection<CrawlURI> links = curi.getOutLinks();
if (links != null && links.size() > 0) {
for (Link link: links) {
r.addLabelValue("outlink", link.toString());
for (CrawlURI link: links) {
r.addLabelValue("outlink", link.getURI());
}
}
modules/src/test/java/org/archive/modules/extractor/ExtractorCSSTest.java
+5 -9
View File
@@ -19,22 +19,17 @@
package org.archive.modules.extractor;
import static org.archive.modules.extractor.LinkContext.EMBED_MISC;
import static org.archive.modules.extractor.LinkContext.NAVLINK_MISC;
import java.util.Collection;
import java.util.Collections;
import org.archive.modules.CrawlURI;
import org.archive.modules.extractor.Extractor;
import org.archive.modules.extractor.ExtractorCSS;
import org.archive.modules.extractor.Hop;
import org.archive.modules.extractor.Link;
import org.archive.modules.extractor.StringExtractorTestBase;
import org.archive.net.UURI;
import org.archive.net.UURIFactory;
import org.archive.util.Recorder;
import static org.archive.modules.extractor.LinkContext.EMBED_MISC;
import static org.archive.modules.extractor.LinkContext.NAVLINK_MISC;
/**
* Unit test for ExtractorCSS.
@@ -81,8 +76,9 @@ public class ExtractorCSSTest extends StringExtractorTestBase {
euri.setRecorder(recorder);
euri.setContentSize(content.length());
// TODO: This test was naively modified to account for the abscense of LINK, but no effort was made to confirm that it is actually testing anything useful
UURI dest = UURIFactory.getInstance(uri);
Link link = new Link(src, dest, EMBED_MISC, Hop.EMBED);
CrawlURI link = euri.createCrawlURI(dest, EMBED_MISC, Hop.EMBED);
TestData td = new TestData(euri, link);
return Collections.singleton(td);
}
modules/src/test/java/org/archive/modules/extractor/ExtractorHTMLTest.java
+19 -21
View File
@@ -107,7 +107,7 @@ public class ExtractorHTMLTest extends StringExtractorTestBase {
UURI dest = UURIFactory.getInstance(destURI);
LinkContext context = determineContext(content);
Hop hop = determineHop(content);
Link link = new Link(src, dest, context, hop);
CrawlURI link = euri.createCrawlURI(dest, context, hop);
result.add(new TestData(euri, link));
euri = new CrawlURI(src, null, null, LinkContext.NAVLINK_MISC);
@@ -166,10 +166,10 @@ public class ExtractorHTMLTest extends StringExtractorTestBase {
CrawlURI puri = new CrawlURI(UURIFactory
.getInstance("http://www.example.com"));
getExtractor().extract(puri, source);
Link[] links = puri.getOutLinks().toArray(new Link[0]);
CrawlURI[] links = puri.getOutLinks().toArray(new CrawlURI[0]);
assertTrue("did not find single link",links.length==1);
assertTrue("expected link not found",
links[0].getDestination().toString().equals(expected));
links[0].getURI().equals(expected));
}
/**
@@ -188,9 +188,8 @@ public class ExtractorHTMLTest extends StringExtractorTestBase {
"<form method=\"post\" action=\"http://www.example.com/notok\"> "+
"<form action=\"http://www.example.com/ok3\"> ";
getExtractor().extract(puri, cs);
Link[] links = puri.getOutLinks().toArray(new Link[0]);
// find exactly 3 (not the POST) action URIs
assertTrue("incorrect number of links found",links.length==3);
assertTrue("incorrect number of links found", puri.getOutLinks().size()==3);
}
/**
@@ -207,7 +206,7 @@ public class ExtractorHTMLTest extends StringExtractorTestBase {
getExtractor().extract(puri, cs);
assertEquals("meta robots content not extracted","index,nofollow",
puri.getData().get(ExtractorHTML.A_META_ROBOTS));
Link[] links = puri.getOutLinks().toArray(new Link[0]);
CrawlURI[] links = puri.getOutLinks().toArray(new CrawlURI[0]);
assertTrue("link extracted despite meta robots",links.length==0);
}
@@ -228,9 +227,8 @@ public class ExtractorHTMLTest extends StringExtractorTestBase {
assertTrue(CollectionUtils.exists(curi.getOutLinks(), new Predicate() {
public boolean evaluate(Object object) {
return ((Link) object)
.getDestination()
.toString()
return ((CrawlURI) object)
.getURI()
.indexOf(
"/example.html;jsessionid=deadbeef:deadbeed?parameter=this:value") >= 0;
}
@@ -238,7 +236,7 @@ public class ExtractorHTMLTest extends StringExtractorTestBase {
assertTrue(CollectionUtils.exists(curi.getOutLinks(), new Predicate() {
public boolean evaluate(Object object) {
return ((Link) object).getDestination().toString().indexOf(
return ((CrawlURI) object).getURI().indexOf(
"/example.html?parameter=this:value") >= 0;
}
}));
@@ -262,15 +260,15 @@ public class ExtractorHTMLTest extends StringExtractorTestBase {
assertTrue(CollectionUtils.exists(curi.getOutLinks(), new Predicate() {
public boolean evaluate(Object object) {
System.err.println("comparing: "
+ ((Link) object).getDestination().toString()
+ ((CrawlURI) object).getURI()
+ " and https://www.anotherexample.com/");
return ((Link) object).getDestination().toString().equals(
return ((CrawlURI) object).getURI().equals(
"http://www.anotherexample.com/");
}
}));
assertTrue(CollectionUtils.exists(curi.getOutLinks(), new Predicate() {
public boolean evaluate(Object object) {
return ((Link) object).getDestination().toString().equals(
return ((CrawlURI) object).getURI().equals(
"https://www.example.com/index.html");
}
}));
@@ -310,21 +308,21 @@ public class ExtractorHTMLTest extends StringExtractorTestBase {
"<a href=\"def/another1.html\">" +
"<a href=\"ghi/another2.html\">";
getExtractor().extract(puri, cs);
Link[] links = puri.getOutLinks().toArray(new Link[0]);
CrawlURI[] links = puri.getOutLinks().toArray(new CrawlURI[0]);
Arrays.sort(links);
String dest1 = "http://www.example.com/def/another1.html";
String dest2 = "http://www.example.com/ghi/another2.html";
// ensure outlink from base href
assertEquals("outlink1 from base href",dest1,
links[1].getDestination().toString());
links[1].getURI());
assertEquals("outlink2 from base href",dest2,
links[2].getDestination().toString());
links[2].getURI());
}
protected Predicate destinationContainsPredicate(final String fragment) {
return new Predicate() {
public boolean evaluate(Object object) {
return ((Link) object).getDestination().toString().indexOf(fragment) >= 0;
return ((CrawlURI) object).getURI().indexOf(fragment) >= 0;
}
};
}
@@ -332,7 +330,7 @@ public class ExtractorHTMLTest extends StringExtractorTestBase {
protected Predicate destinationsIsPredicate(final String value) {
return new Predicate() {
public boolean evaluate(Object object) {
return ((Link) object).getDestination().toString().equals(value);
return ((CrawlURI) object).getURI().equals(value);
}
};
}
@@ -397,16 +395,16 @@ public class ExtractorHTMLTest extends StringExtractorTestBase {
getExtractor().extract(curi, cs);
Link[] links = curi.getOutLinks().toArray(new Link[0]);
CrawlURI[] links = curi.getOutLinks().toArray(new CrawlURI[0]);
Arrays.sort(links);
String dest1 = "http://www.example.com/foo.gif";
String dest2 = "http://www.example.com/foo.js";
assertEquals("outlink1 from conditional comment img src",dest1,
links[0].getDestination().toString());
links[0].getURI());
assertEquals("outlink2 from conditional comment script src",dest2,
links[1].getDestination().toString());
links[1].getURI());
}
modules/src/test/java/org/archive/modules/extractor/ExtractorJSTest.java
+1 -1
View File
@@ -141,7 +141,7 @@ public class ExtractorJSTest extends StringExtractorTestBase {
if (destURI != null) {
UURI dest = UURIFactory.getInstance(destURI);
Link link = new Link(src, dest, LinkContext.JS_MISC, Hop.SPECULATIVE);
CrawlURI link = euri.createCrawlURI(dest, LinkContext.JS_MISC, Hop.SPECULATIVE);
result.add(new TestData(euri, link));
} else {
result.add(new TestData(euri, null));
modules/src/test/java/org/archive/modules/extractor/ExtractorMultipleRegexTest.java
+1 -1
View File
@@ -188,7 +188,7 @@ public class ExtractorMultipleRegexTest extends ContentExtractorTestBase {
extractor.process(testUri);
for (String expectedLinkString: EXPECTED_OUTLINKS) {
Link expectedLink = new Link(testUri.getUURI(),
CrawlURI expectedLink = testUri.createCrawlURI(
UURIFactory.getInstance(expectedLinkString),
HTMLLinkContext.INFERRED_MISC, Hop.INFERRED);
assertTrue(testUri.getOutLinks().contains(expectedLink));
modules/src/test/java/org/archive/modules/extractor/ExtractorSWFTest.java
+4 -4
View File
@@ -109,9 +109,9 @@ public class ExtractorSWFTest extends ContentExtractorTestBase {
+ elapsed + "ms to process " + url);
boolean foundIt = false;
for (Link link : curi.getOutLinks()) {
for (CrawlURI link : curi.getOutLinks()) {
logger.info("found link: " + link);
foundIt = foundIt || link.getDestination().toString().endsWith(testUrls.get(url));
foundIt = foundIt || link.getURI().endsWith(testUrls.get(url));
}
assertTrue("failed to extract link \"" + testUrls.get(url)
@@ -165,9 +165,9 @@ public class ExtractorSWFTest extends ContentExtractorTestBase {
+ elapsed + "ms to process " + url);
boolean foundIt = false;
for (Link link : curi.getOutLinks()) {
for (CrawlURI link : curi.getOutLinks()) {
logger.info("found link: " + link);
foundIt = foundIt || link.getDestination().toString().endsWith(testUrls.get(url));
foundIt = foundIt || link.getURI().endsWith(testUrls.get(url));
}
if (!foundIt)
modules/src/test/java/org/archive/modules/extractor/JerichoExtractorHTMLTest.java
+3 -3
View File
@@ -73,7 +73,7 @@ public class JerichoExtractorHTMLTest extends ExtractorHTMLTest {
curi.getOutLinks();
assertTrue(CollectionUtils.exists(curi.getOutLinks(), new Predicate() {
public boolean evaluate(Object object) {
return ((Link) object).getDestination().toString().indexOf(
return ((CrawlURI) object).getURI().indexOf(
"/redirect_me?form=true&checked[]=1&unchecked[]=&selectBox=selectedOption&test=Go")>=0;
}
}));
@@ -101,7 +101,7 @@ public class JerichoExtractorHTMLTest extends ExtractorHTMLTest {
curi.getOutLinks();
assertTrue(!CollectionUtils.exists(curi.getOutLinks(), new Predicate() {
public boolean evaluate(Object object) {
return ((Link) object).getDestination().toString().indexOf(
return ((CrawlURI) object).getURI().indexOf(
"/redirect_me?form=true&checked[]=1&unchecked[]=&selectBox=selectedOption&test=Go")>=0;
}
}));
@@ -130,7 +130,7 @@ public class JerichoExtractorHTMLTest extends ExtractorHTMLTest {
curi.getOutLinks();
assertTrue(CollectionUtils.exists(curi.getOutLinks(), new Predicate() {
public boolean evaluate(Object object) {
return ((Link) object).getDestination().toString().indexOf(
return ((CrawlURI) object).getURI().indexOf(
"/redirect_me?form=true&checked[]=1&unchecked[]=&selectBox=selectedOption&test=Go")>=0;
}
}));