lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-09-25 15:16:24 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #76 from nlevitt/HER-2039

Browse Source 
HER-2039 remove class Link, use CrawlURI

It should be noted that this is an API change that will affect any custom link extractors that people may have built.
This commit is contained in:
Kristinn Sigurðsson
2014-07-10 09:41:00 +00:00
parent c48de14663 ee52f48be9
commit f101bba509
37 changed files with 259 additions and 764 deletions
Download Patch File Download Diff File Expand all files Collapse all files
contrib/src/main/java/org/archive/modules/extractor/ExtractorPDFContent.java
+1 -5
View File
@@ -27,8 +27,6 @@ import java.util.regex.Pattern;
import org.apache.commons.httpclient.URIException;
import org.archive.modules.CrawlURI;
import org.archive.net.UURI;
import org.archive.net.UURIFactory;
import com.itextpdf.text.pdf.PdfReader;
import com.itextpdf.text.pdf.parser.PdfReaderContentParser;
@@ -140,11 +138,9 @@ public class ExtractorPDFContent extends ContentExtractor {
for (String uri: uris) {
try {
UURI src = curi.getUURI();
UURI dest = UURIFactory.getInstance(uri);
LinkContext lc = LinkContext.NAVLINK_MISC;
Hop hop = Hop.NAVLINK;
Link out = new Link(src, dest, lc, hop);
CrawlURI out = curi.createCrawlURI(uri, lc, hop);
curi.getOutLinks().add(out);
} catch (URIException e1) {
logUriError(e1, curi.getUURI(), uri);
contrib/src/test/java/org/archive/modules/extractor/ExtractorPDFContentTest.java
+9 -11
View File
@@ -43,14 +43,14 @@ public class ExtractorPDFContentTest extends ContentExtractorTestBase {
CrawlURI testUri = createTestUri("http://www.example.com/fake.pdf", TEST_RESOURCE_FILE_1);
extractor.process(testUri);
Set<Link> expected = makeLinkSet(testUri, new String[]{"http://www.businessdictionary.com/definition/supervisor.html","http://management.about.com/od/policiesandprocedures/g/supervisor1.html"});
Set<CrawlURI> expected = makeLinkSet(testUri, new String[]{"http://www.businessdictionary.com/definition/supervisor.html","http://management.about.com/od/policiesandprocedures/g/supervisor1.html"});
assertTrue(testUri.getOutLinks().containsAll(expected));
}
public void testEndingInDot() throws URIException, UnsupportedEncodingException, IOException, InterruptedException{
CrawlURI testUri = createTestUri("http://www.example.com/fake.pdf", TEST_RESOURCE_FILE_2);
extractor.process(testUri);
Set<Link> expected = makeLinkSet(testUri, new String[]{"http://www.fec.gov/data/CommitteeSummary.do",
Set<CrawlURI> expected = makeLinkSet(testUri, new String[]{"http://www.fec.gov/data/CommitteeSummary.do",
"http://www.opensecrets.org/bigpicture/elec_stats.php",
"http://www.opensecrets.org/pacs"});
assertTrue(testUri.getOutLinks().containsAll(expected));
@@ -59,21 +59,21 @@ public class ExtractorPDFContentTest extends ContentExtractorTestBase {
CrawlURI testUri = createTestUri("http://www.example.com/fake.pdf", TEST_RESOURCE_FILE_3);
extractor.process(testUri);
Set<Link> expected = makeLinkSet(testUri, new String[]{"http://www.dot.gov/sites/dot.dev/files/docs/2014_February_ATCR.pdf"});
Set<CrawlURI> expected = makeLinkSet(testUri, new String[]{"http://www.dot.gov/sites/dot.dev/files/docs/2014_February_ATCR.pdf"});
assertTrue(testUri.getOutLinks().containsAll(expected));
}
public void testParenthesis() throws URIException, UnsupportedEncodingException, IOException, InterruptedException{
CrawlURI testUri = createTestUri("http://www.example.com/fake.pdf", TEST_RESOURCE_FILE_4);
extractor.process(testUri);
Set<Link> expected = makeLinkSet(testUri, new String[]{"http://www.unisys.com","http://www.myserver.mycorp.com/images/exttest.jpg","http://www.adobe.com/intro?100,200","http://www.w3.org/1999/xhtml","http://www.xfa.org/schema/xfa-data/1.0","http://www.adobe.com","http://www.adobe.com/getacro.gif","http://www.example.com/testOpeningParen"});
Set<CrawlURI> expected = makeLinkSet(testUri, new String[]{"http://www.unisys.com","http://www.myserver.mycorp.com/images/exttest.jpg","http://www.adobe.com/intro?100,200","http://www.w3.org/1999/xhtml","http://www.xfa.org/schema/xfa-data/1.0","http://www.adobe.com","http://www.adobe.com/getacro.gif","http://www.example.com/testOpeningParen"});
assertTrue(testUri.getOutLinks().containsAll(expected));
}
public void testNewlineSeparatedURIs() throws URIException, UnsupportedEncodingException, IOException, InterruptedException{
CrawlURI testUri = createTestUri("http://www.example.com/fake.pdf", TEST_RESOURCE_FILE_4);
extractor.process(testUri);
Set<Link> expected = makeLinkSet(testUri, new String[]{"http://www.unisys.com","http://www.myserver.mycorp.com/images/exttest.jpg","http://www.example.com/test","http://www.adobe.com/intro?100,200","http://www.w3.org/1999/xhtml","http://www.xfa.org/schema/xfa-data/1.0","http://www.adobe.com","http://www.adobe.com/getacro.gif"});
Set<CrawlURI> expected = makeLinkSet(testUri, new String[]{"http://www.unisys.com","http://www.myserver.mycorp.com/images/exttest.jpg","http://www.example.com/test","http://www.adobe.com/intro?100,200","http://www.w3.org/1999/xhtml","http://www.xfa.org/schema/xfa-data/1.0","http://www.adobe.com","http://www.adobe.com/getacro.gif"});
assertTrue(testUri.getOutLinks().containsAll(expected));
}
@@ -86,13 +86,11 @@ public class ExtractorPDFContentTest extends ContentExtractorTestBase {
result.setLoggerModule(ulm);
return (Extractor)result;
}
private Set<Link> makeLinkSet(CrawlURI sourceUri, String[] urlStrs) throws URIException {
HashSet<Link> linkSet = new HashSet<Link>();
private Set<CrawlURI> makeLinkSet(CrawlURI sourceUri, String[] urlStrs) throws URIException {
HashSet<CrawlURI> linkSet = new HashSet<CrawlURI>();
for (String urlStr : urlStrs) {
linkSet.add(new Link(sourceUri.getUURI(),
UURIFactory.getInstance(urlStr),
HTMLLinkContext.NAVLINK_MISC, Hop.NAVLINK)
);
CrawlURI link = sourceUri.createCrawlURI(urlStr, HTMLLinkContext.NAVLINK_MISC, Hop.NAVLINK);
linkSet.add(link);
}
return linkSet;
}
contrib/src/test/java/org/archive/modules/extractor/ExtractorYoutubeFormatStreamTest.java
+8 -10
View File
@@ -95,7 +95,7 @@ public class ExtractorYoutubeFormatStreamTest extends ContentExtractorTestBase {
extractor.process(testUri);
Set<Link> expected = makeLinkSet(testUri, getExpectedOutlinksAllInItagPriority());
Set<CrawlURI> expected = makeLinkSet(testUri, getExpectedOutlinksAllInItagPriority());
assertEquals(expected, testUri.getOutLinks());
}
@@ -105,7 +105,7 @@ public class ExtractorYoutubeFormatStreamTest extends ContentExtractorTestBase {
extractor().setExtractLimit(0);
extractor.process(testUri);
Set<Link> expected = makeLinkSet(testUri, getExpectedOutlinksAll());
Set<CrawlURI> expected = makeLinkSet(testUri, getExpectedOutlinksAll());
assertEquals(expected, testUri.getOutLinks());
}
@@ -165,7 +165,7 @@ public class ExtractorYoutubeFormatStreamTest extends ContentExtractorTestBase {
extractor.process(testUri);
Set<Link> expected = makeLinkSet(testUri, getExpectedOutlinksSubset());
Set<CrawlURI> expected = makeLinkSet(testUri, getExpectedOutlinksSubset());
assertEquals(expected, testUri.getOutLinks());
}
@@ -187,17 +187,15 @@ public class ExtractorYoutubeFormatStreamTest extends ContentExtractorTestBase {
extractor.process(testUri);
Set<Link> expected = makeLinkSet(testUri, getExpectedSingleDefaultOutlink());
Set<CrawlURI> expected = makeLinkSet(testUri, getExpectedSingleDefaultOutlink());
assertEquals(expected, testUri.getOutLinks());
}
private Set<Link> makeLinkSet(CrawlURI sourceUri, String[] urlStrs) throws URIException {
HashSet<Link> linkSet = new HashSet<Link>();
private Set<CrawlURI> makeLinkSet(CrawlURI sourceUri, String[] urlStrs) throws URIException {
HashSet<CrawlURI> linkSet = new HashSet<CrawlURI>();
for (String urlStr : urlStrs) {
linkSet.add(new Link(sourceUri.getUURI(),
UURIFactory.getInstance(urlStr),
HTMLLinkContext.EMBED_MISC, Hop.EMBED)
);
CrawlURI link = sourceUri.createCrawlURI(urlStr, HTMLLinkContext.EMBED_MISC, Hop.EMBED);
linkSet.add(link);
}
return linkSet;
}
engine/src/main/java/org/archive/crawler/postprocessor/CandidatesProcessor.java
+1 -23
View File
@@ -23,7 +23,6 @@ package org.archive.crawler.postprocessor;
import static org.archive.modules.fetcher.FetchStatusCodes.S_DEFERRED;
import static org.archive.modules.fetcher.FetchStatusCodes.S_PREREQUISITE_UNSCHEDULABLE_FAILURE;
import org.apache.commons.httpclient.URIException;
import org.archive.crawler.framework.Frontier;
import org.archive.crawler.reporting.CrawlerLoggerModule;
import org.archive.crawler.spring.SheetOverlaysManager;
@@ -32,7 +31,6 @@ import org.archive.modules.CrawlURI;
import org.archive.modules.Processor;
import org.archive.modules.SchedulingConstants;
import org.archive.modules.extractor.Hop;
import org.archive.modules.extractor.Link;
import org.archive.modules.seeds.SeedModule;
import org.archive.spring.KeyedProperties;
import org.springframework.beans.factory.annotation.Autowired;
@@ -219,15 +217,6 @@ public class CandidatesProcessor extends Processor {
return;
}
// (2) NEW: also (and before-outlinks) run outCandidates (usually empty;
// only current use is a form-submission CrawlURI; could
// potentially take over prerequisite duties for consistency
for(CrawlURI candidate : curi.getOutCandidates()) {
runCandidateChain(candidate, curi);
}
// Only consider candidate links of error pages if configured to do so
if (!getProcessErrorOutlinks()
&& (curi.getFetchStatus() < 200 || curi.getFetchStatus() >= 400)) {
@@ -236,21 +225,10 @@ public class CandidatesProcessor extends Processor {
}
// (3) Handle outlinks (usual bulk of discoveries)
for (Link wref: curi.getOutLinks()) {
CrawlURI candidate;
try {
candidate = curi.createCrawlURI(curi.getBaseURI(),wref);
} catch (URIException e) {
loggerModule.logUriError(e, curi.getUURI(),
wref.getDestination().toString());
continue;
}
for (CrawlURI candidate: curi.getOutLinks()) {
runCandidateChain(candidate, curi);
// TODO: evaluate if this necessary (anyone uses?); wise (bloat?)
curi.getOutCandidates().add(candidate);
}
curi.getOutLinks().clear();
}
engine/src/main/java/org/archive/crawler/postprocessor/LinksScoper.java
-242
View File
@@ -1,242 +0,0 @@
/*
* This file is part of the Heritrix web crawler (crawler.archive.org).
*
* Licensed to the Internet Archive (IA) by one or more individual
* contributors.
*
* The IA licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.archive.crawler.postprocessor;
import static org.archive.modules.SchedulingConstants.HIGH;
import static org.archive.modules.SchedulingConstants.MEDIUM;
import static org.archive.modules.SchedulingConstants.NORMAL;
import static org.archive.modules.fetcher.FetchStatusCodes.S_PREREQUISITE_UNSCHEDULABLE_FAILURE;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.apache.commons.httpclient.URIException;
import org.archive.crawler.framework.Scoper;
import org.archive.modules.CrawlURI;
import org.archive.modules.deciderules.DecideResult;
import org.archive.modules.deciderules.DecideRule;
import org.archive.modules.deciderules.RejectDecideRule;
import org.archive.modules.extractor.Hop;
import org.archive.modules.extractor.Link;
/**
* Determine which extracted links are within scope.
* TODO: To test scope, requires that Link be converted to
* a CrawlURI. Make it so don't have to make a CrawlURI to test
* if Link is in scope.
* <p>Since this scoper has to create CrawlURIs, no sense
* discarding them since later in the processing chain CrawlURIs rather
* than Links are whats needed scheduling extracted links w/ the
* Frontier (Frontier#schedule expects CrawlURI, not Link). This class
* replaces Links w/ the CrawlURI that wraps the Link in the CrawlURI.
*
* @deprecated Use CandidatesProcessor and CandidateChain/CandidateScoper instead
*
* @author gojomo
* @author stack
*/
public class LinksScoper extends Scoper {
@SuppressWarnings("unused")
private static final long serialVersionUID = -3L;
private static Logger LOGGER =
Logger.getLogger(LinksScoper.class.getName());
/**
* If enabled, any URL found because a seed redirected to it (original seed
* returned 301 or 302), will also be treated as a seed.
*/
{
setSeedsRedirectNewSeeds(true);
}
public boolean getSeedsRedirectNewSeeds() {
return (Boolean) kp.get("seedsRedirectNewSeeds");
}
public void setSeedsRedirectNewSeeds(boolean redirect) {
kp.put("seedsRedirectNewSeeds",redirect);
}
/**
* DecideRules applied after an URI has been rejected. If the rules return
* {@link DecideResult#ACCEPT}, the URI is logged (if the logging level is
* INFO). Depends on {@link Scoper#OVERRIDE_LOGGER} being enabled.
*/
{
setLogRejectsRule(new RejectDecideRule());
}
public DecideRule getLogRejectsRule() {
return (DecideRule) kp.get("logRejectsRule");
}
public void setLogRejectsRule(DecideRule rule) {
kp.put("logRejectsRule", rule);
}
/**
* Number of hops (of any sort) from a seed up to which a URI has higher
* priority scheduling than any remaining seed. For example, if set to 1
* items one hop (link, embed, redirect, etc.) away from a seed will be
* scheduled with HIGH priority. If set to -1, no preferencing will occur,
* and a breadth-first search with seeds processed before discovered links
* will proceed. If set to zero, a purely depth-first search will proceed,
* with all discovered links processed before remaining seeds. Seed
* redirects are treated as one hop from a seed.
*/
{
setPreferenceDepthHops(-1); // no limit
}
public int getPreferenceDepthHops() {
return (Integer) kp.get("preferenceDepthHops");
}
public void setPreferenceDepthHops(int depth) {
kp.put("preferenceDepthHops",depth);
}
/**
* @param name Name of this filter.
*/
public LinksScoper() {
super();
}
@Override
protected boolean shouldProcess(CrawlURI puri) {
if (!(puri instanceof CrawlURI)) {
return false;
}
CrawlURI curi = (CrawlURI)puri;
// If prerequisites, nothing to be done in here.
if (curi.hasPrerequisiteUri()) {
handlePrerequisite(curi);
return false;
}
// Don't extract links of error pages.
if (curi.getFetchStatus() < 200 || curi.getFetchStatus() >= 400) {
curi.getOutLinks().clear();
return false;
}
if (curi.getOutLinks().isEmpty()) {
// No outlinks to process.
return false;
}
return true;
}
@Override
protected void innerProcess(final CrawlURI puri) {
CrawlURI curi = (CrawlURI)puri;
final boolean redirectsNewSeeds = getSeedsRedirectNewSeeds();
int preferenceDepthHops = getPreferenceDepthHops();
for (Link wref: curi.getOutLinks()) try {
int directive = getSchedulingFor(curi, wref, preferenceDepthHops);
CrawlURI caURI = curi.createCrawlURI(curi.getBaseURI(),
wref, directive,
considerAsSeed(curi, wref, redirectsNewSeeds));
if (isInScope(caURI)) {
curi.getOutCandidates().add(caURI);
}
} catch (URIException e) {
loggerModule.logUriError(e, curi.getUURI(),
wref.getDestination().toString());
}
curi.getOutLinks().clear();
}
/**
* The CrawlURI has a prerequisite; apply scoping and update
* Link to CrawlURI in manner analogous to outlink handling.
* @param curi CrawlURI with prereq to consider
*/
protected void handlePrerequisite(CrawlURI curi) {
CrawlURI caUri = curi.getPrerequisiteUri();
// FIXME!!! getController().setStateProvider(caUri);
if(isInScope(caUri)) {
// replace link with CrawlURI
curi.setPrerequisiteUri(caUri);
} else {
// prerequisite is out-of-scope; mark CrawlURI as error,
// preventing normal S_DEFERRED handling
curi.clearPrerequisiteUri();
curi.setFetchStatus(S_PREREQUISITE_UNSCHEDULABLE_FAILURE);
}
}
// TODO: move similar outOfScope logging to CandidatesProcessor/CandidateChain/CandidateScoper
protected void outOfScope(CrawlURI caUri) {
super.outOfScope(caUri);
if (!LOGGER.isLoggable(Level.INFO)) {
return;
}
DecideRule seq = getLogRejectsRule();
if (seq.decisionFor(caUri) == DecideResult.ACCEPT) {
LOGGER.info(caUri.getUURI().toString());
}
}
private boolean considerAsSeed(final CrawlURI curi, final Link wref,
final boolean redirectsNewSeeds) {
return redirectsNewSeeds && curi.isSeed()
&& wref.getHopType() == Hop.REFER;
}
/**
* Determine scheduling for the <code>curi</code>.
* As with the LinksScoper in general, this only handles extracted links,
* seeds do not pass through here, but are given MEDIUM priority.
* Imports into the frontier similarly do not pass through here,
* but are given NORMAL priority.
*/
protected int getSchedulingFor(final CrawlURI curi, final Link wref,
final int preferenceDepthHops) {
final Hop c = wref.getHopType();
if (LOGGER.isLoggable(Level.FINEST)) {
LOGGER.finest(curi + " with path=" + curi.getPathFromSeed() +
" isSeed=" + curi.isSeed() + " with fetchStatus=" +
curi.getFetchStatus() + " -> " + wref.getDestination() +
" type " + c + " with context=" + wref.getContext());
}
switch (c) {
case REFER:
// Treat redirects somewhat urgently
// This also ensures seed redirects remain seed priority
return (preferenceDepthHops >= 0 ? HIGH : MEDIUM);
default:
if (preferenceDepthHops == 0)
return HIGH;
// this implies seed redirects are treated as path
// length 1, which I belive is standard.
// curi.getPathFromSeed() can never be null here, because
// we're processing a link extracted from curi
if (preferenceDepthHops > 0 &&
curi.getPathFromSeed().length() + 1 <= preferenceDepthHops)
return HIGH;
// Everything else normal (at least for now)
return NORMAL;
}
}
}
engine/src/main/java/org/archive/crawler/postprocessor/SupplementaryLinksScoper.java
+1 -1
View File
@@ -87,7 +87,7 @@ public class SupplementaryLinksScoper extends Scoper {
}
// Collection<CrawlURI> inScopeLinks = new HashSet<CrawlURI>();
Iterator<CrawlURI> iter = curi.getOutCandidates().iterator();
Iterator<CrawlURI> iter = curi.getOutLinks().iterator();
while (iter.hasNext()) {
CrawlURI cauri = iter.next();
if (!isInScope(cauri)) {
engine/src/main/java/org/archive/crawler/processor/CrawlMapper.java
+1 -1
View File
@@ -215,7 +215,7 @@ public abstract class CrawlMapper extends Processor implements Lifecycle {
if (getCheckOutlinks()) {
// consider outlinks for mapping
Iterator<CrawlURI> iter = curi.getOutCandidates().iterator();
Iterator<CrawlURI> iter = curi.getOutLinks().iterator();
while(iter.hasNext()) {
CrawlURI cauri = iter.next();
if (decideToMapOutlink(cauri)) {
engine/src/main/java/org/archive/crawler/reporting/SeedRecord.java
+1 -1
View File
@@ -110,7 +110,7 @@ public class SeedRecord implements CoreAttributeConstants, Serializable, Identit
this.statusCode = curi.getFetchStatus();
this.disposition = disposition;
if (statusCode==301 || statusCode == 302) {
for (CrawlURI cauri: curi.getOutCandidates()) {
for (CrawlURI cauri: curi.getOutLinks()) {
if("location:".equalsIgnoreCase(cauri.getViaContext().
toString())) {
redirectUri = cauri.toString();
engine/src/test/java/org/archive/crawler/postprocessor/LinksScoperTest.java
-34
View File
@@ -1,34 +0,0 @@
/*
* This file is part of the Heritrix web crawler (crawler.archive.org).
*
* Licensed to the Internet Archive (IA) by one or more individual
* contributors.
*
* The IA licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.archive.crawler.postprocessor;
import org.archive.crawler.framework.CrawlerProcessorTestBase;
/**
* Unit test for {@link LinksScoper}.
*
* @author pjack
*/
public class LinksScoperTest extends CrawlerProcessorTestBase {
// TODO TESTME!
}
modules/src/main/java/org/archive/modules/CrawlURI.java
+103 -91
View File
@@ -86,7 +86,6 @@ import org.archive.modules.credential.Credential;
import org.archive.modules.credential.HttpAuthenticationCredential;
import org.archive.modules.extractor.HTMLLinkContext;
import org.archive.modules.extractor.Hop;
import org.archive.modules.extractor.Link;
import org.archive.modules.extractor.LinkContext;
import org.archive.net.UURI;
import org.archive.net.UURIFactory;
@@ -119,8 +118,8 @@ import org.json.JSONObject;
* @author Gordon Mohr
*/
public class CrawlURI
implements Reporter, Serializable, OverlayContext {
private static final long serialVersionUID = 3L;
implements Reporter, Serializable, OverlayContext, Comparable<CrawlURI> {
private static final long serialVersionUID = 4L;
private static final Logger logger =
Logger.getLogger(CrawlURI.class.getName());
@@ -276,6 +275,7 @@ implements Reporter, Serializable, OverlayContext {
*/
public CrawlURI(UURI uuri) {
this.uuri = uuri;
this.pathFromSeed = "";
}
public static CrawlURI fromHopsViaString(String uriHopsViaContext) throws URIException {
@@ -302,7 +302,11 @@ implements Reporter, Serializable, OverlayContext {
public CrawlURI(UURI u, String pathFromSeed, UURI via,
LinkContext viaContext) {
this.uuri = u;
this.pathFromSeed = pathFromSeed;
if (pathFromSeed != null) {
this.pathFromSeed = pathFromSeed;
} else {
this.pathFromSeed = "";
}
this.via = via;
this.viaContext = viaContext;
}
@@ -868,7 +872,6 @@ implements Reporter, Serializable, OverlayContext {
this.data = getPersistentDataMap();
extraInfo = null;
outCandidates = null;
outLinks = null;
// XXX er uh surprised this wasn't here before?
@@ -1080,38 +1083,25 @@ implements Reporter, Serializable, OverlayContext {
}
/**
* All discovered outbound Links (navlinks, embeds, etc.)
* Can either contain Link instances or CrawlURI instances, or both.
* The LinksScoper processor converts Link instances in this collection
* to CrawlURI instances.
* All discovered outbound urls as CrawlURIs (navlinks, embeds, etc.)
*/
protected transient Collection<Link> outLinks = new LinkedHashSet<Link>();
protected transient Collection<CrawlURI> outCandidates = new LinkedHashSet<CrawlURI>();
protected transient Collection<CrawlURI> outLinks;
/**
* Returns discovered links. The returned collection might be empty if
* no links were discovered, or if something like LinksScoper promoted
* the links to CrawlURIs.
*
* @return Collection of all discovered outbound Links
* @return Collection of all discovered outbound links
*/
public Collection<Link> getOutLinks() {
public Collection<CrawlURI> getOutLinks() {
if (outLinks==null) {
outLinks = new LinkedHashSet<CrawlURI>();
}
return outLinks;
// return Transform.subclasses(outLinks, Link.class);
}
/**
* Returns discovered candidate URIs. The returned collection will be
* emtpy until something like LinksScoper promotes discovered Links
* into CrawlURIs.
*
* @return Collection of candidate URIs
*/
public Collection<CrawlURI> getOutCandidates() {
return outCandidates;
}
/**
* Set the (HTML) Base URI used for derelativizing internal URIs.
*
@@ -1178,8 +1168,6 @@ implements Reporter, Serializable, OverlayContext {
@SuppressWarnings("unchecked")
Map<String,Object> temp = (Map<String,Object>)stream.readObject();
this.data = temp;
outLinks = new HashSet<Link>();
outCandidates = new HashSet<CrawlURI>();
}
/**
@@ -1397,28 +1385,6 @@ implements Reporter, Serializable, OverlayContext {
}
// public void setStateProvider(SheetManager manager) {
// if(this.provider!=null) {
// return;
// }
// this.manager = manager;
//// this.provider = manager.findConfig(SURT.fromURI(toString()));
// }
//
//
// public StateProvider getStateProvider() {
// return provider;
// }
// public <T> T get(Object module, Key<T> key) {
// if (provider == null) {
// throw new AssertionError("ToeThread never set up CrawlURI's sheet.");
// }
// return provider.get(module, key);
// }
//
// Reporter implementation
//
@@ -1617,25 +1583,30 @@ implements Reporter, Serializable, OverlayContext {
}
/**
* Utility method for creation of CandidateURIs found extracting
* links from this CrawlURI.
* @param baseUURI BaseUURI for <code>link</code>.
* @param link Link to wrap CandidateURI in.
* @return New candidateURI wrapper around <code>link</code>.
* Utility method for creating CrawlURIs that were found as out links from the current CrawlURI
* links from this CrawlURI.
* <p>
* Any relative URIs will be treated as relative to this CrawlURI's UURI.
* @param destination The new URI, possibly a relative URI
* @param context
* @param hop
* @return New CrawlURI with the current CrawlURI set as the one it inherits from
* @throws URIException
*/
public CrawlURI createCrawlURI(UURI baseUURI, Link link)
throws URIException {
UURI u = (link.getDestination() instanceof UURI)?
(UURI)link.getDestination():
UURIFactory.getInstance(baseUURI,
link.getDestination().toString());
CrawlURI newCaURI = new CrawlURI(u,
extendHopsPath(getPathFromSeed(),link.getHopType().getHopChar()),
getUURI(), link.getContext());
if (link.hasData()) {
newCaURI.data = link.getData();
}
public CrawlURI createCrawlURI(UURI destination, LinkContext context, Hop hop)
throws URIException {
return createCrawlURI(destination.toString(), context, hop);
}
public CrawlURI createCrawlURI(String destination, LinkContext context, Hop hop)
throws URIException {
UURI u = UURIFactory.getInstance(this.getBaseURI(), destination);
CrawlURI newCaURI = new CrawlURI(
u,
extendHopsPath(getPathFromSeed(),
hop.getHopChar()),
this.getUURI(),
context);
newCaURI.inheritFrom(this);
return newCaURI;
}
@@ -1660,19 +1631,19 @@ implements Reporter, Serializable, OverlayContext {
}
/**
* Utility method for creation of CandidateURIs found extracting
* Utility method for creation of CrawlURIs found extracting
* links from this CrawlURI.
* @param baseUURI BaseUURI for <code>link</code>.
* @param link Link to wrap CandidateURI in.
* TODO: Fix JavaDoc
* @param scheduling How new CandidateURI should be scheduled.
* @param seed True if this CandidateURI is a seed.
* @return New candidateURI wrapper around <code>link</code>.
* @throws URIException
*/
public CrawlURI createCrawlURI(UURI baseUURI, Link link,
public CrawlURI createCrawlURI(UURI destination, LinkContext context, Hop hop,
int scheduling, boolean seed)
throws URIException {
final CrawlURI caURI = createCrawlURI(baseUURI, link);
final CrawlURI caURI = createCrawlURI(destination, context, hop);
caURI.setSchedulingDirective(scheduling);
caURI.setSeed(seed);
return caURI;
@@ -1862,7 +1833,7 @@ implements Reporter, Serializable, OverlayContext {
*/
public CrawlURI markPrerequisite(String preq)
throws URIException {
CrawlURI caUri = makeConsequentCandidate(preq, LinkContext.PREREQ_MISC, Hop.PREREQ);
CrawlURI caUri = createCrawlURI(preq, LinkContext.PREREQ_MISC, Hop.PREREQ);
caUri.setPrerequisite(true);
// TODO: consider moving some of this to configurable candidate-handling
int prereqPriority = getSchedulingDirective() - 1;
@@ -1879,25 +1850,6 @@ implements Reporter, Serializable, OverlayContext {
return caUri;
}
/**
* Create a consequent CrawlURI from this one, given the
* additional parameters
*
* @param destination URI string
* @param lc LinkContext
* @param hop Hop
* @return the newly created prerequisite CrawlURI
* @throws URIException
*/
public CrawlURI makeConsequentCandidate(String destination, LinkContext lc, Hop hop)
throws URIException {
UURI src = getUURI();
UURI dest = UURIFactory.getInstance(getBaseURI(),destination);
Link link = new Link(src, dest, lc, hop);
CrawlURI caUri = createCrawlURI(getBaseURI(), link);
return caUri;
}
public boolean containsContentTypeCharsetDeclaration() {
// TODO can this regex be improved? should the test consider if its legal?
return getContentType().matches("(?i).*charset=.*");
@@ -1959,4 +1911,64 @@ implements Reporter, Serializable, OverlayContext {
public boolean hasContentDigestHistory() {
return getData().get(A_CONTENT_DIGEST_HISTORY) != null;
}
// brought over from old Link class
@Override
public int compareTo(CrawlURI o) {
int cmp = compare(via.toString(), o.via.toString());
if (cmp == 0) {
cmp = compare(uuri.toString(), o.uuri.toString());
}
if (cmp == 0) {
cmp = compare(viaContext.toString(), o.viaContext.toString());
}
if (cmp == 0) {
cmp = compare(pathFromSeed, o.pathFromSeed);
}
return cmp;
}
// brought over from old Link class
@Override
public int hashCode() {
int r = 37;
return r ^ hash(via.toString()) ^ hash(uuri.toString())
^ hash(viaContext.toString()) ^ hash(pathFromSeed.toString());
}
// handles nulls
private static int hash(String a) {
return a == null ? 0 : a.hashCode();
}
// handles nulls
private static boolean equals(Object a, Object b) {
return a == null ? b == null : a.equals(b);
}
// handles nulls
private static int compare(String a, String b) {
if (a == null && b == null) {
return 0;
} else if (a == null && b != null) {
return -1;
} else if (a != null && b != null) {
return 1;
} else {
return a.compareTo(b);
}
}
// brought over from old Link class
@Override
public boolean equals(Object o) {
if (!(o instanceof CrawlURI)) {
return false;
}
CrawlURI u = (CrawlURI) o;
return equals(via, u.via) && equals(uuri, u.uuri)
&& equals(viaContext, u.viaContext)
&& equals(pathFromSeed, u.pathFromSeed);
}
}
modules/src/main/java/org/archive/modules/extractor/Extractor.java
+53 -2
View File
@@ -43,7 +43,8 @@ import org.springframework.beans.factory.annotation.Autowired;
* @author pjack
*/
public abstract class Extractor extends Processor {
private static final Logger LOGGER = Logger.getLogger(Extractor.class.getName());
protected AtomicLong numberOfLinksExtracted = new AtomicLong(0);
/** Logger. */
@@ -143,13 +144,23 @@ public abstract class Extractor extends Processor {
Hop hop) {
try {
UURI dest = UURIFactory.getInstance(curi.getUURI(), uri);
Link link = new Link(curi.getUURI(), dest, context, hop);
CrawlURI link = curi.createCrawlURI(dest, context, hop);
curi.getOutLinks().add(link);
} catch (URIException e) {
logUriError(e, curi.getUURI(), uri);
}
}
protected void addOutlink(CrawlURI curi, UURI uuri, LinkContext context,
Hop hop) {
try {
CrawlURI link = curi.createCrawlURI(uuri, context, hop);
curi.getOutLinks().add(link);
} catch (URIException e) {
logUriError(e, curi.getUURI(), uuri.toString());
}
}
public void logUriError(URIException e, UURI uuri,
CharSequence l) {
loggerModule.logUriError(e, uuri, l);
@@ -174,5 +185,45 @@ public abstract class Extractor extends Processor {
ret.append(" " + numberOfLinksExtracted + " links from " + getURICount() +" CrawlURIs\n");
return ret.toString();
}
public static CrawlURI addRelativeToBase(CrawlURI uri, int max,
String newUri, LinkContext context, Hop hop) throws URIException {
UURI dest = UURIFactory.getInstance(uri.getBaseURI(), newUri);
return add2(uri, max, dest, context, hop);
}
public static CrawlURI addRelativeToVia(CrawlURI uri, int max, String newUri,
LinkContext context, Hop hop) throws URIException {
UURI relTo = uri.getVia();
if (relTo == null) {
if (!uri.getAnnotations().contains("usedBaseForVia")) {
LOGGER.info("no via where expected; using base instead: " + uri);
uri.getAnnotations().add("usedBaseForVia");
}
relTo = uri.getBaseURI();
}
UURI dest = UURIFactory.getInstance(relTo, newUri);
return add2(uri, max, dest, context, hop);
}
public static void add(CrawlURI uri, int max, String newUri,
LinkContext context, Hop hop) throws URIException {
UURI dest = UURIFactory.getInstance(newUri);
add2(uri, max, dest, context, hop);
}
private static CrawlURI add2(CrawlURI curi, int max, UURI dest,
LinkContext context, Hop hop) throws URIException {
if (curi.getOutLinks().size() < max) {
CrawlURI link = curi.createCrawlURI(dest, context, hop);
curi.getOutLinks().add(link);
return link;
} else {
curi.incrementDiscardedOutLinks();
return null;
}
}
}
modules/src/main/java/org/archive/modules/extractor/ExtractorCSS.java
+1 -1
View File
@@ -136,7 +136,7 @@ public class ExtractorCSS extends ContentExtractor {
foundLinks++;
int max = ext.getExtractorParameters().getMaxOutlinks();
try {
Link.addRelativeToBase(curi, max, cssUri,
addRelativeToBase(curi, max, cssUri,
LinkContext.EMBED_MISC, Hop.EMBED);
} catch (URIException e) {
ext.logUriError(e, curi.getUURI(), cssUri);
modules/src/main/java/org/archive/modules/extractor/ExtractorDOC.java
+1 -2
View File
@@ -114,8 +114,7 @@ public class ExtractorDOC extends ContentExtractor {
try {
UURI dest = UURIFactory.getInstance(curi.getUURI(), hyperlink);
LinkContext lc = LinkContext.NAVLINK_MISC;
Link link = new Link(curi.getUURI(), dest, lc, Hop.NAVLINK);
curi.getOutLinks().add(link);
addOutlink(curi, hyperlink, lc, Hop.NAVLINK);
} catch (URIException e1) {
logUriError(e1, curi.getUURI(), hyperlink);
}
modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java
+2 -2
View File
@@ -656,7 +656,7 @@ public class ExtractorHTML extends ContentExtractor implements InitializingBean
// ReplayCharSequence.
HTMLLinkContext hc = HTMLLinkContext.get(context.toString());
int max = getExtractorParameters().getMaxOutlinks();
Link.addRelativeToBase(curi, max, uri.toString(), hc, hop);
addRelativeToBase(curi, max, uri.toString(), hc, hop);
} catch (URIException e) {
logUriError(e, curi.getUURI(), uri);
}
@@ -954,7 +954,7 @@ public class ExtractorHTML extends ContentExtractor implements InitializingBean
String refreshUri = content.substring(urlIndex);
try {
int max = getExtractorParameters().getMaxOutlinks();
Link.addRelativeToBase(curi, max, refreshUri,
addRelativeToBase(curi, max, refreshUri,
HTMLLinkContext.META, Hop.REFER);
} catch (URIException e) {
logUriError(e, curi.getUURI(), refreshUri);
modules/src/main/java/org/archive/modules/extractor/ExtractorHTTP.java
+2 -3
View File
@@ -95,9 +95,8 @@ public class ExtractorHTTP extends Extractor {
protected void addHeaderLink(CrawlURI curi, String headerName, String url) {
try {
UURI dest = UURIFactory.getInstance(curi.getUURI(), url);
LinkContext lc = HTMLLinkContext.get(headerName+":");
Link link = new Link(curi.getUURI(), dest, lc, Hop.REFER);
curi.getOutLinks().add(link);
LinkContext lc = HTMLLinkContext.get(headerName+":");
addOutlink(curi, dest.toString(), lc, Hop.REFER);
numberOfLinksExtracted.incrementAndGet();
} catch (URIException e) {
logUriError(e, curi.getUURI(), url);
modules/src/main/java/org/archive/modules/extractor/ExtractorImpliedURI.java
+8 -10
View File
@@ -121,36 +121,34 @@ public class ExtractorImpliedURI extends Extractor {
*/
@Override
public void extract(CrawlURI curi) {
List<Link> links = new ArrayList<Link>(curi.getOutLinks());
List<CrawlURI> links = new ArrayList<CrawlURI>(curi.getOutLinks());
int max = links.size();
for (int i = 0; i < max; i++) {
Link link = links.get(i);
CrawlURI link = links.get(i);
Pattern trigger = getRegex();
String build = getFormat();
CharSequence dest = link.getDestination();
CharSequence dest = link.getUURI();
String implied = extractImplied(dest, trigger, build);
if (implied != null) {
try {
UURI src = curi.getUURI();
UURI target = UURIFactory.getInstance(implied);
LinkContext lc = LinkContext.INFERRED_MISC;
Hop hop = Hop.INFERRED;
Link out = new Link(src, target, lc, hop);
curi.getOutLinks().add(out);
addOutlink(curi, target, lc, hop);
numberOfLinksExtracted.incrementAndGet();
boolean removeTriggerURI = getRemoveTriggerUris();
// remove trigger URI from the outlinks if configured so.
if (removeTriggerURI) {
if (curi.getOutLinks().remove(link)) {
LOGGER.log(Level.FINE, link.getDestination() +
LOGGER.log(Level.FINE, link.getURI() +
" has been removed from " +
link.getSource() + " outlinks list.");
curi.getURI() + " outlinks list.");
numberOfLinksExtracted.decrementAndGet();
} else {
LOGGER.log(Level.FINE, "Failed to remove " +
link.getDestination() + " from " +
link.getSource()+ " outlinks list.");
link.getURI() + " from " +
curi.getURI() + " outlinks list.");
}
}
} catch (URIException e) {
modules/src/main/java/org/archive/modules/extractor/ExtractorJS.java
+2 -2
View File
@@ -175,11 +175,11 @@ public class ExtractorJS extends ContentExtractor {
try {
int max = ext.getExtractorParameters().getMaxOutlinks();
if (handlingJSFile) {
Link.addRelativeToVia(curi, max, candidate, JS_MISC,
addRelativeToVia(curi, max, candidate, JS_MISC,
SPECULATIVE);
return true;
} else {
Link.addRelativeToBase(curi, max, candidate, JS_MISC,
addRelativeToBase(curi, max, candidate, JS_MISC,
SPECULATIVE);
return true;
}
modules/src/main/java/org/archive/modules/extractor/ExtractorMultipleRegex.java
+1 -1
View File
@@ -280,7 +280,7 @@ public class ExtractorMultipleRegex extends Extractor {
String outlinkUri = groovyTemplate().make(bindings).toString();
try {
Link.addRelativeToBase(curi,
addRelativeToBase(curi,
getExtractorParameters().getMaxOutlinks(), outlinkUri,
HTMLLinkContext.INFERRED_MISC, Hop.INFERRED);
} catch (URIException e) {
modules/src/main/java/org/archive/modules/extractor/ExtractorPDF.java
+1 -2
View File
@@ -117,8 +117,7 @@ public class ExtractorPDF extends ContentExtractor {
UURI dest = UURIFactory.getInstance(uri);
LinkContext lc = LinkContext.NAVLINK_MISC;
Hop hop = Hop.NAVLINK;
Link out = new Link(src, dest, lc, hop);
curi.getOutLinks().add(out);
addOutlink(curi, dest, lc, hop);
} catch (URIException e1) {
// There may not be a controller (e.g. If we're being run
// by the extractor tool).
modules/src/main/java/org/archive/modules/extractor/ExtractorSWF.java
+6 -6
View File
@@ -334,18 +334,18 @@ public class ExtractorSWF extends ContentExtractor {
}
} else {
int max = ext.getExtractorParameters().getMaxOutlinks();
Link relToVia = Link.addRelativeToVia(curi, max, url,
CrawlURI relToVia = addRelativeToVia(curi, max, url,
LinkContext.EMBED_MISC, Hop.EMBED);
Link relToBase = Link.addRelativeToBase(curi, max, url,
CrawlURI relToBase = addRelativeToBase(curi, max, url,
LinkContext.EMBED_MISC, Hop.EMBED);
addAnnotations(relToVia, relToBase);
linkCount++;
}
}
protected void addAnnotations(Link relToVia, Link relToBase) {
protected void addAnnotations(CrawlURI relToVia, CrawlURI relToBase) {
if (relToVia != null && relToBase != null
&& relToVia.getDestination().equals(relToBase.getDestination())) {
&& relToVia.getUURI().equals(relToBase.getUURI())) {
relToVia.getAnnotations().add("extractorSWFRelToBoth");
relToBase.getAnnotations().add("extractorSWFRelToBoth");
} else {
@@ -361,9 +361,9 @@ public class ExtractorSWF extends ContentExtractor {
public void considerStringAsUri(String str) throws IOException {
if (UriUtils.isVeryLikelyUri(str)) {
int max = ext.getExtractorParameters().getMaxOutlinks();
Link relToVia = Link.addRelativeToVia(curi, max, str,
CrawlURI relToVia = addRelativeToVia(curi, max, str,
LinkContext.SPECULATIVE_MISC, Hop.SPECULATIVE);
Link relToBase = Link.addRelativeToBase(curi, max, str,
CrawlURI relToBase = addRelativeToBase(curi, max, str,
LinkContext.SPECULATIVE_MISC, Hop.SPECULATIVE);
addAnnotations(relToVia, relToBase);
linkCount++;
modules/src/main/java/org/archive/modules/extractor/ExtractorURI.java
+5 -10
View File
@@ -77,11 +77,8 @@ public class ExtractorURI extends Extractor {
*/
@Override
public void extract(CrawlURI curi) {
List<Link> links = new ArrayList<Link>(curi.getOutLinks());
int max = links.size();
for (int i = 0; i < max; i++) {
Link wref = links.get(i);
extractLink(curi, wref);
for (CrawlURI link : curi.getOutLinks()) {
extractLink(curi, link);
}
}
@@ -91,10 +88,10 @@ public class ExtractorURI extends Extractor {
* @param curi CrawlURI to add discoveries to
* @param wref Link to examine for internal URIs
*/
protected void extractLink(CrawlURI curi, Link wref) {
protected void extractLink(CrawlURI curi, CrawlURI wref) {
UURI source = null;
try {
source = UURIFactory.getInstance(wref.getDestination().toString());
source = UURIFactory.getInstance(wref.getURI());
} catch (URIException e) {
LOGGER.log(Level.FINE,"bad URI",e);
}
@@ -105,13 +102,11 @@ public class ExtractorURI extends Extractor {
List<String> found = extractQueryStringLinks(source);
for (String uri : found) {
try {
UURI src = curi.getUURI();
UURI dest = UURIFactory.getInstance(uri);
LinkContext lc = LinkContext.SPECULATIVE_MISC;
Hop hop = Hop.SPECULATIVE;
Link link = new Link(src, dest, lc, hop);
addOutlink(curi, dest, lc, hop);
numberOfLinksExtracted.incrementAndGet();
curi.getOutLinks().add(link);
} catch (URIException e) {
LOGGER.log(Level.FINE, "bad URI", e);
}
modules/src/main/java/org/archive/modules/extractor/ExtractorUniversal.java
+1 -3
View File
@@ -405,12 +405,10 @@ public class ExtractorUniversal extends ContentExtractor {
// And add the URL to speculative embeds.
numberOfLinksExtracted.incrementAndGet();
UURI src = curi.getUURI();
UURI dest = UURIFactory.getInstance(newURL);
LinkContext lc = LinkContext.SPECULATIVE_MISC;
Hop hop = Hop.SPECULATIVE;
Link link = new Link(src, dest, lc, hop);
curi.getOutLinks().add(link);
addOutlink(curi, dest, lc, hop);
}
// Reset lookat for next string.
lookat = new StringBuffer();
modules/src/main/java/org/archive/modules/extractor/ExtractorXML.java
+1 -1
View File
@@ -159,7 +159,7 @@ public class ExtractorXML extends ContentExtractor {
// intends to create a followable/fetchable URI is
// unknown
int max = ext.getExtractorParameters().getMaxOutlinks();
Link.addRelativeToBase(curi, max, xmlUri,
addRelativeToBase(curi, max, xmlUri,
LinkContext.SPECULATIVE_MISC, Hop.SPECULATIVE);
} catch (URIException e) {
// There may not be a controller (e.g. If we're being run
modules/src/main/java/org/archive/modules/extractor/JerichoExtractorHTML.java
+1 -1
View File
@@ -319,7 +319,7 @@ public class JerichoExtractorHTML extends ExtractorHTML {
String refreshUri = content.substring(content.indexOf("=") + 1);
try {
int max = getExtractorParameters().getMaxOutlinks();
Link.addRelativeToBase(curi, max, refreshUri,
addRelativeToBase(curi, max, refreshUri,
HTMLLinkContext.META, Hop.REFER);
} catch (URIException e) {
logUriError(e, curi.getUURI(), refreshUri);
modules/src/main/java/org/archive/modules/extractor/Link.java
-242
View File
@@ -1,242 +0,0 @@
/*
* This file is part of the Heritrix web crawler (crawler.archive.org).
*
* Licensed to the Internet Archive (IA) by one or more individual
* contributors.
*
* The IA licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.archive.modules.extractor;
import static org.archive.modules.CoreAttributeConstants.A_ANNOTATIONS;
import java.io.Serializable;
import java.util.Collection;
import java.util.HashMap;
import java.util.LinkedHashSet;
import java.util.Map;
import java.util.logging.Logger;
import org.apache.commons.httpclient.URIException;
import org.archive.modules.CoreAttributeConstants;
import org.archive.modules.CrawlURI;
import org.archive.net.UURI;
import org.archive.net.UURIFactory;
/**
* Link represents one discovered "edge" of the web graph: the source
* URI, the destination URI, and the type of reference (represented by the
* context in which it was found).
*
* As such, it is a suitably generic item to returned from generic
* link-extraction utility code.
*
* @author gojomo
*/
public class Link implements Serializable, Comparable<Link> {
private static final Logger LOGGER = Logger.getLogger(Link.class.getName());
private static final long serialVersionUID = 3L;
/** URI where this Link was discovered */
private CharSequence source;
/** URI (absolute) where this Link points */
private CharSequence destination;
/** context of discovery -- will be an XPath-like element[/@attribute]
* fragment for HTML URIs, a header name with trailing ':' for header
* values, or one of the stand-in constants when other context is
* unavailable */
private LinkContext context;
/** hop-type */
private Hop hop;
/**
* Flexible dynamic attributes list.
* <p>
* See further {@link Link#getData()}
*/
protected Map<String,Object> data;
/**
* Create a Link with the given fields.
* @param source
* @param destination
* @param context
* @param hopType
*/
public Link(CharSequence source, CharSequence destination,
LinkContext context, Hop hop) {
super();
this.source = source;
this.destination = destination;
this.context = context;
this.hop = hop;
}
/**
* @return Returns the context.
*/
public LinkContext getContext() {
return context;
}
/**
* @return Returns the destination.
*/
public CharSequence getDestination() {
return destination;
}
/**
* @return Returns the source.
*/
public CharSequence getSource() {
return source;
}
/**
* @return char hopType
*/
public Hop getHopType() {
return hop;
}
@Override
public String toString() {
return this.destination + " " + hop.getHopChar() + " " + this.context;
}
/**
* Attribute list
* <p>
* By convention the attribute list is keyed by constants found in the
* {@link CoreAttributeConstants} interface. Use this list to carry
* data or state produced by custom processors rather change the
* classes {@link CrawlURI} or this class, CrawlURI.
* <p>
* This list becomes {@link CrawlURI#getData()} when the Link is promoted to CrawlURI via
* {@link CrawlURI#createCrawlURI(UURI, Link)}
* <p>
* If the list is null when this method is invoked, a new instance will be created and returned.
*
* @returns a flexible map of key/value pairs for storing
* status of this URI for use by other processors.
*/
public Map<String, Object> getData() {
if (data == null) {
data = new HashMap<String,Object>();
}
return data;
}
/**
* @return true if data map is not null
*/
public boolean hasData() {
return data != null;
}
@Override
public boolean equals(Object o) {
if (!(o instanceof Link)) {
return false;
}
Link l = (Link)o;
return l.source.equals(source) && l.destination.equals(destination)
&& l.context.equals(context) && l.hop.equals(hop);
}
@Override
public int hashCode() {
int r = 37;
return r ^ source.hashCode() ^ destination.hashCode()
^ context.hashCode() ^ hop.hashCode();
}
public static Link addRelativeToBase(CrawlURI uri, int max,
String newUri, LinkContext context, Hop hop) throws URIException {
UURI dest = UURIFactory.getInstance(uri.getBaseURI(), newUri);
return addOrDiscard(uri, max, dest, context, hop);
}
public static Link addRelativeToVia(CrawlURI uri, int max, String newUri,
LinkContext context, Hop hop) throws URIException {
UURI relTo = uri.getVia();
if (relTo == null) {
if (!uri.getAnnotations().contains("usedBaseForVia")) {
LOGGER.info("no via where expected; using base instead: " + uri);
uri.getAnnotations().add("usedBaseForVia");
}
relTo = uri.getBaseURI();
}
UURI dest = UURIFactory.getInstance(relTo, newUri);
return addOrDiscard(uri, max, dest, context, hop);
}
public static Link add(CrawlURI uri, int max, String newUri,
LinkContext context, Hop hop) throws URIException {
UURI dest = UURIFactory.getInstance(newUri);
return addOrDiscard(uri, max, dest, context, hop);
}
private static Link addOrDiscard(CrawlURI uri, int max, UURI dest,
LinkContext context, Hop hop) throws URIException {
if (uri.getOutLinks().size() < max) {
UURI src = uri.getUURI();
Link link = new Link(src, dest, context, hop);
uri.getOutLinks().add(link);
return link;
} else {
uri.incrementDiscardedOutLinks();
return null;
}
}
public int compareTo(Link o) {
int cmp = source.toString().compareTo(o.source.toString());
if (cmp==0) {
cmp = destination.toString().compareTo(o.destination.toString());
}
if (cmp==0) {
cmp = context.toString().compareTo(o.context.toString());
}
if (cmp==0) {
cmp = hop.toString().compareTo(o.hop.toString());
}
return cmp;
}
/**
* Get the annotations set for this uri.
*
* @return the annotations set for this uri.
*/
// XXX copied from CrawlURI :-\ let's get HER-2039 in there
public Collection<String> getAnnotations() {
@SuppressWarnings("unchecked")
Collection<String> annotations = (Collection<String>)getData().get(A_ANNOTATIONS);
if (annotations == null) {
annotations = new LinkedHashSet<String>();
getData().put(A_ANNOTATIONS, annotations);
}
return annotations;
}
}
modules/src/main/java/org/archive/modules/extractor/StringExtractorTestBase.java
+3 -3
View File
@@ -29,9 +29,9 @@ public abstract class StringExtractorTestBase extends ContentExtractorTestBase {
public static class TestData {
public CrawlURI uri;
public Link expectedResult;
public CrawlURI expectedResult;
public TestData(CrawlURI uri, Link expectedResult) {
public TestData(CrawlURI uri, CrawlURI expectedResult) {
this.uri = uri;
this.expectedResult = expectedResult;
}
@@ -80,7 +80,7 @@ public abstract class StringExtractorTestBase extends ContentExtractorTestBase {
Collection<TestData> testDataCol = makeData(text, expectedURL);
for (TestData testData: testDataCol) {
extractor.process(testData.uri);
HashSet<Link> expected = new HashSet<Link>();
HashSet<CrawlURI> expected = new HashSet<CrawlURI>();
if (testData.expectedResult != null) {
expected.add(testData.expectedResult);
}
modules/src/main/java/org/archive/modules/fetcher/FetchFTP.java
+2 -3
View File
@@ -44,7 +44,6 @@ import org.archive.io.ReplayCharSequence;
import org.archive.modules.CrawlURI;
import org.archive.modules.Processor;
import org.archive.modules.extractor.Hop;
import org.archive.modules.extractor.Link;
import org.archive.modules.extractor.LinkContext;
import org.archive.net.ClientFTP;
import org.archive.net.UURI;
@@ -573,7 +572,7 @@ public class FetchFTP extends Processor {
}
try {
UURI n = UURIFactory.getInstance(base + "/" + file);
Link link = new Link(curi.getUURI(), n, LinkContext.NAVLINK_MISC, Hop.NAVLINK);
CrawlURI link = curi.createCrawlURI(n, LinkContext.NAVLINK_MISC, Hop.NAVLINK);
curi.getOutLinks().add(link);
} catch (URIException e) {
logger.log(Level.WARNING, "URI error during extraction.", e);
@@ -610,7 +609,7 @@ public class FetchFTP extends Processor {
String path = uuri.getEscapedCurrentHierPath();
UURI parent = UURIFactory.getInstance(scheme + "://" + auth + path);
Link link = new Link(uuri, parent, LinkContext.NAVLINK_MISC,
CrawlURI link = curi.createCrawlURI(parent, LinkContext.NAVLINK_MISC,
Hop.NAVLINK);
curi.getOutLinks().add(link);
} catch (URIException e) {
modules/src/main/java/org/archive/modules/fetcher/FetchWhois.java
+2 -2
View File
@@ -37,8 +37,8 @@ import org.archive.modules.CoreAttributeConstants;
import org.archive.modules.CrawlURI;
import org.archive.modules.ProcessResult;
import org.archive.modules.Processor;
import org.archive.modules.extractor.Extractor;
import org.archive.modules.extractor.Hop;
import org.archive.modules.extractor.Link;
import org.archive.modules.extractor.LinkContext;
import org.archive.modules.net.CrawlHost;
import org.archive.modules.net.ServerCache;
@@ -427,7 +427,7 @@ public class FetchWhois extends Processor implements CoreAttributeConstants,
protected void addWhoisLink(CrawlURI curi, String query) {
String whoisUrl = "whois:" + query;
try {
Link.add(curi, Integer.MAX_VALUE, whoisUrl, LinkContext.INFERRED_MISC, Hop.INFERRED);
Extractor.add(curi, Integer.MAX_VALUE, whoisUrl, LinkContext.INFERRED_MISC, Hop.INFERRED);
} catch (URIException e) {
logger.log(Level.WARNING, "problem with url " + whoisUrl, e);
}
modules/src/main/java/org/archive/modules/forms/FormLoginProcessor.java
+2 -2
View File
@@ -261,7 +261,7 @@ public class FormLoginProcessor extends Processor implements Checkpointable {
protected void createFormSubmissionAttempt(CrawlURI curi, HTMLForm templateForm, String formProvince) {
LinkContext lc = new LinkContext.SimpleLinkContext("form/@action");
try {
CrawlURI submitCuri = curi.makeConsequentCandidate(templateForm.getAction(),lc, Hop.SUBMIT);
CrawlURI submitCuri = curi.createCrawlURI(templateForm.getAction(), lc, Hop.SUBMIT);
submitCuri.setFetchType(FetchType.HTTP_POST);
submitCuri.getData().put(
CoreAttributeConstants.A_SUBMIT_DATA,
@@ -270,7 +270,7 @@ public class FormLoginProcessor extends Processor implements Checkpointable {
getLoginPassword()));
submitCuri.setSchedulingDirective(SchedulingConstants.HIGH);
submitCuri.setForceFetch(true);
curi.getOutCandidates().add(submitCuri);
curi.getOutLinks().add(submitCuri);
curi.getAnnotations().add("submit:"+templateForm.getAction());
} catch (URIException ue) {
loggerModule.logUriError(ue,curi.getUURI(),templateForm.getAction());
modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java
+3 -4
View File
@@ -88,7 +88,6 @@ import org.archive.modules.CrawlMetadata;
import org.archive.modules.CrawlURI;
import org.archive.modules.ProcessResult;
import org.archive.modules.deciderules.recrawl.IdenticalDigestDecideRule;
import org.archive.modules.extractor.Link;
import org.archive.spring.ConfigPath;
import org.archive.uid.RecordIDGenerator;
import org.archive.uid.UUIDGenerator;
@@ -848,10 +847,10 @@ public class WARCWriterProcessor extends WriterPoolProcessor implements WARCWrit
}
// Add outlinks though they are effectively useless without anchor text.
Collection<Link> links = curi.getOutLinks();
Collection<CrawlURI> links = curi.getOutLinks();
if (links != null && links.size() > 0) {
for (Link link: links) {
r.addLabelValue("outlink", link.toString());
for (CrawlURI link: links) {
r.addLabelValue("outlink", link.getURI());
}
}
modules/src/test/java/org/archive/modules/extractor/ExtractorCSSTest.java
+5 -9
View File
@@ -19,22 +19,17 @@
package org.archive.modules.extractor;
import static org.archive.modules.extractor.LinkContext.EMBED_MISC;
import static org.archive.modules.extractor.LinkContext.NAVLINK_MISC;
import java.util.Collection;
import java.util.Collections;
import org.archive.modules.CrawlURI;
import org.archive.modules.extractor.Extractor;
import org.archive.modules.extractor.ExtractorCSS;
import org.archive.modules.extractor.Hop;
import org.archive.modules.extractor.Link;
import org.archive.modules.extractor.StringExtractorTestBase;
import org.archive.net.UURI;
import org.archive.net.UURIFactory;
import org.archive.util.Recorder;
import static org.archive.modules.extractor.LinkContext.EMBED_MISC;
import static org.archive.modules.extractor.LinkContext.NAVLINK_MISC;
/**
* Unit test for ExtractorCSS.
@@ -81,8 +76,9 @@ public class ExtractorCSSTest extends StringExtractorTestBase {
euri.setRecorder(recorder);
euri.setContentSize(content.length());
// TODO: This test was naively modified to account for the abscense of LINK, but no effort was made to confirm that it is actually testing anything useful
UURI dest = UURIFactory.getInstance(uri);
Link link = new Link(src, dest, EMBED_MISC, Hop.EMBED);
CrawlURI link = euri.createCrawlURI(dest, EMBED_MISC, Hop.EMBED);
TestData td = new TestData(euri, link);
return Collections.singleton(td);
}
modules/src/test/java/org/archive/modules/extractor/ExtractorHTMLTest.java
+19 -21
View File
@@ -107,7 +107,7 @@ public class ExtractorHTMLTest extends StringExtractorTestBase {
UURI dest = UURIFactory.getInstance(destURI);
LinkContext context = determineContext(content);
Hop hop = determineHop(content);
Link link = new Link(src, dest, context, hop);
CrawlURI link = euri.createCrawlURI(dest, context, hop);
result.add(new TestData(euri, link));
euri = new CrawlURI(src, null, null, LinkContext.NAVLINK_MISC);
@@ -166,10 +166,10 @@ public class ExtractorHTMLTest extends StringExtractorTestBase {
CrawlURI puri = new CrawlURI(UURIFactory
.getInstance("http://www.example.com"));
getExtractor().extract(puri, source);
Link[] links = puri.getOutLinks().toArray(new Link[0]);
CrawlURI[] links = puri.getOutLinks().toArray(new CrawlURI[0]);
assertTrue("did not find single link",links.length==1);
assertTrue("expected link not found",
links[0].getDestination().toString().equals(expected));
links[0].getURI().equals(expected));
}
/**
@@ -188,9 +188,8 @@ public class ExtractorHTMLTest extends StringExtractorTestBase {
"<form method=\"post\" action=\"http://www.example.com/notok\"> "+
"<form action=\"http://www.example.com/ok3\"> ";
getExtractor().extract(puri, cs);
Link[] links = puri.getOutLinks().toArray(new Link[0]);
// find exactly 3 (not the POST) action URIs
assertTrue("incorrect number of links found",links.length==3);
assertTrue("incorrect number of links found", puri.getOutLinks().size()==3);
}
/**
@@ -207,7 +206,7 @@ public class ExtractorHTMLTest extends StringExtractorTestBase {
getExtractor().extract(puri, cs);
assertEquals("meta robots content not extracted","index,nofollow",
puri.getData().get(ExtractorHTML.A_META_ROBOTS));
Link[] links = puri.getOutLinks().toArray(new Link[0]);
CrawlURI[] links = puri.getOutLinks().toArray(new CrawlURI[0]);
assertTrue("link extracted despite meta robots",links.length==0);
}
@@ -228,9 +227,8 @@ public class ExtractorHTMLTest extends StringExtractorTestBase {
assertTrue(CollectionUtils.exists(curi.getOutLinks(), new Predicate() {
public boolean evaluate(Object object) {
return ((Link) object)
.getDestination()
.toString()
return ((CrawlURI) object)
.getURI()
.indexOf(
"/example.html;jsessionid=deadbeef:deadbeed?parameter=this:value") >= 0;
}
@@ -238,7 +236,7 @@ public class ExtractorHTMLTest extends StringExtractorTestBase {
assertTrue(CollectionUtils.exists(curi.getOutLinks(), new Predicate() {
public boolean evaluate(Object object) {
return ((Link) object).getDestination().toString().indexOf(
return ((CrawlURI) object).getURI().indexOf(
"/example.html?parameter=this:value") >= 0;
}
}));
@@ -262,15 +260,15 @@ public class ExtractorHTMLTest extends StringExtractorTestBase {
assertTrue(CollectionUtils.exists(curi.getOutLinks(), new Predicate() {
public boolean evaluate(Object object) {
System.err.println("comparing: "
+ ((Link) object).getDestination().toString()
+ ((CrawlURI) object).getURI()
+ " and https://www.anotherexample.com/");
return ((Link) object).getDestination().toString().equals(
return ((CrawlURI) object).getURI().equals(
"http://www.anotherexample.com/");
}
}));
assertTrue(CollectionUtils.exists(curi.getOutLinks(), new Predicate() {
public boolean evaluate(Object object) {
return ((Link) object).getDestination().toString().equals(
return ((CrawlURI) object).getURI().equals(
"https://www.example.com/index.html");
}
}));
@@ -310,21 +308,21 @@ public class ExtractorHTMLTest extends StringExtractorTestBase {
"<a href=\"def/another1.html\">" +
"<a href=\"ghi/another2.html\">";
getExtractor().extract(puri, cs);
Link[] links = puri.getOutLinks().toArray(new Link[0]);
CrawlURI[] links = puri.getOutLinks().toArray(new CrawlURI[0]);
Arrays.sort(links);
String dest1 = "http://www.example.com/def/another1.html";
String dest2 = "http://www.example.com/ghi/another2.html";
// ensure outlink from base href
assertEquals("outlink1 from base href",dest1,
links[1].getDestination().toString());
links[1].getURI());
assertEquals("outlink2 from base href",dest2,
links[2].getDestination().toString());
links[2].getURI());
}
protected Predicate destinationContainsPredicate(final String fragment) {
return new Predicate() {
public boolean evaluate(Object object) {
return ((Link) object).getDestination().toString().indexOf(fragment) >= 0;
return ((CrawlURI) object).getURI().indexOf(fragment) >= 0;
}
};
}
@@ -332,7 +330,7 @@ public class ExtractorHTMLTest extends StringExtractorTestBase {
protected Predicate destinationsIsPredicate(final String value) {
return new Predicate() {
public boolean evaluate(Object object) {
return ((Link) object).getDestination().toString().equals(value);
return ((CrawlURI) object).getURI().equals(value);
}
};
}
@@ -397,16 +395,16 @@ public class ExtractorHTMLTest extends StringExtractorTestBase {
getExtractor().extract(curi, cs);
Link[] links = curi.getOutLinks().toArray(new Link[0]);
CrawlURI[] links = curi.getOutLinks().toArray(new CrawlURI[0]);
Arrays.sort(links);
String dest1 = "http://www.example.com/foo.gif";
String dest2 = "http://www.example.com/foo.js";
assertEquals("outlink1 from conditional comment img src",dest1,
links[0].getDestination().toString());
links[0].getURI());
assertEquals("outlink2 from conditional comment script src",dest2,
links[1].getDestination().toString());
links[1].getURI());
}
modules/src/test/java/org/archive/modules/extractor/ExtractorJSTest.java
+1 -1
View File
@@ -141,7 +141,7 @@ public class ExtractorJSTest extends StringExtractorTestBase {
if (destURI != null) {
UURI dest = UURIFactory.getInstance(destURI);
Link link = new Link(src, dest, LinkContext.JS_MISC, Hop.SPECULATIVE);
CrawlURI link = euri.createCrawlURI(dest, LinkContext.JS_MISC, Hop.SPECULATIVE);
result.add(new TestData(euri, link));
} else {
result.add(new TestData(euri, null));
modules/src/test/java/org/archive/modules/extractor/ExtractorMultipleRegexTest.java
+1 -1
View File
@@ -188,7 +188,7 @@ public class ExtractorMultipleRegexTest extends ContentExtractorTestBase {
extractor.process(testUri);
for (String expectedLinkString: EXPECTED_OUTLINKS) {
Link expectedLink = new Link(testUri.getUURI(),
CrawlURI expectedLink = testUri.createCrawlURI(
UURIFactory.getInstance(expectedLinkString),
HTMLLinkContext.INFERRED_MISC, Hop.INFERRED);
assertTrue(testUri.getOutLinks().contains(expectedLink));
modules/src/test/java/org/archive/modules/extractor/ExtractorSWFTest.java
+6 -6
View File
@@ -109,9 +109,9 @@ public class ExtractorSWFTest extends ContentExtractorTestBase {
+ elapsed + "ms to process " + url);
boolean foundIt = false;
for (Link link : curi.getOutLinks()) {
for (CrawlURI link : curi.getOutLinks()) {
logger.info("found link: " + link);
foundIt = foundIt || link.getDestination().toString().endsWith(testUrls.get(url));
foundIt = foundIt || link.getURI().endsWith(testUrls.get(url));
}
assertTrue("failed to extract link \"" + testUrls.get(url)
@@ -165,9 +165,9 @@ public class ExtractorSWFTest extends ContentExtractorTestBase {
+ elapsed + "ms to process " + url);
boolean foundIt = false;
for (Link link : curi.getOutLinks()) {
for (CrawlURI link : curi.getOutLinks()) {
logger.info("found link: " + link);
foundIt = foundIt || link.getDestination().toString().endsWith(testUrls.get(url));
foundIt = foundIt || link.getURI().endsWith(testUrls.get(url));
}
if (!foundIt)
@@ -198,11 +198,11 @@ public class ExtractorSWFTest extends ContentExtractorTestBase {
expected.put("http://nyumedecs.kk5.org/containermain.swf", "extractorSWFRelToVia");
expected.put("https://wayback.archive-it.org/3771/20131119163257/http://nyumedecs.kk5.org/_app/28727/en/resources/containermain.swf", "extractorSWFRelToBase");
for (Link link: curi.getOutLinks()) {
for (CrawlURI link: curi.getOutLinks()) {
System.out.println(link + " " + link.getData());
assertEquals(1, link.getAnnotations().size());
String dest = link.getDestination().toString();
String dest = link.toString();
assertTrue(expected.containsKey(dest));
// remove the entry, so at the end the map should be empty, confirming that we found all the expected links
modules/src/test/java/org/archive/modules/extractor/JerichoExtractorHTMLTest.java
+3 -3
View File
@@ -73,7 +73,7 @@ public class JerichoExtractorHTMLTest extends ExtractorHTMLTest {
curi.getOutLinks();
assertTrue(CollectionUtils.exists(curi.getOutLinks(), new Predicate() {
public boolean evaluate(Object object) {
return ((Link) object).getDestination().toString().indexOf(
return ((CrawlURI) object).getURI().indexOf(
"/redirect_me?form=true&checked[]=1&unchecked[]=&selectBox=selectedOption&test=Go")>=0;
}
}));
@@ -101,7 +101,7 @@ public class JerichoExtractorHTMLTest extends ExtractorHTMLTest {
curi.getOutLinks();
assertTrue(!CollectionUtils.exists(curi.getOutLinks(), new Predicate() {
public boolean evaluate(Object object) {
return ((Link) object).getDestination().toString().indexOf(
return ((CrawlURI) object).getURI().indexOf(
"/redirect_me?form=true&checked[]=1&unchecked[]=&selectBox=selectedOption&test=Go")>=0;
}
}));
@@ -130,7 +130,7 @@ public class JerichoExtractorHTMLTest extends ExtractorHTMLTest {
curi.getOutLinks();
assertTrue(CollectionUtils.exists(curi.getOutLinks(), new Predicate() {
public boolean evaluate(Object object) {
return ((Link) object).getDestination().toString().indexOf(
return ((CrawlURI) object).getURI().indexOf(
"/redirect_me?form=true&checked[]=1&unchecked[]=&selectBox=selectedOption&test=Go")>=0;
}
}));
modules/src/test/java/org/archive/modules/forms/FormLoginProcessorTest.java
+2 -2
View File
@@ -62,8 +62,8 @@ public class FormLoginProcessorTest extends ProcessorTestBase {
assertEquals("WARC-Simple-Form-Province-Status: 0,0,http://(com,example,)", curi.getDataList(A_WARC_RESPONSE_HEADERS).get(0));
assertTrue(curi.getAnnotations().contains("submit:/login"));
assertEquals(1, curi.getOutCandidates().size());
CrawlURI submitCuri = curi.getOutCandidates().toArray(new CrawlURI[0])[0];
assertEquals(1, curi.getOutLinks().size());
CrawlURI submitCuri = curi.getOutLinks().toArray(new CrawlURI[0])[0];
assertEquals("http://example.com/login", submitCuri.toString());
assertEquals(FetchType.HTTP_POST, submitCuri.getFetchType());
String queryString = (String) submitCuri.getData().get(CoreAttributeConstants.A_SUBMIT_DATA);