diff --git a/engine/src/main/java/org/archive/crawler/postprocessor/CandidatesProcessor.java b/engine/src/main/java/org/archive/crawler/postprocessor/CandidatesProcessor.java index d70d41eb..e2e909a0 100644 --- a/engine/src/main/java/org/archive/crawler/postprocessor/CandidatesProcessor.java +++ b/engine/src/main/java/org/archive/crawler/postprocessor/CandidatesProcessor.java @@ -23,7 +23,6 @@ package org.archive.crawler.postprocessor; import static org.archive.modules.fetcher.FetchStatusCodes.S_DEFERRED; import static org.archive.modules.fetcher.FetchStatusCodes.S_PREREQUISITE_UNSCHEDULABLE_FAILURE; -import org.apache.commons.httpclient.URIException; import org.archive.crawler.framework.Frontier; import org.archive.crawler.reporting.CrawlerLoggerModule; import org.archive.crawler.spring.SheetOverlaysManager; @@ -32,7 +31,6 @@ import org.archive.modules.CrawlURI; import org.archive.modules.Processor; import org.archive.modules.SchedulingConstants; import org.archive.modules.extractor.Hop; -import org.archive.modules.extractor.Link; import org.archive.modules.seeds.SeedModule; import org.archive.spring.KeyedProperties; import org.springframework.beans.factory.annotation.Autowired; @@ -219,15 +217,6 @@ public class CandidatesProcessor extends Processor { return; } - // (2) NEW: also (and before-outlinks) run outCandidates (usually empty; - // only current use is a form-submission CrawlURI; could - // potentially take over prerequisite duties for consistency - for(CrawlURI candidate : curi.getOutCandidates()) { - - runCandidateChain(candidate, curi); - - } - // Only consider candidate links of error pages if configured to do so if (!getProcessErrorOutlinks() && (curi.getFetchStatus() < 200 || curi.getFetchStatus() >= 400)) { @@ -236,21 +225,10 @@ public class CandidatesProcessor extends Processor { } // (3) Handle outlinks (usual bulk of discoveries) - for (Link wref: curi.getOutLinks()) { - CrawlURI candidate; - try { - candidate = curi.createCrawlURI(curi.getBaseURI(),wref); - } catch (URIException e) { - loggerModule.logUriError(e, curi.getUURI(), - wref.getDestination().toString()); - continue; - } + for (CrawlURI candidate: curi.getOutLinks()) { runCandidateChain(candidate, curi); - // TODO: evaluate if this necessary (anyone uses?); wise (bloat?) - curi.getOutCandidates().add(candidate); - } curi.getOutLinks().clear(); } diff --git a/engine/src/main/java/org/archive/crawler/postprocessor/LinksScoper.java b/engine/src/main/java/org/archive/crawler/postprocessor/LinksScoper.java index 7db90061..0cdf48e5 100644 --- a/engine/src/main/java/org/archive/crawler/postprocessor/LinksScoper.java +++ b/engine/src/main/java/org/archive/crawler/postprocessor/LinksScoper.java @@ -27,14 +27,12 @@ import static org.archive.modules.fetcher.FetchStatusCodes.S_PREREQUISITE_UNSCHE import java.util.logging.Level; import java.util.logging.Logger; -import org.apache.commons.httpclient.URIException; import org.archive.crawler.framework.Scoper; import org.archive.modules.CrawlURI; import org.archive.modules.deciderules.DecideResult; import org.archive.modules.deciderules.DecideRule; import org.archive.modules.deciderules.RejectDecideRule; import org.archive.modules.extractor.Hop; -import org.archive.modules.extractor.Link; /** * Determine which extracted links are within scope. @@ -151,18 +149,11 @@ public class LinksScoper extends Scoper { final boolean redirectsNewSeeds = getSeedsRedirectNewSeeds(); int preferenceDepthHops = getPreferenceDepthHops(); - for (Link wref: curi.getOutLinks()) try { - int directive = getSchedulingFor(curi, wref, preferenceDepthHops); - CrawlURI caURI = curi.createCrawlURI(curi.getBaseURI(), - wref, directive, - considerAsSeed(curi, wref, redirectsNewSeeds)); - if (isInScope(caURI)) { - curi.getOutCandidates().add(caURI); - } - } catch (URIException e) { - loggerModule.logUriError(e, curi.getUURI(), - wref.getDestination().toString()); - } + for (CrawlURI caURI: curi.getOutLinks()) { + int directive = getSchedulingFor(curi, caURI, preferenceDepthHops); + caURI.setSchedulingDirective(directive ); + caURI.setSeed(considerAsSeed(curi, caURI, redirectsNewSeeds)); + } curi.getOutLinks().clear(); } @@ -197,10 +188,10 @@ public class LinksScoper extends Scoper { } } - private boolean considerAsSeed(final CrawlURI curi, final Link wref, + private boolean considerAsSeed(final CrawlURI curi, final CrawlURI link, final boolean redirectsNewSeeds) { return redirectsNewSeeds && curi.isSeed() - && wref.getHopType() == Hop.REFER; + && link.getLastHopType() == Hop.REFER; } /** @@ -210,14 +201,14 @@ public class LinksScoper extends Scoper { * Imports into the frontier similarly do not pass through here, * but are given NORMAL priority. */ - protected int getSchedulingFor(final CrawlURI curi, final Link wref, + protected int getSchedulingFor(final CrawlURI curi, final CrawlURI link, final int preferenceDepthHops) { - final Hop c = wref.getHopType(); + final Hop c = link.getLastHopType(); if (LOGGER.isLoggable(Level.FINEST)) { LOGGER.finest(curi + " with path=" + curi.getPathFromSeed() + " isSeed=" + curi.isSeed() + " with fetchStatus=" + - curi.getFetchStatus() + " -> " + wref.getDestination() + - " type " + c + " with context=" + wref.getContext()); + curi.getFetchStatus() + " -> " + link.getURI() + + " type " + c + " with context=" + link.getViaContext()); } switch (c) { diff --git a/engine/src/main/java/org/archive/crawler/postprocessor/SupplementaryLinksScoper.java b/engine/src/main/java/org/archive/crawler/postprocessor/SupplementaryLinksScoper.java index bd78cf78..d3de5d20 100644 --- a/engine/src/main/java/org/archive/crawler/postprocessor/SupplementaryLinksScoper.java +++ b/engine/src/main/java/org/archive/crawler/postprocessor/SupplementaryLinksScoper.java @@ -87,7 +87,7 @@ public class SupplementaryLinksScoper extends Scoper { } // Collection inScopeLinks = new HashSet(); - Iterator iter = curi.getOutCandidates().iterator(); + Iterator iter = curi.getOutLinks().iterator(); while (iter.hasNext()) { CrawlURI cauri = iter.next(); if (!isInScope(cauri)) { diff --git a/engine/src/main/java/org/archive/crawler/processor/CrawlMapper.java b/engine/src/main/java/org/archive/crawler/processor/CrawlMapper.java index 9e56689e..2771db37 100644 --- a/engine/src/main/java/org/archive/crawler/processor/CrawlMapper.java +++ b/engine/src/main/java/org/archive/crawler/processor/CrawlMapper.java @@ -215,7 +215,7 @@ public abstract class CrawlMapper extends Processor implements Lifecycle { if (getCheckOutlinks()) { // consider outlinks for mapping - Iterator iter = curi.getOutCandidates().iterator(); + Iterator iter = curi.getOutLinks().iterator(); while(iter.hasNext()) { CrawlURI cauri = iter.next(); if (decideToMapOutlink(cauri)) { diff --git a/engine/src/main/java/org/archive/crawler/reporting/SeedRecord.java b/engine/src/main/java/org/archive/crawler/reporting/SeedRecord.java index edc8a15d..d0b9ba5a 100644 --- a/engine/src/main/java/org/archive/crawler/reporting/SeedRecord.java +++ b/engine/src/main/java/org/archive/crawler/reporting/SeedRecord.java @@ -110,7 +110,7 @@ public class SeedRecord implements CoreAttributeConstants, Serializable, Identit this.statusCode = curi.getFetchStatus(); this.disposition = disposition; if (statusCode==301 || statusCode == 302) { - for (CrawlURI cauri: curi.getOutCandidates()) { + for (CrawlURI cauri: curi.getOutLinks()) { if("location:".equalsIgnoreCase(cauri.getViaContext(). toString())) { redirectUri = cauri.toString(); diff --git a/modules/src/main/java/org/archive/modules/CrawlURI.java b/modules/src/main/java/org/archive/modules/CrawlURI.java index e08ecf9e..8b5003f7 100644 --- a/modules/src/main/java/org/archive/modules/CrawlURI.java +++ b/modules/src/main/java/org/archive/modules/CrawlURI.java @@ -88,7 +88,6 @@ import org.archive.modules.credential.Credential; import org.archive.modules.credential.HttpAuthenticationCredential; import org.archive.modules.extractor.HTMLLinkContext; import org.archive.modules.extractor.Hop; -import org.archive.modules.extractor.Link; import org.archive.modules.extractor.LinkContext; import org.archive.net.UURI; import org.archive.net.UURIFactory; @@ -869,7 +868,6 @@ implements Reporter, Serializable, OverlayContext { this.data = getPersistentDataMap(); extraInfo = null; - outCandidates = null; outLinks = null; method = null; } @@ -1080,38 +1078,25 @@ implements Reporter, Serializable, OverlayContext { } /** - * All discovered outbound Links (navlinks, embeds, etc.) - * Can either contain Link instances or CrawlURI instances, or both. - * The LinksScoper processor converts Link instances in this collection - * to CrawlURI instances. + * All discovered outbound urls as CrawlURIs (navlinks, embeds, etc.) */ - protected transient Collection outLinks = new LinkedHashSet(); - - protected transient Collection outCandidates = new LinkedHashSet(); + protected transient Collection outLinks; /** * Returns discovered links. The returned collection might be empty if * no links were discovered, or if something like LinksScoper promoted * the links to CrawlURIs. * - * @return Collection of all discovered outbound Links + * @return Collection of all discovered outbound links */ - public Collection getOutLinks() { + public Collection getOutLinks() { + if (outLinks==null) { + outLinks = new LinkedHashSet(); + } return outLinks; -// return Transform.subclasses(outLinks, Link.class); - } - - /** - * Returns discovered candidate URIs. The returned collection will be - * emtpy until something like LinksScoper promotes discovered Links - * into CrawlURIs. - * - * @return Collection of candidate URIs - */ - public Collection getOutCandidates() { - return outCandidates; } + /** * Set the (HTML) Base URI used for derelativizing internal URIs. * @@ -1178,8 +1163,6 @@ implements Reporter, Serializable, OverlayContext { @SuppressWarnings("unchecked") Map temp = (Map)stream.readObject(); this.data = temp; - outLinks = new HashSet(); - outCandidates = new HashSet(); } /** @@ -1379,6 +1362,11 @@ implements Reporter, Serializable, OverlayContext { return StringUtils.isEmpty(pathFromSeed) ? "" : pathFromSeed.substring(pathFromSeed.length()-1); } + public Hop getLastHopType() { + return Hop.getForChar(getLastHop().charAt(0)); + } + + /** * @return URI via which this one was discovered */ @@ -1412,28 +1400,6 @@ implements Reporter, Serializable, OverlayContext { } -// public void setStateProvider(SheetManager manager) { -// if(this.provider!=null) { -// return; -// } -// this.manager = manager; -//// this.provider = manager.findConfig(SURT.fromURI(toString())); -// } -// -// -// public StateProvider getStateProvider() { -// return provider; -// } - - -// public T get(Object module, Key key) { -// if (provider == null) { -// throw new AssertionError("ToeThread never set up CrawlURI's sheet."); -// } -// return provider.get(module, key); -// } - - // // Reporter implementation // @@ -1632,26 +1598,31 @@ implements Reporter, Serializable, OverlayContext { } /** - * Utility method for creation of CandidateURIs found extracting - * links from this CrawlURI. - * @param baseUURI BaseUURI for link. - * @param link Link to wrap CandidateURI in. - * @return New candidateURI wrapper around link. + * Utility method for creating CrawlURIs that were found as out links from the current CrawlURI + * links from this CrawlURI. + *

+ * Any relative URIs will be treated as relative to this CrawlURI's UURI. + * @param destination The new URI, possibly a relative URI + * @param context + * @param hop + * @return New CrawlURI with the current CrawlURI set as the one it inherits from * @throws URIException */ - public CrawlURI createCrawlURI(UURI baseUURI, Link link) - throws URIException { - UURI u = (link.getDestination() instanceof UURI)? - (UURI)link.getDestination(): - UURIFactory.getInstance(baseUURI, - link.getDestination().toString()); - CrawlURI newCaURI = new CrawlURI(u, - extendHopsPath(getPathFromSeed(),link.getHopType().getHopChar()), - getUURI(), link.getContext()); + public CrawlURI createCrawlURI(UURI destination, LinkContext context, Hop hop) + throws URIException { + return createCrawlURI(destination.toString(), context, hop); + } + + public CrawlURI createCrawlURI(String destination, LinkContext context, Hop hop) + throws URIException { + UURI u = UURIFactory.getInstance(this.getBaseURI(), destination); + CrawlURI newCaURI = new CrawlURI( + u, + extendHopsPath(getPathFromSeed(), + hop.getHopChar()), + this.getUURI(), + context); newCaURI.inheritFrom(this); - if (link.hasData()) { - newCaURI.data = link.getData(); - } return newCaURI; } @@ -1675,19 +1646,19 @@ implements Reporter, Serializable, OverlayContext { } /** - * Utility method for creation of CandidateURIs found extracting + * Utility method for creation of CrawlURIs found extracting * links from this CrawlURI. * @param baseUURI BaseUURI for link. - * @param link Link to wrap CandidateURI in. + * TODO: Fix JavaDoc * @param scheduling How new CandidateURI should be scheduled. * @param seed True if this CandidateURI is a seed. * @return New candidateURI wrapper around link. * @throws URIException */ - public CrawlURI createCrawlURI(UURI baseUURI, Link link, + public CrawlURI createCrawlURI(UURI destination, LinkContext context, Hop hop, int scheduling, boolean seed) throws URIException { - final CrawlURI caURI = createCrawlURI(baseUURI, link); + final CrawlURI caURI = createCrawlURI(destination, context, hop); caURI.setSchedulingDirective(scheduling); caURI.setSeed(seed); return caURI; @@ -1877,7 +1848,7 @@ implements Reporter, Serializable, OverlayContext { */ public CrawlURI markPrerequisite(String preq) throws URIException { - CrawlURI caUri = makeConsequentCandidate(preq, LinkContext.PREREQ_MISC, Hop.PREREQ); + CrawlURI caUri = createCrawlURI(preq, LinkContext.PREREQ_MISC, Hop.PREREQ); caUri.setPrerequisite(true); // TODO: consider moving some of this to configurable candidate-handling int prereqPriority = getSchedulingDirective() - 1; @@ -1894,25 +1865,6 @@ implements Reporter, Serializable, OverlayContext { return caUri; } - /** - * Create a consequent CrawlURI from this one, given the - * additional parameters - * - * @param destination URI string - * @param lc LinkContext - * @param hop Hop - * @return the newly created prerequisite CrawlURI - * @throws URIException - */ - public CrawlURI makeConsequentCandidate(String destination, LinkContext lc, Hop hop) - throws URIException { - UURI src = getUURI(); - UURI dest = UURIFactory.getInstance(getBaseURI(),destination); - Link link = new Link(src, dest, lc, hop); - CrawlURI caUri = createCrawlURI(getBaseURI(), link); - return caUri; - } - public boolean containsContentTypeCharsetDeclaration() { // TODO can this regex be improved? should the test consider if its legal? return getContentType().matches("(?i).*charset=.*"); diff --git a/modules/src/main/java/org/archive/modules/extractor/Extractor.java b/modules/src/main/java/org/archive/modules/extractor/Extractor.java index 44cfd9c6..d177b5a7 100644 --- a/modules/src/main/java/org/archive/modules/extractor/Extractor.java +++ b/modules/src/main/java/org/archive/modules/extractor/Extractor.java @@ -43,7 +43,8 @@ import org.springframework.beans.factory.annotation.Autowired; * @author pjack */ public abstract class Extractor extends Processor { - + private static final Logger LOGGER = Logger.getLogger(Extractor.class.getName()); + protected AtomicLong numberOfLinksExtracted = new AtomicLong(0); /** Logger. */ @@ -143,13 +144,23 @@ public abstract class Extractor extends Processor { Hop hop) { try { UURI dest = UURIFactory.getInstance(curi.getUURI(), uri); - Link link = new Link(curi.getUURI(), dest, context, hop); + CrawlURI link = curi.createCrawlURI(dest, context, hop); curi.getOutLinks().add(link); } catch (URIException e) { logUriError(e, curi.getUURI(), uri); } } + protected void addOutlink(CrawlURI curi, UURI uuri, LinkContext context, + Hop hop) { + try { + CrawlURI link = curi.createCrawlURI(uuri, context, hop); + curi.getOutLinks().add(link); + } catch (URIException e) { + logUriError(e, curi.getUURI(), uuri.toString()); + } + } + public void logUriError(URIException e, UURI uuri, CharSequence l) { loggerModule.logUriError(e, uuri, l); @@ -174,5 +185,43 @@ public abstract class Extractor extends Processor { ret.append(" " + numberOfLinksExtracted + " links from " + getURICount() +" CrawlURIs\n"); return ret.toString(); } + + public static void addRelativeToBase(CrawlURI uri, int max, + String newUri, LinkContext context, Hop hop) throws URIException { + UURI dest = UURIFactory.getInstance(uri.getBaseURI(), newUri); + add2(uri, max, dest, context, hop); + } + + + public static void addRelativeToVia(CrawlURI uri, int max, String newUri, + LinkContext context, Hop hop) throws URIException { + UURI relTo = uri.getVia(); + if (relTo == null) { + if (!uri.getAnnotations().contains("usedBaseForVia")) { + LOGGER.info("no via where expected; using base instead: " + uri); + uri.getAnnotations().add("usedBaseForVia"); + } + relTo = uri.getBaseURI(); + } + UURI dest = UURIFactory.getInstance(relTo, newUri); + add2(uri, max, dest, context, hop); + } + + public static void add(CrawlURI uri, int max, String newUri, + LinkContext context, Hop hop) throws URIException { + UURI dest = UURIFactory.getInstance(newUri); + add2(uri, max, dest, context, hop); + } + + + private static void add2(CrawlURI curi, int max, UURI dest, + LinkContext context, Hop hop) throws URIException { + if (curi.getOutLinks().size() < max) { + CrawlURI link = curi.createCrawlURI(dest, context, hop); + curi.getOutLinks().add(link); + } else { + curi.incrementDiscardedOutLinks(); + } + } } diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorCSS.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorCSS.java index 60d13e2f..c0bec497 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorCSS.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorCSS.java @@ -136,7 +136,7 @@ public class ExtractorCSS extends ContentExtractor { foundLinks++; int max = ext.getExtractorParameters().getMaxOutlinks(); try { - Link.addRelativeToBase(curi, max, cssUri, + addRelativeToBase(curi, max, cssUri, LinkContext.EMBED_MISC, Hop.EMBED); } catch (URIException e) { ext.logUriError(e, curi.getUURI(), cssUri); diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorDOC.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorDOC.java index dc61f77d..438d30fb 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorDOC.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorDOC.java @@ -114,8 +114,7 @@ public class ExtractorDOC extends ContentExtractor { try { UURI dest = UURIFactory.getInstance(curi.getUURI(), hyperlink); LinkContext lc = LinkContext.NAVLINK_MISC; - Link link = new Link(curi.getUURI(), dest, lc, Hop.NAVLINK); - curi.getOutLinks().add(link); + addOutlink(curi, hyperlink, lc, Hop.NAVLINK); } catch (URIException e1) { logUriError(e1, curi.getUURI(), hyperlink); } diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java index 0c93cd38..baffd76b 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java @@ -656,7 +656,7 @@ public class ExtractorHTML extends ContentExtractor implements InitializingBean // ReplayCharSequence. HTMLLinkContext hc = HTMLLinkContext.get(context.toString()); int max = getExtractorParameters().getMaxOutlinks(); - Link.addRelativeToBase(curi, max, uri.toString(), hc, hop); + addRelativeToBase(curi, max, uri.toString(), hc, hop); } catch (URIException e) { logUriError(e, curi.getUURI(), uri); } @@ -954,7 +954,7 @@ public class ExtractorHTML extends ContentExtractor implements InitializingBean String refreshUri = content.substring(urlIndex); try { int max = getExtractorParameters().getMaxOutlinks(); - Link.addRelativeToBase(curi, max, refreshUri, + addRelativeToBase(curi, max, refreshUri, HTMLLinkContext.META, Hop.REFER); } catch (URIException e) { logUriError(e, curi.getUURI(), refreshUri); diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorHTTP.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorHTTP.java index 23b230fb..3cab5881 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorHTTP.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorHTTP.java @@ -83,8 +83,7 @@ public class ExtractorHTTP extends Extractor { try { UURI dest = UURIFactory.getInstance(curi.getUURI(), loc.getValue()); LinkContext lc = HTMLLinkContext.get(loc.getName()+":"); - Link link = new Link(curi.getUURI(), dest, lc, Hop.REFER); - curi.getOutLinks().add(link); + addOutlink(curi, dest.toString(), lc, Hop.REFER); numberOfLinksExtracted.incrementAndGet(); } catch (URIException e) { logUriError(e, curi.getUURI(), loc.getValue()); diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorImpliedURI.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorImpliedURI.java index b64cedb4..7d4812ca 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorImpliedURI.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorImpliedURI.java @@ -121,36 +121,34 @@ public class ExtractorImpliedURI extends Extractor { */ @Override public void extract(CrawlURI curi) { - List links = new ArrayList(curi.getOutLinks()); + List links = new ArrayList(curi.getOutLinks()); int max = links.size(); for (int i = 0; i < max; i++) { - Link link = links.get(i); + CrawlURI link = links.get(i); Pattern trigger = getRegex(); String build = getFormat(); - CharSequence dest = link.getDestination(); + CharSequence dest = link.getUURI(); String implied = extractImplied(dest, trigger, build); if (implied != null) { try { - UURI src = curi.getUURI(); UURI target = UURIFactory.getInstance(implied); LinkContext lc = LinkContext.INFERRED_MISC; Hop hop = Hop.INFERRED; - Link out = new Link(src, target, lc, hop); - curi.getOutLinks().add(out); + addOutlink(curi, target, lc, hop); numberOfLinksExtracted.incrementAndGet(); boolean removeTriggerURI = getRemoveTriggerUris(); // remove trigger URI from the outlinks if configured so. if (removeTriggerURI) { if (curi.getOutLinks().remove(link)) { - LOGGER.log(Level.FINE, link.getDestination() + + LOGGER.log(Level.FINE, link.getURI() + " has been removed from " + - link.getSource() + " outlinks list."); + curi.getURI() + " outlinks list."); numberOfLinksExtracted.decrementAndGet(); } else { LOGGER.log(Level.FINE, "Failed to remove " + - link.getDestination() + " from " + - link.getSource()+ " outlinks list."); + link.getURI() + " from " + + curi.getURI() + " outlinks list."); } } } catch (URIException e) { diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorJS.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorJS.java index cb4809b4..7f9af920 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorJS.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorJS.java @@ -171,11 +171,11 @@ public class ExtractorJS extends ContentExtractor { try { int max = ext.getExtractorParameters().getMaxOutlinks(); if (handlingJSFile) { - Link.addRelativeToVia(curi, max, candidate, JS_MISC, + addRelativeToVia(curi, max, candidate, JS_MISC, SPECULATIVE); return true; } else { - Link.addRelativeToBase(curi, max, candidate, JS_MISC, + addRelativeToBase(curi, max, candidate, JS_MISC, SPECULATIVE); return true; } diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorMultipleRegex.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorMultipleRegex.java index 30479f24..d669cd2f 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorMultipleRegex.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorMultipleRegex.java @@ -280,7 +280,7 @@ public class ExtractorMultipleRegex extends Extractor { String outlinkUri = groovyTemplate().make(bindings).toString(); try { - Link.addRelativeToBase(curi, + addRelativeToBase(curi, getExtractorParameters().getMaxOutlinks(), outlinkUri, HTMLLinkContext.INFERRED_MISC, Hop.INFERRED); } catch (URIException e) { diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorPDF.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorPDF.java index d15e2240..ccd840f3 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorPDF.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorPDF.java @@ -117,8 +117,7 @@ public class ExtractorPDF extends ContentExtractor { UURI dest = UURIFactory.getInstance(uri); LinkContext lc = LinkContext.NAVLINK_MISC; Hop hop = Hop.NAVLINK; - Link out = new Link(src, dest, lc, hop); - curi.getOutLinks().add(out); + addOutlink(curi, dest, lc, hop); } catch (URIException e1) { // There may not be a controller (e.g. If we're being run // by the extractor tool). diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorSWF.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorSWF.java index 70ab4a57..299d15ff 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorSWF.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorSWF.java @@ -334,7 +334,7 @@ public class ExtractorSWF extends ContentExtractor { } } else { int max = ext.getExtractorParameters().getMaxOutlinks(); - Link.addRelativeToVia(curi, max, url, LinkContext.EMBED_MISC, + addRelativeToVia(curi, max, url, LinkContext.EMBED_MISC, Hop.EMBED); linkCount++; } @@ -343,7 +343,7 @@ public class ExtractorSWF extends ContentExtractor { public void considerStringAsUri(String str) throws IOException { if (UriUtils.isLikelyUri(str)) { int max = ext.getExtractorParameters().getMaxOutlinks(); - Link.addRelativeToVia(curi, max, str, + addRelativeToVia(curi, max, str, LinkContext.SPECULATIVE_MISC, Hop.SPECULATIVE); linkCount++; } diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorURI.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorURI.java index b856f95b..3fcac20a 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorURI.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorURI.java @@ -77,11 +77,8 @@ public class ExtractorURI extends Extractor { */ @Override public void extract(CrawlURI curi) { - List links = new ArrayList(curi.getOutLinks()); - int max = links.size(); - for (int i = 0; i < max; i++) { - Link wref = links.get(i); - extractLink(curi, wref); + for (CrawlURI link : curi.getOutLinks()) { + extractLink(curi, link); } } @@ -91,10 +88,10 @@ public class ExtractorURI extends Extractor { * @param curi CrawlURI to add discoveries to * @param wref Link to examine for internal URIs */ - protected void extractLink(CrawlURI curi, Link wref) { + protected void extractLink(CrawlURI curi, CrawlURI wref) { UURI source = null; try { - source = UURIFactory.getInstance(wref.getDestination().toString()); + source = UURIFactory.getInstance(wref.getURI()); } catch (URIException e) { LOGGER.log(Level.FINE,"bad URI",e); } @@ -105,13 +102,11 @@ public class ExtractorURI extends Extractor { List found = extractQueryStringLinks(source); for (String uri : found) { try { - UURI src = curi.getUURI(); UURI dest = UURIFactory.getInstance(uri); LinkContext lc = LinkContext.SPECULATIVE_MISC; Hop hop = Hop.SPECULATIVE; - Link link = new Link(src, dest, lc, hop); + addOutlink(curi, dest, lc, hop); numberOfLinksExtracted.incrementAndGet(); - curi.getOutLinks().add(link); } catch (URIException e) { LOGGER.log(Level.FINE, "bad URI", e); } diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorUniversal.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorUniversal.java index 9b8cbba3..b0bbf7db 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorUniversal.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorUniversal.java @@ -405,12 +405,10 @@ public class ExtractorUniversal extends ContentExtractor { // And add the URL to speculative embeds. numberOfLinksExtracted.incrementAndGet(); - UURI src = curi.getUURI(); UURI dest = UURIFactory.getInstance(newURL); LinkContext lc = LinkContext.SPECULATIVE_MISC; Hop hop = Hop.SPECULATIVE; - Link link = new Link(src, dest, lc, hop); - curi.getOutLinks().add(link); + addOutlink(curi, dest, lc, hop); } // Reset lookat for next string. lookat = new StringBuffer(); diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorXML.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorXML.java index 903abfbd..9127716c 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorXML.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorXML.java @@ -159,7 +159,7 @@ public class ExtractorXML extends ContentExtractor { // intends to create a followable/fetchable URI is // unknown int max = ext.getExtractorParameters().getMaxOutlinks(); - Link.addRelativeToBase(curi, max, xmlUri, + addRelativeToBase(curi, max, xmlUri, LinkContext.SPECULATIVE_MISC, Hop.SPECULATIVE); } catch (URIException e) { // There may not be a controller (e.g. If we're being run diff --git a/modules/src/main/java/org/archive/modules/extractor/Hop.java b/modules/src/main/java/org/archive/modules/extractor/Hop.java index 23f4c688..4659ca9b 100644 --- a/modules/src/main/java/org/archive/modules/extractor/Hop.java +++ b/modules/src/main/java/org/archive/modules/extractor/Hop.java @@ -87,4 +87,18 @@ public enum Hop { public String getHopString() { return hopString; } + + /** + * Turns a character into the appropriate Hop enum. + * @param hopChar + * @return A Hop enum represented by the supplied char or null if char does not match any known hop type + */ + public static final Hop getForChar(char hopChar) { + for (Hop h : Hop.values()) { + if (h.hopChar==hopChar) { + return h; + } + } + return null; + } } diff --git a/modules/src/main/java/org/archive/modules/extractor/JerichoExtractorHTML.java b/modules/src/main/java/org/archive/modules/extractor/JerichoExtractorHTML.java index eb1f6e42..53770b3a 100644 --- a/modules/src/main/java/org/archive/modules/extractor/JerichoExtractorHTML.java +++ b/modules/src/main/java/org/archive/modules/extractor/JerichoExtractorHTML.java @@ -319,7 +319,7 @@ public class JerichoExtractorHTML extends ExtractorHTML { String refreshUri = content.substring(content.indexOf("=") + 1); try { int max = getExtractorParameters().getMaxOutlinks(); - Link.addRelativeToBase(curi, max, refreshUri, + addRelativeToBase(curi, max, refreshUri, HTMLLinkContext.META, Hop.REFER); } catch (URIException e) { logUriError(e, curi.getUURI(), refreshUri); diff --git a/modules/src/main/java/org/archive/modules/extractor/Link.java b/modules/src/main/java/org/archive/modules/extractor/Link.java deleted file mode 100644 index 26d66187..00000000 --- a/modules/src/main/java/org/archive/modules/extractor/Link.java +++ /dev/null @@ -1,222 +0,0 @@ -/* - * This file is part of the Heritrix web crawler (crawler.archive.org). - * - * Licensed to the Internet Archive (IA) by one or more individual - * contributors. - * - * The IA licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.archive.modules.extractor; - -import java.io.Serializable; -import java.util.HashMap; -import java.util.Map; -import java.util.logging.Logger; - -import org.apache.commons.httpclient.URIException; -import org.archive.modules.CoreAttributeConstants; -import org.archive.modules.CrawlURI; -import org.archive.net.UURI; -import org.archive.net.UURIFactory; - - -/** - * Link represents one discovered "edge" of the web graph: the source - * URI, the destination URI, and the type of reference (represented by the - * context in which it was found). - * - * As such, it is a suitably generic item to returned from generic - * link-extraction utility code. - * - * @author gojomo - */ -public class Link implements Serializable, Comparable { - private static final Logger LOGGER = Logger.getLogger(Link.class.getName()); - - private static final long serialVersionUID = 2L; - - - /** URI where this Link was discovered */ - private CharSequence source; - - /** URI (absolute) where this Link points */ - private CharSequence destination; - - /** context of discovery -- will be an XPath-like element[/@attribute] - * fragment for HTML URIs, a header name with trailing ':' for header - * values, or one of the stand-in constants when other context is - * unavailable */ - private LinkContext context; - - /** hop-type */ - private Hop hop; - - /** - * Flexible dynamic attributes list. - *

- * See further {@link Link#getData()} - */ - protected Map data; - - - /** - * Create a Link with the given fields. - * @param source - * @param destination - * @param context - * @param hopType - */ - public Link(CharSequence source, CharSequence destination, - LinkContext context, Hop hop) { - super(); - this.source = source; - this.destination = destination; - this.context = context; - this.hop = hop; - } - - - /** - * @return Returns the context. - */ - public LinkContext getContext() { - return context; - } - /** - * @return Returns the destination. - */ - public CharSequence getDestination() { - return destination; - } - /** - * @return Returns the source. - */ - public CharSequence getSource() { - return source; - } - - /** - * @return char hopType - */ - public Hop getHopType() { - return hop; - } - - - @Override - public String toString() { - return this.destination + " " + hop.getHopChar() + " " + this.context; - } - /** - * Attribute list - *

- * By convention the attribute list is keyed by constants found in the - * {@link CoreAttributeConstants} interface. Use this list to carry - * data or state produced by custom processors rather change the - * classes {@link CrawlURI} or this class, CrawlURI. - *

- * This list becomes {@link CrawlURI#getData()} when the Link is promoted to CrawlURI via - * {@link CrawlURI#createCrawlURI(UURI, Link)} - *

- * If the list is null when this method is invoked, a new instance will be created and returned. - * - * @returns a flexible map of key/value pairs for storing - * status of this URI for use by other processors. - */ - public Map getData() { - if (data == null) { - data = new HashMap(); - } - return data; - } - - /** - * @return true if data map is not null - */ - public boolean hasData() { - return data != null; - } - - @Override - public boolean equals(Object o) { - if (!(o instanceof Link)) { - return false; - } - Link l = (Link)o; - return l.source.equals(source) && l.destination.equals(destination) - && l.context.equals(context) && l.hop.equals(hop); - } - - - @Override - public int hashCode() { - int r = 37; - return r ^ source.hashCode() ^ destination.hashCode() - ^ context.hashCode() ^ hop.hashCode(); - } - - public static void addRelativeToBase(CrawlURI uri, int max, - String newUri, LinkContext context, Hop hop) throws URIException { - UURI dest = UURIFactory.getInstance(uri.getBaseURI(), newUri); - add2(uri, max, dest, context, hop); - } - - - public static void addRelativeToVia(CrawlURI uri, int max, String newUri, - LinkContext context, Hop hop) throws URIException { - UURI relTo = uri.getVia(); - if (relTo == null) { - if (!uri.getAnnotations().contains("usedBaseForVia")) { - LOGGER.info("no via where expected; using base instead: " + uri); - uri.getAnnotations().add("usedBaseForVia"); - } - relTo = uri.getBaseURI(); - } - UURI dest = UURIFactory.getInstance(relTo, newUri); - add2(uri, max, dest, context, hop); - } - - public static void add(CrawlURI uri, int max, String newUri, - LinkContext context, Hop hop) throws URIException { - UURI dest = UURIFactory.getInstance(newUri); - add2(uri, max, dest, context, hop); - } - - - private static void add2(CrawlURI uri, int max, UURI dest, - LinkContext context, Hop hop) throws URIException { - if (uri.getOutLinks().size() < max) { - UURI src = uri.getUURI(); - Link link = new Link(src, dest, context, hop); - uri.getOutLinks().add(link); -// return link; - } else { - uri.incrementDiscardedOutLinks(); - } - } - - public int compareTo(Link o) { - int cmp = source.toString().compareTo(o.source.toString()); - if (cmp==0) { - cmp = destination.toString().compareTo(o.destination.toString()); - } - if (cmp==0) { - cmp = context.toString().compareTo(o.context.toString()); - } - if (cmp==0) { - cmp = hop.toString().compareTo(o.hop.toString()); - } - return cmp; - } - -} diff --git a/modules/src/main/java/org/archive/modules/extractor/StringExtractorTestBase.java b/modules/src/main/java/org/archive/modules/extractor/StringExtractorTestBase.java index 6a97176f..3c87c2b7 100644 --- a/modules/src/main/java/org/archive/modules/extractor/StringExtractorTestBase.java +++ b/modules/src/main/java/org/archive/modules/extractor/StringExtractorTestBase.java @@ -29,9 +29,9 @@ public abstract class StringExtractorTestBase extends ContentExtractorTestBase { public static class TestData { public CrawlURI uri; - public Link expectedResult; + public CrawlURI expectedResult; - public TestData(CrawlURI uri, Link expectedResult) { + public TestData(CrawlURI uri, CrawlURI expectedResult) { this.uri = uri; this.expectedResult = expectedResult; } @@ -80,7 +80,7 @@ public abstract class StringExtractorTestBase extends ContentExtractorTestBase { Collection testDataCol = makeData(text, expectedURL); for (TestData testData: testDataCol) { extractor.process(testData.uri); - HashSet expected = new HashSet(); + HashSet expected = new HashSet(); if (testData.expectedResult != null) { expected.add(testData.expectedResult); } diff --git a/modules/src/main/java/org/archive/modules/fetcher/FetchFTP.java b/modules/src/main/java/org/archive/modules/fetcher/FetchFTP.java index 0cc1742d..b89ff729 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/FetchFTP.java +++ b/modules/src/main/java/org/archive/modules/fetcher/FetchFTP.java @@ -44,7 +44,6 @@ import org.archive.io.ReplayCharSequence; import org.archive.modules.CrawlURI; import org.archive.modules.Processor; import org.archive.modules.extractor.Hop; -import org.archive.modules.extractor.Link; import org.archive.modules.extractor.LinkContext; import org.archive.net.ClientFTP; import org.archive.net.UURI; @@ -573,7 +572,7 @@ public class FetchFTP extends Processor { } try { UURI n = UURIFactory.getInstance(base + "/" + file); - Link link = new Link(curi.getUURI(), n, LinkContext.NAVLINK_MISC, Hop.NAVLINK); + CrawlURI link = curi.createCrawlURI(n, LinkContext.NAVLINK_MISC, Hop.NAVLINK); curi.getOutLinks().add(link); } catch (URIException e) { logger.log(Level.WARNING, "URI error during extraction.", e); @@ -610,7 +609,7 @@ public class FetchFTP extends Processor { String path = uuri.getEscapedCurrentHierPath(); UURI parent = UURIFactory.getInstance(scheme + "://" + auth + path); - Link link = new Link(uuri, parent, LinkContext.NAVLINK_MISC, + CrawlURI link = curi.createCrawlURI(parent, LinkContext.NAVLINK_MISC, Hop.NAVLINK); curi.getOutLinks().add(link); } catch (URIException e) { diff --git a/modules/src/main/java/org/archive/modules/fetcher/FetchWhois.java b/modules/src/main/java/org/archive/modules/fetcher/FetchWhois.java index 77c89889..6cf9fb18 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/FetchWhois.java +++ b/modules/src/main/java/org/archive/modules/fetcher/FetchWhois.java @@ -37,8 +37,8 @@ import org.archive.modules.CoreAttributeConstants; import org.archive.modules.CrawlURI; import org.archive.modules.ProcessResult; import org.archive.modules.Processor; +import org.archive.modules.extractor.Extractor; import org.archive.modules.extractor.Hop; -import org.archive.modules.extractor.Link; import org.archive.modules.extractor.LinkContext; import org.archive.modules.net.CrawlHost; import org.archive.modules.net.ServerCache; @@ -423,7 +423,7 @@ public class FetchWhois extends Processor implements CoreAttributeConstants, protected void addWhoisLink(CrawlURI curi, String query) { String whoisUrl = "whois:" + query; try { - Link.add(curi, Integer.MAX_VALUE, whoisUrl, LinkContext.INFERRED_MISC, Hop.INFERRED); + Extractor.add(curi, Integer.MAX_VALUE, whoisUrl, LinkContext.INFERRED_MISC, Hop.INFERRED); } catch (URIException e) { logger.log(Level.WARNING, "problem with url " + whoisUrl, e); } diff --git a/modules/src/main/java/org/archive/modules/forms/FormLoginProcessor.java b/modules/src/main/java/org/archive/modules/forms/FormLoginProcessor.java index e9f1be04..d6f47b9f 100644 --- a/modules/src/main/java/org/archive/modules/forms/FormLoginProcessor.java +++ b/modules/src/main/java/org/archive/modules/forms/FormLoginProcessor.java @@ -254,7 +254,7 @@ public class FormLoginProcessor extends Processor implements Checkpointable { protected void createFormSubmissionAttempt(CrawlURI curi, HTMLForm templateForm, String formProvince) { LinkContext lc = new LinkContext.SimpleLinkContext("form/@action"); try { - CrawlURI submitCuri = curi.makeConsequentCandidate(templateForm.getAction(),lc, Hop.SUBMIT); + CrawlURI submitCuri = curi.createCrawlURI(templateForm.getAction(), lc, Hop.SUBMIT); submitCuri.setFetchType(FetchType.HTTP_POST); submitCuri.getData().put( CoreAttributeConstants.A_SUBMIT_DATA, @@ -264,7 +264,7 @@ public class FormLoginProcessor extends Processor implements Checkpointable { //submitCuri.setSchedulingDirective(Math.max(curi.getSchedulingDirective()-1, 0)); submitCuri.setSchedulingDirective(SchedulingConstants.HIGH); submitCuri.setForceFetch(true); - curi.getOutCandidates().add(submitCuri); + curi.getOutLinks().add(submitCuri); curi.getAnnotations().add("submit:"+templateForm.getAction()); } catch (URIException ue) { loggerModule.logUriError(ue,curi.getUURI(),templateForm.getAction()); diff --git a/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java b/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java index c1aaa1ed..83621f7d 100644 --- a/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java +++ b/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java @@ -19,36 +19,36 @@ package org.archive.modules.writer; -import static org.archive.io.warc.WARCConstants.FTP_CONTROL_CONVERSATION_MIMETYPE; -import static org.archive.io.warc.WARCConstants.HEADER_KEY_CONCURRENT_TO; -import static org.archive.io.warc.WARCConstants.HEADER_KEY_ETAG; -import static org.archive.io.warc.WARCConstants.HEADER_KEY_IP; -import static org.archive.io.warc.WARCConstants.HEADER_KEY_LAST_MODIFIED; -import static org.archive.io.warc.WARCConstants.HEADER_KEY_PAYLOAD_DIGEST; -import static org.archive.io.warc.WARCConstants.HEADER_KEY_PROFILE; -import static org.archive.io.warc.WARCConstants.HEADER_KEY_REFERS_TO; -import static org.archive.io.warc.WARCConstants.HEADER_KEY_REFERS_TO_DATE; -import static org.archive.io.warc.WARCConstants.HEADER_KEY_REFERS_TO_FILENAME; -import static org.archive.io.warc.WARCConstants.HEADER_KEY_REFERS_TO_FILE_OFFSET; -import static org.archive.io.warc.WARCConstants.HEADER_KEY_REFERS_TO_TARGET_URI; -import static org.archive.io.warc.WARCConstants.HEADER_KEY_TRUNCATED; -import static org.archive.io.warc.WARCConstants.HTTP_REQUEST_MIMETYPE; -import static org.archive.io.warc.WARCConstants.HTTP_RESPONSE_MIMETYPE; -import static org.archive.io.warc.WARCConstants.NAMED_FIELD_TRUNCATED_VALUE_HEAD; -import static org.archive.io.warc.WARCConstants.NAMED_FIELD_TRUNCATED_VALUE_LENGTH; -import static org.archive.io.warc.WARCConstants.NAMED_FIELD_TRUNCATED_VALUE_TIME; -import static org.archive.io.warc.WARCConstants.PROFILE_REVISIT_IDENTICAL_DIGEST; -import static org.archive.io.warc.WARCConstants.PROFILE_REVISIT_NOT_MODIFIED; -import static org.archive.io.warc.WARCConstants.PROFILE_REVISIT_URI_AGNOSTIC_IDENTICAL_DIGEST; -import static org.archive.io.warc.WARCConstants.TYPE; +import static org.archive.format.warc.WARCConstants.FTP_CONTROL_CONVERSATION_MIMETYPE; +import static org.archive.format.warc.WARCConstants.HEADER_KEY_CONCURRENT_TO; +import static org.archive.format.warc.WARCConstants.HEADER_KEY_ETAG; +import static org.archive.format.warc.WARCConstants.HEADER_KEY_IP; +import static org.archive.format.warc.WARCConstants.HEADER_KEY_LAST_MODIFIED; +import static org.archive.format.warc.WARCConstants.HEADER_KEY_PAYLOAD_DIGEST; +import static org.archive.format.warc.WARCConstants.HEADER_KEY_PROFILE; +import static org.archive.format.warc.WARCConstants.HEADER_KEY_REFERS_TO; +import static org.archive.format.warc.WARCConstants.HEADER_KEY_REFERS_TO_DATE; +import static org.archive.format.warc.WARCConstants.HEADER_KEY_REFERS_TO_FILENAME; +import static org.archive.format.warc.WARCConstants.HEADER_KEY_REFERS_TO_FILE_OFFSET; +import static org.archive.format.warc.WARCConstants.HEADER_KEY_REFERS_TO_TARGET_URI; +import static org.archive.format.warc.WARCConstants.HEADER_KEY_TRUNCATED; +import static org.archive.format.warc.WARCConstants.HTTP_REQUEST_MIMETYPE; +import static org.archive.format.warc.WARCConstants.HTTP_RESPONSE_MIMETYPE; +import static org.archive.format.warc.WARCConstants.NAMED_FIELD_TRUNCATED_VALUE_HEAD; +import static org.archive.format.warc.WARCConstants.NAMED_FIELD_TRUNCATED_VALUE_LENGTH; +import static org.archive.format.warc.WARCConstants.NAMED_FIELD_TRUNCATED_VALUE_TIME; +import static org.archive.format.warc.WARCConstants.PROFILE_REVISIT_IDENTICAL_DIGEST; +import static org.archive.format.warc.WARCConstants.PROFILE_REVISIT_NOT_MODIFIED; +import static org.archive.format.warc.WARCConstants.PROFILE_REVISIT_URI_AGNOSTIC_IDENTICAL_DIGEST; +import static org.archive.format.warc.WARCConstants.TYPE; import static org.archive.modules.CoreAttributeConstants.A_DNS_SERVER_IP_LABEL; import static org.archive.modules.CoreAttributeConstants.A_FTP_CONTROL_CONVERSATION; import static org.archive.modules.CoreAttributeConstants.A_FTP_FETCH_STATUS; import static org.archive.modules.CoreAttributeConstants.A_SOURCE_TAG; +import static org.archive.modules.CoreAttributeConstants.A_WARC_RESPONSE_HEADERS; import static org.archive.modules.CoreAttributeConstants.HEADER_TRUNC; import static org.archive.modules.CoreAttributeConstants.LENGTH_TRUNC; import static org.archive.modules.CoreAttributeConstants.TIMER_TRUNC; -import static org.archive.modules.CoreAttributeConstants.A_WARC_RESPONSE_HEADERS; import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_CONTENT_DIGEST_COUNT; import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_ETAG_HEADER; import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_FETCH_HISTORY; @@ -83,8 +83,8 @@ import org.apache.commons.httpclient.HttpMethod; import org.apache.commons.httpclient.HttpStatus; import org.apache.commons.io.IOUtils; import org.apache.commons.lang.StringUtils; -import org.archive.io.ReplayInputStream; import org.archive.format.warc.WARCConstants.WARCRecordType; +import org.archive.io.ReplayInputStream; import org.archive.io.warc.WARCRecordInfo; import org.archive.io.warc.WARCWriter; import org.archive.io.warc.WARCWriterPool; @@ -94,7 +94,6 @@ import org.archive.modules.CrawlMetadata; import org.archive.modules.CrawlURI; import org.archive.modules.ProcessResult; import org.archive.modules.deciderules.recrawl.IdenticalDigestDecideRule; -import org.archive.modules.extractor.Link; import org.archive.spring.ConfigPath; import org.archive.uid.RecordIDGenerator; import org.archive.uid.UUIDGenerator; @@ -862,10 +861,10 @@ public class WARCWriterProcessor extends WriterPoolProcessor implements WARCWrit } // Add outlinks though they are effectively useless without anchor text. - Collection links = curi.getOutLinks(); + Collection links = curi.getOutLinks(); if (links != null && links.size() > 0) { - for (Link link: links) { - r.addLabelValue("outlink", link.toString()); + for (CrawlURI link: links) { + r.addLabelValue("outlink", link.getURI()); } } diff --git a/modules/src/test/java/org/archive/modules/extractor/ExtractorCSSTest.java b/modules/src/test/java/org/archive/modules/extractor/ExtractorCSSTest.java index f6775014..faa9c257 100644 --- a/modules/src/test/java/org/archive/modules/extractor/ExtractorCSSTest.java +++ b/modules/src/test/java/org/archive/modules/extractor/ExtractorCSSTest.java @@ -19,22 +19,17 @@ package org.archive.modules.extractor; +import static org.archive.modules.extractor.LinkContext.EMBED_MISC; +import static org.archive.modules.extractor.LinkContext.NAVLINK_MISC; + import java.util.Collection; import java.util.Collections; import org.archive.modules.CrawlURI; -import org.archive.modules.extractor.Extractor; -import org.archive.modules.extractor.ExtractorCSS; -import org.archive.modules.extractor.Hop; -import org.archive.modules.extractor.Link; -import org.archive.modules.extractor.StringExtractorTestBase; import org.archive.net.UURI; import org.archive.net.UURIFactory; import org.archive.util.Recorder; -import static org.archive.modules.extractor.LinkContext.EMBED_MISC; -import static org.archive.modules.extractor.LinkContext.NAVLINK_MISC; - /** * Unit test for ExtractorCSS. @@ -81,8 +76,9 @@ public class ExtractorCSSTest extends StringExtractorTestBase { euri.setRecorder(recorder); euri.setContentSize(content.length()); + // TODO: This test was naively modified to account for the abscense of LINK, but no effort was made to confirm that it is actually testing anything useful UURI dest = UURIFactory.getInstance(uri); - Link link = new Link(src, dest, EMBED_MISC, Hop.EMBED); + CrawlURI link = euri.createCrawlURI(dest, EMBED_MISC, Hop.EMBED); TestData td = new TestData(euri, link); return Collections.singleton(td); } diff --git a/modules/src/test/java/org/archive/modules/extractor/ExtractorHTMLTest.java b/modules/src/test/java/org/archive/modules/extractor/ExtractorHTMLTest.java index 7e5a21cf..4e2abbbd 100644 --- a/modules/src/test/java/org/archive/modules/extractor/ExtractorHTMLTest.java +++ b/modules/src/test/java/org/archive/modules/extractor/ExtractorHTMLTest.java @@ -107,7 +107,7 @@ public class ExtractorHTMLTest extends StringExtractorTestBase { UURI dest = UURIFactory.getInstance(destURI); LinkContext context = determineContext(content); Hop hop = determineHop(content); - Link link = new Link(src, dest, context, hop); + CrawlURI link = euri.createCrawlURI(dest, context, hop); result.add(new TestData(euri, link)); euri = new CrawlURI(src, null, null, LinkContext.NAVLINK_MISC); @@ -166,10 +166,10 @@ public class ExtractorHTMLTest extends StringExtractorTestBase { CrawlURI puri = new CrawlURI(UURIFactory .getInstance("http://www.example.com")); getExtractor().extract(puri, source); - Link[] links = puri.getOutLinks().toArray(new Link[0]); + CrawlURI[] links = puri.getOutLinks().toArray(new CrawlURI[0]); assertTrue("did not find single link",links.length==1); assertTrue("expected link not found", - links[0].getDestination().toString().equals(expected)); + links[0].getURI().equals(expected)); } /** @@ -188,9 +188,8 @@ public class ExtractorHTMLTest extends StringExtractorTestBase { "

"+ " "; getExtractor().extract(puri, cs); - Link[] links = puri.getOutLinks().toArray(new Link[0]); // find exactly 3 (not the POST) action URIs - assertTrue("incorrect number of links found",links.length==3); + assertTrue("incorrect number of links found", puri.getOutLinks().size()==3); } /** @@ -207,7 +206,7 @@ public class ExtractorHTMLTest extends StringExtractorTestBase { getExtractor().extract(puri, cs); assertEquals("meta robots content not extracted","index,nofollow", puri.getData().get(ExtractorHTML.A_META_ROBOTS)); - Link[] links = puri.getOutLinks().toArray(new Link[0]); + CrawlURI[] links = puri.getOutLinks().toArray(new CrawlURI[0]); assertTrue("link extracted despite meta robots",links.length==0); } @@ -228,9 +227,8 @@ public class ExtractorHTMLTest extends StringExtractorTestBase { assertTrue(CollectionUtils.exists(curi.getOutLinks(), new Predicate() { public boolean evaluate(Object object) { - return ((Link) object) - .getDestination() - .toString() + return ((CrawlURI) object) + .getURI() .indexOf( "/example.html;jsessionid=deadbeef:deadbeed?parameter=this:value") >= 0; } @@ -238,7 +236,7 @@ public class ExtractorHTMLTest extends StringExtractorTestBase { assertTrue(CollectionUtils.exists(curi.getOutLinks(), new Predicate() { public boolean evaluate(Object object) { - return ((Link) object).getDestination().toString().indexOf( + return ((CrawlURI) object).getURI().indexOf( "/example.html?parameter=this:value") >= 0; } })); @@ -262,15 +260,15 @@ public class ExtractorHTMLTest extends StringExtractorTestBase { assertTrue(CollectionUtils.exists(curi.getOutLinks(), new Predicate() { public boolean evaluate(Object object) { System.err.println("comparing: " - + ((Link) object).getDestination().toString() + + ((CrawlURI) object).getURI() + " and https://www.anotherexample.com/"); - return ((Link) object).getDestination().toString().equals( + return ((CrawlURI) object).getURI().equals( "http://www.anotherexample.com/"); } })); assertTrue(CollectionUtils.exists(curi.getOutLinks(), new Predicate() { public boolean evaluate(Object object) { - return ((Link) object).getDestination().toString().equals( + return ((CrawlURI) object).getURI().equals( "https://www.example.com/index.html"); } })); @@ -310,21 +308,21 @@ public class ExtractorHTMLTest extends StringExtractorTestBase { "" + ""; getExtractor().extract(puri, cs); - Link[] links = puri.getOutLinks().toArray(new Link[0]); + CrawlURI[] links = puri.getOutLinks().toArray(new CrawlURI[0]); Arrays.sort(links); String dest1 = "http://www.example.com/def/another1.html"; String dest2 = "http://www.example.com/ghi/another2.html"; // ensure outlink from base href assertEquals("outlink1 from base href",dest1, - links[1].getDestination().toString()); + links[1].getURI()); assertEquals("outlink2 from base href",dest2, - links[2].getDestination().toString()); + links[2].getURI()); } protected Predicate destinationContainsPredicate(final String fragment) { return new Predicate() { public boolean evaluate(Object object) { - return ((Link) object).getDestination().toString().indexOf(fragment) >= 0; + return ((CrawlURI) object).getURI().indexOf(fragment) >= 0; } }; } @@ -332,7 +330,7 @@ public class ExtractorHTMLTest extends StringExtractorTestBase { protected Predicate destinationsIsPredicate(final String value) { return new Predicate() { public boolean evaluate(Object object) { - return ((Link) object).getDestination().toString().equals(value); + return ((CrawlURI) object).getURI().equals(value); } }; } @@ -397,16 +395,16 @@ public class ExtractorHTMLTest extends StringExtractorTestBase { getExtractor().extract(curi, cs); - Link[] links = curi.getOutLinks().toArray(new Link[0]); + CrawlURI[] links = curi.getOutLinks().toArray(new CrawlURI[0]); Arrays.sort(links); String dest1 = "http://www.example.com/foo.gif"; String dest2 = "http://www.example.com/foo.js"; assertEquals("outlink1 from conditional comment img src",dest1, - links[0].getDestination().toString()); + links[0].getURI()); assertEquals("outlink2 from conditional comment script src",dest2, - links[1].getDestination().toString()); + links[1].getURI()); } diff --git a/modules/src/test/java/org/archive/modules/extractor/ExtractorJSTest.java b/modules/src/test/java/org/archive/modules/extractor/ExtractorJSTest.java index af0c3efd..c5baafb0 100644 --- a/modules/src/test/java/org/archive/modules/extractor/ExtractorJSTest.java +++ b/modules/src/test/java/org/archive/modules/extractor/ExtractorJSTest.java @@ -141,7 +141,7 @@ public class ExtractorJSTest extends StringExtractorTestBase { if (destURI != null) { UURI dest = UURIFactory.getInstance(destURI); - Link link = new Link(src, dest, LinkContext.JS_MISC, Hop.SPECULATIVE); + CrawlURI link = euri.createCrawlURI(dest, LinkContext.JS_MISC, Hop.SPECULATIVE); result.add(new TestData(euri, link)); } else { result.add(new TestData(euri, null)); diff --git a/modules/src/test/java/org/archive/modules/extractor/ExtractorMultipleRegexTest.java b/modules/src/test/java/org/archive/modules/extractor/ExtractorMultipleRegexTest.java index 308379d7..e65b4e27 100644 --- a/modules/src/test/java/org/archive/modules/extractor/ExtractorMultipleRegexTest.java +++ b/modules/src/test/java/org/archive/modules/extractor/ExtractorMultipleRegexTest.java @@ -188,7 +188,7 @@ public class ExtractorMultipleRegexTest extends ContentExtractorTestBase { extractor.process(testUri); for (String expectedLinkString: EXPECTED_OUTLINKS) { - Link expectedLink = new Link(testUri.getUURI(), + CrawlURI expectedLink = testUri.createCrawlURI( UURIFactory.getInstance(expectedLinkString), HTMLLinkContext.INFERRED_MISC, Hop.INFERRED); assertTrue(testUri.getOutLinks().contains(expectedLink)); diff --git a/modules/src/test/java/org/archive/modules/extractor/ExtractorSWFTest.java b/modules/src/test/java/org/archive/modules/extractor/ExtractorSWFTest.java index 5167d8bb..84f5ee0e 100644 --- a/modules/src/test/java/org/archive/modules/extractor/ExtractorSWFTest.java +++ b/modules/src/test/java/org/archive/modules/extractor/ExtractorSWFTest.java @@ -109,9 +109,9 @@ public class ExtractorSWFTest extends ContentExtractorTestBase { + elapsed + "ms to process " + url); boolean foundIt = false; - for (Link link : curi.getOutLinks()) { + for (CrawlURI link : curi.getOutLinks()) { logger.info("found link: " + link); - foundIt = foundIt || link.getDestination().toString().endsWith(testUrls.get(url)); + foundIt = foundIt || link.getURI().endsWith(testUrls.get(url)); } assertTrue("failed to extract link \"" + testUrls.get(url) @@ -165,9 +165,9 @@ public class ExtractorSWFTest extends ContentExtractorTestBase { + elapsed + "ms to process " + url); boolean foundIt = false; - for (Link link : curi.getOutLinks()) { + for (CrawlURI link : curi.getOutLinks()) { logger.info("found link: " + link); - foundIt = foundIt || link.getDestination().toString().endsWith(testUrls.get(url)); + foundIt = foundIt || link.getURI().endsWith(testUrls.get(url)); } if (!foundIt) diff --git a/modules/src/test/java/org/archive/modules/extractor/JerichoExtractorHTMLTest.java b/modules/src/test/java/org/archive/modules/extractor/JerichoExtractorHTMLTest.java index a4d17345..c6bcee6b 100644 --- a/modules/src/test/java/org/archive/modules/extractor/JerichoExtractorHTMLTest.java +++ b/modules/src/test/java/org/archive/modules/extractor/JerichoExtractorHTMLTest.java @@ -73,7 +73,7 @@ public class JerichoExtractorHTMLTest extends ExtractorHTMLTest { curi.getOutLinks(); assertTrue(CollectionUtils.exists(curi.getOutLinks(), new Predicate() { public boolean evaluate(Object object) { - return ((Link) object).getDestination().toString().indexOf( + return ((CrawlURI) object).getURI().indexOf( "/redirect_me?form=true&checked[]=1&unchecked[]=&selectBox=selectedOption&test=Go")>=0; } })); @@ -101,7 +101,7 @@ public class JerichoExtractorHTMLTest extends ExtractorHTMLTest { curi.getOutLinks(); assertTrue(!CollectionUtils.exists(curi.getOutLinks(), new Predicate() { public boolean evaluate(Object object) { - return ((Link) object).getDestination().toString().indexOf( + return ((CrawlURI) object).getURI().indexOf( "/redirect_me?form=true&checked[]=1&unchecked[]=&selectBox=selectedOption&test=Go")>=0; } })); @@ -130,7 +130,7 @@ public class JerichoExtractorHTMLTest extends ExtractorHTMLTest { curi.getOutLinks(); assertTrue(CollectionUtils.exists(curi.getOutLinks(), new Predicate() { public boolean evaluate(Object object) { - return ((Link) object).getDestination().toString().indexOf( + return ((CrawlURI) object).getURI().indexOf( "/redirect_me?form=true&checked[]=1&unchecked[]=&selectBox=selectedOption&test=Go")>=0; } }));