diff --git a/contrib/src/main/java/org/archive/modules/extractor/ExtractorPDFContent.java b/contrib/src/main/java/org/archive/modules/extractor/ExtractorPDFContent.java index 082517ca..447b8bd2 100644 --- a/contrib/src/main/java/org/archive/modules/extractor/ExtractorPDFContent.java +++ b/contrib/src/main/java/org/archive/modules/extractor/ExtractorPDFContent.java @@ -27,8 +27,6 @@ import java.util.regex.Pattern; import org.apache.commons.httpclient.URIException; import org.archive.modules.CrawlURI; -import org.archive.net.UURI; -import org.archive.net.UURIFactory; import com.itextpdf.text.pdf.PdfReader; import com.itextpdf.text.pdf.parser.PdfReaderContentParser; @@ -140,11 +138,9 @@ public class ExtractorPDFContent extends ContentExtractor { for (String uri: uris) { try { - UURI src = curi.getUURI(); - UURI dest = UURIFactory.getInstance(uri); LinkContext lc = LinkContext.NAVLINK_MISC; Hop hop = Hop.NAVLINK; - Link out = new Link(src, dest, lc, hop); + CrawlURI out = curi.createCrawlURI(uri, lc, hop); curi.getOutLinks().add(out); } catch (URIException e1) { logUriError(e1, curi.getUURI(), uri); diff --git a/contrib/src/test/java/org/archive/modules/extractor/ExtractorPDFContentTest.java b/contrib/src/test/java/org/archive/modules/extractor/ExtractorPDFContentTest.java index 32701bfa..3bcca2a1 100644 --- a/contrib/src/test/java/org/archive/modules/extractor/ExtractorPDFContentTest.java +++ b/contrib/src/test/java/org/archive/modules/extractor/ExtractorPDFContentTest.java @@ -43,14 +43,14 @@ public class ExtractorPDFContentTest extends ContentExtractorTestBase { CrawlURI testUri = createTestUri("http://www.example.com/fake.pdf", TEST_RESOURCE_FILE_1); extractor.process(testUri); - Set expected = makeLinkSet(testUri, new String[]{"http://www.businessdictionary.com/definition/supervisor.html","http://management.about.com/od/policiesandprocedures/g/supervisor1.html"}); + Set expected = makeLinkSet(testUri, new String[]{"http://www.businessdictionary.com/definition/supervisor.html","http://management.about.com/od/policiesandprocedures/g/supervisor1.html"}); assertTrue(testUri.getOutLinks().containsAll(expected)); } public void testEndingInDot() throws URIException, UnsupportedEncodingException, IOException, InterruptedException{ CrawlURI testUri = createTestUri("http://www.example.com/fake.pdf", TEST_RESOURCE_FILE_2); extractor.process(testUri); - Set expected = makeLinkSet(testUri, new String[]{"http://www.fec.gov/data/CommitteeSummary.do", + Set expected = makeLinkSet(testUri, new String[]{"http://www.fec.gov/data/CommitteeSummary.do", "http://www.opensecrets.org/bigpicture/elec_stats.php", "http://www.opensecrets.org/pacs"}); assertTrue(testUri.getOutLinks().containsAll(expected)); @@ -59,21 +59,21 @@ public class ExtractorPDFContentTest extends ContentExtractorTestBase { CrawlURI testUri = createTestUri("http://www.example.com/fake.pdf", TEST_RESOURCE_FILE_3); extractor.process(testUri); - Set expected = makeLinkSet(testUri, new String[]{"http://www.dot.gov/sites/dot.dev/files/docs/2014_February_ATCR.pdf"}); + Set expected = makeLinkSet(testUri, new String[]{"http://www.dot.gov/sites/dot.dev/files/docs/2014_February_ATCR.pdf"}); assertTrue(testUri.getOutLinks().containsAll(expected)); } public void testParenthesis() throws URIException, UnsupportedEncodingException, IOException, InterruptedException{ CrawlURI testUri = createTestUri("http://www.example.com/fake.pdf", TEST_RESOURCE_FILE_4); extractor.process(testUri); - Set expected = makeLinkSet(testUri, new String[]{"http://www.unisys.com","http://www.myserver.mycorp.com/images/exttest.jpg","http://www.adobe.com/intro?100,200","http://www.w3.org/1999/xhtml","http://www.xfa.org/schema/xfa-data/1.0","http://www.adobe.com","http://www.adobe.com/getacro.gif","http://www.example.com/testOpeningParen"}); + Set expected = makeLinkSet(testUri, new String[]{"http://www.unisys.com","http://www.myserver.mycorp.com/images/exttest.jpg","http://www.adobe.com/intro?100,200","http://www.w3.org/1999/xhtml","http://www.xfa.org/schema/xfa-data/1.0","http://www.adobe.com","http://www.adobe.com/getacro.gif","http://www.example.com/testOpeningParen"}); assertTrue(testUri.getOutLinks().containsAll(expected)); } public void testNewlineSeparatedURIs() throws URIException, UnsupportedEncodingException, IOException, InterruptedException{ CrawlURI testUri = createTestUri("http://www.example.com/fake.pdf", TEST_RESOURCE_FILE_4); extractor.process(testUri); - Set expected = makeLinkSet(testUri, new String[]{"http://www.unisys.com","http://www.myserver.mycorp.com/images/exttest.jpg","http://www.example.com/test","http://www.adobe.com/intro?100,200","http://www.w3.org/1999/xhtml","http://www.xfa.org/schema/xfa-data/1.0","http://www.adobe.com","http://www.adobe.com/getacro.gif"}); + Set expected = makeLinkSet(testUri, new String[]{"http://www.unisys.com","http://www.myserver.mycorp.com/images/exttest.jpg","http://www.example.com/test","http://www.adobe.com/intro?100,200","http://www.w3.org/1999/xhtml","http://www.xfa.org/schema/xfa-data/1.0","http://www.adobe.com","http://www.adobe.com/getacro.gif"}); assertTrue(testUri.getOutLinks().containsAll(expected)); } @@ -86,13 +86,11 @@ public class ExtractorPDFContentTest extends ContentExtractorTestBase { result.setLoggerModule(ulm); return (Extractor)result; } - private Set makeLinkSet(CrawlURI sourceUri, String[] urlStrs) throws URIException { - HashSet linkSet = new HashSet(); + private Set makeLinkSet(CrawlURI sourceUri, String[] urlStrs) throws URIException { + HashSet linkSet = new HashSet(); for (String urlStr : urlStrs) { - linkSet.add(new Link(sourceUri.getUURI(), - UURIFactory.getInstance(urlStr), - HTMLLinkContext.NAVLINK_MISC, Hop.NAVLINK) - ); + CrawlURI link = sourceUri.createCrawlURI(urlStr, HTMLLinkContext.NAVLINK_MISC, Hop.NAVLINK); + linkSet.add(link); } return linkSet; } diff --git a/contrib/src/test/java/org/archive/modules/extractor/ExtractorYoutubeFormatStreamTest.java b/contrib/src/test/java/org/archive/modules/extractor/ExtractorYoutubeFormatStreamTest.java index 6d09d03e..c71137c1 100644 --- a/contrib/src/test/java/org/archive/modules/extractor/ExtractorYoutubeFormatStreamTest.java +++ b/contrib/src/test/java/org/archive/modules/extractor/ExtractorYoutubeFormatStreamTest.java @@ -95,7 +95,7 @@ public class ExtractorYoutubeFormatStreamTest extends ContentExtractorTestBase { extractor.process(testUri); - Set expected = makeLinkSet(testUri, getExpectedOutlinksAllInItagPriority()); + Set expected = makeLinkSet(testUri, getExpectedOutlinksAllInItagPriority()); assertEquals(expected, testUri.getOutLinks()); } @@ -105,7 +105,7 @@ public class ExtractorYoutubeFormatStreamTest extends ContentExtractorTestBase { extractor().setExtractLimit(0); extractor.process(testUri); - Set expected = makeLinkSet(testUri, getExpectedOutlinksAll()); + Set expected = makeLinkSet(testUri, getExpectedOutlinksAll()); assertEquals(expected, testUri.getOutLinks()); } @@ -165,7 +165,7 @@ public class ExtractorYoutubeFormatStreamTest extends ContentExtractorTestBase { extractor.process(testUri); - Set expected = makeLinkSet(testUri, getExpectedOutlinksSubset()); + Set expected = makeLinkSet(testUri, getExpectedOutlinksSubset()); assertEquals(expected, testUri.getOutLinks()); } @@ -187,17 +187,15 @@ public class ExtractorYoutubeFormatStreamTest extends ContentExtractorTestBase { extractor.process(testUri); - Set expected = makeLinkSet(testUri, getExpectedSingleDefaultOutlink()); + Set expected = makeLinkSet(testUri, getExpectedSingleDefaultOutlink()); assertEquals(expected, testUri.getOutLinks()); } - private Set makeLinkSet(CrawlURI sourceUri, String[] urlStrs) throws URIException { - HashSet linkSet = new HashSet(); + private Set makeLinkSet(CrawlURI sourceUri, String[] urlStrs) throws URIException { + HashSet linkSet = new HashSet(); for (String urlStr : urlStrs) { - linkSet.add(new Link(sourceUri.getUURI(), - UURIFactory.getInstance(urlStr), - HTMLLinkContext.EMBED_MISC, Hop.EMBED) - ); + CrawlURI link = sourceUri.createCrawlURI(urlStr, HTMLLinkContext.EMBED_MISC, Hop.EMBED); + linkSet.add(link); } return linkSet; } diff --git a/engine/src/main/java/org/archive/crawler/postprocessor/CandidatesProcessor.java b/engine/src/main/java/org/archive/crawler/postprocessor/CandidatesProcessor.java index d70d41eb..e2e909a0 100644 --- a/engine/src/main/java/org/archive/crawler/postprocessor/CandidatesProcessor.java +++ b/engine/src/main/java/org/archive/crawler/postprocessor/CandidatesProcessor.java @@ -23,7 +23,6 @@ package org.archive.crawler.postprocessor; import static org.archive.modules.fetcher.FetchStatusCodes.S_DEFERRED; import static org.archive.modules.fetcher.FetchStatusCodes.S_PREREQUISITE_UNSCHEDULABLE_FAILURE; -import org.apache.commons.httpclient.URIException; import org.archive.crawler.framework.Frontier; import org.archive.crawler.reporting.CrawlerLoggerModule; import org.archive.crawler.spring.SheetOverlaysManager; @@ -32,7 +31,6 @@ import org.archive.modules.CrawlURI; import org.archive.modules.Processor; import org.archive.modules.SchedulingConstants; import org.archive.modules.extractor.Hop; -import org.archive.modules.extractor.Link; import org.archive.modules.seeds.SeedModule; import org.archive.spring.KeyedProperties; import org.springframework.beans.factory.annotation.Autowired; @@ -219,15 +217,6 @@ public class CandidatesProcessor extends Processor { return; } - // (2) NEW: also (and before-outlinks) run outCandidates (usually empty; - // only current use is a form-submission CrawlURI; could - // potentially take over prerequisite duties for consistency - for(CrawlURI candidate : curi.getOutCandidates()) { - - runCandidateChain(candidate, curi); - - } - // Only consider candidate links of error pages if configured to do so if (!getProcessErrorOutlinks() && (curi.getFetchStatus() < 200 || curi.getFetchStatus() >= 400)) { @@ -236,21 +225,10 @@ public class CandidatesProcessor extends Processor { } // (3) Handle outlinks (usual bulk of discoveries) - for (Link wref: curi.getOutLinks()) { - CrawlURI candidate; - try { - candidate = curi.createCrawlURI(curi.getBaseURI(),wref); - } catch (URIException e) { - loggerModule.logUriError(e, curi.getUURI(), - wref.getDestination().toString()); - continue; - } + for (CrawlURI candidate: curi.getOutLinks()) { runCandidateChain(candidate, curi); - // TODO: evaluate if this necessary (anyone uses?); wise (bloat?) - curi.getOutCandidates().add(candidate); - } curi.getOutLinks().clear(); } diff --git a/engine/src/main/java/org/archive/crawler/postprocessor/LinksScoper.java b/engine/src/main/java/org/archive/crawler/postprocessor/LinksScoper.java deleted file mode 100644 index 7db90061..00000000 --- a/engine/src/main/java/org/archive/crawler/postprocessor/LinksScoper.java +++ /dev/null @@ -1,242 +0,0 @@ -/* - * This file is part of the Heritrix web crawler (crawler.archive.org). - * - * Licensed to the Internet Archive (IA) by one or more individual - * contributors. - * - * The IA licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.archive.crawler.postprocessor; - -import static org.archive.modules.SchedulingConstants.HIGH; -import static org.archive.modules.SchedulingConstants.MEDIUM; -import static org.archive.modules.SchedulingConstants.NORMAL; -import static org.archive.modules.fetcher.FetchStatusCodes.S_PREREQUISITE_UNSCHEDULABLE_FAILURE; - -import java.util.logging.Level; -import java.util.logging.Logger; - -import org.apache.commons.httpclient.URIException; -import org.archive.crawler.framework.Scoper; -import org.archive.modules.CrawlURI; -import org.archive.modules.deciderules.DecideResult; -import org.archive.modules.deciderules.DecideRule; -import org.archive.modules.deciderules.RejectDecideRule; -import org.archive.modules.extractor.Hop; -import org.archive.modules.extractor.Link; - -/** - * Determine which extracted links are within scope. - * TODO: To test scope, requires that Link be converted to - * a CrawlURI. Make it so don't have to make a CrawlURI to test - * if Link is in scope. - *

Since this scoper has to create CrawlURIs, no sense - * discarding them since later in the processing chain CrawlURIs rather - * than Links are whats needed scheduling extracted links w/ the - * Frontier (Frontier#schedule expects CrawlURI, not Link). This class - * replaces Links w/ the CrawlURI that wraps the Link in the CrawlURI. - * - * @deprecated Use CandidatesProcessor and CandidateChain/CandidateScoper instead - * - * @author gojomo - * @author stack - */ -public class LinksScoper extends Scoper { - - @SuppressWarnings("unused") - private static final long serialVersionUID = -3L; - - private static Logger LOGGER = - Logger.getLogger(LinksScoper.class.getName()); - - /** - * If enabled, any URL found because a seed redirected to it (original seed - * returned 301 or 302), will also be treated as a seed. - */ - { - setSeedsRedirectNewSeeds(true); - } - public boolean getSeedsRedirectNewSeeds() { - return (Boolean) kp.get("seedsRedirectNewSeeds"); - } - public void setSeedsRedirectNewSeeds(boolean redirect) { - kp.put("seedsRedirectNewSeeds",redirect); - } - - /** - * DecideRules applied after an URI has been rejected. If the rules return - * {@link DecideResult#ACCEPT}, the URI is logged (if the logging level is - * INFO). Depends on {@link Scoper#OVERRIDE_LOGGER} being enabled. - */ - { - setLogRejectsRule(new RejectDecideRule()); - } - public DecideRule getLogRejectsRule() { - return (DecideRule) kp.get("logRejectsRule"); - } - public void setLogRejectsRule(DecideRule rule) { - kp.put("logRejectsRule", rule); - } - - /** - * Number of hops (of any sort) from a seed up to which a URI has higher - * priority scheduling than any remaining seed. For example, if set to 1 - * items one hop (link, embed, redirect, etc.) away from a seed will be - * scheduled with HIGH priority. If set to -1, no preferencing will occur, - * and a breadth-first search with seeds processed before discovered links - * will proceed. If set to zero, a purely depth-first search will proceed, - * with all discovered links processed before remaining seeds. Seed - * redirects are treated as one hop from a seed. - */ - { - setPreferenceDepthHops(-1); // no limit - } - public int getPreferenceDepthHops() { - return (Integer) kp.get("preferenceDepthHops"); - } - public void setPreferenceDepthHops(int depth) { - kp.put("preferenceDepthHops",depth); - } - - /** - * @param name Name of this filter. - */ - public LinksScoper() { - super(); - } - - - @Override - protected boolean shouldProcess(CrawlURI puri) { - if (!(puri instanceof CrawlURI)) { - return false; - } - CrawlURI curi = (CrawlURI)puri; - - // If prerequisites, nothing to be done in here. - if (curi.hasPrerequisiteUri()) { - handlePrerequisite(curi); - return false; - } - - // Don't extract links of error pages. - if (curi.getFetchStatus() < 200 || curi.getFetchStatus() >= 400) { - curi.getOutLinks().clear(); - return false; - } - - if (curi.getOutLinks().isEmpty()) { - // No outlinks to process. - return false; - } - - return true; - } - - - @Override - protected void innerProcess(final CrawlURI puri) { - CrawlURI curi = (CrawlURI)puri; - final boolean redirectsNewSeeds = getSeedsRedirectNewSeeds(); - int preferenceDepthHops = getPreferenceDepthHops(); - - for (Link wref: curi.getOutLinks()) try { - int directive = getSchedulingFor(curi, wref, preferenceDepthHops); - CrawlURI caURI = curi.createCrawlURI(curi.getBaseURI(), - wref, directive, - considerAsSeed(curi, wref, redirectsNewSeeds)); - if (isInScope(caURI)) { - curi.getOutCandidates().add(caURI); - } - } catch (URIException e) { - loggerModule.logUriError(e, curi.getUURI(), - wref.getDestination().toString()); - } - curi.getOutLinks().clear(); - } - - /** - * The CrawlURI has a prerequisite; apply scoping and update - * Link to CrawlURI in manner analogous to outlink handling. - * @param curi CrawlURI with prereq to consider - */ - protected void handlePrerequisite(CrawlURI curi) { - CrawlURI caUri = curi.getPrerequisiteUri(); -// FIXME!!! getController().setStateProvider(caUri); - if(isInScope(caUri)) { - // replace link with CrawlURI - curi.setPrerequisiteUri(caUri); - } else { - // prerequisite is out-of-scope; mark CrawlURI as error, - // preventing normal S_DEFERRED handling - curi.clearPrerequisiteUri(); - curi.setFetchStatus(S_PREREQUISITE_UNSCHEDULABLE_FAILURE); - } - } - - // TODO: move similar outOfScope logging to CandidatesProcessor/CandidateChain/CandidateScoper - protected void outOfScope(CrawlURI caUri) { - super.outOfScope(caUri); - if (!LOGGER.isLoggable(Level.INFO)) { - return; - } - DecideRule seq = getLogRejectsRule(); - if (seq.decisionFor(caUri) == DecideResult.ACCEPT) { - LOGGER.info(caUri.getUURI().toString()); - } - } - - private boolean considerAsSeed(final CrawlURI curi, final Link wref, - final boolean redirectsNewSeeds) { - return redirectsNewSeeds && curi.isSeed() - && wref.getHopType() == Hop.REFER; - } - - /** - * Determine scheduling for the curi. - * As with the LinksScoper in general, this only handles extracted links, - * seeds do not pass through here, but are given MEDIUM priority. - * Imports into the frontier similarly do not pass through here, - * but are given NORMAL priority. - */ - protected int getSchedulingFor(final CrawlURI curi, final Link wref, - final int preferenceDepthHops) { - final Hop c = wref.getHopType(); - if (LOGGER.isLoggable(Level.FINEST)) { - LOGGER.finest(curi + " with path=" + curi.getPathFromSeed() + - " isSeed=" + curi.isSeed() + " with fetchStatus=" + - curi.getFetchStatus() + " -> " + wref.getDestination() + - " type " + c + " with context=" + wref.getContext()); - } - - switch (c) { - case REFER: - // Treat redirects somewhat urgently - // This also ensures seed redirects remain seed priority - return (preferenceDepthHops >= 0 ? HIGH : MEDIUM); - default: - if (preferenceDepthHops == 0) - return HIGH; - // this implies seed redirects are treated as path - // length 1, which I belive is standard. - // curi.getPathFromSeed() can never be null here, because - // we're processing a link extracted from curi - if (preferenceDepthHops > 0 && - curi.getPathFromSeed().length() + 1 <= preferenceDepthHops) - return HIGH; - // Everything else normal (at least for now) - return NORMAL; - } - } -} diff --git a/engine/src/main/java/org/archive/crawler/postprocessor/SupplementaryLinksScoper.java b/engine/src/main/java/org/archive/crawler/postprocessor/SupplementaryLinksScoper.java index bd78cf78..d3de5d20 100644 --- a/engine/src/main/java/org/archive/crawler/postprocessor/SupplementaryLinksScoper.java +++ b/engine/src/main/java/org/archive/crawler/postprocessor/SupplementaryLinksScoper.java @@ -87,7 +87,7 @@ public class SupplementaryLinksScoper extends Scoper { } // Collection inScopeLinks = new HashSet(); - Iterator iter = curi.getOutCandidates().iterator(); + Iterator iter = curi.getOutLinks().iterator(); while (iter.hasNext()) { CrawlURI cauri = iter.next(); if (!isInScope(cauri)) { diff --git a/engine/src/main/java/org/archive/crawler/processor/CrawlMapper.java b/engine/src/main/java/org/archive/crawler/processor/CrawlMapper.java index 9e56689e..2771db37 100644 --- a/engine/src/main/java/org/archive/crawler/processor/CrawlMapper.java +++ b/engine/src/main/java/org/archive/crawler/processor/CrawlMapper.java @@ -215,7 +215,7 @@ public abstract class CrawlMapper extends Processor implements Lifecycle { if (getCheckOutlinks()) { // consider outlinks for mapping - Iterator iter = curi.getOutCandidates().iterator(); + Iterator iter = curi.getOutLinks().iterator(); while(iter.hasNext()) { CrawlURI cauri = iter.next(); if (decideToMapOutlink(cauri)) { diff --git a/engine/src/main/java/org/archive/crawler/reporting/SeedRecord.java b/engine/src/main/java/org/archive/crawler/reporting/SeedRecord.java index edc8a15d..d0b9ba5a 100644 --- a/engine/src/main/java/org/archive/crawler/reporting/SeedRecord.java +++ b/engine/src/main/java/org/archive/crawler/reporting/SeedRecord.java @@ -110,7 +110,7 @@ public class SeedRecord implements CoreAttributeConstants, Serializable, Identit this.statusCode = curi.getFetchStatus(); this.disposition = disposition; if (statusCode==301 || statusCode == 302) { - for (CrawlURI cauri: curi.getOutCandidates()) { + for (CrawlURI cauri: curi.getOutLinks()) { if("location:".equalsIgnoreCase(cauri.getViaContext(). toString())) { redirectUri = cauri.toString(); diff --git a/engine/src/test/java/org/archive/crawler/postprocessor/LinksScoperTest.java b/engine/src/test/java/org/archive/crawler/postprocessor/LinksScoperTest.java deleted file mode 100644 index e48de3cc..00000000 --- a/engine/src/test/java/org/archive/crawler/postprocessor/LinksScoperTest.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * This file is part of the Heritrix web crawler (crawler.archive.org). - * - * Licensed to the Internet Archive (IA) by one or more individual - * contributors. - * - * The IA licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.archive.crawler.postprocessor; - - -import org.archive.crawler.framework.CrawlerProcessorTestBase; - - -/** - * Unit test for {@link LinksScoper}. - * - * @author pjack - */ -public class LinksScoperTest extends CrawlerProcessorTestBase { - - // TODO TESTME! - -} diff --git a/modules/src/main/java/org/archive/modules/CrawlURI.java b/modules/src/main/java/org/archive/modules/CrawlURI.java index 4d97db01..7221c86c 100644 --- a/modules/src/main/java/org/archive/modules/CrawlURI.java +++ b/modules/src/main/java/org/archive/modules/CrawlURI.java @@ -86,7 +86,6 @@ import org.archive.modules.credential.Credential; import org.archive.modules.credential.HttpAuthenticationCredential; import org.archive.modules.extractor.HTMLLinkContext; import org.archive.modules.extractor.Hop; -import org.archive.modules.extractor.Link; import org.archive.modules.extractor.LinkContext; import org.archive.net.UURI; import org.archive.net.UURIFactory; @@ -119,8 +118,8 @@ import org.json.JSONObject; * @author Gordon Mohr */ public class CrawlURI -implements Reporter, Serializable, OverlayContext { - private static final long serialVersionUID = 3L; +implements Reporter, Serializable, OverlayContext, Comparable { + private static final long serialVersionUID = 4L; private static final Logger logger = Logger.getLogger(CrawlURI.class.getName()); @@ -276,6 +275,7 @@ implements Reporter, Serializable, OverlayContext { */ public CrawlURI(UURI uuri) { this.uuri = uuri; + this.pathFromSeed = ""; } public static CrawlURI fromHopsViaString(String uriHopsViaContext) throws URIException { @@ -302,7 +302,11 @@ implements Reporter, Serializable, OverlayContext { public CrawlURI(UURI u, String pathFromSeed, UURI via, LinkContext viaContext) { this.uuri = u; - this.pathFromSeed = pathFromSeed; + if (pathFromSeed != null) { + this.pathFromSeed = pathFromSeed; + } else { + this.pathFromSeed = ""; + } this.via = via; this.viaContext = viaContext; } @@ -868,7 +872,6 @@ implements Reporter, Serializable, OverlayContext { this.data = getPersistentDataMap(); extraInfo = null; - outCandidates = null; outLinks = null; // XXX er uh surprised this wasn't here before? @@ -1080,38 +1083,25 @@ implements Reporter, Serializable, OverlayContext { } /** - * All discovered outbound Links (navlinks, embeds, etc.) - * Can either contain Link instances or CrawlURI instances, or both. - * The LinksScoper processor converts Link instances in this collection - * to CrawlURI instances. + * All discovered outbound urls as CrawlURIs (navlinks, embeds, etc.) */ - protected transient Collection outLinks = new LinkedHashSet(); - - protected transient Collection outCandidates = new LinkedHashSet(); + protected transient Collection outLinks; /** * Returns discovered links. The returned collection might be empty if * no links were discovered, or if something like LinksScoper promoted * the links to CrawlURIs. * - * @return Collection of all discovered outbound Links + * @return Collection of all discovered outbound links */ - public Collection getOutLinks() { + public Collection getOutLinks() { + if (outLinks==null) { + outLinks = new LinkedHashSet(); + } return outLinks; -// return Transform.subclasses(outLinks, Link.class); - } - - /** - * Returns discovered candidate URIs. The returned collection will be - * emtpy until something like LinksScoper promotes discovered Links - * into CrawlURIs. - * - * @return Collection of candidate URIs - */ - public Collection getOutCandidates() { - return outCandidates; } + /** * Set the (HTML) Base URI used for derelativizing internal URIs. * @@ -1178,8 +1168,6 @@ implements Reporter, Serializable, OverlayContext { @SuppressWarnings("unchecked") Map temp = (Map)stream.readObject(); this.data = temp; - outLinks = new HashSet(); - outCandidates = new HashSet(); } /** @@ -1397,28 +1385,6 @@ implements Reporter, Serializable, OverlayContext { } -// public void setStateProvider(SheetManager manager) { -// if(this.provider!=null) { -// return; -// } -// this.manager = manager; -//// this.provider = manager.findConfig(SURT.fromURI(toString())); -// } -// -// -// public StateProvider getStateProvider() { -// return provider; -// } - - -// public T get(Object module, Key key) { -// if (provider == null) { -// throw new AssertionError("ToeThread never set up CrawlURI's sheet."); -// } -// return provider.get(module, key); -// } - - // // Reporter implementation // @@ -1617,25 +1583,30 @@ implements Reporter, Serializable, OverlayContext { } /** - * Utility method for creation of CandidateURIs found extracting - * links from this CrawlURI. - * @param baseUURI BaseUURI for link. - * @param link Link to wrap CandidateURI in. - * @return New candidateURI wrapper around link. + * Utility method for creating CrawlURIs that were found as out links from the current CrawlURI + * links from this CrawlURI. + *

+ * Any relative URIs will be treated as relative to this CrawlURI's UURI. + * @param destination The new URI, possibly a relative URI + * @param context + * @param hop + * @return New CrawlURI with the current CrawlURI set as the one it inherits from * @throws URIException */ - public CrawlURI createCrawlURI(UURI baseUURI, Link link) - throws URIException { - UURI u = (link.getDestination() instanceof UURI)? - (UURI)link.getDestination(): - UURIFactory.getInstance(baseUURI, - link.getDestination().toString()); - CrawlURI newCaURI = new CrawlURI(u, - extendHopsPath(getPathFromSeed(),link.getHopType().getHopChar()), - getUURI(), link.getContext()); - if (link.hasData()) { - newCaURI.data = link.getData(); - } + public CrawlURI createCrawlURI(UURI destination, LinkContext context, Hop hop) + throws URIException { + return createCrawlURI(destination.toString(), context, hop); + } + + public CrawlURI createCrawlURI(String destination, LinkContext context, Hop hop) + throws URIException { + UURI u = UURIFactory.getInstance(this.getBaseURI(), destination); + CrawlURI newCaURI = new CrawlURI( + u, + extendHopsPath(getPathFromSeed(), + hop.getHopChar()), + this.getUURI(), + context); newCaURI.inheritFrom(this); return newCaURI; } @@ -1660,19 +1631,19 @@ implements Reporter, Serializable, OverlayContext { } /** - * Utility method for creation of CandidateURIs found extracting + * Utility method for creation of CrawlURIs found extracting * links from this CrawlURI. * @param baseUURI BaseUURI for link. - * @param link Link to wrap CandidateURI in. + * TODO: Fix JavaDoc * @param scheduling How new CandidateURI should be scheduled. * @param seed True if this CandidateURI is a seed. * @return New candidateURI wrapper around link. * @throws URIException */ - public CrawlURI createCrawlURI(UURI baseUURI, Link link, + public CrawlURI createCrawlURI(UURI destination, LinkContext context, Hop hop, int scheduling, boolean seed) throws URIException { - final CrawlURI caURI = createCrawlURI(baseUURI, link); + final CrawlURI caURI = createCrawlURI(destination, context, hop); caURI.setSchedulingDirective(scheduling); caURI.setSeed(seed); return caURI; @@ -1862,7 +1833,7 @@ implements Reporter, Serializable, OverlayContext { */ public CrawlURI markPrerequisite(String preq) throws URIException { - CrawlURI caUri = makeConsequentCandidate(preq, LinkContext.PREREQ_MISC, Hop.PREREQ); + CrawlURI caUri = createCrawlURI(preq, LinkContext.PREREQ_MISC, Hop.PREREQ); caUri.setPrerequisite(true); // TODO: consider moving some of this to configurable candidate-handling int prereqPriority = getSchedulingDirective() - 1; @@ -1879,25 +1850,6 @@ implements Reporter, Serializable, OverlayContext { return caUri; } - /** - * Create a consequent CrawlURI from this one, given the - * additional parameters - * - * @param destination URI string - * @param lc LinkContext - * @param hop Hop - * @return the newly created prerequisite CrawlURI - * @throws URIException - */ - public CrawlURI makeConsequentCandidate(String destination, LinkContext lc, Hop hop) - throws URIException { - UURI src = getUURI(); - UURI dest = UURIFactory.getInstance(getBaseURI(),destination); - Link link = new Link(src, dest, lc, hop); - CrawlURI caUri = createCrawlURI(getBaseURI(), link); - return caUri; - } - public boolean containsContentTypeCharsetDeclaration() { // TODO can this regex be improved? should the test consider if its legal? return getContentType().matches("(?i).*charset=.*"); @@ -1959,4 +1911,64 @@ implements Reporter, Serializable, OverlayContext { public boolean hasContentDigestHistory() { return getData().get(A_CONTENT_DIGEST_HISTORY) != null; } + + // brought over from old Link class + @Override + public int compareTo(CrawlURI o) { + int cmp = compare(via.toString(), o.via.toString()); + if (cmp == 0) { + cmp = compare(uuri.toString(), o.uuri.toString()); + } + if (cmp == 0) { + cmp = compare(viaContext.toString(), o.viaContext.toString()); + } + if (cmp == 0) { + cmp = compare(pathFromSeed, o.pathFromSeed); + } + return cmp; + } + + // brought over from old Link class + @Override + public int hashCode() { + int r = 37; + return r ^ hash(via.toString()) ^ hash(uuri.toString()) + ^ hash(viaContext.toString()) ^ hash(pathFromSeed.toString()); + } + + // handles nulls + private static int hash(String a) { + return a == null ? 0 : a.hashCode(); + } + + // handles nulls + private static boolean equals(Object a, Object b) { + return a == null ? b == null : a.equals(b); + } + + // handles nulls + private static int compare(String a, String b) { + if (a == null && b == null) { + return 0; + } else if (a == null && b != null) { + return -1; + } else if (a != null && b != null) { + return 1; + } else { + return a.compareTo(b); + } + } + + // brought over from old Link class + @Override + public boolean equals(Object o) { + if (!(o instanceof CrawlURI)) { + return false; + } + CrawlURI u = (CrawlURI) o; + return equals(via, u.via) && equals(uuri, u.uuri) + && equals(viaContext, u.viaContext) + && equals(pathFromSeed, u.pathFromSeed); + } + } diff --git a/modules/src/main/java/org/archive/modules/extractor/Extractor.java b/modules/src/main/java/org/archive/modules/extractor/Extractor.java index 44cfd9c6..651c74cb 100644 --- a/modules/src/main/java/org/archive/modules/extractor/Extractor.java +++ b/modules/src/main/java/org/archive/modules/extractor/Extractor.java @@ -43,7 +43,8 @@ import org.springframework.beans.factory.annotation.Autowired; * @author pjack */ public abstract class Extractor extends Processor { - + private static final Logger LOGGER = Logger.getLogger(Extractor.class.getName()); + protected AtomicLong numberOfLinksExtracted = new AtomicLong(0); /** Logger. */ @@ -143,13 +144,23 @@ public abstract class Extractor extends Processor { Hop hop) { try { UURI dest = UURIFactory.getInstance(curi.getUURI(), uri); - Link link = new Link(curi.getUURI(), dest, context, hop); + CrawlURI link = curi.createCrawlURI(dest, context, hop); curi.getOutLinks().add(link); } catch (URIException e) { logUriError(e, curi.getUURI(), uri); } } + protected void addOutlink(CrawlURI curi, UURI uuri, LinkContext context, + Hop hop) { + try { + CrawlURI link = curi.createCrawlURI(uuri, context, hop); + curi.getOutLinks().add(link); + } catch (URIException e) { + logUriError(e, curi.getUURI(), uuri.toString()); + } + } + public void logUriError(URIException e, UURI uuri, CharSequence l) { loggerModule.logUriError(e, uuri, l); @@ -174,5 +185,45 @@ public abstract class Extractor extends Processor { ret.append(" " + numberOfLinksExtracted + " links from " + getURICount() +" CrawlURIs\n"); return ret.toString(); } + + public static CrawlURI addRelativeToBase(CrawlURI uri, int max, + String newUri, LinkContext context, Hop hop) throws URIException { + UURI dest = UURIFactory.getInstance(uri.getBaseURI(), newUri); + return add2(uri, max, dest, context, hop); + } + + + public static CrawlURI addRelativeToVia(CrawlURI uri, int max, String newUri, + LinkContext context, Hop hop) throws URIException { + UURI relTo = uri.getVia(); + if (relTo == null) { + if (!uri.getAnnotations().contains("usedBaseForVia")) { + LOGGER.info("no via where expected; using base instead: " + uri); + uri.getAnnotations().add("usedBaseForVia"); + } + relTo = uri.getBaseURI(); + } + UURI dest = UURIFactory.getInstance(relTo, newUri); + return add2(uri, max, dest, context, hop); + } + + public static void add(CrawlURI uri, int max, String newUri, + LinkContext context, Hop hop) throws URIException { + UURI dest = UURIFactory.getInstance(newUri); + add2(uri, max, dest, context, hop); + } + + + private static CrawlURI add2(CrawlURI curi, int max, UURI dest, + LinkContext context, Hop hop) throws URIException { + if (curi.getOutLinks().size() < max) { + CrawlURI link = curi.createCrawlURI(dest, context, hop); + curi.getOutLinks().add(link); + return link; + } else { + curi.incrementDiscardedOutLinks(); + return null; + } + } } diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorCSS.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorCSS.java index 60d13e2f..c0bec497 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorCSS.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorCSS.java @@ -136,7 +136,7 @@ public class ExtractorCSS extends ContentExtractor { foundLinks++; int max = ext.getExtractorParameters().getMaxOutlinks(); try { - Link.addRelativeToBase(curi, max, cssUri, + addRelativeToBase(curi, max, cssUri, LinkContext.EMBED_MISC, Hop.EMBED); } catch (URIException e) { ext.logUriError(e, curi.getUURI(), cssUri); diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorDOC.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorDOC.java index dc61f77d..438d30fb 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorDOC.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorDOC.java @@ -114,8 +114,7 @@ public class ExtractorDOC extends ContentExtractor { try { UURI dest = UURIFactory.getInstance(curi.getUURI(), hyperlink); LinkContext lc = LinkContext.NAVLINK_MISC; - Link link = new Link(curi.getUURI(), dest, lc, Hop.NAVLINK); - curi.getOutLinks().add(link); + addOutlink(curi, hyperlink, lc, Hop.NAVLINK); } catch (URIException e1) { logUriError(e1, curi.getUURI(), hyperlink); } diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java index e76f7f17..c5f710c8 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java @@ -656,7 +656,7 @@ public class ExtractorHTML extends ContentExtractor implements InitializingBean // ReplayCharSequence. HTMLLinkContext hc = HTMLLinkContext.get(context.toString()); int max = getExtractorParameters().getMaxOutlinks(); - Link.addRelativeToBase(curi, max, uri.toString(), hc, hop); + addRelativeToBase(curi, max, uri.toString(), hc, hop); } catch (URIException e) { logUriError(e, curi.getUURI(), uri); } @@ -954,7 +954,7 @@ public class ExtractorHTML extends ContentExtractor implements InitializingBean String refreshUri = content.substring(urlIndex); try { int max = getExtractorParameters().getMaxOutlinks(); - Link.addRelativeToBase(curi, max, refreshUri, + addRelativeToBase(curi, max, refreshUri, HTMLLinkContext.META, Hop.REFER); } catch (URIException e) { logUriError(e, curi.getUURI(), refreshUri); diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorHTTP.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorHTTP.java index 65694933..0e7fd78a 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorHTTP.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorHTTP.java @@ -95,9 +95,8 @@ public class ExtractorHTTP extends Extractor { protected void addHeaderLink(CrawlURI curi, String headerName, String url) { try { UURI dest = UURIFactory.getInstance(curi.getUURI(), url); - LinkContext lc = HTMLLinkContext.get(headerName+":"); - Link link = new Link(curi.getUURI(), dest, lc, Hop.REFER); - curi.getOutLinks().add(link); + LinkContext lc = HTMLLinkContext.get(headerName+":"); + addOutlink(curi, dest.toString(), lc, Hop.REFER); numberOfLinksExtracted.incrementAndGet(); } catch (URIException e) { logUriError(e, curi.getUURI(), url); diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorImpliedURI.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorImpliedURI.java index b64cedb4..7d4812ca 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorImpliedURI.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorImpliedURI.java @@ -121,36 +121,34 @@ public class ExtractorImpliedURI extends Extractor { */ @Override public void extract(CrawlURI curi) { - List links = new ArrayList(curi.getOutLinks()); + List links = new ArrayList(curi.getOutLinks()); int max = links.size(); for (int i = 0; i < max; i++) { - Link link = links.get(i); + CrawlURI link = links.get(i); Pattern trigger = getRegex(); String build = getFormat(); - CharSequence dest = link.getDestination(); + CharSequence dest = link.getUURI(); String implied = extractImplied(dest, trigger, build); if (implied != null) { try { - UURI src = curi.getUURI(); UURI target = UURIFactory.getInstance(implied); LinkContext lc = LinkContext.INFERRED_MISC; Hop hop = Hop.INFERRED; - Link out = new Link(src, target, lc, hop); - curi.getOutLinks().add(out); + addOutlink(curi, target, lc, hop); numberOfLinksExtracted.incrementAndGet(); boolean removeTriggerURI = getRemoveTriggerUris(); // remove trigger URI from the outlinks if configured so. if (removeTriggerURI) { if (curi.getOutLinks().remove(link)) { - LOGGER.log(Level.FINE, link.getDestination() + + LOGGER.log(Level.FINE, link.getURI() + " has been removed from " + - link.getSource() + " outlinks list."); + curi.getURI() + " outlinks list."); numberOfLinksExtracted.decrementAndGet(); } else { LOGGER.log(Level.FINE, "Failed to remove " + - link.getDestination() + " from " + - link.getSource()+ " outlinks list."); + link.getURI() + " from " + + curi.getURI() + " outlinks list."); } } } catch (URIException e) { diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorJS.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorJS.java index 61bf36c0..1fa9672b 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorJS.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorJS.java @@ -175,11 +175,11 @@ public class ExtractorJS extends ContentExtractor { try { int max = ext.getExtractorParameters().getMaxOutlinks(); if (handlingJSFile) { - Link.addRelativeToVia(curi, max, candidate, JS_MISC, + addRelativeToVia(curi, max, candidate, JS_MISC, SPECULATIVE); return true; } else { - Link.addRelativeToBase(curi, max, candidate, JS_MISC, + addRelativeToBase(curi, max, candidate, JS_MISC, SPECULATIVE); return true; } diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorMultipleRegex.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorMultipleRegex.java index 30479f24..d669cd2f 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorMultipleRegex.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorMultipleRegex.java @@ -280,7 +280,7 @@ public class ExtractorMultipleRegex extends Extractor { String outlinkUri = groovyTemplate().make(bindings).toString(); try { - Link.addRelativeToBase(curi, + addRelativeToBase(curi, getExtractorParameters().getMaxOutlinks(), outlinkUri, HTMLLinkContext.INFERRED_MISC, Hop.INFERRED); } catch (URIException e) { diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorPDF.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorPDF.java index d15e2240..ccd840f3 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorPDF.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorPDF.java @@ -117,8 +117,7 @@ public class ExtractorPDF extends ContentExtractor { UURI dest = UURIFactory.getInstance(uri); LinkContext lc = LinkContext.NAVLINK_MISC; Hop hop = Hop.NAVLINK; - Link out = new Link(src, dest, lc, hop); - curi.getOutLinks().add(out); + addOutlink(curi, dest, lc, hop); } catch (URIException e1) { // There may not be a controller (e.g. If we're being run // by the extractor tool). diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorSWF.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorSWF.java index 4a8bc6e8..14a019f3 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorSWF.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorSWF.java @@ -334,18 +334,18 @@ public class ExtractorSWF extends ContentExtractor { } } else { int max = ext.getExtractorParameters().getMaxOutlinks(); - Link relToVia = Link.addRelativeToVia(curi, max, url, + CrawlURI relToVia = addRelativeToVia(curi, max, url, LinkContext.EMBED_MISC, Hop.EMBED); - Link relToBase = Link.addRelativeToBase(curi, max, url, + CrawlURI relToBase = addRelativeToBase(curi, max, url, LinkContext.EMBED_MISC, Hop.EMBED); addAnnotations(relToVia, relToBase); linkCount++; } } - protected void addAnnotations(Link relToVia, Link relToBase) { + protected void addAnnotations(CrawlURI relToVia, CrawlURI relToBase) { if (relToVia != null && relToBase != null - && relToVia.getDestination().equals(relToBase.getDestination())) { + && relToVia.getUURI().equals(relToBase.getUURI())) { relToVia.getAnnotations().add("extractorSWFRelToBoth"); relToBase.getAnnotations().add("extractorSWFRelToBoth"); } else { @@ -361,9 +361,9 @@ public class ExtractorSWF extends ContentExtractor { public void considerStringAsUri(String str) throws IOException { if (UriUtils.isVeryLikelyUri(str)) { int max = ext.getExtractorParameters().getMaxOutlinks(); - Link relToVia = Link.addRelativeToVia(curi, max, str, + CrawlURI relToVia = addRelativeToVia(curi, max, str, LinkContext.SPECULATIVE_MISC, Hop.SPECULATIVE); - Link relToBase = Link.addRelativeToBase(curi, max, str, + CrawlURI relToBase = addRelativeToBase(curi, max, str, LinkContext.SPECULATIVE_MISC, Hop.SPECULATIVE); addAnnotations(relToVia, relToBase); linkCount++; diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorURI.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorURI.java index b856f95b..3fcac20a 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorURI.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorURI.java @@ -77,11 +77,8 @@ public class ExtractorURI extends Extractor { */ @Override public void extract(CrawlURI curi) { - List links = new ArrayList(curi.getOutLinks()); - int max = links.size(); - for (int i = 0; i < max; i++) { - Link wref = links.get(i); - extractLink(curi, wref); + for (CrawlURI link : curi.getOutLinks()) { + extractLink(curi, link); } } @@ -91,10 +88,10 @@ public class ExtractorURI extends Extractor { * @param curi CrawlURI to add discoveries to * @param wref Link to examine for internal URIs */ - protected void extractLink(CrawlURI curi, Link wref) { + protected void extractLink(CrawlURI curi, CrawlURI wref) { UURI source = null; try { - source = UURIFactory.getInstance(wref.getDestination().toString()); + source = UURIFactory.getInstance(wref.getURI()); } catch (URIException e) { LOGGER.log(Level.FINE,"bad URI",e); } @@ -105,13 +102,11 @@ public class ExtractorURI extends Extractor { List found = extractQueryStringLinks(source); for (String uri : found) { try { - UURI src = curi.getUURI(); UURI dest = UURIFactory.getInstance(uri); LinkContext lc = LinkContext.SPECULATIVE_MISC; Hop hop = Hop.SPECULATIVE; - Link link = new Link(src, dest, lc, hop); + addOutlink(curi, dest, lc, hop); numberOfLinksExtracted.incrementAndGet(); - curi.getOutLinks().add(link); } catch (URIException e) { LOGGER.log(Level.FINE, "bad URI", e); } diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorUniversal.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorUniversal.java index 9b8cbba3..b0bbf7db 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorUniversal.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorUniversal.java @@ -405,12 +405,10 @@ public class ExtractorUniversal extends ContentExtractor { // And add the URL to speculative embeds. numberOfLinksExtracted.incrementAndGet(); - UURI src = curi.getUURI(); UURI dest = UURIFactory.getInstance(newURL); LinkContext lc = LinkContext.SPECULATIVE_MISC; Hop hop = Hop.SPECULATIVE; - Link link = new Link(src, dest, lc, hop); - curi.getOutLinks().add(link); + addOutlink(curi, dest, lc, hop); } // Reset lookat for next string. lookat = new StringBuffer(); diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorXML.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorXML.java index dbf97d59..64a1d500 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorXML.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorXML.java @@ -159,7 +159,7 @@ public class ExtractorXML extends ContentExtractor { // intends to create a followable/fetchable URI is // unknown int max = ext.getExtractorParameters().getMaxOutlinks(); - Link.addRelativeToBase(curi, max, xmlUri, + addRelativeToBase(curi, max, xmlUri, LinkContext.SPECULATIVE_MISC, Hop.SPECULATIVE); } catch (URIException e) { // There may not be a controller (e.g. If we're being run diff --git a/modules/src/main/java/org/archive/modules/extractor/JerichoExtractorHTML.java b/modules/src/main/java/org/archive/modules/extractor/JerichoExtractorHTML.java index eb1f6e42..53770b3a 100644 --- a/modules/src/main/java/org/archive/modules/extractor/JerichoExtractorHTML.java +++ b/modules/src/main/java/org/archive/modules/extractor/JerichoExtractorHTML.java @@ -319,7 +319,7 @@ public class JerichoExtractorHTML extends ExtractorHTML { String refreshUri = content.substring(content.indexOf("=") + 1); try { int max = getExtractorParameters().getMaxOutlinks(); - Link.addRelativeToBase(curi, max, refreshUri, + addRelativeToBase(curi, max, refreshUri, HTMLLinkContext.META, Hop.REFER); } catch (URIException e) { logUriError(e, curi.getUURI(), refreshUri); diff --git a/modules/src/main/java/org/archive/modules/extractor/Link.java b/modules/src/main/java/org/archive/modules/extractor/Link.java deleted file mode 100644 index 3f50a1f3..00000000 --- a/modules/src/main/java/org/archive/modules/extractor/Link.java +++ /dev/null @@ -1,242 +0,0 @@ -/* - * This file is part of the Heritrix web crawler (crawler.archive.org). - * - * Licensed to the Internet Archive (IA) by one or more individual - * contributors. - * - * The IA licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.archive.modules.extractor; - -import static org.archive.modules.CoreAttributeConstants.A_ANNOTATIONS; - -import java.io.Serializable; -import java.util.Collection; -import java.util.HashMap; -import java.util.LinkedHashSet; -import java.util.Map; -import java.util.logging.Logger; - -import org.apache.commons.httpclient.URIException; -import org.archive.modules.CoreAttributeConstants; -import org.archive.modules.CrawlURI; -import org.archive.net.UURI; -import org.archive.net.UURIFactory; - - -/** - * Link represents one discovered "edge" of the web graph: the source - * URI, the destination URI, and the type of reference (represented by the - * context in which it was found). - * - * As such, it is a suitably generic item to returned from generic - * link-extraction utility code. - * - * @author gojomo - */ -public class Link implements Serializable, Comparable { - private static final Logger LOGGER = Logger.getLogger(Link.class.getName()); - - private static final long serialVersionUID = 3L; - - - /** URI where this Link was discovered */ - private CharSequence source; - - /** URI (absolute) where this Link points */ - private CharSequence destination; - - /** context of discovery -- will be an XPath-like element[/@attribute] - * fragment for HTML URIs, a header name with trailing ':' for header - * values, or one of the stand-in constants when other context is - * unavailable */ - private LinkContext context; - - /** hop-type */ - private Hop hop; - - /** - * Flexible dynamic attributes list. - *

- * See further {@link Link#getData()} - */ - protected Map data; - - - /** - * Create a Link with the given fields. - * @param source - * @param destination - * @param context - * @param hopType - */ - public Link(CharSequence source, CharSequence destination, - LinkContext context, Hop hop) { - super(); - this.source = source; - this.destination = destination; - this.context = context; - this.hop = hop; - } - - - /** - * @return Returns the context. - */ - public LinkContext getContext() { - return context; - } - /** - * @return Returns the destination. - */ - public CharSequence getDestination() { - return destination; - } - /** - * @return Returns the source. - */ - public CharSequence getSource() { - return source; - } - - /** - * @return char hopType - */ - public Hop getHopType() { - return hop; - } - - - @Override - public String toString() { - return this.destination + " " + hop.getHopChar() + " " + this.context; - } - /** - * Attribute list - *

- * By convention the attribute list is keyed by constants found in the - * {@link CoreAttributeConstants} interface. Use this list to carry - * data or state produced by custom processors rather change the - * classes {@link CrawlURI} or this class, CrawlURI. - *

- * This list becomes {@link CrawlURI#getData()} when the Link is promoted to CrawlURI via - * {@link CrawlURI#createCrawlURI(UURI, Link)} - *

- * If the list is null when this method is invoked, a new instance will be created and returned. - * - * @returns a flexible map of key/value pairs for storing - * status of this URI for use by other processors. - */ - public Map getData() { - if (data == null) { - data = new HashMap(); - } - return data; - } - - /** - * @return true if data map is not null - */ - public boolean hasData() { - return data != null; - } - - @Override - public boolean equals(Object o) { - if (!(o instanceof Link)) { - return false; - } - Link l = (Link)o; - return l.source.equals(source) && l.destination.equals(destination) - && l.context.equals(context) && l.hop.equals(hop); - } - - - @Override - public int hashCode() { - int r = 37; - return r ^ source.hashCode() ^ destination.hashCode() - ^ context.hashCode() ^ hop.hashCode(); - } - - public static Link addRelativeToBase(CrawlURI uri, int max, - String newUri, LinkContext context, Hop hop) throws URIException { - UURI dest = UURIFactory.getInstance(uri.getBaseURI(), newUri); - return addOrDiscard(uri, max, dest, context, hop); - } - - - public static Link addRelativeToVia(CrawlURI uri, int max, String newUri, - LinkContext context, Hop hop) throws URIException { - UURI relTo = uri.getVia(); - if (relTo == null) { - if (!uri.getAnnotations().contains("usedBaseForVia")) { - LOGGER.info("no via where expected; using base instead: " + uri); - uri.getAnnotations().add("usedBaseForVia"); - } - relTo = uri.getBaseURI(); - } - UURI dest = UURIFactory.getInstance(relTo, newUri); - return addOrDiscard(uri, max, dest, context, hop); - } - - public static Link add(CrawlURI uri, int max, String newUri, - LinkContext context, Hop hop) throws URIException { - UURI dest = UURIFactory.getInstance(newUri); - return addOrDiscard(uri, max, dest, context, hop); - } - - - private static Link addOrDiscard(CrawlURI uri, int max, UURI dest, - LinkContext context, Hop hop) throws URIException { - if (uri.getOutLinks().size() < max) { - UURI src = uri.getUURI(); - Link link = new Link(src, dest, context, hop); - uri.getOutLinks().add(link); - return link; - } else { - uri.incrementDiscardedOutLinks(); - return null; - } - } - - public int compareTo(Link o) { - int cmp = source.toString().compareTo(o.source.toString()); - if (cmp==0) { - cmp = destination.toString().compareTo(o.destination.toString()); - } - if (cmp==0) { - cmp = context.toString().compareTo(o.context.toString()); - } - if (cmp==0) { - cmp = hop.toString().compareTo(o.hop.toString()); - } - return cmp; - } - - /** - * Get the annotations set for this uri. - * - * @return the annotations set for this uri. - */ - // XXX copied from CrawlURI :-\ let's get HER-2039 in there - public Collection getAnnotations() { - @SuppressWarnings("unchecked") - Collection annotations = (Collection)getData().get(A_ANNOTATIONS); - if (annotations == null) { - annotations = new LinkedHashSet(); - getData().put(A_ANNOTATIONS, annotations); - } - return annotations; - } -} diff --git a/modules/src/main/java/org/archive/modules/extractor/StringExtractorTestBase.java b/modules/src/main/java/org/archive/modules/extractor/StringExtractorTestBase.java index 6a97176f..3c87c2b7 100644 --- a/modules/src/main/java/org/archive/modules/extractor/StringExtractorTestBase.java +++ b/modules/src/main/java/org/archive/modules/extractor/StringExtractorTestBase.java @@ -29,9 +29,9 @@ public abstract class StringExtractorTestBase extends ContentExtractorTestBase { public static class TestData { public CrawlURI uri; - public Link expectedResult; + public CrawlURI expectedResult; - public TestData(CrawlURI uri, Link expectedResult) { + public TestData(CrawlURI uri, CrawlURI expectedResult) { this.uri = uri; this.expectedResult = expectedResult; } @@ -80,7 +80,7 @@ public abstract class StringExtractorTestBase extends ContentExtractorTestBase { Collection testDataCol = makeData(text, expectedURL); for (TestData testData: testDataCol) { extractor.process(testData.uri); - HashSet expected = new HashSet(); + HashSet expected = new HashSet(); if (testData.expectedResult != null) { expected.add(testData.expectedResult); } diff --git a/modules/src/main/java/org/archive/modules/fetcher/FetchFTP.java b/modules/src/main/java/org/archive/modules/fetcher/FetchFTP.java index 0cc1742d..b89ff729 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/FetchFTP.java +++ b/modules/src/main/java/org/archive/modules/fetcher/FetchFTP.java @@ -44,7 +44,6 @@ import org.archive.io.ReplayCharSequence; import org.archive.modules.CrawlURI; import org.archive.modules.Processor; import org.archive.modules.extractor.Hop; -import org.archive.modules.extractor.Link; import org.archive.modules.extractor.LinkContext; import org.archive.net.ClientFTP; import org.archive.net.UURI; @@ -573,7 +572,7 @@ public class FetchFTP extends Processor { } try { UURI n = UURIFactory.getInstance(base + "/" + file); - Link link = new Link(curi.getUURI(), n, LinkContext.NAVLINK_MISC, Hop.NAVLINK); + CrawlURI link = curi.createCrawlURI(n, LinkContext.NAVLINK_MISC, Hop.NAVLINK); curi.getOutLinks().add(link); } catch (URIException e) { logger.log(Level.WARNING, "URI error during extraction.", e); @@ -610,7 +609,7 @@ public class FetchFTP extends Processor { String path = uuri.getEscapedCurrentHierPath(); UURI parent = UURIFactory.getInstance(scheme + "://" + auth + path); - Link link = new Link(uuri, parent, LinkContext.NAVLINK_MISC, + CrawlURI link = curi.createCrawlURI(parent, LinkContext.NAVLINK_MISC, Hop.NAVLINK); curi.getOutLinks().add(link); } catch (URIException e) { diff --git a/modules/src/main/java/org/archive/modules/fetcher/FetchWhois.java b/modules/src/main/java/org/archive/modules/fetcher/FetchWhois.java index ef7add90..ca34a9f3 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/FetchWhois.java +++ b/modules/src/main/java/org/archive/modules/fetcher/FetchWhois.java @@ -37,8 +37,8 @@ import org.archive.modules.CoreAttributeConstants; import org.archive.modules.CrawlURI; import org.archive.modules.ProcessResult; import org.archive.modules.Processor; +import org.archive.modules.extractor.Extractor; import org.archive.modules.extractor.Hop; -import org.archive.modules.extractor.Link; import org.archive.modules.extractor.LinkContext; import org.archive.modules.net.CrawlHost; import org.archive.modules.net.ServerCache; @@ -427,7 +427,7 @@ public class FetchWhois extends Processor implements CoreAttributeConstants, protected void addWhoisLink(CrawlURI curi, String query) { String whoisUrl = "whois:" + query; try { - Link.add(curi, Integer.MAX_VALUE, whoisUrl, LinkContext.INFERRED_MISC, Hop.INFERRED); + Extractor.add(curi, Integer.MAX_VALUE, whoisUrl, LinkContext.INFERRED_MISC, Hop.INFERRED); } catch (URIException e) { logger.log(Level.WARNING, "problem with url " + whoisUrl, e); } diff --git a/modules/src/main/java/org/archive/modules/forms/FormLoginProcessor.java b/modules/src/main/java/org/archive/modules/forms/FormLoginProcessor.java index 27bad191..84f02b6e 100644 --- a/modules/src/main/java/org/archive/modules/forms/FormLoginProcessor.java +++ b/modules/src/main/java/org/archive/modules/forms/FormLoginProcessor.java @@ -261,7 +261,7 @@ public class FormLoginProcessor extends Processor implements Checkpointable { protected void createFormSubmissionAttempt(CrawlURI curi, HTMLForm templateForm, String formProvince) { LinkContext lc = new LinkContext.SimpleLinkContext("form/@action"); try { - CrawlURI submitCuri = curi.makeConsequentCandidate(templateForm.getAction(),lc, Hop.SUBMIT); + CrawlURI submitCuri = curi.createCrawlURI(templateForm.getAction(), lc, Hop.SUBMIT); submitCuri.setFetchType(FetchType.HTTP_POST); submitCuri.getData().put( CoreAttributeConstants.A_SUBMIT_DATA, @@ -270,7 +270,7 @@ public class FormLoginProcessor extends Processor implements Checkpointable { getLoginPassword())); submitCuri.setSchedulingDirective(SchedulingConstants.HIGH); submitCuri.setForceFetch(true); - curi.getOutCandidates().add(submitCuri); + curi.getOutLinks().add(submitCuri); curi.getAnnotations().add("submit:"+templateForm.getAction()); } catch (URIException ue) { loggerModule.logUriError(ue,curi.getUURI(),templateForm.getAction()); diff --git a/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java b/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java index c60c96b9..17424a37 100644 --- a/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java +++ b/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java @@ -88,7 +88,6 @@ import org.archive.modules.CrawlMetadata; import org.archive.modules.CrawlURI; import org.archive.modules.ProcessResult; import org.archive.modules.deciderules.recrawl.IdenticalDigestDecideRule; -import org.archive.modules.extractor.Link; import org.archive.spring.ConfigPath; import org.archive.uid.RecordIDGenerator; import org.archive.uid.UUIDGenerator; @@ -848,10 +847,10 @@ public class WARCWriterProcessor extends WriterPoolProcessor implements WARCWrit } // Add outlinks though they are effectively useless without anchor text. - Collection links = curi.getOutLinks(); + Collection links = curi.getOutLinks(); if (links != null && links.size() > 0) { - for (Link link: links) { - r.addLabelValue("outlink", link.toString()); + for (CrawlURI link: links) { + r.addLabelValue("outlink", link.getURI()); } } diff --git a/modules/src/test/java/org/archive/modules/extractor/ExtractorCSSTest.java b/modules/src/test/java/org/archive/modules/extractor/ExtractorCSSTest.java index f6775014..faa9c257 100644 --- a/modules/src/test/java/org/archive/modules/extractor/ExtractorCSSTest.java +++ b/modules/src/test/java/org/archive/modules/extractor/ExtractorCSSTest.java @@ -19,22 +19,17 @@ package org.archive.modules.extractor; +import static org.archive.modules.extractor.LinkContext.EMBED_MISC; +import static org.archive.modules.extractor.LinkContext.NAVLINK_MISC; + import java.util.Collection; import java.util.Collections; import org.archive.modules.CrawlURI; -import org.archive.modules.extractor.Extractor; -import org.archive.modules.extractor.ExtractorCSS; -import org.archive.modules.extractor.Hop; -import org.archive.modules.extractor.Link; -import org.archive.modules.extractor.StringExtractorTestBase; import org.archive.net.UURI; import org.archive.net.UURIFactory; import org.archive.util.Recorder; -import static org.archive.modules.extractor.LinkContext.EMBED_MISC; -import static org.archive.modules.extractor.LinkContext.NAVLINK_MISC; - /** * Unit test for ExtractorCSS. @@ -81,8 +76,9 @@ public class ExtractorCSSTest extends StringExtractorTestBase { euri.setRecorder(recorder); euri.setContentSize(content.length()); + // TODO: This test was naively modified to account for the abscense of LINK, but no effort was made to confirm that it is actually testing anything useful UURI dest = UURIFactory.getInstance(uri); - Link link = new Link(src, dest, EMBED_MISC, Hop.EMBED); + CrawlURI link = euri.createCrawlURI(dest, EMBED_MISC, Hop.EMBED); TestData td = new TestData(euri, link); return Collections.singleton(td); } diff --git a/modules/src/test/java/org/archive/modules/extractor/ExtractorHTMLTest.java b/modules/src/test/java/org/archive/modules/extractor/ExtractorHTMLTest.java index 7e5a21cf..4e2abbbd 100644 --- a/modules/src/test/java/org/archive/modules/extractor/ExtractorHTMLTest.java +++ b/modules/src/test/java/org/archive/modules/extractor/ExtractorHTMLTest.java @@ -107,7 +107,7 @@ public class ExtractorHTMLTest extends StringExtractorTestBase { UURI dest = UURIFactory.getInstance(destURI); LinkContext context = determineContext(content); Hop hop = determineHop(content); - Link link = new Link(src, dest, context, hop); + CrawlURI link = euri.createCrawlURI(dest, context, hop); result.add(new TestData(euri, link)); euri = new CrawlURI(src, null, null, LinkContext.NAVLINK_MISC); @@ -166,10 +166,10 @@ public class ExtractorHTMLTest extends StringExtractorTestBase { CrawlURI puri = new CrawlURI(UURIFactory .getInstance("http://www.example.com")); getExtractor().extract(puri, source); - Link[] links = puri.getOutLinks().toArray(new Link[0]); + CrawlURI[] links = puri.getOutLinks().toArray(new CrawlURI[0]); assertTrue("did not find single link",links.length==1); assertTrue("expected link not found", - links[0].getDestination().toString().equals(expected)); + links[0].getURI().equals(expected)); } /** @@ -188,9 +188,8 @@ public class ExtractorHTMLTest extends StringExtractorTestBase { "

"+ " "; getExtractor().extract(puri, cs); - Link[] links = puri.getOutLinks().toArray(new Link[0]); // find exactly 3 (not the POST) action URIs - assertTrue("incorrect number of links found",links.length==3); + assertTrue("incorrect number of links found", puri.getOutLinks().size()==3); } /** @@ -207,7 +206,7 @@ public class ExtractorHTMLTest extends StringExtractorTestBase { getExtractor().extract(puri, cs); assertEquals("meta robots content not extracted","index,nofollow", puri.getData().get(ExtractorHTML.A_META_ROBOTS)); - Link[] links = puri.getOutLinks().toArray(new Link[0]); + CrawlURI[] links = puri.getOutLinks().toArray(new CrawlURI[0]); assertTrue("link extracted despite meta robots",links.length==0); } @@ -228,9 +227,8 @@ public class ExtractorHTMLTest extends StringExtractorTestBase { assertTrue(CollectionUtils.exists(curi.getOutLinks(), new Predicate() { public boolean evaluate(Object object) { - return ((Link) object) - .getDestination() - .toString() + return ((CrawlURI) object) + .getURI() .indexOf( "/example.html;jsessionid=deadbeef:deadbeed?parameter=this:value") >= 0; } @@ -238,7 +236,7 @@ public class ExtractorHTMLTest extends StringExtractorTestBase { assertTrue(CollectionUtils.exists(curi.getOutLinks(), new Predicate() { public boolean evaluate(Object object) { - return ((Link) object).getDestination().toString().indexOf( + return ((CrawlURI) object).getURI().indexOf( "/example.html?parameter=this:value") >= 0; } })); @@ -262,15 +260,15 @@ public class ExtractorHTMLTest extends StringExtractorTestBase { assertTrue(CollectionUtils.exists(curi.getOutLinks(), new Predicate() { public boolean evaluate(Object object) { System.err.println("comparing: " - + ((Link) object).getDestination().toString() + + ((CrawlURI) object).getURI() + " and https://www.anotherexample.com/"); - return ((Link) object).getDestination().toString().equals( + return ((CrawlURI) object).getURI().equals( "http://www.anotherexample.com/"); } })); assertTrue(CollectionUtils.exists(curi.getOutLinks(), new Predicate() { public boolean evaluate(Object object) { - return ((Link) object).getDestination().toString().equals( + return ((CrawlURI) object).getURI().equals( "https://www.example.com/index.html"); } })); @@ -310,21 +308,21 @@ public class ExtractorHTMLTest extends StringExtractorTestBase { "" + ""; getExtractor().extract(puri, cs); - Link[] links = puri.getOutLinks().toArray(new Link[0]); + CrawlURI[] links = puri.getOutLinks().toArray(new CrawlURI[0]); Arrays.sort(links); String dest1 = "http://www.example.com/def/another1.html"; String dest2 = "http://www.example.com/ghi/another2.html"; // ensure outlink from base href assertEquals("outlink1 from base href",dest1, - links[1].getDestination().toString()); + links[1].getURI()); assertEquals("outlink2 from base href",dest2, - links[2].getDestination().toString()); + links[2].getURI()); } protected Predicate destinationContainsPredicate(final String fragment) { return new Predicate() { public boolean evaluate(Object object) { - return ((Link) object).getDestination().toString().indexOf(fragment) >= 0; + return ((CrawlURI) object).getURI().indexOf(fragment) >= 0; } }; } @@ -332,7 +330,7 @@ public class ExtractorHTMLTest extends StringExtractorTestBase { protected Predicate destinationsIsPredicate(final String value) { return new Predicate() { public boolean evaluate(Object object) { - return ((Link) object).getDestination().toString().equals(value); + return ((CrawlURI) object).getURI().equals(value); } }; } @@ -397,16 +395,16 @@ public class ExtractorHTMLTest extends StringExtractorTestBase { getExtractor().extract(curi, cs); - Link[] links = curi.getOutLinks().toArray(new Link[0]); + CrawlURI[] links = curi.getOutLinks().toArray(new CrawlURI[0]); Arrays.sort(links); String dest1 = "http://www.example.com/foo.gif"; String dest2 = "http://www.example.com/foo.js"; assertEquals("outlink1 from conditional comment img src",dest1, - links[0].getDestination().toString()); + links[0].getURI()); assertEquals("outlink2 from conditional comment script src",dest2, - links[1].getDestination().toString()); + links[1].getURI()); } diff --git a/modules/src/test/java/org/archive/modules/extractor/ExtractorJSTest.java b/modules/src/test/java/org/archive/modules/extractor/ExtractorJSTest.java index 755c4c34..cc095078 100644 --- a/modules/src/test/java/org/archive/modules/extractor/ExtractorJSTest.java +++ b/modules/src/test/java/org/archive/modules/extractor/ExtractorJSTest.java @@ -141,7 +141,7 @@ public class ExtractorJSTest extends StringExtractorTestBase { if (destURI != null) { UURI dest = UURIFactory.getInstance(destURI); - Link link = new Link(src, dest, LinkContext.JS_MISC, Hop.SPECULATIVE); + CrawlURI link = euri.createCrawlURI(dest, LinkContext.JS_MISC, Hop.SPECULATIVE); result.add(new TestData(euri, link)); } else { result.add(new TestData(euri, null)); diff --git a/modules/src/test/java/org/archive/modules/extractor/ExtractorMultipleRegexTest.java b/modules/src/test/java/org/archive/modules/extractor/ExtractorMultipleRegexTest.java index 308379d7..e65b4e27 100644 --- a/modules/src/test/java/org/archive/modules/extractor/ExtractorMultipleRegexTest.java +++ b/modules/src/test/java/org/archive/modules/extractor/ExtractorMultipleRegexTest.java @@ -188,7 +188,7 @@ public class ExtractorMultipleRegexTest extends ContentExtractorTestBase { extractor.process(testUri); for (String expectedLinkString: EXPECTED_OUTLINKS) { - Link expectedLink = new Link(testUri.getUURI(), + CrawlURI expectedLink = testUri.createCrawlURI( UURIFactory.getInstance(expectedLinkString), HTMLLinkContext.INFERRED_MISC, Hop.INFERRED); assertTrue(testUri.getOutLinks().contains(expectedLink)); diff --git a/modules/src/test/java/org/archive/modules/extractor/ExtractorSWFTest.java b/modules/src/test/java/org/archive/modules/extractor/ExtractorSWFTest.java index 9ed57b08..df1713d5 100644 --- a/modules/src/test/java/org/archive/modules/extractor/ExtractorSWFTest.java +++ b/modules/src/test/java/org/archive/modules/extractor/ExtractorSWFTest.java @@ -109,9 +109,9 @@ public class ExtractorSWFTest extends ContentExtractorTestBase { + elapsed + "ms to process " + url); boolean foundIt = false; - for (Link link : curi.getOutLinks()) { + for (CrawlURI link : curi.getOutLinks()) { logger.info("found link: " + link); - foundIt = foundIt || link.getDestination().toString().endsWith(testUrls.get(url)); + foundIt = foundIt || link.getURI().endsWith(testUrls.get(url)); } assertTrue("failed to extract link \"" + testUrls.get(url) @@ -165,9 +165,9 @@ public class ExtractorSWFTest extends ContentExtractorTestBase { + elapsed + "ms to process " + url); boolean foundIt = false; - for (Link link : curi.getOutLinks()) { + for (CrawlURI link : curi.getOutLinks()) { logger.info("found link: " + link); - foundIt = foundIt || link.getDestination().toString().endsWith(testUrls.get(url)); + foundIt = foundIt || link.getURI().endsWith(testUrls.get(url)); } if (!foundIt) @@ -198,11 +198,11 @@ public class ExtractorSWFTest extends ContentExtractorTestBase { expected.put("http://nyumedecs.kk5.org/containermain.swf", "extractorSWFRelToVia"); expected.put("https://wayback.archive-it.org/3771/20131119163257/http://nyumedecs.kk5.org/_app/28727/en/resources/containermain.swf", "extractorSWFRelToBase"); - for (Link link: curi.getOutLinks()) { + for (CrawlURI link: curi.getOutLinks()) { System.out.println(link + " " + link.getData()); assertEquals(1, link.getAnnotations().size()); - String dest = link.getDestination().toString(); + String dest = link.toString(); assertTrue(expected.containsKey(dest)); // remove the entry, so at the end the map should be empty, confirming that we found all the expected links diff --git a/modules/src/test/java/org/archive/modules/extractor/JerichoExtractorHTMLTest.java b/modules/src/test/java/org/archive/modules/extractor/JerichoExtractorHTMLTest.java index a4d17345..c6bcee6b 100644 --- a/modules/src/test/java/org/archive/modules/extractor/JerichoExtractorHTMLTest.java +++ b/modules/src/test/java/org/archive/modules/extractor/JerichoExtractorHTMLTest.java @@ -73,7 +73,7 @@ public class JerichoExtractorHTMLTest extends ExtractorHTMLTest { curi.getOutLinks(); assertTrue(CollectionUtils.exists(curi.getOutLinks(), new Predicate() { public boolean evaluate(Object object) { - return ((Link) object).getDestination().toString().indexOf( + return ((CrawlURI) object).getURI().indexOf( "/redirect_me?form=true&checked[]=1&unchecked[]=&selectBox=selectedOption&test=Go")>=0; } })); @@ -101,7 +101,7 @@ public class JerichoExtractorHTMLTest extends ExtractorHTMLTest { curi.getOutLinks(); assertTrue(!CollectionUtils.exists(curi.getOutLinks(), new Predicate() { public boolean evaluate(Object object) { - return ((Link) object).getDestination().toString().indexOf( + return ((CrawlURI) object).getURI().indexOf( "/redirect_me?form=true&checked[]=1&unchecked[]=&selectBox=selectedOption&test=Go")>=0; } })); @@ -130,7 +130,7 @@ public class JerichoExtractorHTMLTest extends ExtractorHTMLTest { curi.getOutLinks(); assertTrue(CollectionUtils.exists(curi.getOutLinks(), new Predicate() { public boolean evaluate(Object object) { - return ((Link) object).getDestination().toString().indexOf( + return ((CrawlURI) object).getURI().indexOf( "/redirect_me?form=true&checked[]=1&unchecked[]=&selectBox=selectedOption&test=Go")>=0; } })); diff --git a/modules/src/test/java/org/archive/modules/forms/FormLoginProcessorTest.java b/modules/src/test/java/org/archive/modules/forms/FormLoginProcessorTest.java index b1d3a5c4..0fc297b5 100644 --- a/modules/src/test/java/org/archive/modules/forms/FormLoginProcessorTest.java +++ b/modules/src/test/java/org/archive/modules/forms/FormLoginProcessorTest.java @@ -62,8 +62,8 @@ public class FormLoginProcessorTest extends ProcessorTestBase { assertEquals("WARC-Simple-Form-Province-Status: 0,0,http://(com,example,)", curi.getDataList(A_WARC_RESPONSE_HEADERS).get(0)); assertTrue(curi.getAnnotations().contains("submit:/login")); - assertEquals(1, curi.getOutCandidates().size()); - CrawlURI submitCuri = curi.getOutCandidates().toArray(new CrawlURI[0])[0]; + assertEquals(1, curi.getOutLinks().size()); + CrawlURI submitCuri = curi.getOutLinks().toArray(new CrawlURI[0])[0]; assertEquals("http://example.com/login", submitCuri.toString()); assertEquals(FetchType.HTTP_POST, submitCuri.getFetchType()); String queryString = (String) submitCuri.getData().get(CoreAttributeConstants.A_SUBMIT_DATA);