diff --git a/contrib/src/main/java/org/archive/crawler/frontier/AMQPUrlReceiver.java b/contrib/src/main/java/org/archive/crawler/frontier/AMQPUrlReceiver.java index 4074a842..112d9d8c 100644 --- a/contrib/src/main/java/org/archive/crawler/frontier/AMQPUrlReceiver.java +++ b/contrib/src/main/java/org/archive/crawler/frontier/AMQPUrlReceiver.java @@ -112,9 +112,9 @@ public class AMQPUrlReceiver implements Lifecycle, ApplicationListener getSurtPrefixes() { + return new ArrayList(surtPrefixes); + } + @Required + public void setSurtPrefixes(List surtPrefixes) { + this.surtPrefixes.clear(); + + if(surtPrefixes!=null) { + for(String surt : surtPrefixes) { + this.surtPrefixes.considerAsAddDirective(surt); + } + } + } + + /** + * Evaluate whether given object's surt form + * matches one of the supplied surts + * + * @param object + * @return true if a surt prefix matches + */ + @Override + protected boolean evaluate(CrawlURI uri) { + if (uri.getVia() != null && getSurtPrefixes() !=null){ + return surtPrefixes.containsPrefixOf(SurtPrefixSet.getCandidateSurt(uri.getVia())); + } + else + return false; + } + +} diff --git a/modules/src/test/java/org/archive/modules/deciderules/ViaSurtPrefixedDecideRuleTest.java b/modules/src/test/java/org/archive/modules/deciderules/ViaSurtPrefixedDecideRuleTest.java new file mode 100644 index 00000000..6a520f7a --- /dev/null +++ b/modules/src/test/java/org/archive/modules/deciderules/ViaSurtPrefixedDecideRuleTest.java @@ -0,0 +1,107 @@ +package org.archive.modules.deciderules; + + +import java.util.ArrayList; +import java.util.List; + +import org.apache.commons.httpclient.URIException; +import org.archive.modules.CrawlURI; +import org.archive.modules.extractor.LinkContext; +import org.archive.net.UURI; +import org.archive.net.UURIFactory; +import org.archive.state.ModuleTestBase; + +public class ViaSurtPrefixedDecideRuleTest extends ModuleTestBase { + + public void testNoVia() throws Exception { + ViaSurtPrefixedDecideRule dr = new ViaSurtPrefixedDecideRule(); + List surtPrefixes = new ArrayList(); + surtPrefixes.add("http://(org,archive,"); + dr.setSurtPrefixes(surtPrefixes); + CrawlURI testUri = createTestUri("http://example.com"); + + assertFalse(dr.evaluate(testUri)); + } + public void testNoSurts() throws Exception { + ViaSurtPrefixedDecideRule dr = new ViaSurtPrefixedDecideRule(); + List surtPrefixes = new ArrayList(); + dr.setSurtPrefixes(surtPrefixes); + CrawlURI testUri = createTestUri("http://example.com"); + + assertFalse(dr.evaluate(testUri)); + } + public void testNullSurts() throws Exception { + ViaSurtPrefixedDecideRule dr = new ViaSurtPrefixedDecideRule(); + dr.setSurtPrefixes(null); + CrawlURI testUri = createTestUri("http://example.com"); + + assertFalse(dr.evaluate(testUri)); + } + public void testPositiveSingleSurt() throws Exception { + ViaSurtPrefixedDecideRule dr = new ViaSurtPrefixedDecideRule(); + List surtPrefixes = new ArrayList(); + surtPrefixes.add("http://(org,archive,"); + dr.setSurtPrefixes(surtPrefixes); + CrawlURI testUri = createTestUri("http://example.com","http://archive.org"); + + assertTrue(dr.evaluate(testUri)); + } + public void testNegativeSingleSurt() throws Exception { + ViaSurtPrefixedDecideRule dr = new ViaSurtPrefixedDecideRule(); + List surtPrefixes = new ArrayList(); + surtPrefixes.add("http://(org,archive,"); + dr.setSurtPrefixes(surtPrefixes); + CrawlURI testUri = createTestUri("http://example.com","http://google.com"); + + assertFalse(dr.evaluate(testUri)); + } + public void testPositiveMultipleSurts() throws Exception { + ViaSurtPrefixedDecideRule dr = new ViaSurtPrefixedDecideRule(); + List surtPrefixes = new ArrayList(); + surtPrefixes.add("http://(org,archive,"); + surtPrefixes.add("http://(com,test,"); + surtPrefixes.add("http://(com,google,"); + dr.setSurtPrefixes(surtPrefixes); + CrawlURI testUri = createTestUri("http://example.com","http://google.com"); + + assertTrue(dr.evaluate(testUri)); + } + public void testPositiveMultipleSurts2() throws Exception { + ViaSurtPrefixedDecideRule dr = new ViaSurtPrefixedDecideRule(); + List surtPrefixes = new ArrayList(); + surtPrefixes.add("http://(org,archive,"); + surtPrefixes.add("http://(com,google,"); + surtPrefixes.add("http://(com,test,"); + dr.setSurtPrefixes(surtPrefixes); + CrawlURI testUri = createTestUri("http://example.com","http://google.com"); + + assertTrue(dr.evaluate(testUri)); + } + public void testNegativeMultipleSurts() throws Exception { + ViaSurtPrefixedDecideRule dr = new ViaSurtPrefixedDecideRule(); + List surtPrefixes = new ArrayList(); + surtPrefixes.add("http://(org,archive,"); + surtPrefixes.add("http://(com,test,"); + surtPrefixes.add("http://(com,google,"); + dr.setSurtPrefixes(surtPrefixes); + CrawlURI testUri = createTestUri("http://example.com","http://negativeexample.com"); + + assertFalse(dr.evaluate(testUri)); + } + + + private CrawlURI createTestUri(String urlStr) throws URIException { + UURI testUuri = UURIFactory.getInstance(urlStr); + CrawlURI testUri = new CrawlURI(testUuri, null, null, LinkContext.NAVLINK_MISC); + + return testUri; + } + private CrawlURI createTestUri(String urlStr, String via) throws URIException { + + UURI testViaUuri = UURIFactory.getInstance(via); + CrawlURI testUri = createTestUri(urlStr); + testUri.setVia(testViaUuri); + + return testUri; + } +}