From 753404f662446a1fbaf1823b164db960c587539c Mon Sep 17 00:00:00 2001 From: Adam Miller Date: Mon, 3 Mar 2014 16:51:11 -0800 Subject: [PATCH 1/5] Adding ViaSurtPrefixedDecideRule. Returns true if a surtform 'via' matches any surt in a specified list --- .../ViaSurtPrefixedDecideRule.java | 68 +++++++++++ .../ViaSurtPrefixedDecideRuleTest.java | 108 ++++++++++++++++++ 2 files changed, 176 insertions(+) create mode 100644 modules/src/main/java/org/archive/modules/deciderules/ViaSurtPrefixedDecideRule.java create mode 100644 modules/src/test/java/org/archive/modules/deciderules/ViaSurtPrefixedDecideRuleTest.java diff --git a/modules/src/main/java/org/archive/modules/deciderules/ViaSurtPrefixedDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/ViaSurtPrefixedDecideRule.java new file mode 100644 index 00000000..238aaf27 --- /dev/null +++ b/modules/src/main/java/org/archive/modules/deciderules/ViaSurtPrefixedDecideRule.java @@ -0,0 +1,68 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.modules.deciderules; + +import java.util.List; + +import org.archive.modules.CrawlURI; +import org.springframework.beans.factory.annotation.Required; + +/** + * Rule applies the configured decision for any URI which has a 'via' whose + * surtform matches any surt specified in the surtPrefixes list + * + * + * @author adam-miller + */ +public class ViaSurtPrefixedDecideRule extends PredicatedDecideRule { + + private static final long serialVersionUID = 3L; + + protected List surtPrefixes; + + public List getSurtPrefixes() { + return surtPrefixes; + } + @Required + public void setSurtPrefixes(List surtPrefixes) { + this.surtPrefixes = surtPrefixes; + } + + /** + * Evaluate whether given object's surt form + * matches one of the supplied surts + * + * @param object + * @return true if a surt prefix matches + */ + @Override + protected boolean evaluate(CrawlURI uri) { + if (uri.getVia() != null && getSurtPrefixes() !=null){ + for(String surt : getSurtPrefixes()){ + if(uri.getVia().getSurtForm().startsWith(surt)) + return true; + } + return false; + } + else + return false; + } + +} diff --git a/modules/src/test/java/org/archive/modules/deciderules/ViaSurtPrefixedDecideRuleTest.java b/modules/src/test/java/org/archive/modules/deciderules/ViaSurtPrefixedDecideRuleTest.java new file mode 100644 index 00000000..d0ec2924 --- /dev/null +++ b/modules/src/test/java/org/archive/modules/deciderules/ViaSurtPrefixedDecideRuleTest.java @@ -0,0 +1,108 @@ +package org.archive.modules.deciderules; + + +import java.util.ArrayList; +import java.util.List; + +import org.apache.commons.httpclient.URIException; +import org.archive.modules.CrawlURI; + +import org.archive.modules.extractor.LinkContext; +import org.archive.net.UURI; +import org.archive.net.UURIFactory; +import org.archive.state.ModuleTestBase; + +public class ViaSurtPrefixedDecideRuleTest extends ModuleTestBase { + + public void testNoVia() throws Exception { + ViaSurtPrefixedDecideRule dr = new ViaSurtPrefixedDecideRule(); + List surtPrefixes = new ArrayList(); + surtPrefixes.add("http://(org,archive,"); + dr.setSurtPrefixes(surtPrefixes); + CrawlURI testUri = createTestUri("http://example.com"); + + assertFalse(dr.evaluate(testUri)); + } + public void testNoSurts() throws Exception { + ViaSurtPrefixedDecideRule dr = new ViaSurtPrefixedDecideRule(); + List surtPrefixes = new ArrayList(); + dr.setSurtPrefixes(surtPrefixes); + CrawlURI testUri = createTestUri("http://example.com"); + + assertFalse(dr.evaluate(testUri)); + } + public void testNullSurts() throws Exception { + ViaSurtPrefixedDecideRule dr = new ViaSurtPrefixedDecideRule(); + dr.setSurtPrefixes(null); + CrawlURI testUri = createTestUri("http://example.com"); + + assertFalse(dr.evaluate(testUri)); + } + public void testPositiveSingleSurt() throws Exception { + ViaSurtPrefixedDecideRule dr = new ViaSurtPrefixedDecideRule(); + List surtPrefixes = new ArrayList(); + surtPrefixes.add("http://(org,archive,"); + dr.setSurtPrefixes(surtPrefixes); + CrawlURI testUri = createTestUri("http://example.com","http://archive.org"); + + assertTrue(dr.evaluate(testUri)); + } + public void testNegativeSingleSurt() throws Exception { + ViaSurtPrefixedDecideRule dr = new ViaSurtPrefixedDecideRule(); + List surtPrefixes = new ArrayList(); + surtPrefixes.add("http://(org,archive,"); + dr.setSurtPrefixes(surtPrefixes); + CrawlURI testUri = createTestUri("http://example.com","http://google.com"); + + assertFalse(dr.evaluate(testUri)); + } + public void testPositiveMultipleSurts() throws Exception { + ViaSurtPrefixedDecideRule dr = new ViaSurtPrefixedDecideRule(); + List surtPrefixes = new ArrayList(); + surtPrefixes.add("http://(org,archive,"); + surtPrefixes.add("http://(com,test,"); + surtPrefixes.add("http://(com,google,"); + dr.setSurtPrefixes(surtPrefixes); + CrawlURI testUri = createTestUri("http://example.com","http://google.com"); + + assertTrue(dr.evaluate(testUri)); + } + public void testPositiveMultipleSurts2() throws Exception { + ViaSurtPrefixedDecideRule dr = new ViaSurtPrefixedDecideRule(); + List surtPrefixes = new ArrayList(); + surtPrefixes.add("http://(org,archive,"); + surtPrefixes.add("http://(com,google,"); + surtPrefixes.add("http://(com,test,"); + dr.setSurtPrefixes(surtPrefixes); + CrawlURI testUri = createTestUri("http://example.com","http://google.com"); + + assertTrue(dr.evaluate(testUri)); + } + public void testNegativeMultipleSurts() throws Exception { + ViaSurtPrefixedDecideRule dr = new ViaSurtPrefixedDecideRule(); + List surtPrefixes = new ArrayList(); + surtPrefixes.add("http://(org,archive,"); + surtPrefixes.add("http://(com,test,"); + surtPrefixes.add("http://(com,google,"); + dr.setSurtPrefixes(surtPrefixes); + CrawlURI testUri = createTestUri("http://example.com","http://negativeexample.com"); + + assertFalse(dr.evaluate(testUri)); + } + + + private CrawlURI createTestUri(String urlStr) throws URIException { + UURI testUuri = UURIFactory.getInstance(urlStr); + CrawlURI testUri = new CrawlURI(testUuri, null, null, LinkContext.NAVLINK_MISC); + + return testUri; + } + private CrawlURI createTestUri(String urlStr, String via) throws URIException { + + UURI testViaUuri = UURIFactory.getInstance(via); + CrawlURI testUri = createTestUri(urlStr); + testUri.setVia(testViaUuri); + + return testUri; + } +} From f2bebd41b013284e131ad40123a5a8c933e4a95d Mon Sep 17 00:00:00 2001 From: Adam Miller Date: Thu, 1 May 2014 12:36:05 -0700 Subject: [PATCH 2/5] Cleanup / Optimization --- .../deciderules/ViaSurtPrefixedDecideRule.java | 17 +++++++++-------- .../ViaSurtPrefixedDecideRuleTest.java | 3 +-- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/modules/src/main/java/org/archive/modules/deciderules/ViaSurtPrefixedDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/ViaSurtPrefixedDecideRule.java index 238aaf27..53f3023a 100644 --- a/modules/src/main/java/org/archive/modules/deciderules/ViaSurtPrefixedDecideRule.java +++ b/modules/src/main/java/org/archive/modules/deciderules/ViaSurtPrefixedDecideRule.java @@ -19,9 +19,11 @@ package org.archive.modules.deciderules; +import java.util.ArrayList; import java.util.List; import org.archive.modules.CrawlURI; +import org.archive.util.SurtPrefixSet; import org.springframework.beans.factory.annotation.Required; /** @@ -35,14 +37,17 @@ public class ViaSurtPrefixedDecideRule extends PredicatedDecideRule { private static final long serialVersionUID = 3L; - protected List surtPrefixes; + protected SurtPrefixSet surtPrefixes = new SurtPrefixSet(); public List getSurtPrefixes() { - return surtPrefixes; + return new ArrayList(surtPrefixes); } @Required public void setSurtPrefixes(List surtPrefixes) { - this.surtPrefixes = surtPrefixes; + this.surtPrefixes.clear(); + if(surtPrefixes!=null) + for(String surt : surtPrefixes) + this.surtPrefixes.considerAsAddDirective(surt); } /** @@ -55,11 +60,7 @@ public class ViaSurtPrefixedDecideRule extends PredicatedDecideRule { @Override protected boolean evaluate(CrawlURI uri) { if (uri.getVia() != null && getSurtPrefixes() !=null){ - for(String surt : getSurtPrefixes()){ - if(uri.getVia().getSurtForm().startsWith(surt)) - return true; - } - return false; + return surtPrefixes.containsPrefixOf(SurtPrefixSet.getCandidateSurt(uri.getVia())); } else return false; diff --git a/modules/src/test/java/org/archive/modules/deciderules/ViaSurtPrefixedDecideRuleTest.java b/modules/src/test/java/org/archive/modules/deciderules/ViaSurtPrefixedDecideRuleTest.java index d0ec2924..6a520f7a 100644 --- a/modules/src/test/java/org/archive/modules/deciderules/ViaSurtPrefixedDecideRuleTest.java +++ b/modules/src/test/java/org/archive/modules/deciderules/ViaSurtPrefixedDecideRuleTest.java @@ -6,7 +6,6 @@ import java.util.List; import org.apache.commons.httpclient.URIException; import org.archive.modules.CrawlURI; - import org.archive.modules.extractor.LinkContext; import org.archive.net.UURI; import org.archive.net.UURIFactory; @@ -20,7 +19,7 @@ public class ViaSurtPrefixedDecideRuleTest extends ModuleTestBase { surtPrefixes.add("http://(org,archive,"); dr.setSurtPrefixes(surtPrefixes); CrawlURI testUri = createTestUri("http://example.com"); - + assertFalse(dr.evaluate(testUri)); } public void testNoSurts() throws Exception { From ae878d29cf01e56f4ba126043f6e4a46ba87e05d Mon Sep 17 00:00:00 2001 From: Adam Miller Date: Thu, 1 May 2014 15:22:16 -0700 Subject: [PATCH 3/5] Code cleanup --- .../modules/deciderules/ViaSurtPrefixedDecideRule.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/modules/src/main/java/org/archive/modules/deciderules/ViaSurtPrefixedDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/ViaSurtPrefixedDecideRule.java index 53f3023a..4c5e9d4c 100644 --- a/modules/src/main/java/org/archive/modules/deciderules/ViaSurtPrefixedDecideRule.java +++ b/modules/src/main/java/org/archive/modules/deciderules/ViaSurtPrefixedDecideRule.java @@ -35,7 +35,7 @@ import org.springframework.beans.factory.annotation.Required; */ public class ViaSurtPrefixedDecideRule extends PredicatedDecideRule { - private static final long serialVersionUID = 3L; + private static final long serialVersionUID = 1L; protected SurtPrefixSet surtPrefixes = new SurtPrefixSet(); @@ -45,9 +45,12 @@ public class ViaSurtPrefixedDecideRule extends PredicatedDecideRule { @Required public void setSurtPrefixes(List surtPrefixes) { this.surtPrefixes.clear(); - if(surtPrefixes!=null) - for(String surt : surtPrefixes) + + if(surtPrefixes!=null) { + for(String surt : surtPrefixes) { this.surtPrefixes.considerAsAddDirective(surt); + } + } } /** From 949c9f086caf2832f3fc81e48dd34c8aea3c108c Mon Sep 17 00:00:00 2001 From: Noah Levitt Date: Fri, 2 May 2014 15:42:29 -0700 Subject: [PATCH 4/5] shut down the starter-restarter thread at crawl finish!! --- .../crawler/frontier/AMQPUrlReceiver.java | 23 ++++++++++++++----- 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/contrib/src/main/java/org/archive/crawler/frontier/AMQPUrlReceiver.java b/contrib/src/main/java/org/archive/crawler/frontier/AMQPUrlReceiver.java index 4074a842..112d9d8c 100644 --- a/contrib/src/main/java/org/archive/crawler/frontier/AMQPUrlReceiver.java +++ b/contrib/src/main/java/org/archive/crawler/frontier/AMQPUrlReceiver.java @@ -112,9 +112,9 @@ public class AMQPUrlReceiver implements Lifecycle, ApplicationListener Date: Wed, 7 May 2014 10:01:47 -0700 Subject: [PATCH 5/5] oops, make "exit java process" button work again --- .../main/java/org/archive/crawler/restlet/EngineResource.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engine/src/main/java/org/archive/crawler/restlet/EngineResource.java b/engine/src/main/java/org/archive/crawler/restlet/EngineResource.java index 1b482788..495aec87 100644 --- a/engine/src/main/java/org/archive/crawler/restlet/EngineResource.java +++ b/engine/src/main/java/org/archive/crawler/restlet/EngineResource.java @@ -154,7 +154,7 @@ public class EngineResource extends BaseResource { } } } - } else if ("Exit Java Process".equals(action)) { + } else if ("exit java process".equals(action)) { boolean cancel = false; if(!"on".equals(form.getFirstValue("im_sure"))) { Flash.addFlash(