lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-09-21 13:16:03 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' of git://github.com/internetarchive/heritrix3 into ExtractorPDFContent

Browse Source
This commit is contained in:
Adam Miller
2014-05-13 14:04:29 -07:00
parent fb42c07c92 ab7a60f9e3
commit aaecd6d78c
4 changed files with 197 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
contrib/src/main/java/org/archive/crawler/frontier/AMQPUrlReceiver.java
+17 -6
View File
@@ -112,9 +112,9 @@ public class AMQPUrlReceiver implements Lifecycle, ApplicationListener<CrawlStat
@Override
public void run() {
while (true) {
synchronized (AMQPUrlReceiver.this) {
if (!isRunning) {
while (!Thread.interrupted()) {
if (!isRunning) {
synchronized (AMQPUrlReceiver.this) {
try {
Consumer consumer = new UrlConsumer(channel());
channel.queueDeclare(getQueueName(), false, false, true, null);
@@ -130,18 +130,21 @@ public class AMQPUrlReceiver implements Lifecycle, ApplicationListener<CrawlStat
try {
Thread.sleep(30000);
} catch (InterruptedException e1) {
} catch (InterruptedException e) {
return;
}
}
}
}
transient private StarterRestarter starterRestarter;
@Override
synchronized public void start() {
// spawn off a thread to start up the amqp consumer, and try to restart it if it dies
if (!isRunning) {
StarterRestarter t = new StarterRestarter(AMQPUrlReceiver.class.getSimpleName() + "-starter-restarter");
t.start();
starterRestarter = new StarterRestarter(AMQPUrlReceiver.class.getSimpleName() + "-starter-restarter");
starterRestarter.start();
}
}
@@ -155,6 +158,14 @@ public class AMQPUrlReceiver implements Lifecycle, ApplicationListener<CrawlStat
logger.log(Level.SEVERE, "problem closing AMQP connection", e);
}
}
if (starterRestarter != null && starterRestarter.isAlive()) {
starterRestarter.interrupt();
try {
starterRestarter.join();
} catch (InterruptedException e) {
}
}
starterRestarter = null;
connection = null;
channel = null;
isRunning = false;
engine/src/main/java/org/archive/crawler/restlet/EngineResource.java
+1 -1
View File
@@ -154,7 +154,7 @@ public class EngineResource extends BaseResource {
}
}
}
} else if ("Exit Java Process".equals(action)) {
} else if ("exit java process".equals(action)) {
boolean cancel = false;
if(!"on".equals(form.getFirstValue("im_sure"))) {
Flash.addFlash(
modules/src/main/java/org/archive/modules/deciderules/ViaSurtPrefixedDecideRule.java
+72
View File
@@ -0,0 +1,72 @@
/*
* This file is part of the Heritrix web crawler (crawler.archive.org).
*
* Licensed to the Internet Archive (IA) by one or more individual
* contributors.
*
* The IA licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.archive.modules.deciderules;
import java.util.ArrayList;
import java.util.List;
import org.archive.modules.CrawlURI;
import org.archive.util.SurtPrefixSet;
import org.springframework.beans.factory.annotation.Required;
/**
* Rule applies the configured decision for any URI which has a 'via' whose
* surtform matches any surt specified in the surtPrefixes list
*
*
* @author adam-miller
*/
public class ViaSurtPrefixedDecideRule extends PredicatedDecideRule {
private static final long serialVersionUID = 1L;
protected SurtPrefixSet surtPrefixes = new SurtPrefixSet();
public List<String> getSurtPrefixes() {
return new ArrayList<String>(surtPrefixes);
}
@Required
public void setSurtPrefixes(List<String> surtPrefixes) {
this.surtPrefixes.clear();
if(surtPrefixes!=null) {
for(String surt : surtPrefixes) {
this.surtPrefixes.considerAsAddDirective(surt);
}
}
}
/**
* Evaluate whether given object's surt form
* matches one of the supplied surts
*
* @param object
* @return true if a surt prefix matches
*/
@Override
protected boolean evaluate(CrawlURI uri) {
if (uri.getVia() != null && getSurtPrefixes() !=null){
return surtPrefixes.containsPrefixOf(SurtPrefixSet.getCandidateSurt(uri.getVia()));
}
else
return false;
}
}
modules/src/test/java/org/archive/modules/deciderules/ViaSurtPrefixedDecideRuleTest.java
+107
View File
@@ -0,0 +1,107 @@
package org.archive.modules.deciderules;
import java.util.ArrayList;
import java.util.List;
import org.apache.commons.httpclient.URIException;
import org.archive.modules.CrawlURI;
import org.archive.modules.extractor.LinkContext;
import org.archive.net.UURI;
import org.archive.net.UURIFactory;
import org.archive.state.ModuleTestBase;
public class ViaSurtPrefixedDecideRuleTest extends ModuleTestBase {
public void testNoVia() throws Exception {
ViaSurtPrefixedDecideRule dr = new ViaSurtPrefixedDecideRule();
List<String> surtPrefixes = new ArrayList<String>();
surtPrefixes.add("http://(org,archive,");
dr.setSurtPrefixes(surtPrefixes);
CrawlURI testUri = createTestUri("http://example.com");
assertFalse(dr.evaluate(testUri));
}
public void testNoSurts() throws Exception {
ViaSurtPrefixedDecideRule dr = new ViaSurtPrefixedDecideRule();
List<String> surtPrefixes = new ArrayList<String>();
dr.setSurtPrefixes(surtPrefixes);
CrawlURI testUri = createTestUri("http://example.com");
assertFalse(dr.evaluate(testUri));
}
public void testNullSurts() throws Exception {
ViaSurtPrefixedDecideRule dr = new ViaSurtPrefixedDecideRule();
dr.setSurtPrefixes(null);
CrawlURI testUri = createTestUri("http://example.com");
assertFalse(dr.evaluate(testUri));
}
public void testPositiveSingleSurt() throws Exception {
ViaSurtPrefixedDecideRule dr = new ViaSurtPrefixedDecideRule();
List<String> surtPrefixes = new ArrayList<String>();
surtPrefixes.add("http://(org,archive,");
dr.setSurtPrefixes(surtPrefixes);
CrawlURI testUri = createTestUri("http://example.com","http://archive.org");
assertTrue(dr.evaluate(testUri));
}
public void testNegativeSingleSurt() throws Exception {
ViaSurtPrefixedDecideRule dr = new ViaSurtPrefixedDecideRule();
List<String> surtPrefixes = new ArrayList<String>();
surtPrefixes.add("http://(org,archive,");
dr.setSurtPrefixes(surtPrefixes);
CrawlURI testUri = createTestUri("http://example.com","http://google.com");
assertFalse(dr.evaluate(testUri));
}
public void testPositiveMultipleSurts() throws Exception {
ViaSurtPrefixedDecideRule dr = new ViaSurtPrefixedDecideRule();
List<String> surtPrefixes = new ArrayList<String>();
surtPrefixes.add("http://(org,archive,");
surtPrefixes.add("http://(com,test,");
surtPrefixes.add("http://(com,google,");
dr.setSurtPrefixes(surtPrefixes);
CrawlURI testUri = createTestUri("http://example.com","http://google.com");
assertTrue(dr.evaluate(testUri));
}
public void testPositiveMultipleSurts2() throws Exception {
ViaSurtPrefixedDecideRule dr = new ViaSurtPrefixedDecideRule();
List<String> surtPrefixes = new ArrayList<String>();
surtPrefixes.add("http://(org,archive,");
surtPrefixes.add("http://(com,google,");
surtPrefixes.add("http://(com,test,");
dr.setSurtPrefixes(surtPrefixes);
CrawlURI testUri = createTestUri("http://example.com","http://google.com");
assertTrue(dr.evaluate(testUri));
}
public void testNegativeMultipleSurts() throws Exception {
ViaSurtPrefixedDecideRule dr = new ViaSurtPrefixedDecideRule();
List<String> surtPrefixes = new ArrayList<String>();
surtPrefixes.add("http://(org,archive,");
surtPrefixes.add("http://(com,test,");
surtPrefixes.add("http://(com,google,");
dr.setSurtPrefixes(surtPrefixes);
CrawlURI testUri = createTestUri("http://example.com","http://negativeexample.com");
assertFalse(dr.evaluate(testUri));
}
private CrawlURI createTestUri(String urlStr) throws URIException {
UURI testUuri = UURIFactory.getInstance(urlStr);
CrawlURI testUri = new CrawlURI(testUuri, null, null, LinkContext.NAVLINK_MISC);
return testUri;
}
private CrawlURI createTestUri(String urlStr, String via) throws URIException {
UURI testViaUuri = UURIFactory.getInstance(via);
CrawlURI testUri = createTestUri(urlStr);
testUri.setVia(testViaUuri);
return testUri;
}
}