diff --git a/dist/src/main/conf/jobs/profile-defaults/profile-crawler-beans.cxml b/dist/src/main/conf/jobs/profile-defaults/profile-crawler-beans.cxml index dcf74cdd..63dbedaa 100644 --- a/dist/src/main/conf/jobs/profile-defaults/profile-crawler-beans.cxml +++ b/dist/src/main/conf/jobs/profile-defaults/profile-crawler-beans.cxml @@ -368,8 +368,18 @@ http://example.example/example - - + + diff --git a/engine/src/main/java/org/archive/crawler/frontier/AbstractFrontier.java b/engine/src/main/java/org/archive/crawler/frontier/AbstractFrontier.java index 851ccd7b..e54ad283 100644 --- a/engine/src/main/java/org/archive/crawler/frontier/AbstractFrontier.java +++ b/engine/src/main/java/org/archive/crawler/frontier/AbstractFrontier.java @@ -43,6 +43,7 @@ import java.io.Serializable; import java.util.Collection; import java.util.Iterator; import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.BlockingQueue; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.locks.ReentrantReadWriteLock; @@ -145,23 +146,23 @@ public abstract class AbstractFrontier kp.put("maxOutlinks", max); } - /** size of the 'outbound' mediation queue between manager thread - * and toethreads */ - int outboundQueueCapacity = 200; - public int getOutboundQueueCapacity() { - return this.outboundQueueCapacity; - } - public void setOutboundQueueCapacity(int capacity) { - this.outboundQueueCapacity = capacity; - } - /** size of the inbound queue as multiple of the outbound queue */ - int inboundQueueMultiple = 10; - public int getInboundQueueMultiple() { - return this.inboundQueueMultiple; + /** inbound updates: URIs to be scheduled, finished; requested state changes */ + transient protected BlockingQueue inbound; + public BlockingQueue getInbound() { + return inbound; } - public void setInboundQueueMultiple(int multiple) { - this.inboundQueueMultiple = multiple; + public void setInbound(BlockingQueue inbound) { + this.inbound = inbound; + } + + /** outbound URIs */ + transient protected BlockingQueue outbound; + public BlockingQueue getOutbound() { + return outbound; + } + public void setOutbound(BlockingQueue outbound) { + this.outbound = outbound; } public boolean isRunning() { @@ -286,17 +287,6 @@ public abstract class AbstractFrontier /** reusable no-op inbound event, to force reeval of state/eligible URIs */ transient protected InEvent NOOP = new InEvent() { public void process() {} }; - /** inbound updates: URIs to be scheduled, finished; requested state changes */ - transient protected ArrayBlockingQueue inbound; - /** outbound URIs */ - transient protected ArrayBlockingQueue outbound; - - /** Capacity of the inbound queue. */ - private int inboundCapacity; - - /** Capacity of the outbound queue. */ - private int outboundCapacity; - /** * lock to allow holding all worker ToeThreads from taking URIs already * on the outbound queue; they acquire read permission before take()ing; @@ -343,11 +333,12 @@ public abstract class AbstractFrontier throw new IllegalStateException(e); } - this.outboundCapacity = getOutboundQueueCapacity(); - this.inboundCapacity = outboundCapacity * - getInboundQueueMultiple(); - outbound = new ArrayBlockingQueue(outboundCapacity, true); - inbound = new ArrayBlockingQueue(inboundCapacity, true); + if(outbound==null) { + outbound = new ArrayBlockingQueue(200, true); + } + if(inbound==null) { + inbound = new ArrayBlockingQueue(40000, true); + } pause(); startManagerThread(); } diff --git a/engine/src/main/resources/org/archive/crawler/restlet/profile-crawler-beans.cxml b/engine/src/main/resources/org/archive/crawler/restlet/profile-crawler-beans.cxml index 3e6f90ad..f8457494 100644 --- a/engine/src/main/resources/org/archive/crawler/restlet/profile-crawler-beans.cxml +++ b/engine/src/main/resources/org/archive/crawler/restlet/profile-crawler-beans.cxml @@ -368,8 +368,18 @@ http://example.example/example - - + +