From b7631e2b02059f1d58f50e504f31f4ab8d809987 Mon Sep 17 00:00:00 2001 From: gojomo Date: Wed, 7 Oct 2009 04:22:05 +0000 Subject: [PATCH] Misc fixes from H1-to-H3 testing * BdbModule.java be more robust about 0/-1 as 'don't care' values * AbstractFrontier.java reorder settable properties for ease of discovery * FetchHTTP.java correct type to allow setting property --- .../main/java/org/archive/bdb/BdbModule.java | 8 +- .../crawler/frontier/AbstractFrontier.java | 145 +++++++++--------- .../archive/modules/fetcher/FetchHTTP.java | 2 +- 3 files changed, 78 insertions(+), 77 deletions(-) diff --git a/commons/src/main/java/org/archive/bdb/BdbModule.java b/commons/src/main/java/org/archive/bdb/BdbModule.java index 7fdd54fb..f9086c71 100644 --- a/commons/src/main/java/org/archive/bdb/BdbModule.java +++ b/commons/src/main/java/org/archive/bdb/BdbModule.java @@ -162,7 +162,7 @@ Serializable, Closeable { this.dir = dir; } - int cachePercent = 60; + int cachePercent = -1; public int getCachePercent() { return cachePercent; } @@ -243,8 +243,10 @@ Serializable, Closeable { throws DatabaseException { EnvironmentConfig config = new EnvironmentConfig(); config.setAllowCreate(create); - config.setLockTimeout(5000000); - config.setCachePercent(cachePercent); + config.setLockTimeout(5000000); + if(cachePercent>0) { + config.setCachePercent(cachePercent); + } config.setSharedCache(sharedCache); // we take the advice literally from... diff --git a/engine/src/main/java/org/archive/crawler/frontier/AbstractFrontier.java b/engine/src/main/java/org/archive/crawler/frontier/AbstractFrontier.java index 33149df0..1d1b2baf 100644 --- a/engine/src/main/java/org/archive/crawler/frontier/AbstractFrontier.java +++ b/engine/src/main/java/org/archive/crawler/frontier/AbstractFrontier.java @@ -104,6 +104,77 @@ public abstract class AbstractFrontier return kp; } + /** for retryable problems, seconds to wait before a retry */ + { + setRetryDelaySeconds(900); + } + public int getRetryDelaySeconds() { + return (Integer) kp.get("retryDelaySeconds"); + } + public void setRetryDelaySeconds(int delay) { + kp.put("retryDelaySeconds",delay); + } + + /** maximum times to emit a CrawlURI without final disposition */ + { + setMaxRetries(30); + } + public int getMaxRetries() { + return (Integer) kp.get("maxRetries"); + } + public void setMaxRetries(int maxRetries) { + kp.put("maxRetries",maxRetries); + } + + protected ConfigPath recoveryDir = new ConfigPath("recovery subdirectory","logs"); + public ConfigPath getRecoveryDir() { + return recoveryDir; + } + public void setRecoveryDir(ConfigPath recoveryDir) { + this.recoveryDir = recoveryDir; + } + + /** + * Recover log on or off attribute. + */ + { + setRecoveryLogEnabled(true); + } + public boolean getRecoveryLogEnabled() { + return (Boolean) kp.get("recoveryLogEnabled"); + } + public void setRecoveryLogEnabled(boolean enabled) { + kp.put("recoveryLogEnabled",enabled); + } + + { + setMaxOutlinks(6000); + } + public int getMaxOutlinks() { + return (Integer) kp.get("maxOutlinks"); + } + public void setMaxOutlinks(int max) { + kp.put("maxOutlinks", max); + } + + /** size of the 'outbound' mediation queue between manager thread + * and toethreads */ + int outboundQueueCapacity = 50; + public int getOutboundQueueCapacity() { + return this.outboundQueueCapacity; + } + public void setOutboundQueueCapacity(int capacity) { + this.outboundQueueCapacity = capacity; + } + + /** size of the inbound queue as multiple of the outbound queue */ + int inboundQueueMultiple = 3; + public int getInboundQueueMultiple() { + return this.inboundQueueMultiple; + } + public void setInboundQueueMultiple(int multiple) { + this.inboundQueueMultiple = multiple; + } public boolean isRunning() { return managerThread!=null && managerThread.isAlive(); @@ -170,14 +241,6 @@ public abstract class AbstractFrontier public void setScope(DecideRule scope) { this.scope = scope; } - - protected ConfigPath recoveryDir = new ConfigPath("recovery subdirectory","logs"); - public ConfigPath getRecoveryDir() { - return recoveryDir; - } - public void setRecoveryDir(ConfigPath recoveryDir) { - this.recoveryDir = recoveryDir; - } FrontierPreparer preparer; public FrontierPreparer getFrontierPreparer() { @@ -196,71 +259,7 @@ public abstract class AbstractFrontier assert KeyedProperties.overridesActiveFrom(curi); return preparer.getClassKey(curi); } - - /** for retryable problems, seconds to wait before a retry */ - { - setRetryDelaySeconds(900); - } - public int getRetryDelaySeconds() { - return (Integer) kp.get("retryDelaySeconds"); - } - public void setRetryDelaySeconds(int delay) { - kp.put("retryDelaySeconds",delay); - } - - /** maximum times to emit a CrawlURI without final disposition */ - { - setMaxRetries(30); - } - public int getMaxRetries() { - return (Integer) kp.get("maxRetries"); - } - public void setMaxRetries(int maxRetries) { - kp.put("maxRetries",maxRetries); - } - - /** size of the 'outbound' mediation queue between manager thread - * and toethreads */ - int outboundQueueCapacity = 50; - public int getOutboundQueueCapacity() { - return this.outboundQueueCapacity; - } - public void setOutboundQueueCapacity(int capacity) { - this.outboundQueueCapacity = capacity; - } - - /** size of the inbound queue as multiple of the outbound queue */ - int inboundQueueMultiple = 3; - public int getInboundQueueMultiple() { - return this.inboundQueueMultiple; - } - public void setInboundQueueMultiple(int multiple) { - this.inboundQueueMultiple = multiple; - } - - /** - * Recover log on or off attribute. - */ - { - setRecoveryLogEnabled(true); - } - public boolean getRecoveryLogEnabled() { - return (Boolean) kp.get("recoveryLogEnabled"); - } - public void setRecoveryLogEnabled(boolean enabled) { - kp.put("recoveryLogEnabled",enabled); - } - - { - setMaxOutlinks(6000); - } - public int getMaxOutlinks() { - return (Integer) kp.get("maxOutlinks"); - } - public void setMaxOutlinks(int max) { - kp.put("maxOutlinks", max); - } - + // top-level stats /** total URIs queued to be visited */ protected AtomicLong queuedUriCount = new AtomicLong(0); diff --git a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java index a258ce42..30f7b451 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java +++ b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java @@ -159,7 +159,7 @@ public class FetchHTTP extends Processor implements Lifecycle { public int getTimeoutSeconds() { return (Integer) kp.get("timeoutSeconds"); } - public void setTimeoutSeconds(Integer timeout) { + public void setTimeoutSeconds(int timeout) { kp.put("timeoutSeconds",timeout); }