Misc fixes from H1-to-H3 testing

* BdbModule.java
    be more robust about 0/-1 as 'don't care' values
* AbstractFrontier.java
    reorder settable properties for ease of discovery
* FetchHTTP.java
    correct type to allow setting property
This commit is contained in:
gojomo committed 2009-10-07 04:22:05 +00:00
1 parent 07adde3a73
commit b7631e2b02
3 files changed
+78 -77

No files matched your search

@@ -104,6 +104,77 @@ public abstract class AbstractFrontier
return kp;
}
/** for retryable problems, seconds to wait before a retry */
{
setRetryDelaySeconds(900);
}
public int getRetryDelaySeconds() {
return (Integer) kp.get("retryDelaySeconds");
}
public void setRetryDelaySeconds(int delay) {
kp.put("retryDelaySeconds",delay);
}
/** maximum times to emit a CrawlURI without final disposition */
{
setMaxRetries(30);
}
public int getMaxRetries() {
return (Integer) kp.get("maxRetries");
}
public void setMaxRetries(int maxRetries) {
kp.put("maxRetries",maxRetries);
}
protected ConfigPath recoveryDir = new ConfigPath("recovery subdirectory","logs");
public ConfigPath getRecoveryDir() {
return recoveryDir;
}
public void setRecoveryDir(ConfigPath recoveryDir) {
this.recoveryDir = recoveryDir;
}
/**
* Recover log on or off attribute.
*/
{
setRecoveryLogEnabled(true);
}
public boolean getRecoveryLogEnabled() {
return (Boolean) kp.get("recoveryLogEnabled");
}
public void setRecoveryLogEnabled(boolean enabled) {
kp.put("recoveryLogEnabled",enabled);
}
{
setMaxOutlinks(6000);
}
public int getMaxOutlinks() {
return (Integer) kp.get("maxOutlinks");
}
public void setMaxOutlinks(int max) {
kp.put("maxOutlinks", max);
}
/** size of the 'outbound' mediation queue between manager thread
* and toethreads */
int outboundQueueCapacity = 50;
public int getOutboundQueueCapacity() {
return this.outboundQueueCapacity;
}
public void setOutboundQueueCapacity(int capacity) {
this.outboundQueueCapacity = capacity;
}
/** size of the inbound queue as multiple of the outbound queue */
int inboundQueueMultiple = 3;
public int getInboundQueueMultiple() {
return this.inboundQueueMultiple;
}
public void setInboundQueueMultiple(int multiple) {
this.inboundQueueMultiple = multiple;
}
public boolean isRunning() {
return managerThread!=null && managerThread.isAlive();
@@ -170,14 +241,6 @@ public abstract class AbstractFrontier
public void setScope(DecideRule scope) {
this.scope = scope;
}
protected ConfigPath recoveryDir = new ConfigPath("recovery subdirectory","logs");
public ConfigPath getRecoveryDir() {
return recoveryDir;
}
public void setRecoveryDir(ConfigPath recoveryDir) {
this.recoveryDir = recoveryDir;
}
FrontierPreparer preparer;
public FrontierPreparer getFrontierPreparer() {
@@ -196,71 +259,7 @@ public abstract class AbstractFrontier
assert KeyedProperties.overridesActiveFrom(curi);
return preparer.getClassKey(curi);
}
/** for retryable problems, seconds to wait before a retry */
{
setRetryDelaySeconds(900);
}
public int getRetryDelaySeconds() {
return (Integer) kp.get("retryDelaySeconds");
}
public void setRetryDelaySeconds(int delay) {
kp.put("retryDelaySeconds",delay);
}
/** maximum times to emit a CrawlURI without final disposition */
{
setMaxRetries(30);
}
public int getMaxRetries() {
return (Integer) kp.get("maxRetries");
}
public void setMaxRetries(int maxRetries) {
kp.put("maxRetries",maxRetries);
}
/** size of the 'outbound' mediation queue between manager thread
* and toethreads */
int outboundQueueCapacity = 50;
public int getOutboundQueueCapacity() {
return this.outboundQueueCapacity;
}
public void setOutboundQueueCapacity(int capacity) {
this.outboundQueueCapacity = capacity;
}
/** size of the inbound queue as multiple of the outbound queue */
int inboundQueueMultiple = 3;
public int getInboundQueueMultiple() {
return this.inboundQueueMultiple;
}
public void setInboundQueueMultiple(int multiple) {
this.inboundQueueMultiple = multiple;
}
/**
* Recover log on or off attribute.
*/
{
setRecoveryLogEnabled(true);
}
public boolean getRecoveryLogEnabled() {
return (Boolean) kp.get("recoveryLogEnabled");
}
public void setRecoveryLogEnabled(boolean enabled) {
kp.put("recoveryLogEnabled",enabled);
}
{
setMaxOutlinks(6000);
}
public int getMaxOutlinks() {
return (Integer) kp.get("maxOutlinks");
}
public void setMaxOutlinks(int max) {
kp.put("maxOutlinks", max);
}
// top-level stats
/** total URIs queued to be visited */
protected AtomicLong queuedUriCount = new AtomicLong(0);