[HER-1676] improve bundled profiles to match best-practice, be self-documenting

* profile-crawler-beans.cxml
    add comments for all unstated default values an operator might want to change
* (many) 
    reorder, refactor, rename to better minimize/match simple configuration

engine/src/main/java/org/archive/crawler/framework/ActionDirectory.java

@@ -84,20 +84,20 @@ public class ActionDirectory implements ApplicationContextAware, Lifecycle, Runn
     ScheduledExecutorService executor;
 
     /** how long after crawl start to first scan action directory */
-    protected int initialDelay = 10;
-    public int getInitialDelay() {
-        return initialDelay;
+    protected int initialDelaySeconds = 10;
+    public int getInitialDelaySeconds() {
+        return initialDelaySeconds;
     }
-    public void setInitialDelay(int initialDelay) {
-        this.initialDelay = initialDelay;
+    public void setInitialDelaySeconds(int initialDelay) {
+        this.initialDelaySeconds = initialDelay;
     }
     /** delay between scans of actionDirectory for new files */
-    protected int delay = 30;
-    public int getDelay() {
-        return delay;
+    protected int delaySeconds = 30;
+    public int getDelaySeconds() {
+        return delaySeconds;
     }
-    public void setDelay(int delay) {
-        this.delay = delay;
+    public void setDelaySeconds(int delay) {
+        this.delaySeconds = delay;
     }
     
     /**
@@ -161,7 +161,7 @@ public class ActionDirectory implements ApplicationContextAware, Lifecycle, Runn
         getDoneDir().getFile().mkdirs();
         // start background executor
         executor = Executors.newSingleThreadScheduledExecutor();
-        executor.scheduleWithFixedDelay(this, getInitialDelay(), getDelay(), TimeUnit.SECONDS);
+        executor.scheduleWithFixedDelay(this, getInitialDelaySeconds(), getDelaySeconds(), TimeUnit.SECONDS);
     }
     
     public void stop() {

engine/src/main/java/org/archive/crawler/framework/Scoper.java

@@ -43,6 +43,15 @@ public abstract class Scoper extends Processor implements Lifecycle {
     private static Logger LOGGER =
         Logger.getLogger(Scoper.class.getName());
     
+    protected DecideRule scope;
+    public DecideRule getScope() {
+        return this.scope;
+    }
+    @Autowired
+    public void setScope(DecideRule scope) {
+        this.scope = scope;
+    }
+    
     FileHandler fileLogger = null; 
     
     /**
@@ -72,15 +81,6 @@ public abstract class Scoper extends Processor implements Lifecycle {
         this.loggerModule = loggerModule;
     }
     
-    protected DecideRule scope;
-    public DecideRule getScope() {
-        return this.scope;
-    }
-    @Autowired
-    public void setScope(DecideRule scope) {
-        this.scope = scope;
-    }
-    
     // FIXME: Weirdo log overriding might not work on a per-subclass basis,
     // we may need to cut and paste it to the three subclasses, or eliminate
     // it in favor of java.util.logging best practice.

engine/src/main/java/org/archive/crawler/postprocessor/DispositionProcessor.java

@@ -31,7 +31,6 @@ import org.archive.modules.CrawlMetadata;
 import org.archive.modules.CrawlURI;
 import org.archive.modules.ModuleAttributeConstants;
 import org.archive.modules.Processor;
-import org.archive.modules.fetcher.UserAgentProvider;
 import org.archive.modules.net.CrawlHost;
 import org.archive.modules.net.CrawlServer;
 import org.archive.modules.net.RobotsExclusionPolicy;
@@ -142,15 +141,6 @@ public class DispositionProcessor extends Processor {
         this.metadata = provider;
     }
 
-    
-    public UserAgentProvider getUserAgentProvider() {
-        return (UserAgentProvider) kp.get("userAgentProvider");
-    }
-    @Autowired
-    public void setUserAgentProvider(UserAgentProvider provider) {
-        kp.put("userAgentProvider",provider);
-    }
-    
     public DispositionProcessor() {
         super();
     }
@@ -234,10 +224,9 @@ public class DispositionProcessor extends Processor {
                 // may need to extend wait
                 CrawlServer s = ServerCacheUtil.getServerFor(
                         getServerCache(),curi.getUURI());
-                UserAgentProvider uap = getUserAgentProvider();
                 String ua = curi.getUserAgent();
                 if (ua == null) {
-                    ua = uap.getUserAgent();
+                    ua = metadata.getUserAgent();
                 }
                 RobotsExclusionPolicy rep = s.getRobots();
                 if (rep != null) {

engine/src/main/java/org/archive/crawler/prefetch/PreconditionEnforcer.java

@@ -57,21 +57,10 @@ import org.springframework.beans.factory.annotation.Autowired;
  * @author gojomo
  */
 public class PreconditionEnforcer extends Processor  {
-
     private static final long serialVersionUID = 3L;
-
     private static final Logger logger =
         Logger.getLogger(PreconditionEnforcer.class.getName());
 
-
-    public UserAgentProvider getUserAgentProvider() {
-        return (UserAgentProvider) kp.get("userAgentProvider");
-    }
-    @Autowired
-    public void setUserAgentProvider(UserAgentProvider provider) {
-        kp.put("userAgentProvider",provider);
-    }
-    
     /**
      * The minimum interval for which a dns-record will be considered 
      * valid (in seconds). If the record's DNS TTL is larger, that will 
@@ -113,10 +102,30 @@ public class PreconditionEnforcer extends Processor  {
     public boolean getCalculateRobotsOnly() {
         return (Boolean) kp.get("calculateRobotsOnly");
     }
-    public void setCalculateRobotsOnly(boolean recheck) {
-        kp.put("calculateRobotsOnly",recheck);
+    public void setCalculateRobotsOnly(boolean calcOnly) {
+        kp.put("calculateRobotsOnly",calcOnly);
     }   
     
+    public UserAgentProvider getUserAgentProvider() {
+        return (UserAgentProvider) kp.get("userAgentProvider");
+    }
+    @Autowired
+    public void setUserAgentProvider(UserAgentProvider provider) {
+        kp.put("userAgentProvider",provider);
+    }
+    
+    {
+        // initialize with empty store so declaration not required
+        setCredentialStore(new CredentialStore());
+    }
+    public CredentialStore getCredentialStore() {
+        return (CredentialStore) kp.get("credentialStore");
+    }
+    @Autowired(required=false)
+    public void setCredentialStore(CredentialStore credentials) {
+        kp.put("credentialStore",credentials);
+    }
+    
     protected ServerCache serverCache;
     public ServerCache getServerCache() {
         return this.serverCache;
@@ -126,14 +135,6 @@ public class PreconditionEnforcer extends Processor  {
         this.serverCache = serverCache;
     }
     
-    public CredentialStore getCredentialStore() {
-        return (CredentialStore) kp.get("credentialStore");
-    }
-    @Autowired
-    public void setCredentialStore(CredentialStore credentials) {
-        kp.put("credentialStore",credentials);
-    }
-    
     protected CrawlerLoggerModule loggerModule;
     public CrawlerLoggerModule getLoggerModule() {
         return this.loggerModule;