diff --git a/commons/src/main/java/org/archive/spring/ConfigPathConfigurer.java b/commons/src/main/java/org/archive/spring/ConfigPathConfigurer.java index 938c0ec5..e252f4e2 100644 --- a/commons/src/main/java/org/archive/spring/ConfigPathConfigurer.java +++ b/commons/src/main/java/org/archive/spring/ConfigPathConfigurer.java @@ -57,6 +57,16 @@ implements Ordered { Map allBeans = new HashMap(); + + //// BEAN PROPERTIES + + /** 'home' directory for all other paths to be resolved + * relative to; defaults to directory of primary XML config file */ + ConfigPath path; + public ConfigPath getPath() { + return path; + } + //// BEANPOSTPROCESSOR IMPLEMENTATION /** * Remember all beans for later fixup. @@ -117,19 +127,7 @@ implements } return bean; } - - //// BEAN PROPERTIES - - /** 'home' directory for all other paths to be resolved - * relative to; defaults to directory of primary XML config file */ - ConfigPath path; - public ConfigPath getPath() { - return path; - } - public void setPath(ConfigPath p) { - path = p; - } - + //// APPLICATIONCONTEXTAWARE IMPLEMENTATION AbstractApplicationContext appCtx; diff --git a/dist/src/main/conf/jobs/profile-defaults/profile-crawler-beans.cxml b/dist/src/main/conf/jobs/profile-defaults/profile-crawler-beans.cxml index cdcc13ed..0e6d194a 100644 --- a/dist/src/main/conf/jobs/profile-defaults/profile-crawler-beans.cxml +++ b/dist/src/main/conf/jobs/profile-defaults/profile-crawler-beans.cxml @@ -1,4 +1,7 @@ + + + +# This Properties map is specified in the Java 'property list' text format +# http://java.sun.com/javase/6/docs/api/java/util/Properties.html#load%28java.io.Reader%29 metadata.operatorContactUrl=ENTER_AN_URL_WITH_YOUR_CONTACT_INFO_HERE_FOR_WEBMASTERS_AFFECTED_BY_YOUR_CRAWL -metadata.description=basic crawl metadata.jobName=basic +metadata.description=Basic crawl starting with useful defaults +##..more?..## + @@ -36,24 +54,52 @@ http://example.example/example - + - - - + + + + + + + + + - + + + + + + +# [see override above] + + + + + + + + + - + @@ -61,27 +107,42 @@ http://example.example/example - + + + + + + + - - - + + + + + + + + + @@ -90,18 +151,106 @@ http://example.example/example - - + + + + + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -109,109 +258,245 @@ http://example.example/example - - - - - - - - - - - - - - - - - - - - - - - - - + + - + + + + + + + + + - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + + + + + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + + + + + + + + + - + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/engine/src/main/java/org/archive/crawler/framework/ActionDirectory.java b/engine/src/main/java/org/archive/crawler/framework/ActionDirectory.java index dc0513f3..0cfed86b 100644 --- a/engine/src/main/java/org/archive/crawler/framework/ActionDirectory.java +++ b/engine/src/main/java/org/archive/crawler/framework/ActionDirectory.java @@ -84,20 +84,20 @@ public class ActionDirectory implements ApplicationContextAware, Lifecycle, Runn ScheduledExecutorService executor; /** how long after crawl start to first scan action directory */ - protected int initialDelay = 10; - public int getInitialDelay() { - return initialDelay; + protected int initialDelaySeconds = 10; + public int getInitialDelaySeconds() { + return initialDelaySeconds; } - public void setInitialDelay(int initialDelay) { - this.initialDelay = initialDelay; + public void setInitialDelaySeconds(int initialDelay) { + this.initialDelaySeconds = initialDelay; } /** delay between scans of actionDirectory for new files */ - protected int delay = 30; - public int getDelay() { - return delay; + protected int delaySeconds = 30; + public int getDelaySeconds() { + return delaySeconds; } - public void setDelay(int delay) { - this.delay = delay; + public void setDelaySeconds(int delay) { + this.delaySeconds = delay; } /** @@ -161,7 +161,7 @@ public class ActionDirectory implements ApplicationContextAware, Lifecycle, Runn getDoneDir().getFile().mkdirs(); // start background executor executor = Executors.newSingleThreadScheduledExecutor(); - executor.scheduleWithFixedDelay(this, getInitialDelay(), getDelay(), TimeUnit.SECONDS); + executor.scheduleWithFixedDelay(this, getInitialDelaySeconds(), getDelaySeconds(), TimeUnit.SECONDS); } public void stop() { diff --git a/engine/src/main/java/org/archive/crawler/framework/Scoper.java b/engine/src/main/java/org/archive/crawler/framework/Scoper.java index 71ca2466..1eb9e61e 100644 --- a/engine/src/main/java/org/archive/crawler/framework/Scoper.java +++ b/engine/src/main/java/org/archive/crawler/framework/Scoper.java @@ -43,6 +43,15 @@ public abstract class Scoper extends Processor implements Lifecycle { private static Logger LOGGER = Logger.getLogger(Scoper.class.getName()); + protected DecideRule scope; + public DecideRule getScope() { + return this.scope; + } + @Autowired + public void setScope(DecideRule scope) { + this.scope = scope; + } + FileHandler fileLogger = null; /** @@ -72,15 +81,6 @@ public abstract class Scoper extends Processor implements Lifecycle { this.loggerModule = loggerModule; } - protected DecideRule scope; - public DecideRule getScope() { - return this.scope; - } - @Autowired - public void setScope(DecideRule scope) { - this.scope = scope; - } - // FIXME: Weirdo log overriding might not work on a per-subclass basis, // we may need to cut and paste it to the three subclasses, or eliminate // it in favor of java.util.logging best practice. diff --git a/engine/src/main/java/org/archive/crawler/postprocessor/DispositionProcessor.java b/engine/src/main/java/org/archive/crawler/postprocessor/DispositionProcessor.java index 0b96b3a2..b2d27bab 100644 --- a/engine/src/main/java/org/archive/crawler/postprocessor/DispositionProcessor.java +++ b/engine/src/main/java/org/archive/crawler/postprocessor/DispositionProcessor.java @@ -31,7 +31,6 @@ import org.archive.modules.CrawlMetadata; import org.archive.modules.CrawlURI; import org.archive.modules.ModuleAttributeConstants; import org.archive.modules.Processor; -import org.archive.modules.fetcher.UserAgentProvider; import org.archive.modules.net.CrawlHost; import org.archive.modules.net.CrawlServer; import org.archive.modules.net.RobotsExclusionPolicy; @@ -142,15 +141,6 @@ public class DispositionProcessor extends Processor { this.metadata = provider; } - - public UserAgentProvider getUserAgentProvider() { - return (UserAgentProvider) kp.get("userAgentProvider"); - } - @Autowired - public void setUserAgentProvider(UserAgentProvider provider) { - kp.put("userAgentProvider",provider); - } - public DispositionProcessor() { super(); } @@ -234,10 +224,9 @@ public class DispositionProcessor extends Processor { // may need to extend wait CrawlServer s = ServerCacheUtil.getServerFor( getServerCache(),curi.getUURI()); - UserAgentProvider uap = getUserAgentProvider(); String ua = curi.getUserAgent(); if (ua == null) { - ua = uap.getUserAgent(); + ua = metadata.getUserAgent(); } RobotsExclusionPolicy rep = s.getRobots(); if (rep != null) { diff --git a/engine/src/main/java/org/archive/crawler/prefetch/PreconditionEnforcer.java b/engine/src/main/java/org/archive/crawler/prefetch/PreconditionEnforcer.java index 5a25a18e..faba4218 100644 --- a/engine/src/main/java/org/archive/crawler/prefetch/PreconditionEnforcer.java +++ b/engine/src/main/java/org/archive/crawler/prefetch/PreconditionEnforcer.java @@ -57,21 +57,10 @@ import org.springframework.beans.factory.annotation.Autowired; * @author gojomo */ public class PreconditionEnforcer extends Processor { - private static final long serialVersionUID = 3L; - private static final Logger logger = Logger.getLogger(PreconditionEnforcer.class.getName()); - - public UserAgentProvider getUserAgentProvider() { - return (UserAgentProvider) kp.get("userAgentProvider"); - } - @Autowired - public void setUserAgentProvider(UserAgentProvider provider) { - kp.put("userAgentProvider",provider); - } - /** * The minimum interval for which a dns-record will be considered * valid (in seconds). If the record's DNS TTL is larger, that will @@ -113,10 +102,30 @@ public class PreconditionEnforcer extends Processor { public boolean getCalculateRobotsOnly() { return (Boolean) kp.get("calculateRobotsOnly"); } - public void setCalculateRobotsOnly(boolean recheck) { - kp.put("calculateRobotsOnly",recheck); + public void setCalculateRobotsOnly(boolean calcOnly) { + kp.put("calculateRobotsOnly",calcOnly); } + public UserAgentProvider getUserAgentProvider() { + return (UserAgentProvider) kp.get("userAgentProvider"); + } + @Autowired + public void setUserAgentProvider(UserAgentProvider provider) { + kp.put("userAgentProvider",provider); + } + + { + // initialize with empty store so declaration not required + setCredentialStore(new CredentialStore()); + } + public CredentialStore getCredentialStore() { + return (CredentialStore) kp.get("credentialStore"); + } + @Autowired(required=false) + public void setCredentialStore(CredentialStore credentials) { + kp.put("credentialStore",credentials); + } + protected ServerCache serverCache; public ServerCache getServerCache() { return this.serverCache; @@ -126,14 +135,6 @@ public class PreconditionEnforcer extends Processor { this.serverCache = serverCache; } - public CredentialStore getCredentialStore() { - return (CredentialStore) kp.get("credentialStore"); - } - @Autowired - public void setCredentialStore(CredentialStore credentials) { - kp.put("credentialStore",credentials); - } - protected CrawlerLoggerModule loggerModule; public CrawlerLoggerModule getLoggerModule() { return this.loggerModule; diff --git a/modules/src/main/java/org/archive/modules/deciderules/surt/SurtPrefixedDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/surt/SurtPrefixedDecideRule.java index 94972258..5a1322d9 100644 --- a/modules/src/main/java/org/archive/modules/deciderules/surt/SurtPrefixedDecideRule.java +++ b/modules/src/main/java/org/archive/modules/deciderules/surt/SurtPrefixedDecideRule.java @@ -95,7 +95,7 @@ implements * Dump file to save SURT prefixes actually used: Useful debugging SURTs. */ protected ConfigFile surtsDumpFile = - new ConfigFile("surtsDumpFile",""); + new ConfigFile("surtsDumpFile","surts.dump"); public ConfigFile getSurtsDumpFile() { return surtsDumpFile; } diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java index 7d0c22e3..f9abe922 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java @@ -120,28 +120,28 @@ public class ExtractorHTML extends ContentExtractor implements InitializingBean // "(?is)(\\w+)\\s+.*?(?:(robots))\\s*=(?:(?:\\s*\"(.+)\")|(?:\\s*'(.+)')|(\\S+))"); { - setMaxAttrNameLength(1024); // 1K + setMaxAttributeNameLength(1024); // 1K } - public int getMaxAttrNameLength() { - return (Integer) kp.get("maxAttrNameLength"); + public int getMaxAttributeNameLength() { + return (Integer) kp.get("maxAttributeNameLength"); } - public void setMaxAttrNameLength(int max) { - kp.put("maxAttrNameLength", max); + public void setMaxAttributeNameLength(int max) { + kp.put("maxAttributeNameLength", max); } { - setMaxAttrValLength(16384); // 16K + setMaxAttributeValLength(16384); // 16K } - public int getMaxAttrValLength() { - return (Integer) kp.get("maxAttrValLength"); + public int getMaxAttributeValLength() { + return (Integer) kp.get("maxAttributeValLength"); } - public void setMaxAttrValLength(int max) { - kp.put("maxAttrValLength", max); + public void setMaxAttributeValLength(int max) { + kp.put("maxAttributeValLength", max); } // TODO: perhaps cut to near MAX_URI_LENGTH @@ -317,9 +317,9 @@ public class ExtractorHTML extends ContentExtractor implements InitializingBean regex = EACH_ATTRIBUTE_EXTRACTOR; regex = regex.replace(MAX_ATTR_NAME_REPLACE, - Integer.toString(getMaxAttrNameLength())); + Integer.toString(getMaxAttributeNameLength())); regex = regex.replace(MAX_ATTR_VAL_REPLACE, - Integer.toString(getMaxAttrValLength())); + Integer.toString(getMaxAttributeValLength())); this.eachAttributeExtractor = Pattern.compile(regex); } diff --git a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java index 723d5157..a258ce42 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java +++ b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java @@ -113,9 +113,10 @@ import org.springframework.context.Lifecycle; * href="http://jakarta.apache.org/commons/httpclient/">Apache Jakarta Commons * HttpClient library. * - * @author Gordon Mohr - * @author Igor Ranitovic - * @author others + * @contributor gojomo + * @contributor Igor Ranitovic + * @contributor stack + * @contributor others * @version $Id$ */ public class FetchHTTP extends Processor implements Lifecycle { @@ -171,7 +172,7 @@ public class FetchHTTP extends Processor implements Lifecycle { * retry read. */ { - setSoTimeoutMs(20*1000); // 20 minutes + setSoTimeoutMs(20*1000); // 20 seconds } public int getSoTimeoutMs() { return (Integer) kp.get("soTimeoutMs"); @@ -278,11 +279,14 @@ public class FetchHTTP extends Processor implements Lifecycle { * 'normal' (all valid certificates not including selfsigned) to 'strict' * (Cert is valid and DN must match servername). */ - public TrustLevel getTrustLevel() { + { + setSslTrustLevel(TrustLevel.OPEN); + } + public TrustLevel getSslTrustLevel() { return (TrustLevel) kp.get("trustLevel"); } - public void getTrustLevel(TrustLevel trustLevel) { - kp.put("trustLevel",trustLevel); + public void setSslTrustLevel(TrustLevel trustLevel) { + kp.put("sslTrustLevel",trustLevel); } private transient HttpClient http = null; @@ -452,10 +456,14 @@ public class FetchHTTP extends Processor implements Lifecycle { /** * Used to store credentials. */ + { + // initialize with empty store so declaration not required + setCredentialStore(new CredentialStore()); + } public CredentialStore getCredentialStore() { return (CredentialStore) kp.get("credentialStore"); } - @Autowired + @Autowired(required=false) public void setCredentialStore(CredentialStore credentials) { kp.put("credentialStore",credentials); } @@ -1311,7 +1319,7 @@ public class FetchHTTP extends Processor implements Lifecycle { SSLContext context = SSLContext.getInstance("SSL"); context.init(null, new TrustManager[] { new ConfigurableX509TrustManager( - getTrustLevel()) }, null); + getSslTrustLevel()) }, null); this.sslfactory = context.getSocketFactory(); } catch (Exception e) { logger.log(Level.WARNING, "Failed configure of ssl context " diff --git a/modules/src/main/java/org/archive/modules/seeds/SeedModule.java b/modules/src/main/java/org/archive/modules/seeds/SeedModule.java index ce57cea3..944255f8 100644 --- a/modules/src/main/java/org/archive/modules/seeds/SeedModule.java +++ b/modules/src/main/java/org/archive/modules/seeds/SeedModule.java @@ -40,7 +40,7 @@ public abstract class SeedModule implements Serializable * from that seed. When present, such source tags appear in the * second-to-last crawl.log field. */ - boolean sourceTagSeeds; + boolean sourceTagSeeds = false; public boolean getSourceTagSeeds() { return sourceTagSeeds; } @@ -48,20 +48,6 @@ public abstract class SeedModule implements Serializable this.sourceTagSeeds = sourceTagSeeds; } - /** - * Whether to reread the seeds specification, whether it has changed or not, - * every time any configuration change occurs. If true, seeds are reread - * even when (for example) new domain overrides are set. Rereading the seeds - * can take a long time with large seed lists. - */ - protected boolean rereadSeedsOnConfig = true; - public boolean getRereadSeedsOnConfig() { - return rereadSeedsOnConfig; - } - public void setRereadSeedsOnConfig(boolean rereadSeedsOnConfig) { - this.rereadSeedsOnConfig = rereadSeedsOnConfig; - } - protected Set seedListeners = new HashSet(); public Set getSeedListeners() { diff --git a/modules/src/main/java/org/archive/modules/writer/ARCWriterProcessor.java b/modules/src/main/java/org/archive/modules/writer/ARCWriterProcessor.java index 356fc099..cb0b9685 100644 --- a/modules/src/main/java/org/archive/modules/writer/ARCWriterProcessor.java +++ b/modules/src/main/java/org/archive/modules/writer/ARCWriterProcessor.java @@ -78,7 +78,7 @@ public class ARCWriterProcessor extends WriterPoolProcessor { @Override protected void setupPool(AtomicInteger serialNo) { WriterPoolSettings wps = getWriterPoolSettings(); - setPool(new ARCWriterPool(serialNo, wps, getPoolMaxActive(), getPoolMaxWait())); + setPool(new ARCWriterPool(serialNo, wps, getPoolMaxActive(), getPoolMaxWaitMs())); } /** diff --git a/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java b/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java index c4772c80..36bc801d 100644 --- a/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java +++ b/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java @@ -164,7 +164,7 @@ public class WARCWriterProcessor extends WriterPoolProcessor { @Override protected void setupPool(final AtomicInteger serialNo) { WriterPoolSettings wps = getWriterPoolSettings(); - setPool(new WARCWriterPool(serialNo, wps, getPoolMaxActive(), getPoolMaxWait())); + setPool(new WARCWriterPool(serialNo, wps, getPoolMaxActive(), getPoolMaxWaitMs())); } /** diff --git a/modules/src/main/java/org/archive/modules/writer/WriterPoolProcessor.java b/modules/src/main/java/org/archive/modules/writer/WriterPoolProcessor.java index d4e04c4c..9f827d38 100644 --- a/modules/src/main/java/org/archive/modules/writer/WriterPoolProcessor.java +++ b/modules/src/main/java/org/archive/modules/writer/WriterPoolProcessor.java @@ -129,12 +129,12 @@ implements Lifecycle { * Maximum time to wait on pool element (milliseconds). This setting cannot * be varied over the life of a crawl. */ - int poolMaxWait = WriterPool.DEFAULT_MAXIMUM_WAIT; - public int getPoolMaxWait() { - return poolMaxWait; + int poolMaxWaitMs = WriterPool.DEFAULT_MAXIMUM_WAIT; + public int getPoolMaxWaitMs() { + return poolMaxWaitMs; } - public void setPoolMaxWait(int poolMaxWait) { - this.poolMaxWait = poolMaxWait; + public void setPoolMaxWaitMs(int poolMaxWait) { + this.poolMaxWaitMs = poolMaxWait; } /**