diff --git a/commons/src/main/java/org/archive/spring/ConfigPathConfigurer.java b/commons/src/main/java/org/archive/spring/ConfigPathConfigurer.java index 0856094a..72c003fc 100644 --- a/commons/src/main/java/org/archive/spring/ConfigPathConfigurer.java +++ b/commons/src/main/java/org/archive/spring/ConfigPathConfigurer.java @@ -22,9 +22,11 @@ package org.archive.spring; import java.beans.PropertyDescriptor; import java.io.File; import java.util.HashMap; +import java.util.List; import java.util.Map; import org.apache.commons.lang.StringUtils; +import org.archive.modules.writer.WriterPoolProcessor; import org.springframework.beans.BeanWrapperImpl; import org.springframework.beans.BeansException; import org.springframework.beans.factory.config.BeanPostProcessor; @@ -125,6 +127,16 @@ implements } } } + + // we want to remember writer pool store paths for later interpolation + // (but their base dirs are taken care of elsewhere) + if (bean instanceof WriterPoolProcessor) { + List storePaths = ((WriterPoolProcessor) bean).getStorePaths(); + for (int i = 0; i < storePaths.size(); i++) { + remember(beanName + "." + "storePaths[" + i + "]", storePaths.get(i)); + } + } + if(bean instanceof PathFixupListener) { ((PathFixupListener)bean).pathsFixedUp(); } diff --git a/modules/src/main/java/org/archive/modules/writer/ARCWriterProcessor.java b/modules/src/main/java/org/archive/modules/writer/ARCWriterProcessor.java index 416c0765..55b141ef 100644 --- a/modules/src/main/java/org/archive/modules/writer/ARCWriterProcessor.java +++ b/modules/src/main/java/org/archive/modules/writer/ARCWriterProcessor.java @@ -38,8 +38,9 @@ import org.archive.io.ReplayInputStream; import org.archive.io.WriterPoolMember; import org.archive.io.arc.ARCWriter; import org.archive.io.arc.ARCWriterPool; -import org.archive.modules.ProcessResult; import org.archive.modules.CrawlURI; +import org.archive.modules.ProcessResult; +import org.archive.spring.ConfigPath; import org.archive.util.ArchiveUtils; /** @@ -64,9 +65,9 @@ public class ARCWriterProcessor extends WriterPoolProcessor { public long getDefaultMaxFileSize() { return 100000000L; // 100 SI mega-bytes (10^8 bytes) } - public List getDefaultStorePaths() { - List paths = new ArrayList(); - paths.add("arcs"); + public List getDefaultStorePaths() { + List paths = new ArrayList(); + paths.add(new ConfigPath("arcs default store path", "arcs")); return paths; } diff --git a/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java b/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java index 293b1f2c..81959d2c 100644 --- a/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java +++ b/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java @@ -81,6 +81,7 @@ import org.archive.modules.CrawlURI; import org.archive.modules.ProcessResult; import org.archive.modules.deciderules.recrawl.IdenticalDigestDecideRule; import org.archive.modules.extractor.Link; +import org.archive.spring.ConfigPath; import org.archive.uid.RecordIDGenerator; import org.archive.uid.UUIDGenerator; import org.archive.util.ArchiveUtils; @@ -107,9 +108,9 @@ public class WARCWriterProcessor extends WriterPoolProcessor implements WARCWrit public long getDefaultMaxFileSize() { return 1000000000L; // 1 SI giga-byte (10^9 bytes), per WARC appendix A } - public List getDefaultStorePaths() { - List paths = new ArrayList(); - paths.add("warcs"); + public List getDefaultStorePaths() { + List paths = new ArrayList(); + paths.add(new ConfigPath("warcs default store path", "warcs")); return paths; } diff --git a/modules/src/main/java/org/archive/modules/writer/WriterPoolProcessor.java b/modules/src/main/java/org/archive/modules/writer/WriterPoolProcessor.java index adc6f65a..a3fa9863 100644 --- a/modules/src/main/java/org/archive/modules/writer/WriterPoolProcessor.java +++ b/modules/src/main/java/org/archive/modules/writer/WriterPoolProcessor.java @@ -241,12 +241,12 @@ implements Lifecycle, Checkpointable, WriterPoolSettings { * setting is safe to change midcrawl (You can remove and add new * dirs as the crawler progresses). */ - List storePaths = getDefaultStorePaths(); - abstract List getDefaultStorePaths(); - public List getStorePaths() { + List storePaths = getDefaultStorePaths(); + abstract List getDefaultStorePaths(); + public List getStorePaths() { return storePaths; } - public void setStorePaths(List paths) { + public void setStorePaths(List paths) { this.storePaths = paths; } @@ -430,12 +430,11 @@ implements Lifecycle, Checkpointable, WriterPoolSettings { public abstract List getMetadata(); public List getOutputDirs() { - List list = getStorePaths(); + List list = getStorePaths(); ArrayList results = new ArrayList(); - for (String path: list) { - File f = new File( - path.startsWith("/") ? null : getDirectory().getFile(), - path); + for (ConfigPath path: list) { + path.setBase(getDirectory()); + File f = path.getFile(); if (!f.exists()) { try { f.mkdirs();