diff --git a/commons/src/main/java/org/archive/io/WriterPoolMember.java b/commons/src/main/java/org/archive/io/WriterPoolMember.java index c130b95f..6ea6b295 100644 --- a/commons/src/main/java/org/archive/io/WriterPoolMember.java +++ b/commons/src/main/java/org/archive/io/WriterPoolMember.java @@ -201,7 +201,7 @@ public abstract class WriterPoolMember implements ArchiveFileConstants { String name = currentBasename + '.' + this.extension + ((settings.getCompress())? DOT_COMPRESSED_FILE_EXTENSION: "") + OCCUPIED_SUFFIX; - File dir = getNextDirectory(settings.getOutputDirs()); + File dir = getNextDirectory(settings.calcOutputDirs()); return createFile(new File(dir, name)); } diff --git a/commons/src/main/java/org/archive/io/WriterPoolSettings.java b/commons/src/main/java/org/archive/io/WriterPoolSettings.java index 25071ddb..d0805cdc 100644 --- a/commons/src/main/java/org/archive/io/WriterPoolSettings.java +++ b/commons/src/main/java/org/archive/io/WriterPoolSettings.java @@ -31,7 +31,7 @@ public interface WriterPoolSettings { public long getMaxFileSizeBytes(); public String getPrefix(); public String getTemplate(); - public List getOutputDirs(); + public List calcOutputDirs(); public boolean getCompress(); public List getMetadata(); public boolean getFrequentFlushes(); diff --git a/commons/src/main/java/org/archive/io/arc/WriterPoolSettingsData.java b/commons/src/main/java/org/archive/io/arc/WriterPoolSettingsData.java index 173f84a6..33490b4a 100644 --- a/commons/src/main/java/org/archive/io/arc/WriterPoolSettingsData.java +++ b/commons/src/main/java/org/archive/io/arc/WriterPoolSettingsData.java @@ -58,7 +58,7 @@ public class WriterPoolSettingsData implements WriterPoolSettings { return metadata; } @Override - public List getOutputDirs() { + public List calcOutputDirs() { return outputDirs; } @Override diff --git a/commons/src/main/java/org/archive/spring/ConfigFile.java b/commons/src/main/java/org/archive/spring/ConfigFile.java index 27c0cd94..e0293dbb 100644 --- a/commons/src/main/java/org/archive/spring/ConfigFile.java +++ b/commons/src/main/java/org/archive/spring/ConfigFile.java @@ -26,7 +26,6 @@ import java.io.OutputStreamWriter; import java.io.Reader; import java.io.Writer; -import org.apache.commons.io.FileUtils; import org.archive.io.ReadSource; /** @@ -50,8 +49,7 @@ public class ConfigFile extends ConfigPath implements ReadSource, WriteTarget { getFile().createNewFile(); } - // snapshot file to launch directory - FileUtils.copyFileToDirectory(getFile(), currentLaunchDir); + configurer.snapshotToLaunchDir(getFile()); return new InputStreamReader( new FileInputStream(getFile()), diff --git a/commons/src/main/java/org/archive/spring/ConfigPath.java b/commons/src/main/java/org/archive/spring/ConfigPath.java index 3596d095..142994f7 100644 --- a/commons/src/main/java/org/archive/spring/ConfigPath.java +++ b/commons/src/main/java/org/archive/spring/ConfigPath.java @@ -58,9 +58,6 @@ public class ConfigPath implements Serializable { String path; ConfigPath base; - transient String interpolatedPath; - transient File currentLaunchDir; - public ConfigPath() { super(); } @@ -69,7 +66,6 @@ public class ConfigPath implements Serializable { super(); this.name = name; this.path = path; - this.interpolatedPath = path; } public ConfigPath getBase() { @@ -95,13 +91,19 @@ public class ConfigPath implements Serializable { @Required public void setPath(String path) { this.path = path; - this.interpolatedPath = path; } public File getFile() { - return (base == null || interpolatedPath.startsWith("/")) - ? new File(interpolatedPath) - : new File(base.getFile(), interpolatedPath); + String interpolatedPath; + if (configurer != null) { + interpolatedPath = configurer.interpolate(path); + } else { + interpolatedPath = path; + } + + return base == null || interpolatedPath.startsWith("/") + ? new File(interpolatedPath) + : new File(base.getFile(), interpolatedPath); } /** @@ -122,9 +124,9 @@ public class ConfigPath implements Serializable { return this; } - public void informOfLaunch(String currentLaunchId, File currentLaunchDir) { - this.currentLaunchDir = currentLaunchDir; - // could use PropertyUtils.interpolateWithProperties(String, Properties...), but no real need - interpolatedPath = path.replace("${launch-id}", currentLaunchId); + ConfigPathConfigurer configurer; + public void setConfigurer(ConfigPathConfigurer configPathConfigurer) { + this.configurer = configPathConfigurer; } + } diff --git a/commons/src/main/java/org/archive/spring/ConfigPathConfigurer.java b/commons/src/main/java/org/archive/spring/ConfigPathConfigurer.java index 2d033c16..cf8a4880 100644 --- a/commons/src/main/java/org/archive/spring/ConfigPathConfigurer.java +++ b/commons/src/main/java/org/archive/spring/ConfigPathConfigurer.java @@ -21,12 +21,12 @@ package org.archive.spring; import java.beans.PropertyDescriptor; import java.io.File; +import java.io.IOException; import java.util.HashMap; -import java.util.List; import java.util.Map; +import org.apache.commons.io.FileUtils; import org.apache.commons.lang.StringUtils; -import org.archive.modules.writer.WriterPoolProcessor; import org.springframework.beans.BeanWrapperImpl; import org.springframework.beans.BeansException; import org.springframework.beans.factory.config.BeanPostProcessor; @@ -35,7 +35,6 @@ import org.springframework.context.ApplicationContextAware; import org.springframework.context.ApplicationEvent; import org.springframework.context.ApplicationListener; import org.springframework.context.event.ContextRefreshedEvent; -import org.springframework.context.support.AbstractApplicationContext; import org.springframework.core.Ordered; /** @@ -88,24 +87,13 @@ implements public void onApplicationEvent(ApplicationEvent event) { if(event instanceof ContextRefreshedEvent) { for(String beanName: allBeans.keySet()) { - Object bean = allBeans.get(beanName); - - fixupPaths(bean, beanName); - - // we want to remember writer pool store paths for later interpolation - // (their base dirs are taken care of elsewhere) - if (bean instanceof WriterPoolProcessor) { - List storePaths = ((WriterPoolProcessor) bean).getStorePaths(); - for (int i = 0; i < storePaths.size(); i++) { - remember(beanName + "." + "storePaths[" + i + "]", storePaths.get(i)); - } - } + fixupPaths(allBeans.get(beanName), beanName); } allBeans.clear(); // forget } // ignore all others } - + /** * Find any ConfigPath properties in the passed bean; ensure that * if they have a null 'base', that is replaced with the job home @@ -119,55 +107,66 @@ implements protected Object fixupPaths(Object bean, String beanName) { BeanWrapperImpl wrapper = new BeanWrapperImpl(bean); for(PropertyDescriptor d : wrapper.getPropertyDescriptors()) { - if(d.getPropertyType().isAssignableFrom(ConfigPath.class) - || d.getPropertyType().isAssignableFrom(ConfigFile.class)) { + if (d.getPropertyType().isAssignableFrom(ConfigPath.class) + || d.getPropertyType().isAssignableFrom(ConfigFile.class)) { Object value = wrapper.getPropertyValue(d.getName()); - if(ConfigPath.class.isInstance(value)) { - ConfigPath cp = (ConfigPath) value; - if(cp==null) { - continue; + if (value != null && value instanceof ConfigPath) { + String patchName = beanName+"."+d.getName(); + fixupConfigPath((ConfigPath) value,patchName); + } + } else if (Iterable.class.isAssignableFrom(d.getPropertyType())) { + Iterable iterable = (Iterable) wrapper.getPropertyValue(d.getName()); + if (iterable != null) { + int i = 0; + for (Object candidate : iterable) { + if(candidate!=null && candidate instanceof ConfigPath) { + String patchName = beanName+"."+d.getName()+"["+i+"]"; + fixupConfigPath((ConfigPath) candidate,patchName); + } + i++; } - if(cp.getBase()==null && cp != path) { - cp.setBase(path); - } - String beanPath = beanName+"."+d.getName(); - if(StringUtils.isEmpty(cp.getName())) { - cp.setName(beanPath); - } - remember(beanPath, cp); } } } - if(bean instanceof PathFixupListener) { ((PathFixupListener)bean).pathsFixedUp(); } return bean; } + protected void fixupConfigPath(ConfigPath cp, String patchName) { + if(cp.getBase()==null && cp != path) { + cp.setBase(path); + } + + if(StringUtils.isEmpty(cp.getName())) { + cp.setName(patchName); + } + cp.setConfigurer(this); + remember(patchName, cp); + } + //// APPLICATIONCONTEXTAWARE IMPLEMENTATION - AbstractApplicationContext appCtx; + PathSharingContext appCtx; /** * Remember ApplicationContext, and if possible primary * configuration file's home directory. + * + * Requires appCtx be a PathSharingContext + * * @see org.springframework.context.ApplicationContextAware#setApplicationContext(org.springframework.context.ApplicationContext) */ public void setApplicationContext(ApplicationContext appCtx) throws BeansException { - this.appCtx = (AbstractApplicationContext)appCtx; + this.appCtx = (PathSharingContext)appCtx; String basePath; if(appCtx instanceof PathSharingContext) { - String primaryConfigurationPath = ((PathSharingContext)appCtx).getPrimaryConfigurationPath(); - if(primaryConfigurationPath.startsWith("file:")) { - // strip URI-scheme if present (as is usual) - primaryConfigurationPath = primaryConfigurationPath.substring(5); - } - File configFile = new File(primaryConfigurationPath); - basePath = configFile.getParent(); + basePath = this.appCtx.getConfigurationFile().getParent(); } else { basePath = "."; } path = new ConfigPath("job base",basePath); + path.setConfigurer(this); } // REMEMBERED CONFIGPATHS @@ -193,4 +192,16 @@ implements public int getOrder() { return Ordered.LOWEST_PRECEDENCE; } + + public void snapshotToLaunchDir(File readFile) throws IOException { + FileUtils.copyFileToDirectory(readFile, appCtx.getCurrentLaunchDir()); + } + + protected String interpolate(String rawPath) { + if (appCtx.getCurrentLaunchId() != null) { + return rawPath.replace("${launchId}", appCtx.getCurrentLaunchId()); + } else { + return rawPath; + } + } } diff --git a/commons/src/main/java/org/archive/spring/PathSharingContext.java b/commons/src/main/java/org/archive/spring/PathSharingContext.java index 36c6ce0c..6d86cecb 100644 --- a/commons/src/main/java/org/archive/spring/PathSharingContext.java +++ b/commons/src/main/java/org/archive/spring/PathSharingContext.java @@ -19,13 +19,19 @@ package org.archive.spring; +import java.io.File; +import java.io.IOException; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.LinkedHashSet; import java.util.Map; +import java.util.logging.Level; import java.util.logging.Logger; +import org.apache.commons.io.FileUtils; +import org.archive.util.ArchiveUtils; +import org.archive.util.FilesystemLinkMaker; import org.springframework.beans.BeansException; import org.springframework.beans.factory.config.ConfigurableListableBeanFactory; import org.springframework.context.ApplicationContext; @@ -39,18 +45,27 @@ import org.springframework.validation.Errors; import org.springframework.validation.Validator; /** - * Spring ApplicationContext extended for Heritrix use. + * Spring ApplicationContext extended for Heritrix use. * * Notable extensions: * - * Remembers its primary XML configuration file, and can report its - * filesystem path. + * Remembers its primary XML configuration file, and can report its filesystem + * path. * - * Propagates lifecycle events (start, stop) without triggering - * loops in the case of circular dependencies. + * Propagates lifecycle events (start, stop) without triggering loops in the + * case of circular dependencies. * * Reports a summary of Errors collected from self-Validating Beans. * + * Generates launchId from timestamp, creates launch directory + * {jobDir}/{launchId}, and snapshots crawl configuration file into the launch + * directory. Other configuration files, if any, are automatically snapshotted + * into the launch directory when they are read (see + * {@link ConfigFile#obtainReader()}). The token ${launchId} will be + * interpolated in configuration-relative paths (see + * {@link ConfigPathConfigurer}) so that launch-specific paths can be used for + * logs, reports, warcs, etc. + * * @contributor gojomo */ public class PathSharingContext extends FileSystemXmlApplicationContext { @@ -91,6 +106,7 @@ public class PathSharingContext extends FileSystemXmlApplicationContext { @SuppressWarnings("unchecked") @Override public void start() { + initLaunchDir(); Map lifecycleBeans = getLifecycleBeans(); for (Iterator it = new HashSet(lifecycleBeans.keySet()).iterator(); it.hasNext();) { String beanName = (String) it.next(); @@ -219,6 +235,54 @@ public class PathSharingContext extends FileSystemXmlApplicationContext { return allErrors; } + protected transient String currentLaunchId; + protected void initLaunchId() { + currentLaunchId = ArchiveUtils.getUnique14DigitDate(); + LOGGER.info("launch id " + currentLaunchId); + } + public String getCurrentLaunchId() { + return currentLaunchId; + } + + protected transient File currentLaunchDir; + public File getCurrentLaunchDir() { + return currentLaunchDir; + } + + protected File getConfigurationFile() { + String primaryConfigurationPath = getPrimaryConfigurationPath(); + if(primaryConfigurationPath.startsWith("file:")) { + // strip URI-scheme if present (as is usual) + primaryConfigurationPath = primaryConfigurationPath.substring(5); + } + File configFile = new File(primaryConfigurationPath); + return configFile; + } + + protected void initLaunchDir() { + initLaunchId(); + try { + currentLaunchDir = new File(getConfigurationFile().getParentFile(), getCurrentLaunchId()); + if (!currentLaunchDir.mkdir()) { + throw new IOException("failed to create directory " + currentLaunchDir); + } + + // copy cxml to launch dir + FileUtils.copyFileToDirectory(getConfigurationFile(), currentLaunchDir); + + // attempt to symlink "latest" to launch dir + File latestSymlink = new File(getConfigurationFile().getParentFile(), "latest"); + latestSymlink.delete(); + boolean success = FilesystemLinkMaker.makeSymbolicLink(currentLaunchDir.getName(), latestSymlink.getPath()); + if (!success) { + LOGGER.warning("failed to create symlink from " + latestSymlink + " to " + currentLaunchDir); + } + } catch (IOException e) { + LOGGER.log(Level.SEVERE, "failed to initialize launch directory: " + e); + currentLaunchDir = null; + } + } + /** * Initialize the LifecycleProcessor. * Uses HeritrixLifecycleProcessor, which prevents an automatic lifecycle diff --git a/engine/src/main/java/org/archive/crawler/framework/ActionDirectory.java b/engine/src/main/java/org/archive/crawler/framework/ActionDirectory.java index 310bc36a..9ef67a2e 100644 --- a/engine/src/main/java/org/archive/crawler/framework/ActionDirectory.java +++ b/engine/src/main/java/org/archive/crawler/framework/ActionDirectory.java @@ -111,7 +111,7 @@ public class ActionDirectory implements ApplicationContextAware, Lifecycle, Runn } protected ConfigPath doneDir = - new ConfigPath("ActionDirectory done directory","${launch-id}/actions-done"); + new ConfigPath("ActionDirectory done directory","${launchId}/actions-done"); public ConfigPath getDoneDir() { return doneDir; } diff --git a/engine/src/main/java/org/archive/crawler/framework/CheckpointService.java b/engine/src/main/java/org/archive/crawler/framework/CheckpointService.java index 7f46a015..f6bd4542 100644 --- a/engine/src/main/java/org/archive/crawler/framework/CheckpointService.java +++ b/engine/src/main/java/org/archive/crawler/framework/CheckpointService.java @@ -37,6 +37,7 @@ import org.archive.checkpointing.Checkpoint; import org.archive.checkpointing.Checkpointable; import org.archive.crawler.reporting.CrawlStatSnapshot; import org.archive.spring.ConfigPath; +import org.archive.spring.ConfigPathConfigurer; import org.archive.spring.HasValidator; import org.springframework.beans.BeansException; import org.springframework.beans.factory.annotation.Autowired; @@ -223,7 +224,6 @@ public class CheckpointService implements Lifecycle, ApplicationContextAware, Ha checkpointInProgress = new Checkpoint(); checkpointInProgress.generateFrom(getCheckpointsDir(),getNextCheckpointNumber()); - @SuppressWarnings("unchecked") Map toCheckpoint = appCtx.getBeansOfType(Checkpointable.class); try { @@ -288,8 +288,8 @@ public class CheckpointService implements Lifecycle, ApplicationContextAware, Ha if(getRecoveryCheckpoint()!=null || isRunning()) { return false; } - return (getAvailableCheckpointDirectories() != null - && getAvailableCheckpointDirectories().size() > 0); + return (findAvailableCheckpointDirectories() != null + && findAvailableCheckpointDirectories().size() > 0); } /** @@ -300,7 +300,7 @@ public class CheckpointService implements Lifecycle, ApplicationContextAware, Ha * @return List of valid checkpoint directory File instances */ @SuppressWarnings("unchecked") - public List getAvailableCheckpointDirectories() { + public List findAvailableCheckpointDirectories() { File[] dirs = getCheckpointsDir().getFile().listFiles((FileFilter)FileFilterUtils.directoryFileFilter()); if (dirs == null) { return Collections.EMPTY_LIST; @@ -325,7 +325,6 @@ public class CheckpointService implements Lifecycle, ApplicationContextAware, Ha * * @param selectedCheckpoint */ - @SuppressWarnings("unchecked") public void setRecoveryCheckpointByName(String selectedCheckpoint) { if(isRunning) { throw new RuntimeException("may not set recovery Checkpoint after launch"); @@ -333,6 +332,7 @@ public class CheckpointService implements Lifecycle, ApplicationContextAware, Ha Checkpoint recoveryCheckpoint = new Checkpoint(); recoveryCheckpoint.getCheckpointDir().setBase(getCheckpointsDir()); recoveryCheckpoint.getCheckpointDir().setPath(selectedCheckpoint); + recoveryCheckpoint.getCheckpointDir().setConfigurer(appCtx.getBean(ConfigPathConfigurer.class)); recoveryCheckpoint.afterPropertiesSet(); setRecoveryCheckpoint(recoveryCheckpoint); Map toSetRecovery = appCtx.getBeansOfType(Checkpointable.class); diff --git a/engine/src/main/java/org/archive/crawler/framework/CrawlJob.java b/engine/src/main/java/org/archive/crawler/framework/CrawlJob.java index 5ed831ff..8eaddd79 100644 --- a/engine/src/main/java/org/archive/crawler/framework/CrawlJob.java +++ b/engine/src/main/java/org/archive/crawler/framework/CrawlJob.java @@ -57,7 +57,6 @@ import org.archive.spring.ConfigPath; import org.archive.spring.ConfigPathConfigurer; import org.archive.spring.PathSharingContext; import org.archive.util.ArchiveUtils; -import org.archive.util.FilesystemLinkMaker; import org.archive.util.TextUtils; import org.joda.time.DateTime; import org.springframework.beans.BeanWrapperImpl; @@ -430,7 +429,6 @@ public class CrawlJob implements Comparable, ApplicationListener, ApplicationListener, ApplicationListener, ApplicationListener, ApplicationListener"); - for(File f : cj.getCheckpointService().getAvailableCheckpointDirectories()) { + for(File f : cj.getCheckpointService().findAvailableCheckpointDirectories()) { pw.println(""); } pw.println(""); diff --git a/engine/src/main/resources/org/archive/crawler/restlet/profile-crawler-beans.cxml b/engine/src/main/resources/org/archive/crawler/restlet/profile-crawler-beans.cxml index f9d3db46..d4c087c4 100644 --- a/engine/src/main/resources/org/archive/crawler/restlet/profile-crawler-beans.cxml +++ b/engine/src/main/resources/org/archive/crawler/restlet/profile-crawler-beans.cxml @@ -123,7 +123,7 @@ http://example.example/example - + - + - + @@ -613,7 +613,7 @@ http://example.example/example - + @@ -623,7 +623,7 @@ http://example.example/example - + diff --git a/modules/src/main/java/org/archive/modules/deciderules/surt/SurtPrefixedDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/surt/SurtPrefixedDecideRule.java index e92b1453..a9e51922 100644 --- a/modules/src/main/java/org/archive/modules/deciderules/surt/SurtPrefixedDecideRule.java +++ b/modules/src/main/java/org/archive/modules/deciderules/surt/SurtPrefixedDecideRule.java @@ -111,7 +111,7 @@ implements * Dump file to save SURT prefixes actually used: Useful debugging SURTs. */ protected ConfigFile surtsDumpFile = - new ConfigFile("surtsDumpFile","${launch-id}/surts.dump"); + new ConfigFile("surtsDumpFile","${launchId}/surts.dump"); public ConfigFile getSurtsDumpFile() { return surtsDumpFile; } diff --git a/modules/src/main/java/org/archive/modules/writer/WriterPoolProcessor.java b/modules/src/main/java/org/archive/modules/writer/WriterPoolProcessor.java index a3fa9863..73c5ff99 100644 --- a/modules/src/main/java/org/archive/modules/writer/WriterPoolProcessor.java +++ b/modules/src/main/java/org/archive/modules/writer/WriterPoolProcessor.java @@ -225,7 +225,7 @@ implements Lifecycle, Checkpointable, WriterPoolSettings { this.serverCache = serverCache; } - protected ConfigPath directory = new ConfigPath("writer base path", "${launch-id}"); + protected ConfigPath directory = new ConfigPath("writer base path", "${launchId}"); public ConfigPath getDirectory() { return directory; } @@ -429,7 +429,7 @@ implements Lifecycle, Checkpointable, WriterPoolSettings { public abstract List getMetadata(); - public List getOutputDirs() { + public List calcOutputDirs() { List list = getStorePaths(); ArrayList results = new ArrayList(); for (ConfigPath path: list) {