From 4250e07de98281f6ddc92fb34322bfa3519fa2eb Mon Sep 17 00:00:00 2001 From: nlevitt Date: Wed, 13 Jul 2011 19:18:55 +0000 Subject: [PATCH] HER-1901 timestamped subdirectory for each launch * HardLinker.java renamed FilesystemLinkMaker.java * FilesystemLinkMaker.java add support for symbolic links * CLibrary.java new method symlink() * BdbModule.java use new class name FilesystemLinkMaker * CrawlJob.java at crawl launch, create launch directory launch-{timestamp17}, copy cxml there, symlink "current" to launch dir, inform ConfigPaths * ConfigPath.java interpolate ${launch-id} in configured paths * ConfigFile.java obtainReader() - snapshot config files to launch dir when they are read * ActionDirectory.java default doneDir now ${launch-id}/actions-done actOn() - symlink from old style done dir action/done to done files * SurtPrefixedDecideRule.java default surtsDumpFile now ${launch-id}/surts.dump pathsFixedUp() - this gets called at build time, but we don't want anything written to disk until launch time, so remove call to dumpSurtPrefixSet() here * CrawlerLoggerModule.java default logs dir now ${launch-id}/logs * StatisticsTracker.java default reports dir now ${launch-id}/reports * WriterPoolProcessor.java default writer base path now ${launch-id} * profile-crawler-beans.cxml update to reflect new default paths under launch dirs * PropertyUtils.java fix javadoc typo --- .../main/java/org/archive/bdb/BdbModule.java | 4 +- .../java/org/archive/spring/ConfigFile.java | 5 ++ .../java/org/archive/spring/ConfigPath.java | 19 +++++-- .../main/java/org/archive/util/CLibrary.java | 3 +- ...rdLinker.java => FilesystemLinkMaker.java} | 35 ++++++++++-- .../java/org/archive/util/PropertyUtils.java | 2 +- .../crawler/framework/ActionDirectory.java | 27 ++++++---- .../archive/crawler/framework/CrawlJob.java | 53 +++++++++++++++++-- .../reporting/CrawlerLoggerModule.java | 2 +- .../crawler/reporting/StatisticsTracker.java | 6 +-- .../restlet/profile-crawler-beans.cxml | 11 ++-- .../surt/SurtPrefixedDecideRule.java | 3 +- .../modules/writer/WriterPoolProcessor.java | 2 +- 13 files changed, 134 insertions(+), 38 deletions(-) rename commons/src/main/java/org/archive/util/{HardLinker.java => FilesystemLinkMaker.java} (70%) diff --git a/commons/src/main/java/org/archive/bdb/BdbModule.java b/commons/src/main/java/org/archive/bdb/BdbModule.java index 214ba591..5d854f15 100644 --- a/commons/src/main/java/org/archive/bdb/BdbModule.java +++ b/commons/src/main/java/org/archive/bdb/BdbModule.java @@ -43,7 +43,7 @@ import org.archive.checkpointing.Checkpoint; import org.archive.checkpointing.Checkpointable; import org.archive.spring.ConfigPath; import org.archive.util.CLibrary; -import org.archive.util.HardLinker; +import org.archive.util.FilesystemLinkMaker; import org.archive.util.IdentityCacheable; import org.archive.util.ObjectIdentityBdbManualCache; import org.archive.util.ObjectIdentityCache; @@ -487,7 +487,7 @@ public class BdbModule implements Lifecycle, Checkpointable, Closeable { filedata[i] += ","+f.length(); if(getUseHardLinkCheckpoints()) { File hardLink = new File(envCpDir,filedata[i]); - if (!HardLinker.makeHardLink(f.getAbsolutePath(), hardLink.getAbsolutePath())) { + if (!FilesystemLinkMaker.makeHardLink(f.getAbsolutePath(), hardLink.getAbsolutePath())) { LOGGER.log(Level.SEVERE, "unable to create required checkpoint link "+hardLink); } } diff --git a/commons/src/main/java/org/archive/spring/ConfigFile.java b/commons/src/main/java/org/archive/spring/ConfigFile.java index 5058120e..27c0cd94 100644 --- a/commons/src/main/java/org/archive/spring/ConfigFile.java +++ b/commons/src/main/java/org/archive/spring/ConfigFile.java @@ -26,6 +26,7 @@ import java.io.OutputStreamWriter; import java.io.Reader; import java.io.Writer; +import org.apache.commons.io.FileUtils; import org.archive.io.ReadSource; /** @@ -48,6 +49,10 @@ public class ConfigFile extends ConfigPath implements ReadSource, WriteTarget { if(!getFile().exists()) { getFile().createNewFile(); } + + // snapshot file to launch directory + FileUtils.copyFileToDirectory(getFile(), currentLaunchDir); + return new InputStreamReader( new FileInputStream(getFile()), "UTF-8"); diff --git a/commons/src/main/java/org/archive/spring/ConfigPath.java b/commons/src/main/java/org/archive/spring/ConfigPath.java index 575e7529..3596d095 100644 --- a/commons/src/main/java/org/archive/spring/ConfigPath.java +++ b/commons/src/main/java/org/archive/spring/ConfigPath.java @@ -56,9 +56,10 @@ public class ConfigPath implements Serializable { String name; String path; - ConfigPath base; - transient File resolved; + ConfigPath base; + transient String interpolatedPath; + transient File currentLaunchDir; public ConfigPath() { super(); @@ -68,6 +69,7 @@ public class ConfigPath implements Serializable { super(); this.name = name; this.path = path; + this.interpolatedPath = path; } public ConfigPath getBase() { @@ -93,12 +95,13 @@ public class ConfigPath implements Serializable { @Required public void setPath(String path) { this.path = path; + this.interpolatedPath = path; } public File getFile() { - return (base == null || path.startsWith("/")) - ? new File(path) - : new File(base.getFile(), path); + return (base == null || interpolatedPath.startsWith("/")) + ? new File(interpolatedPath) + : new File(base.getFile(), interpolatedPath); } /** @@ -118,4 +121,10 @@ public class ConfigPath implements Serializable { } return this; } + + public void informOfLaunch(String currentLaunchId, File currentLaunchDir) { + this.currentLaunchDir = currentLaunchDir; + // could use PropertyUtils.interpolateWithProperties(String, Properties...), but no real need + interpolatedPath = path.replace("${launch-id}", currentLaunchId); + } } diff --git a/commons/src/main/java/org/archive/util/CLibrary.java b/commons/src/main/java/org/archive/util/CLibrary.java index 0d5625fc..b47c143a 100644 --- a/commons/src/main/java/org/archive/util/CLibrary.java +++ b/commons/src/main/java/org/archive/util/CLibrary.java @@ -35,5 +35,6 @@ public interface CLibrary extends Library { Native.loadLibrary((Platform.isWindows() ? "msvcrt" : "c"), CLibrary.class); - int link(String fromFile, String toFile); + int link(String existingPath, String newPath); + int symlink(String existingPath, String newPath); } diff --git a/commons/src/main/java/org/archive/util/HardLinker.java b/commons/src/main/java/org/archive/util/FilesystemLinkMaker.java similarity index 70% rename from commons/src/main/java/org/archive/util/HardLinker.java rename to commons/src/main/java/org/archive/util/FilesystemLinkMaker.java index ad784ada..9899a29c 100644 --- a/commons/src/main/java/org/archive/util/HardLinker.java +++ b/commons/src/main/java/org/archive/util/FilesystemLinkMaker.java @@ -32,7 +32,7 @@ import com.sun.jna.win32.StdCallLibrary; * * @see http://stackoverflow.com/questions/783075/creating-a-hard-link-in-java/3023349#3023349 */ -public class HardLinker { +public class FilesystemLinkMaker { // see https://github.com/twall/jna/blob/master/www/GettingStarted.md public interface Kernel32Library extends StdCallLibrary { @@ -63,6 +63,9 @@ public class HardLinker { */ boolean CreateHardLinkA(String newPath, String existingPath, LPSECURITY_ATTRIBUTES lpSecurityAttributes); // boolean CreateHardLinkW(String newPath, String existingPath, LPSECURITY_ATTRIBUTES lpSecurityAttributes); + + // http://msdn.microsoft.com/en-us/library/aa363866%28v=VS.85%29.aspx + boolean CreateSymbolicLinkA(String newPath, String existingPath, LPSECURITY_ATTRIBUTES lpSecurityAttributes); } /** @@ -70,6 +73,7 @@ public class HardLinker { * * @return true on success */ + // XXX could handle errors better (examine errno, throw exception...) public static boolean makeHardLink(String existingPath, String newPath) { if (Platform.isWindows()) { return Kernel32Library.INSTANCE.CreateHardLinkA(newPath, existingPath, null); @@ -78,17 +82,42 @@ public class HardLinker { return status == 0; } } - + + /** + * Wrapper over platform-dependent system calls to create a symboic link. + * + * @return true on success + */ + // XXX could handle errors better (examine errno, throw exception...) + public static boolean makeSymbolicLink(String existingPath, String newPath) { + if (Platform.isWindows()) { + return Kernel32Library.INSTANCE.CreateSymbolicLinkA(newPath, existingPath, null); + } else { + int status = CLibrary.INSTANCE.symlink(existingPath, newPath); + return status == 0; + } + } + public static void main(String[] args) throws IOException { File existingPath = File.createTempFile("heritrixHardLinkTestExistingFile", ".tmp"); File newPath = File.createTempFile("heritrixHardLinkTestNewFile", ".tmp"); newPath.delete(); - if (HardLinker.makeHardLink(existingPath.getAbsolutePath(), newPath.getAbsolutePath())) { + if (FilesystemLinkMaker.makeHardLink(existingPath.getAbsolutePath(), newPath.getAbsolutePath())) { System.out.println("success - made hard link from " + newPath.getAbsolutePath() + " to " + existingPath.getAbsolutePath()); } else { System.out.println("failed to make hard link from " + newPath.getAbsolutePath() + " to " + existingPath.getAbsolutePath()); } + + existingPath = File.createTempFile("heritrixSymlinkTestExistingFile", ".tmp"); + newPath = File.createTempFile("heritrixSymlinkTestNewFile", ".tmp"); + newPath.delete(); + + if (FilesystemLinkMaker.makeSymbolicLink(existingPath.getPath(), newPath.getPath())) { + System.out.println("success - made symlink from " + newPath.getAbsolutePath() + " to " + existingPath.getAbsolutePath()); + } else { + System.out.println("failed to make symlink from " + newPath.getAbsolutePath() + " to " + existingPath.getAbsolutePath()); + } } } diff --git a/commons/src/main/java/org/archive/util/PropertyUtils.java b/commons/src/main/java/org/archive/util/PropertyUtils.java index 0760cea5..955e772f 100644 --- a/commons/src/main/java/org/archive/util/PropertyUtils.java +++ b/commons/src/main/java/org/archive/util/PropertyUtils.java @@ -83,7 +83,7 @@ public class PropertyUtils { * the expression is replaced with the empty-string. * * @param original String - * @param properties Properties to try in order; first value found (if any) is used + * @param props Properties to try in order; first value found (if any) is used * @return modified String */ public static String interpolateWithProperties(String original, diff --git a/engine/src/main/java/org/archive/crawler/framework/ActionDirectory.java b/engine/src/main/java/org/archive/crawler/framework/ActionDirectory.java index 0cfed86b..310bc36a 100644 --- a/engine/src/main/java/org/archive/crawler/framework/ActionDirectory.java +++ b/engine/src/main/java/org/archive/crawler/framework/ActionDirectory.java @@ -41,6 +41,7 @@ import org.apache.commons.lang.StringUtils; import org.archive.modules.seeds.SeedModule; import org.archive.spring.ConfigPath; import org.archive.util.ArchiveUtils; +import org.archive.util.FilesystemLinkMaker; import org.springframework.beans.BeansException; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.ApplicationContext; @@ -100,9 +101,6 @@ public class ActionDirectory implements ApplicationContextAware, Lifecycle, Runn this.delaySeconds = delay; } - /** - * Scratch directory for temporary overflow-to-disk - */ protected ConfigPath actionDir = new ConfigPath("ActionDirectory source directory","action"); public ConfigPath getActionDir() { @@ -112,16 +110,13 @@ public class ActionDirectory implements ApplicationContextAware, Lifecycle, Runn this.actionDir = actionDir; } - /** - * Scratch directory for temporary overflow-to-disk - */ protected ConfigPath doneDir = - new ConfigPath("ActionDirectory done directory","action/done"); + new ConfigPath("ActionDirectory done directory","${launch-id}/actions-done"); public ConfigPath getDoneDir() { return doneDir; } - public void setDoneDir(ConfigPath scratchDir) { - this.doneDir = scratchDir; + public void setDoneDir(ConfigPath doneDir) { + this.doneDir = doneDir; } ApplicationContext appCtx; @@ -258,7 +253,19 @@ public class ActionDirectory implements ApplicationContextAware, Lifecycle, Runn // move file to 'done' area with timestamp prefix while(actionFile.exists()) { try { - FileUtils.moveFile(actionFile, new File(doneDir.getFile(),timestamp+"."+actionFile.getName())); + File doneFile = new File(doneDir.getFile(),timestamp+"."+actionFile.getName()); + FileUtils.moveFile(actionFile, doneFile); + + // attempt to symlink from action/done/ to done file + File actionDoneDirFile = new File(actionDir.getFile(), "done"); + if (!actionDoneDirFile.equals(doneDir.getFile())) { + actionDoneDirFile.mkdirs(); + File doneSymlinkFile = new File(actionDoneDirFile, doneFile.getName()); + boolean success = FilesystemLinkMaker.makeSymbolicLink(doneFile.getPath(), doneSymlinkFile.getPath()); + if (!success) { + LOGGER.warning("failed to create symlink from " + doneSymlinkFile + " to " + doneFile); + } + } } catch (IOException e) { LOGGER.log(Level.SEVERE,"unable to move "+actionFile,e); } diff --git a/engine/src/main/java/org/archive/crawler/framework/CrawlJob.java b/engine/src/main/java/org/archive/crawler/framework/CrawlJob.java index 5067f237..8276960f 100644 --- a/engine/src/main/java/org/archive/crawler/framework/CrawlJob.java +++ b/engine/src/main/java/org/archive/crawler/framework/CrawlJob.java @@ -57,6 +57,7 @@ import org.archive.spring.ConfigPath; import org.archive.spring.ConfigPathConfigurer; import org.archive.spring.PathSharingContext; import org.archive.util.ArchiveUtils; +import org.archive.util.FilesystemLinkMaker; import org.archive.util.TextUtils; import org.joda.time.DateTime; import org.springframework.beans.BeanWrapperImpl; @@ -80,7 +81,7 @@ import org.xml.sax.SAXException; * * @contributor gojomo */ -public class CrawlJob implements Comparable, ApplicationListener { +public class CrawlJob implements Comparable, ApplicationListener { private final static Logger LOGGER = Logger.getLogger(CrawlJob.class.getName()); @@ -97,7 +98,7 @@ public class CrawlJob implements Comparable, ApplicationListener { public CrawlJob(File cxml) { primaryConfig = cxml; isLaunchInfoPartial = false; - scanJobLog(); + scanJobLog(); // XXX look at launch directories instead/first? alertThreadGroup = new AlertThreadGroup(getShortName()); } @@ -428,8 +429,9 @@ public class CrawlJob implements Comparable, ApplicationListener { alertThreadGroup.addLogger(getJobLogger()); Thread launcher = new Thread(alertThreadGroup, getShortName()+" launchthread") { public void run() { - startContext(); CrawlController cc = getCrawlController(); + initLaunchDir(); + startContext(); if(cc!=null) { cc.requestCrawlStart(); } @@ -446,6 +448,50 @@ public class CrawlJob implements Comparable, ApplicationListener { } } + protected transient String currentLaunchId; + public void initLaunchId() { + currentLaunchId = "launch-" + ArchiveUtils.get17DigitDate(); + LOGGER.info("launch id " + currentLaunchId); + } + public String getCurrentLaunchId() { + return currentLaunchId; + } + + protected transient File currentLaunchDir; + public File getCurrentLaunchDir() { + return currentLaunchDir; + } + + protected void initLaunchDir() { + initLaunchId(); + try { + currentLaunchDir = new File(getJobDir(), getCurrentLaunchId()); + if (!currentLaunchDir.mkdir()) { + throw new IOException("failed to create directory " + currentLaunchDir); + } + + // copy cxml to launch dir + FileUtils.copyFileToDirectory(getPrimaryConfig(), currentLaunchDir); + + // attempt to symlink "current" to launch dir + File currentSymlink = new File(getJobDir(), "current"); + currentSymlink.delete(); + boolean success = FilesystemLinkMaker.makeSymbolicLink(currentLaunchDir.getName(), currentSymlink.getPath()); + if (!success) { + LOGGER.warning("failed to create symlink from " + currentSymlink + " to " + currentLaunchDir); + } + } catch (IOException e) { + LOGGER.log(Level.SEVERE, "failed to initialize launch directory: " + e); + currentLaunchDir = null; + } + + // fill in ${launch-id} in all config paths, and let config files know + // where to snapshot themselves + for (ConfigPath configPath: getConfigPaths().values()) { + configPath.informOfLaunch(getCurrentLaunchId(), getCurrentLaunchDir()); + } + } + /** * Start the context, catching and reporting any BeansExceptions. */ @@ -515,7 +561,6 @@ public class CrawlJob implements Comparable, ApplicationListener { * * @return Checkpointer */ - @SuppressWarnings("unchecked") public synchronized CheckpointService getCheckpointService() { if(ac==null) { return null; diff --git a/engine/src/main/java/org/archive/crawler/reporting/CrawlerLoggerModule.java b/engine/src/main/java/org/archive/crawler/reporting/CrawlerLoggerModule.java index 3cf128ac..015001cf 100644 --- a/engine/src/main/java/org/archive/crawler/reporting/CrawlerLoggerModule.java +++ b/engine/src/main/java/org/archive/crawler/reporting/CrawlerLoggerModule.java @@ -62,7 +62,7 @@ public class CrawlerLoggerModule Checkpointable, SimpleFileLoggerProvider { private static final long serialVersionUID = 1L; - protected ConfigPath path = new ConfigPath(Engine.LOGS_DIR_NAME,"logs"); + protected ConfigPath path = new ConfigPath(Engine.LOGS_DIR_NAME,"${launch-id}/logs"); public ConfigPath getPath() { return path; } diff --git a/engine/src/main/java/org/archive/crawler/reporting/StatisticsTracker.java b/engine/src/main/java/org/archive/crawler/reporting/StatisticsTracker.java index e25a4af3..2b725b66 100644 --- a/engine/src/main/java/org/archive/crawler/reporting/StatisticsTracker.java +++ b/engine/src/main/java/org/archive/crawler/reporting/StatisticsTracker.java @@ -130,9 +130,9 @@ import com.sleepycat.je.DatabaseException; public class StatisticsTracker implements ApplicationContextAware, - ApplicationListener, + ApplicationListener, SeedListener, - Lifecycle, + Lifecycle, Runnable, Checkpointable, BeanNameAware { @@ -153,7 +153,7 @@ public class StatisticsTracker this.bdb = bdb; } - protected ConfigPath reportsDir = new ConfigPath(Engine.REPORTS_DIR_NAME,"reports"); + protected ConfigPath reportsDir = new ConfigPath(Engine.REPORTS_DIR_NAME,"${launch-id}/reports"); public ConfigPath getReportsDir() { return reportsDir; } diff --git a/engine/src/main/resources/org/archive/crawler/restlet/profile-crawler-beans.cxml b/engine/src/main/resources/org/archive/crawler/restlet/profile-crawler-beans.cxml index 453c5d84..ef5f6fda 100644 --- a/engine/src/main/resources/org/archive/crawler/restlet/profile-crawler-beans.cxml +++ b/engine/src/main/resources/org/archive/crawler/restlet/profile-crawler-beans.cxml @@ -123,7 +123,7 @@ http://example.example/example - + - + + @@ -612,7 +613,7 @@ http://example.example/example - + @@ -622,7 +623,7 @@ http://example.example/example - + diff --git a/modules/src/main/java/org/archive/modules/deciderules/surt/SurtPrefixedDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/surt/SurtPrefixedDecideRule.java index c82f8e21..e92b1453 100644 --- a/modules/src/main/java/org/archive/modules/deciderules/surt/SurtPrefixedDecideRule.java +++ b/modules/src/main/java/org/archive/modules/deciderules/surt/SurtPrefixedDecideRule.java @@ -111,7 +111,7 @@ implements * Dump file to save SURT prefixes actually used: Useful debugging SURTs. */ protected ConfigFile surtsDumpFile = - new ConfigFile("surtsDumpFile","surts.dump"); + new ConfigFile("surtsDumpFile","${launch-id}/surts.dump"); public ConfigFile getSurtsDumpFile() { return surtsDumpFile; } @@ -156,7 +156,6 @@ implements public void pathsFixedUp() { readPrefixes(); - dumpSurtPrefixSet(); } public void concludedSeedBatch() { diff --git a/modules/src/main/java/org/archive/modules/writer/WriterPoolProcessor.java b/modules/src/main/java/org/archive/modules/writer/WriterPoolProcessor.java index 1b5e33a2..adc6f65a 100644 --- a/modules/src/main/java/org/archive/modules/writer/WriterPoolProcessor.java +++ b/modules/src/main/java/org/archive/modules/writer/WriterPoolProcessor.java @@ -225,7 +225,7 @@ implements Lifecycle, Checkpointable, WriterPoolSettings { this.serverCache = serverCache; } - protected ConfigPath directory = new ConfigPath("writer base path", "."); + protected ConfigPath directory = new ConfigPath("writer base path", "${launch-id}"); public ConfigPath getDirectory() { return directory; }