[HER-1727] (W)ARC filename uniqueness guarantees: adminport option, process_id option(?), JVM-global-repeat-suppression

* Heritrix.java
    make heritrix.hostname, heritrix.pid, heritrix.port available as global properties
* PropertyUtils.java, PropertyUtilsTest.java
    utility methods to interpolate string values from one or several supplied Properties instances
* ArchiveUtils.java
    utility methods to give timestamps guaranteed larger/different than any previously-issued timestamp

* WriterPoolSettings.java, WriterPool.java, ARCWriterPool.java, WARCWriterPool.java, ARCWriterPoolTest.java
    improve field names, change 'suffix' to more general 'template'
* WriterPoolMember.java
    replace 'suffix' with 'template' which is interpolated when specific name is needed
    set default template to pattern extremely unlikely to generate duplicate filenames
    change default prefix to 'WEB'
    centralize creation of new basenames into generateNewBasename() method, which internalizes timetamp/serialNo minting and interpolation
* WriterPoolProcessor.java, ARCWriterProcessor.java, WARCWriterProcessor.java
    serve as own WriterPoolSettings instance
    replace 'suffix' with 'template' 
    
* ARCWriter.java
    accept (but truncate) 17-digit timestamps at creation
* DefaultWriterPoolSettings.java, TimestampSerialNo.java
    delete as superfluous
This commit is contained in:
gojomo committed 2010-08-20 00:21:43 +00:00
1 parent c410c90b76
commit b388479d32
15 files changed
+260 -269

No files matched your search

@@ -27,6 +27,9 @@ import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.PrintStream;
import java.lang.management.ManagementFactory;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.security.KeyStore;
import java.security.MessageDigest;
import java.security.cert.Certificate;
@@ -90,7 +93,6 @@ public class Heritrix {
private static final String ADHOC_KEYSTORE = "adhoc.keystore";
@SuppressWarnings("unused")
private static final Logger logger = Logger.getLogger(Heritrix.class.getName());
/** Name of configuration directory */
@@ -322,6 +324,8 @@ public class Heritrix {
// inside in a container.
TimeZone.setDefault(TimeZone.getTimeZone("GMT"));
setupGlobalProperties(port);
// Start Heritrix.
try {
engine = new Engine(jobsDir);
@@ -372,6 +376,39 @@ public class Heritrix {
}
}
/**
* Setup global system properties that may be of use elsewhere.
*
* @param port
*/
protected void setupGlobalProperties(int port) {
if (System.getProperty("heritrix.port") == null) {
System.setProperty("heritrix.port", port + "");
}
String hostname = "localhost.localdomain";
if (System.getProperty("heritrix.hostname") == null) {
try {
hostname = InetAddress.getLocalHost().getCanonicalHostName();
} catch (UnknownHostException ue) {
logger.warning("Failed getHostAddress for this host: " + ue);
}
System.setProperty("heritrix.hostname", hostname);
}
// while not guaranteed, on our platforms of interest this name
// always seems to be PID@HOSTNAME
String runtimeName = ManagementFactory.getRuntimeMXBean().getName();
if(System.getProperty("heritrix.runtimeName") == null) {
System.setProperty("heritrix.runtimeName", runtimeName);
}
if (System.getProperty("heritrix.pid") == null
&& runtimeName.matches("\\d+@\\S+")) {
System.setProperty("heritrix.pid", runtimeName.substring(0,runtimeName.indexOf("@")));
}
}
/**
* Perform preparation to use an ad-hoc, created-as-necessary
* certificate/keystore for HTTPS access. A keystore with new