[HER-1727] (W)ARC filename uniqueness guarantees: adminport option, process_id option(?), JVM-global-repeat-suppression

* Heritrix.java
    make heritrix.hostname, heritrix.pid, heritrix.port available as global properties
* PropertyUtils.java, PropertyUtilsTest.java
    utility methods to interpolate string values from one or several supplied Properties instances
* ArchiveUtils.java
    utility methods to give timestamps guaranteed larger/different than any previously-issued timestamp

* WriterPoolSettings.java, WriterPool.java, ARCWriterPool.java, WARCWriterPool.java, ARCWriterPoolTest.java
    improve field names, change 'suffix' to more general 'template'
* WriterPoolMember.java
    replace 'suffix' with 'template' which is interpolated when specific name is needed
    set default template to pattern extremely unlikely to generate duplicate filenames
    change default prefix to 'WEB'
    centralize creation of new basenames into generateNewBasename() method, which internalizes timetamp/serialNo minting and interpolation
* WriterPoolProcessor.java, ARCWriterProcessor.java, WARCWriterProcessor.java
    serve as own WriterPoolSettings instance
    replace 'suffix' with 'template' 
    
* ARCWriter.java
    accept (but truncate) 17-digit timestamps at creation
* DefaultWriterPoolSettings.java, TimestampSerialNo.java
    delete as superfluous

commons/src/main/java/org/archive/io/DefaultWriterPoolSettings.java

@@ -1,105 +0,0 @@
-/*
- *  This file is part of the Heritrix web crawler (crawler.archive.org).
- *
- *  Licensed to the Internet Archive (IA) by one or more individual 
- *  contributors. 
- *
- *  The IA licenses this file to You under the Apache License, Version 2.0
- *  (the "License"); you may not use this file except in compliance with
- *  the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.archive.io;
-
-import java.io.File;
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * @author pjack
- */
-public class DefaultWriterPoolSettings 
-implements WriterPoolSettings, Serializable {
-    
-    private static final long serialVersionUID = 1L;
-
-    private long maxSize;
-    private List<String> metadata = new ArrayList<String>();;
-    transient private List<File> outputDirs = new ArrayList<File>();
-    private String prefix;
-    private String suffix;
-    private boolean compressed;
-
-    
-    public DefaultWriterPoolSettings() {
-    }
-    
-    
-    public boolean isCompressed() {
-        return compressed;
-    }
-    
-    
-    public void setCompressed(boolean compressed) {
-        this.compressed = compressed;
-    }
-    
-    
-    public long getMaxSize() {
-        return maxSize;
-    }
-    
-    
-    public void setMaxSize(long maxSize) {
-        this.maxSize = maxSize;
-    }
-    
-    
-    public List<String> getMetadata() {
-        return metadata;
-    }
-    
-    
-    public void setMetadata(List<String> metadata) {
-        this.metadata = metadata;
-    }
-    
-    
-    public List<File> getOutputDirs() {
-        return outputDirs;
-    }
-    
-    
-    public void setOutputDirs(List<File> outputDirs) {
-        this.outputDirs = outputDirs;
-    }
-    
-    
-    public String getPrefix() {
-        return prefix;
-    }
-    
-    
-    public void setPrefix(String prefix) {
-        this.prefix = prefix;
-    }
-    
-    
-    public String getSuffix() {
-        return suffix;
-    }
-    
-    
-    public void setSuffix(String suffix) {
-        this.suffix = suffix;
-    }
-}

commons/src/main/java/org/archive/io/WriterPool.java

@@ -102,9 +102,9 @@ public abstract class WriterPool {
             final int poolMaximumActive, final int poolMaximumWait) {
         logger.info("Initial configuration:" +
                 " prefix=" + settings.getPrefix() +
-                ", suffix=" + settings.getSuffix() +
-                ", compress=" + settings.isCompressed() +
-                ", maxSize=" + settings.getMaxSize() +
+                ", suffix=" + settings.getTemplate() +
+                ", compress=" + settings.getCompress() +
+                ", maxSize=" + settings.getMaxFileSizeBytes() +
                 ", maxActive=" + poolMaximumActive +
                 ", maxWait=" + poolMaximumWait);
         this.settings = settings;

commons/src/main/java/org/archive/io/WriterPoolMember.java

@@ -30,13 +30,14 @@ import java.text.DecimalFormat;
 import java.text.NumberFormat;
 import java.util.Iterator;
 import java.util.List;
+import java.util.Properties;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.logging.Logger;
 import java.util.zip.GZIPOutputStream;
 
 import org.archive.util.ArchiveUtils;
 import org.archive.util.FileUtils;
-import org.archive.util.TimestampSerialno;
+import org.archive.util.PropertyUtils;
 
 
 /**
@@ -53,21 +54,22 @@ public abstract class WriterPoolMember implements ArchiveFileConstants {
     public static final String UTF8 = "UTF-8";
     
     /**
-     * Default file prefix.
+     * Default archival-aggregate filename template. 
+     * 
+     * Under usual assumptions -- hostnames aren't shared among crawling hosts; 
+     * processes have unique PIDs and admin ports; timestamps inside one process
+     * don't repeat (see UniqueTimestampService); clocks are generally 
+     * accurate -- will generate a unique name.
      * 
      * Stands for Internet Archive Heritrix.
      */
-    public static final String DEFAULT_PREFIX = "IAH";
+    public static final String DEFAULT_TEMPLATE = 
+        "${prefix}-${timestamp17}-${serialno}-${heritrix.pid}@${heritrix.hostname}#${heritrix.port}";
     
     /**
-     * Value to interpolate with actual hostname.
+     * Default for file prefix.
      */
-    public static final String HOSTNAME_VARIABLE = "${HOSTNAME}";
-    
-    /**
-     * Default for file suffix.
-     */
-    public static final String DEFAULT_SUFFIX = HOSTNAME_VARIABLE;
+    public static final String DEFAULT_PREFIX = "WEB";
 
     /**
      * Reference to file we're currently writing.
@@ -87,8 +89,8 @@ public abstract class WriterPoolMember implements ArchiveFileConstants {
     
     private final boolean compressed;
     private List<File> writeDirs = null;
+    private String template = DEFAULT_TEMPLATE;
     private String prefix = DEFAULT_PREFIX;
-    private String suffix = DEFAULT_SUFFIX;
     private final long maxSize;
     private final String extension;
 
@@ -96,8 +98,10 @@ public abstract class WriterPoolMember implements ArchiveFileConstants {
      * Creation date for the current file.
      * Set by {@link #createFile()}.
      */
-	private String createTimestamp = "UNSET!!!";
+	protected String currentTimestamp = "UNSET!!!";
     
+	protected String currentBasename; 
+	
     /**
      * A running sequence used making unique file names.
      */
@@ -172,9 +176,9 @@ public abstract class WriterPoolMember implements ArchiveFileConstants {
      */
     public WriterPoolMember(AtomicInteger serialNo,
             final List<File> dirs, final String prefix, 
-            final String suffix, final boolean cmprs,
+            final String template, final boolean cmprs,
             final long maxSize, final String extension) {
-        this.suffix = suffix;
+        this.template = template;
         this.prefix = prefix;
         this.maxSize = maxSize;
         this.writeDirs = dirs;
@@ -211,13 +215,10 @@ public abstract class WriterPoolMember implements ArchiveFileConstants {
      * @throws IOException
      */
     protected String createFile() throws IOException {
-        TimestampSerialno tsn = getTimestampSerialNo();
-        String name = this.prefix + '-' + getUniqueBasename(tsn) +
-            ((this.suffix == null || this.suffix.length() <= 0)?
-                "": "-" + this.suffix) + '.' + this.extension  +
+        generateNewBasename();
+        String name = currentBasename + '.' + this.extension  +
             ((this.compressed)? DOT_COMPRESSED_FILE_EXTENSION: "") +
             OCCUPIED_SUFFIX;
-        this.createTimestamp = tsn.getTimestamp();
         File dir = getNextDirectory(this.writeDirs);
         return createFile(new File(dir, name));
     }
@@ -277,39 +278,28 @@ public abstract class WriterPoolMember implements ArchiveFileConstants {
         }
         return d;
     }
-    
-    protected synchronized TimestampSerialno getTimestampSerialNo() {
-        return getTimestampSerialNo(null);
-    }
-    
+ 
     /**
-     * Do static synchronization around getting of counter and timestamp so
-     * no chance of a thread getting in between the getting of timestamp and
-     * allocation of serial number throwing the two out of alignment.
-     * 
-     * @param timestamp If non-null, use passed timestamp (must be 14 digit
-     * ARC format), else if null, timestamp with now.
-     * @return Instance of data structure that has timestamp and serial no.
+     * Generate a new basename by interpolating values in the configured
+     * template. Values come from local state, other configured values, and 
+     * global system properties. The recommended default template will 
+     * generate a unique basename under reasonable assumptions. 
      */
-    protected synchronized TimestampSerialno
-            getTimestampSerialNo(final String timestamp) {
-        return new TimestampSerialno((timestamp != null)?
-                timestamp: ArchiveUtils.get14DigitDate(),
-                serialNo.getAndIncrement());
-    }
-
-    /**
-     * Return a unique basename.
-     *
-     * Name is timestamp + an every increasing sequence number.
-     *
-     * @param tsn Structure with timestamp and serial number.
-     *
-     * @return Unique basename.
-     */
-    private String getUniqueBasename(TimestampSerialno tsn) {
-        return tsn.getTimestamp() + "-" +
-           WriterPoolMember.serialNoFormatter.format(tsn.getSerialNumber());
+    protected void generateNewBasename() {
+        Properties localProps = new Properties(); 
+        localProps.setProperty("prefix", prefix);
+        synchronized(this.getClass()) {
+            // ensure that serialNo and timestamp are minted together (never inverted sort order)
+            String paddedSerialNumber = WriterPoolMember.serialNoFormatter.format(serialNo.getAndIncrement());
+            String timestamp17 = ArchiveUtils.getUnique17DigitDate(); 
+            String timestamp14 = ArchiveUtils.getUnique14DigitDate(); 
+            currentTimestamp = timestamp17;
+            localProps.setProperty("serialno", paddedSerialNumber);
+            localProps.setProperty("timestamp17", timestamp17);
+            localProps.setProperty("timestamp14", timestamp14);
+        }
+        currentBasename = PropertyUtils.interpolateWithProperties(template, 
+                localProps, System.getProperties());
     }
 
 
@@ -384,7 +374,7 @@ public abstract class WriterPoolMember implements ArchiveFileConstants {
     }
     
 	/**
-     * Postion in current physical file.
+     * Position in current physical file.
      * Used making accounting of bytes written.
 	 * @return Position in underlying file.  Call before or after writing
      * records *only* to be safe.
@@ -482,12 +472,7 @@ public abstract class WriterPoolMember implements ArchiveFileConstants {
     protected OutputStream getOutputStream() {
     	return this.out;
     }
-    
-	protected String getCreateTimestamp() {
-		return createTimestamp;
-	}
-    
-    
+
     /**
      * An override so we get access to underlying output stream.
      * and offer an end() that does not accompany closing underlying

commons/src/main/java/org/archive/io/WriterPoolSettings.java

@@ -28,10 +28,10 @@ import java.util.List;
  * @version $Date$, $Revision$
  */
 public interface WriterPoolSettings {
-    public long getMaxSize();
+    public long getMaxFileSizeBytes();
     public String getPrefix();
-    public String getSuffix(); 
+    public String getTemplate(); 
     public List<File> getOutputDirs();
-    public boolean isCompressed();
+    public boolean getCompress();
     public List<String> getMetadata();
 }

commons/src/main/java/org/archive/io/arc/ARCWriter.java

@@ -186,7 +186,7 @@ public class ARCWriter extends WriterPoolMember implements ARCConstants {
     protected String createFile()
     throws IOException {
         String name = super.createFile();
-        writeFirstRecord(getCreateTimestamp());
+        writeFirstRecord(currentTimestamp);
         return name;
     }
     
@@ -223,13 +223,17 @@ public class ARCWriter extends WriterPoolMember implements ARCConstants {
      * <p>We do things in this roundabout manner because the java
      * GZIPOutputStream does not give access to GZIP header fields.
      *
-     * @param date Date to put into the ARC metadata.
+     * @param date Date to put into the ARC metadata; if 17-digit will be 
+     * truncated to traditional 14-digits
      *
      * @return Byte array filled w/ the arc header.
 	 * @throws IOException
      */
     private byte [] generateARCFileMetaData(String date)
     throws IOException {
+        if(date.length()>14) {
+            date = date.substring(0,14);
+        }
         int metadataBodyLength = getMetadataLength();
         // If metadata body, then the minor part of the version is '1' rather
         // than '0'.

commons/src/main/java/org/archive/io/arc/ARCWriterPool.java

@@ -58,8 +58,8 @@ public class ARCWriterPool extends WriterPool {
     	super(serial, new BasePoolableObjectFactory() {
             public Object makeObject() throws Exception {
                 return new ARCWriter(serial, settings.getOutputDirs(),
-                        settings.getPrefix(), settings.getSuffix(),
-                        settings.isCompressed(), settings.getMaxSize(),
+                        settings.getPrefix(), settings.getTemplate(),
+                        settings.getCompress(), settings.getMaxFileSizeBytes(),
                         settings.getMetadata());
             }
 

commons/src/main/java/org/archive/io/warc/WARCWriterPool.java

@@ -57,8 +57,8 @@ public class WARCWriterPool extends WriterPool {
             public Object makeObject() throws Exception {
                 return new WARCWriter(serial,
                 		settings.getOutputDirs(),
-                        settings.getPrefix(), settings.getSuffix(),
-                        settings.isCompressed(), settings.getMaxSize(),
+                        settings.getPrefix(), settings.getTemplate(),
+                        settings.getCompress(), settings.getMaxFileSizeBytes(),
                         settings.getMetadata());
             }
 

commons/src/main/java/org/archive/util/ArchiveUtils.java

@@ -123,6 +123,31 @@ public class ArchiveUtils {
     public static String get17DigitDate(){
         return TIMESTAMP17.get().format(new Date());
     }
+    
+    static long LAST_UNIQUE_NOW17 = 0;
+    static String LAST_TIMESTAMP17 = ""; 
+    /**
+     * Utility function for creating UNIQUE-from-this-class 
+     * arc-style date stamps in the format yyyMMddHHmmssSSS.
+     * Rather than giving a duplicate datestamp on a 
+     * subsequent call, will increment the milliseconds until a 
+     * unique value is returned. 
+     * 
+     * Date stamps are in the UTC time zone
+     * @return the date stamp
+     */
+    public synchronized static String getUnique17DigitDate(){
+        long effectiveNow = System.currentTimeMillis(); 
+        effectiveNow = Math.max(effectiveNow, LAST_UNIQUE_NOW17+1);
+        String candidate = get17DigitDate(effectiveNow);
+        while(candidate.equals(LAST_TIMESTAMP17)) {
+            effectiveNow++;
+            candidate = get17DigitDate(effectiveNow);
+        }
+        LAST_UNIQUE_NOW17 = effectiveNow;
+        LAST_TIMESTAMP17 = candidate; 
+        return candidate;
+    }
 
     /**
      * Utility function for creating arc-style date stamps
@@ -133,6 +158,31 @@ public class ArchiveUtils {
     public static String get14DigitDate(){
         return TIMESTAMP14.get().format(new Date());
     }
+    
+    static long LAST_UNIQUE_NOW14 = 0;
+    static String LAST_TIMESTAMP14 = ""; 
+    /**
+     * Utility function for creating UNIQUE-from-this-class 
+     * arc-style date stamps in the format yyyMMddHHmmss.
+     * Rather than giving a duplicate datestamp on a 
+     * subsequent call, will increment the seconds until a 
+     * unique value is returned. 
+     * 
+     * Date stamps are in the UTC time zone
+     * @return the date stamp
+     */
+    public synchronized static String getUnique14DigitDate(){
+        long effectiveNow = System.currentTimeMillis(); 
+        effectiveNow = Math.max(effectiveNow, LAST_UNIQUE_NOW14+1);
+        String candidate = get14DigitDate(effectiveNow);
+        while(candidate.equals(LAST_TIMESTAMP14)) {
+            effectiveNow += 1000;
+            candidate = get14DigitDate(effectiveNow);
+        }
+        LAST_UNIQUE_NOW14 = effectiveNow;
+        LAST_TIMESTAMP14 = candidate; 
+        return candidate;
+    }
 
     /**
      * Utility function for creating arc-style date stamps

commons/src/main/java/org/archive/util/PropertyUtils.java

@@ -18,8 +18,16 @@
  */
 package org.archive.util;
 
+import java.util.Properties;
+import java.util.regex.Matcher;
+
+import org.apache.commons.lang.StringUtils;
+
 /**
- * @author stack
+ * Utilities for dealing with Java Properties (incl. System Properties)
+ * 
+ * @contributor stack
+ * @contributor gojomo
  * @version $Date$ $Revision$
  */
 public class PropertyUtils {
@@ -51,4 +59,56 @@ public class PropertyUtils {
         return getPropertyOrNull(key) == null?
                 fallback: Integer.parseInt(getPropertyOrNull(key));
     }
+    
+    /**
+     * Given a string which may contain expressions of the form 
+     * ${key}, replace each expression with the value corresponding to the
+     * given key in System Properties. If no value is present, 
+     * the expression is replaced with the empty-string. 
+     * 
+     * @param original String
+     * @param properties Properties to try in order; first value found (if any) is used
+     * @return modified String
+     */
+    public static String interpolateWithProperties(String original) {
+        return interpolateWithProperties(original,System.getProperties());
+    }
+
+    static String propRefPattern = "\\$\\{([^{}]+)\\}";
+    
+    /**
+     * Given a string which may contain expressions of the form 
+     * ${key}, replace each expression with the value corresponding to the
+     * given key in the supplied Properties instance. If no value is present, 
+     * the expression is replaced with the empty-string. 
+     * 
+     * @param original String
+     * @param properties Properties to try in order; first value found (if any) is used
+     * @return modified String
+     */
+    public static String interpolateWithProperties(String original,
+            Properties... props) {
+        String result = original;
+        // cap number of interpolations as guard against unending loop
+        inter: for(int i =0; i < original.length()*2; i++) {
+            Matcher m = TextUtils.getMatcher(propRefPattern, result);
+            while(m.find()) {
+                String key = m.group(1); 
+                String value = "";
+                for(Properties properties : props) {
+                    value = properties.getProperty(key, "");
+                    if(StringUtils.isNotEmpty(value)) {
+                        break;
+                    }
+                }
+                result = result.substring(0,m.start()) 
+                            + value
+                            + result.substring(m.end());
+                continue inter;
+            }
+            // we only hit here if there were no interpolations last while loop
+            break;
+        }
+        return result; 
+    }
 }

commons/src/main/java/org/archive/util/TimestampSerialno.java

@@ -1,56 +0,0 @@
-/*
- *  This file is part of the Heritrix web crawler (crawler.archive.org).
- *
- *  Licensed to the Internet Archive (IA) by one or more individual 
- *  contributors. 
- *
- *  The IA licenses this file to You under the Apache License, Version 2.0
- *  (the "License"); you may not use this file except in compliance with
- *  the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-package org.archive.util;
-
-/**
- * Immutable data structure that holds a timestamp and an accompanying
- * serial number.
- * 
- * For Igor!
- *
- * @author stack
- */
-public class TimestampSerialno {
-	private final String ts;
-	private final int serialNumber;
-
-	public TimestampSerialno(String ts, int serialNo) {
-		this.ts = ts;
-		this.serialNumber = serialNo;
-	}
-    
-    public TimestampSerialno(int serialNo) {
-        this.ts = ArchiveUtils.get14DigitDate();
-        this.serialNumber = serialNo;
-    }
-
-	/**
-	 * @return Returns the now.
-	 */
-	public String getTimestamp() {
-		return this.ts;
-	}
-
-	/**
-	 * @return Returns the serialNumber.
-	 */
-	public int getSerialNumber() {
-		return this.serialNumber;
-	}
-}