From b388479d32e07aba25754cec1ca6556407cd6155 Mon Sep 17 00:00:00 2001 From: gojomo Date: Fri, 20 Aug 2010 00:21:43 +0000 Subject: [PATCH] [HER-1727] (W)ARC filename uniqueness guarantees: adminport option, process_id option(?), JVM-global-repeat-suppression * Heritrix.java make heritrix.hostname, heritrix.pid, heritrix.port available as global properties * PropertyUtils.java, PropertyUtilsTest.java utility methods to interpolate string values from one or several supplied Properties instances * ArchiveUtils.java utility methods to give timestamps guaranteed larger/different than any previously-issued timestamp * WriterPoolSettings.java, WriterPool.java, ARCWriterPool.java, WARCWriterPool.java, ARCWriterPoolTest.java improve field names, change 'suffix' to more general 'template' * WriterPoolMember.java replace 'suffix' with 'template' which is interpolated when specific name is needed set default template to pattern extremely unlikely to generate duplicate filenames change default prefix to 'WEB' centralize creation of new basenames into generateNewBasename() method, which internalizes timetamp/serialNo minting and interpolation * WriterPoolProcessor.java, ARCWriterProcessor.java, WARCWriterProcessor.java serve as own WriterPoolSettings instance replace 'suffix' with 'template' * ARCWriter.java accept (but truncate) 17-digit timestamps at creation * DefaultWriterPoolSettings.java, TimestampSerialNo.java delete as superfluous --- .../archive/io/DefaultWriterPoolSettings.java | 105 ------------------ .../main/java/org/archive/io/WriterPool.java | 6 +- .../java/org/archive/io/WriterPoolMember.java | 99 +++++++---------- .../org/archive/io/WriterPoolSettings.java | 6 +- .../java/org/archive/io/arc/ARCWriter.java | 8 +- .../org/archive/io/arc/ARCWriterPool.java | 4 +- .../org/archive/io/warc/WARCWriterPool.java | 4 +- .../java/org/archive/util/ArchiveUtils.java | 50 +++++++++ .../java/org/archive/util/PropertyUtils.java | 62 ++++++++++- .../org/archive/io/arc/ARCWriterPoolTest.java | 8 +- .../org/archive/util/PropertyUtilsTest.java} | 51 ++++----- .../java/org/archive/crawler/Heritrix.java | 39 ++++++- .../modules/writer/ARCWriterProcessor.java | 14 +-- .../modules/writer/WARCWriterProcessor.java | 4 +- .../modules/writer/WriterPoolProcessor.java | 69 ++++-------- 15 files changed, 260 insertions(+), 269 deletions(-) delete mode 100644 commons/src/main/java/org/archive/io/DefaultWriterPoolSettings.java rename commons/src/{main/java/org/archive/util/TimestampSerialno.java => test/java/org/archive/util/PropertyUtilsTest.java} (55%) diff --git a/commons/src/main/java/org/archive/io/DefaultWriterPoolSettings.java b/commons/src/main/java/org/archive/io/DefaultWriterPoolSettings.java deleted file mode 100644 index ebcf49d3..00000000 --- a/commons/src/main/java/org/archive/io/DefaultWriterPoolSettings.java +++ /dev/null @@ -1,105 +0,0 @@ -/* - * This file is part of the Heritrix web crawler (crawler.archive.org). - * - * Licensed to the Internet Archive (IA) by one or more individual - * contributors. - * - * The IA licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.archive.io; - -import java.io.File; -import java.io.Serializable; -import java.util.ArrayList; -import java.util.List; - -/** - * @author pjack - */ -public class DefaultWriterPoolSettings -implements WriterPoolSettings, Serializable { - - private static final long serialVersionUID = 1L; - - private long maxSize; - private List metadata = new ArrayList();; - transient private List outputDirs = new ArrayList(); - private String prefix; - private String suffix; - private boolean compressed; - - - public DefaultWriterPoolSettings() { - } - - - public boolean isCompressed() { - return compressed; - } - - - public void setCompressed(boolean compressed) { - this.compressed = compressed; - } - - - public long getMaxSize() { - return maxSize; - } - - - public void setMaxSize(long maxSize) { - this.maxSize = maxSize; - } - - - public List getMetadata() { - return metadata; - } - - - public void setMetadata(List metadata) { - this.metadata = metadata; - } - - - public List getOutputDirs() { - return outputDirs; - } - - - public void setOutputDirs(List outputDirs) { - this.outputDirs = outputDirs; - } - - - public String getPrefix() { - return prefix; - } - - - public void setPrefix(String prefix) { - this.prefix = prefix; - } - - - public String getSuffix() { - return suffix; - } - - - public void setSuffix(String suffix) { - this.suffix = suffix; - } -} diff --git a/commons/src/main/java/org/archive/io/WriterPool.java b/commons/src/main/java/org/archive/io/WriterPool.java index 0c555fa4..f1f8c954 100644 --- a/commons/src/main/java/org/archive/io/WriterPool.java +++ b/commons/src/main/java/org/archive/io/WriterPool.java @@ -102,9 +102,9 @@ public abstract class WriterPool { final int poolMaximumActive, final int poolMaximumWait) { logger.info("Initial configuration:" + " prefix=" + settings.getPrefix() + - ", suffix=" + settings.getSuffix() + - ", compress=" + settings.isCompressed() + - ", maxSize=" + settings.getMaxSize() + + ", suffix=" + settings.getTemplate() + + ", compress=" + settings.getCompress() + + ", maxSize=" + settings.getMaxFileSizeBytes() + ", maxActive=" + poolMaximumActive + ", maxWait=" + poolMaximumWait); this.settings = settings; diff --git a/commons/src/main/java/org/archive/io/WriterPoolMember.java b/commons/src/main/java/org/archive/io/WriterPoolMember.java index d32a2ff8..f609aab0 100644 --- a/commons/src/main/java/org/archive/io/WriterPoolMember.java +++ b/commons/src/main/java/org/archive/io/WriterPoolMember.java @@ -30,13 +30,14 @@ import java.text.DecimalFormat; import java.text.NumberFormat; import java.util.Iterator; import java.util.List; +import java.util.Properties; import java.util.concurrent.atomic.AtomicInteger; import java.util.logging.Logger; import java.util.zip.GZIPOutputStream; import org.archive.util.ArchiveUtils; import org.archive.util.FileUtils; -import org.archive.util.TimestampSerialno; +import org.archive.util.PropertyUtils; /** @@ -53,21 +54,22 @@ public abstract class WriterPoolMember implements ArchiveFileConstants { public static final String UTF8 = "UTF-8"; /** - * Default file prefix. + * Default archival-aggregate filename template. + * + * Under usual assumptions -- hostnames aren't shared among crawling hosts; + * processes have unique PIDs and admin ports; timestamps inside one process + * don't repeat (see UniqueTimestampService); clocks are generally + * accurate -- will generate a unique name. * * Stands for Internet Archive Heritrix. */ - public static final String DEFAULT_PREFIX = "IAH"; + public static final String DEFAULT_TEMPLATE = + "${prefix}-${timestamp17}-${serialno}-${heritrix.pid}@${heritrix.hostname}#${heritrix.port}"; /** - * Value to interpolate with actual hostname. + * Default for file prefix. */ - public static final String HOSTNAME_VARIABLE = "${HOSTNAME}"; - - /** - * Default for file suffix. - */ - public static final String DEFAULT_SUFFIX = HOSTNAME_VARIABLE; + public static final String DEFAULT_PREFIX = "WEB"; /** * Reference to file we're currently writing. @@ -87,8 +89,8 @@ public abstract class WriterPoolMember implements ArchiveFileConstants { private final boolean compressed; private List writeDirs = null; + private String template = DEFAULT_TEMPLATE; private String prefix = DEFAULT_PREFIX; - private String suffix = DEFAULT_SUFFIX; private final long maxSize; private final String extension; @@ -96,8 +98,10 @@ public abstract class WriterPoolMember implements ArchiveFileConstants { * Creation date for the current file. * Set by {@link #createFile()}. */ - private String createTimestamp = "UNSET!!!"; + protected String currentTimestamp = "UNSET!!!"; + protected String currentBasename; + /** * A running sequence used making unique file names. */ @@ -172,9 +176,9 @@ public abstract class WriterPoolMember implements ArchiveFileConstants { */ public WriterPoolMember(AtomicInteger serialNo, final List dirs, final String prefix, - final String suffix, final boolean cmprs, + final String template, final boolean cmprs, final long maxSize, final String extension) { - this.suffix = suffix; + this.template = template; this.prefix = prefix; this.maxSize = maxSize; this.writeDirs = dirs; @@ -211,13 +215,10 @@ public abstract class WriterPoolMember implements ArchiveFileConstants { * @throws IOException */ protected String createFile() throws IOException { - TimestampSerialno tsn = getTimestampSerialNo(); - String name = this.prefix + '-' + getUniqueBasename(tsn) + - ((this.suffix == null || this.suffix.length() <= 0)? - "": "-" + this.suffix) + '.' + this.extension + + generateNewBasename(); + String name = currentBasename + '.' + this.extension + ((this.compressed)? DOT_COMPRESSED_FILE_EXTENSION: "") + OCCUPIED_SUFFIX; - this.createTimestamp = tsn.getTimestamp(); File dir = getNextDirectory(this.writeDirs); return createFile(new File(dir, name)); } @@ -277,39 +278,28 @@ public abstract class WriterPoolMember implements ArchiveFileConstants { } return d; } - - protected synchronized TimestampSerialno getTimestampSerialNo() { - return getTimestampSerialNo(null); - } - + /** - * Do static synchronization around getting of counter and timestamp so - * no chance of a thread getting in between the getting of timestamp and - * allocation of serial number throwing the two out of alignment. - * - * @param timestamp If non-null, use passed timestamp (must be 14 digit - * ARC format), else if null, timestamp with now. - * @return Instance of data structure that has timestamp and serial no. + * Generate a new basename by interpolating values in the configured + * template. Values come from local state, other configured values, and + * global system properties. The recommended default template will + * generate a unique basename under reasonable assumptions. */ - protected synchronized TimestampSerialno - getTimestampSerialNo(final String timestamp) { - return new TimestampSerialno((timestamp != null)? - timestamp: ArchiveUtils.get14DigitDate(), - serialNo.getAndIncrement()); - } - - /** - * Return a unique basename. - * - * Name is timestamp + an every increasing sequence number. - * - * @param tsn Structure with timestamp and serial number. - * - * @return Unique basename. - */ - private String getUniqueBasename(TimestampSerialno tsn) { - return tsn.getTimestamp() + "-" + - WriterPoolMember.serialNoFormatter.format(tsn.getSerialNumber()); + protected void generateNewBasename() { + Properties localProps = new Properties(); + localProps.setProperty("prefix", prefix); + synchronized(this.getClass()) { + // ensure that serialNo and timestamp are minted together (never inverted sort order) + String paddedSerialNumber = WriterPoolMember.serialNoFormatter.format(serialNo.getAndIncrement()); + String timestamp17 = ArchiveUtils.getUnique17DigitDate(); + String timestamp14 = ArchiveUtils.getUnique14DigitDate(); + currentTimestamp = timestamp17; + localProps.setProperty("serialno", paddedSerialNumber); + localProps.setProperty("timestamp17", timestamp17); + localProps.setProperty("timestamp14", timestamp14); + } + currentBasename = PropertyUtils.interpolateWithProperties(template, + localProps, System.getProperties()); } @@ -384,7 +374,7 @@ public abstract class WriterPoolMember implements ArchiveFileConstants { } /** - * Postion in current physical file. + * Position in current physical file. * Used making accounting of bytes written. * @return Position in underlying file. Call before or after writing * records *only* to be safe. @@ -482,12 +472,7 @@ public abstract class WriterPoolMember implements ArchiveFileConstants { protected OutputStream getOutputStream() { return this.out; } - - protected String getCreateTimestamp() { - return createTimestamp; - } - - + /** * An override so we get access to underlying output stream. * and offer an end() that does not accompany closing underlying diff --git a/commons/src/main/java/org/archive/io/WriterPoolSettings.java b/commons/src/main/java/org/archive/io/WriterPoolSettings.java index c9812175..baa36ce7 100644 --- a/commons/src/main/java/org/archive/io/WriterPoolSettings.java +++ b/commons/src/main/java/org/archive/io/WriterPoolSettings.java @@ -28,10 +28,10 @@ import java.util.List; * @version $Date$, $Revision$ */ public interface WriterPoolSettings { - public long getMaxSize(); + public long getMaxFileSizeBytes(); public String getPrefix(); - public String getSuffix(); + public String getTemplate(); public List getOutputDirs(); - public boolean isCompressed(); + public boolean getCompress(); public List getMetadata(); } \ No newline at end of file diff --git a/commons/src/main/java/org/archive/io/arc/ARCWriter.java b/commons/src/main/java/org/archive/io/arc/ARCWriter.java index b58b74a4..10acfe80 100644 --- a/commons/src/main/java/org/archive/io/arc/ARCWriter.java +++ b/commons/src/main/java/org/archive/io/arc/ARCWriter.java @@ -186,7 +186,7 @@ public class ARCWriter extends WriterPoolMember implements ARCConstants { protected String createFile() throws IOException { String name = super.createFile(); - writeFirstRecord(getCreateTimestamp()); + writeFirstRecord(currentTimestamp); return name; } @@ -223,13 +223,17 @@ public class ARCWriter extends WriterPoolMember implements ARCConstants { *

We do things in this roundabout manner because the java * GZIPOutputStream does not give access to GZIP header fields. * - * @param date Date to put into the ARC metadata. + * @param date Date to put into the ARC metadata; if 17-digit will be + * truncated to traditional 14-digits * * @return Byte array filled w/ the arc header. * @throws IOException */ private byte [] generateARCFileMetaData(String date) throws IOException { + if(date.length()>14) { + date = date.substring(0,14); + } int metadataBodyLength = getMetadataLength(); // If metadata body, then the minor part of the version is '1' rather // than '0'. diff --git a/commons/src/main/java/org/archive/io/arc/ARCWriterPool.java b/commons/src/main/java/org/archive/io/arc/ARCWriterPool.java index e06dc086..8e05ddb2 100644 --- a/commons/src/main/java/org/archive/io/arc/ARCWriterPool.java +++ b/commons/src/main/java/org/archive/io/arc/ARCWriterPool.java @@ -58,8 +58,8 @@ public class ARCWriterPool extends WriterPool { super(serial, new BasePoolableObjectFactory() { public Object makeObject() throws Exception { return new ARCWriter(serial, settings.getOutputDirs(), - settings.getPrefix(), settings.getSuffix(), - settings.isCompressed(), settings.getMaxSize(), + settings.getPrefix(), settings.getTemplate(), + settings.getCompress(), settings.getMaxFileSizeBytes(), settings.getMetadata()); } diff --git a/commons/src/main/java/org/archive/io/warc/WARCWriterPool.java b/commons/src/main/java/org/archive/io/warc/WARCWriterPool.java index e9adfb02..414e9da4 100644 --- a/commons/src/main/java/org/archive/io/warc/WARCWriterPool.java +++ b/commons/src/main/java/org/archive/io/warc/WARCWriterPool.java @@ -57,8 +57,8 @@ public class WARCWriterPool extends WriterPool { public Object makeObject() throws Exception { return new WARCWriter(serial, settings.getOutputDirs(), - settings.getPrefix(), settings.getSuffix(), - settings.isCompressed(), settings.getMaxSize(), + settings.getPrefix(), settings.getTemplate(), + settings.getCompress(), settings.getMaxFileSizeBytes(), settings.getMetadata()); } diff --git a/commons/src/main/java/org/archive/util/ArchiveUtils.java b/commons/src/main/java/org/archive/util/ArchiveUtils.java index 2e71fd92..45e5a466 100644 --- a/commons/src/main/java/org/archive/util/ArchiveUtils.java +++ b/commons/src/main/java/org/archive/util/ArchiveUtils.java @@ -123,6 +123,31 @@ public class ArchiveUtils { public static String get17DigitDate(){ return TIMESTAMP17.get().format(new Date()); } + + static long LAST_UNIQUE_NOW17 = 0; + static String LAST_TIMESTAMP17 = ""; + /** + * Utility function for creating UNIQUE-from-this-class + * arc-style date stamps in the format yyyMMddHHmmssSSS. + * Rather than giving a duplicate datestamp on a + * subsequent call, will increment the milliseconds until a + * unique value is returned. + * + * Date stamps are in the UTC time zone + * @return the date stamp + */ + public synchronized static String getUnique17DigitDate(){ + long effectiveNow = System.currentTimeMillis(); + effectiveNow = Math.max(effectiveNow, LAST_UNIQUE_NOW17+1); + String candidate = get17DigitDate(effectiveNow); + while(candidate.equals(LAST_TIMESTAMP17)) { + effectiveNow++; + candidate = get17DigitDate(effectiveNow); + } + LAST_UNIQUE_NOW17 = effectiveNow; + LAST_TIMESTAMP17 = candidate; + return candidate; + } /** * Utility function for creating arc-style date stamps @@ -133,6 +158,31 @@ public class ArchiveUtils { public static String get14DigitDate(){ return TIMESTAMP14.get().format(new Date()); } + + static long LAST_UNIQUE_NOW14 = 0; + static String LAST_TIMESTAMP14 = ""; + /** + * Utility function for creating UNIQUE-from-this-class + * arc-style date stamps in the format yyyMMddHHmmss. + * Rather than giving a duplicate datestamp on a + * subsequent call, will increment the seconds until a + * unique value is returned. + * + * Date stamps are in the UTC time zone + * @return the date stamp + */ + public synchronized static String getUnique14DigitDate(){ + long effectiveNow = System.currentTimeMillis(); + effectiveNow = Math.max(effectiveNow, LAST_UNIQUE_NOW14+1); + String candidate = get14DigitDate(effectiveNow); + while(candidate.equals(LAST_TIMESTAMP14)) { + effectiveNow += 1000; + candidate = get14DigitDate(effectiveNow); + } + LAST_UNIQUE_NOW14 = effectiveNow; + LAST_TIMESTAMP14 = candidate; + return candidate; + } /** * Utility function for creating arc-style date stamps diff --git a/commons/src/main/java/org/archive/util/PropertyUtils.java b/commons/src/main/java/org/archive/util/PropertyUtils.java index ad60a131..0760cea5 100644 --- a/commons/src/main/java/org/archive/util/PropertyUtils.java +++ b/commons/src/main/java/org/archive/util/PropertyUtils.java @@ -18,8 +18,16 @@ */ package org.archive.util; +import java.util.Properties; +import java.util.regex.Matcher; + +import org.apache.commons.lang.StringUtils; + /** - * @author stack + * Utilities for dealing with Java Properties (incl. System Properties) + * + * @contributor stack + * @contributor gojomo * @version $Date$ $Revision$ */ public class PropertyUtils { @@ -51,4 +59,56 @@ public class PropertyUtils { return getPropertyOrNull(key) == null? fallback: Integer.parseInt(getPropertyOrNull(key)); } + + /** + * Given a string which may contain expressions of the form + * ${key}, replace each expression with the value corresponding to the + * given key in System Properties. If no value is present, + * the expression is replaced with the empty-string. + * + * @param original String + * @param properties Properties to try in order; first value found (if any) is used + * @return modified String + */ + public static String interpolateWithProperties(String original) { + return interpolateWithProperties(original,System.getProperties()); + } + + static String propRefPattern = "\\$\\{([^{}]+)\\}"; + + /** + * Given a string which may contain expressions of the form + * ${key}, replace each expression with the value corresponding to the + * given key in the supplied Properties instance. If no value is present, + * the expression is replaced with the empty-string. + * + * @param original String + * @param properties Properties to try in order; first value found (if any) is used + * @return modified String + */ + public static String interpolateWithProperties(String original, + Properties... props) { + String result = original; + // cap number of interpolations as guard against unending loop + inter: for(int i =0; i < original.length()*2; i++) { + Matcher m = TextUtils.getMatcher(propRefPattern, result); + while(m.find()) { + String key = m.group(1); + String value = ""; + for(Properties properties : props) { + value = properties.getProperty(key, ""); + if(StringUtils.isNotEmpty(value)) { + break; + } + } + result = result.substring(0,m.start()) + + value + + result.substring(m.end()); + continue inter; + } + // we only hit here if there were no interpolations last while loop + break; + } + return result; + } } diff --git a/commons/src/test/java/org/archive/io/arc/ARCWriterPoolTest.java b/commons/src/test/java/org/archive/io/arc/ARCWriterPoolTest.java index b0425fc8..c833c078 100644 --- a/commons/src/test/java/org/archive/io/arc/ARCWriterPoolTest.java +++ b/commons/src/test/java/org/archive/io/arc/ARCWriterPoolTest.java @@ -129,7 +129,7 @@ public class ARCWriterPoolTest extends TmpDirTestCase { private WriterPoolSettings getSettings(final boolean isCompressed) { return new WriterPoolSettings() { - public long getMaxSize() { + public long getMaxFileSizeBytes() { return ARCConstants.DEFAULT_MAX_ARC_FILE_SIZE; } @@ -137,8 +137,8 @@ public class ARCWriterPoolTest extends TmpDirTestCase { return PREFIX; } - public String getSuffix() { - return ""; + public String getTemplate() { + return "${prefix}-${timestamp17}-${serialno}-${heritrix.hostname}"; } public List getOutputDirs() { @@ -146,7 +146,7 @@ public class ARCWriterPoolTest extends TmpDirTestCase { return Arrays.asList(files); } - public boolean isCompressed() { + public boolean getCompress() { return isCompressed; } diff --git a/commons/src/main/java/org/archive/util/TimestampSerialno.java b/commons/src/test/java/org/archive/util/PropertyUtilsTest.java similarity index 55% rename from commons/src/main/java/org/archive/util/TimestampSerialno.java rename to commons/src/test/java/org/archive/util/PropertyUtilsTest.java index ed4550d6..0666efb7 100644 --- a/commons/src/main/java/org/archive/util/TimestampSerialno.java +++ b/commons/src/test/java/org/archive/util/PropertyUtilsTest.java @@ -16,41 +16,30 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.archive.util; + +import java.io.IOException; +import java.util.Properties; + +import junit.framework.TestCase; + + /** - * Immutable data structure that holds a timestamp and an accompanying - * serial number. + * PropertyUtils tests. * - * For Igor! - * - * @author stack + * @contributor gojomo + * @version $Date: 2009-11-19 14:39:53 -0800 (Thu, 19 Nov 2009) $, $Revision: 6674 $ */ -public class TimestampSerialno { - private final String ts; - private final int serialNumber; - - public TimestampSerialno(String ts, int serialNo) { - this.ts = ts; - this.serialNumber = serialNo; - } +public class PropertyUtilsTest extends TestCase { - public TimestampSerialno(int serialNo) { - this.ts = ArchiveUtils.get14DigitDate(); - this.serialNumber = serialNo; + public void testSimpleInterpolate() throws IOException { + Properties props = new Properties(); + props.put("foo", "OOF"); + props.put("bar","RAB"); + String original = "FOO|${foo} BAR|${bar}"; + String expected = "FOO|OOF BAR|RAB"; + assertEquals("interpalation problem",expected,PropertyUtils.interpolateWithProperties(original,props)); } - - /** - * @return Returns the now. - */ - public String getTimestamp() { - return this.ts; - } - - /** - * @return Returns the serialNumber. - */ - public int getSerialNumber() { - return this.serialNumber; - } -} \ No newline at end of file +} diff --git a/engine/src/main/java/org/archive/crawler/Heritrix.java b/engine/src/main/java/org/archive/crawler/Heritrix.java index 54341129..5173e349 100644 --- a/engine/src/main/java/org/archive/crawler/Heritrix.java +++ b/engine/src/main/java/org/archive/crawler/Heritrix.java @@ -27,6 +27,9 @@ import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.PrintStream; +import java.lang.management.ManagementFactory; +import java.net.InetAddress; +import java.net.UnknownHostException; import java.security.KeyStore; import java.security.MessageDigest; import java.security.cert.Certificate; @@ -90,7 +93,6 @@ public class Heritrix { private static final String ADHOC_KEYSTORE = "adhoc.keystore"; - @SuppressWarnings("unused") private static final Logger logger = Logger.getLogger(Heritrix.class.getName()); /** Name of configuration directory */ @@ -322,6 +324,8 @@ public class Heritrix { // inside in a container. TimeZone.setDefault(TimeZone.getTimeZone("GMT")); + setupGlobalProperties(port); + // Start Heritrix. try { engine = new Engine(jobsDir); @@ -372,6 +376,39 @@ public class Heritrix { } } + /** + * Setup global system properties that may be of use elsewhere. + * + * @param port + */ + protected void setupGlobalProperties(int port) { + if (System.getProperty("heritrix.port") == null) { + System.setProperty("heritrix.port", port + ""); + } + + String hostname = "localhost.localdomain"; + if (System.getProperty("heritrix.hostname") == null) { + try { + hostname = InetAddress.getLocalHost().getCanonicalHostName(); + } catch (UnknownHostException ue) { + logger.warning("Failed getHostAddress for this host: " + ue); + } + System.setProperty("heritrix.hostname", hostname); + } + + // while not guaranteed, on our platforms of interest this name + // always seems to be PID@HOSTNAME + String runtimeName = ManagementFactory.getRuntimeMXBean().getName(); + if(System.getProperty("heritrix.runtimeName") == null) { + System.setProperty("heritrix.runtimeName", runtimeName); + } + if (System.getProperty("heritrix.pid") == null + && runtimeName.matches("\\d+@\\S+")) { + System.setProperty("heritrix.pid", runtimeName.substring(0,runtimeName.indexOf("@"))); + } + } + + /** * Perform preparation to use an ad-hoc, created-as-necessary * certificate/keystore for HTTPS access. A keystore with new diff --git a/modules/src/main/java/org/archive/modules/writer/ARCWriterProcessor.java b/modules/src/main/java/org/archive/modules/writer/ARCWriterProcessor.java index cb0b9685..24b770f3 100644 --- a/modules/src/main/java/org/archive/modules/writer/ARCWriterProcessor.java +++ b/modules/src/main/java/org/archive/modules/writer/ARCWriterProcessor.java @@ -35,7 +35,6 @@ import org.apache.commons.lang.StringEscapeUtils; import org.apache.commons.lang.StringUtils; import org.archive.io.ReplayInputStream; import org.archive.io.WriterPoolMember; -import org.archive.io.WriterPoolSettings; import org.archive.io.arc.ARCWriter; import org.archive.io.arc.ARCWriterPool; import org.archive.modules.ProcessResult; @@ -54,7 +53,7 @@ import org.archive.util.ArchiveUtils; */ public class ARCWriterProcessor extends WriterPoolProcessor { - final static private String TEMPLATE = readTemplate(); + final static private String METADATA_TEMPLATE = readMetadataTemplate(); private static final long serialVersionUID = 3L; @@ -77,8 +76,7 @@ public class ARCWriterProcessor extends WriterPoolProcessor { @Override protected void setupPool(AtomicInteger serialNo) { - WriterPoolSettings wps = getWriterPoolSettings(); - setPool(new ARCWriterPool(serialNo, wps, getPoolMaxActive(), getPoolMaxWaitMs())); + setPool(new ARCWriterPool(serialNo, this, getPoolMaxActive(), getPoolMaxWaitMs())); } /** @@ -160,8 +158,8 @@ public class ARCWriterProcessor extends WriterPoolProcessor { return checkBytesWritten(); } - protected List getMetadata() { - if (TEMPLATE == null) { + public List getMetadata() { + if (METADATA_TEMPLATE == null) { return null; } @@ -169,7 +167,7 @@ public class ARCWriterProcessor extends WriterPoolProcessor { return cachedMetadata; } - String meta = TEMPLATE; + String meta = METADATA_TEMPLATE; meta = replace(meta, "${VERSION}", ArchiveUtils.VERSION); meta = replace(meta, "${HOST}", getHostName()); meta = replace(meta, "${IP}", getHostAddress()); @@ -225,7 +223,7 @@ public class ARCWriterProcessor extends WriterPoolProcessor { } } - private static String readTemplate() { + private static String readMetadataTemplate() { InputStream input = ARCWriterProcessor.class.getResourceAsStream( "arc_metadata_template.xml"); if (input == null) { diff --git a/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java b/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java index 8a574ea7..33cf4e69 100644 --- a/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java +++ b/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java @@ -70,7 +70,6 @@ import org.apache.commons.io.IOUtils; import org.apache.commons.lang.StringUtils; import org.archive.io.ReplayInputStream; import org.archive.io.WriterPoolMember; -import org.archive.io.WriterPoolSettings; import org.archive.io.warc.WARCWriter; import org.archive.io.warc.WARCWriterPool; import org.archive.modules.CrawlMetadata; @@ -166,8 +165,7 @@ public class WARCWriterProcessor extends WriterPoolProcessor { @Override protected void setupPool(final AtomicInteger serialNo) { - WriterPoolSettings wps = getWriterPoolSettings(); - setPool(new WARCWriterPool(serialNo, wps, getPoolMaxActive(), getPoolMaxWaitMs())); + setPool(new WARCWriterPool(serialNo, this, getPoolMaxActive(), getPoolMaxWaitMs())); } /** diff --git a/modules/src/main/java/org/archive/modules/writer/WriterPoolProcessor.java b/modules/src/main/java/org/archive/modules/writer/WriterPoolProcessor.java index ae5e7027..17099f7d 100644 --- a/modules/src/main/java/org/archive/modules/writer/WriterPoolProcessor.java +++ b/modules/src/main/java/org/archive/modules/writer/WriterPoolProcessor.java @@ -25,7 +25,6 @@ import static org.archive.modules.fetcher.FetchStatusCodes.S_DNS_SUCCESS; import java.io.File; import java.io.IOException; import java.net.InetAddress; -import java.net.UnknownHostException; import java.util.ArrayList; import java.util.List; import java.util.concurrent.atomic.AtomicInteger; @@ -33,7 +32,6 @@ import java.util.logging.Logger; import org.archive.checkpointing.Checkpoint; import org.archive.checkpointing.Checkpointable; -import org.archive.io.DefaultWriterPoolSettings; import org.archive.io.WriterPool; import org.archive.io.WriterPoolMember; import org.archive.io.WriterPoolSettings; @@ -57,8 +55,9 @@ import org.springframework.context.Lifecycle; * @author stack */ public abstract class WriterPoolProcessor extends Processor -implements Lifecycle, Checkpointable { +implements Lifecycle, Checkpointable, WriterPoolSettings { private static final long serialVersionUID = 1L; + @SuppressWarnings("unused") private static final Logger logger = Logger.getLogger(WriterPoolProcessor.class.getName()); @@ -76,9 +75,9 @@ implements Lifecycle, Checkpointable { /** * File prefix. The text supplied here will be used as a prefix naming - * writer files. For example if the prefix is 'IAH', then file names will - * look like IAH-20040808101010-0001-HOSTNAME.arc.gz ...if writing ARCs (The - * prefix will be separated from the date by a hyphen). + * writer files. For example if the prefix is 'WEB', then file names will + * look like WEB-20040808101010-0001-PID@HOSTNAME#PORT.arc.gz ...if + * writing ARCs (The prefix will be separated from the date by a hyphen). */ String prefix = WriterPoolMember.DEFAULT_PREFIX; public String getPrefix() { @@ -90,15 +89,23 @@ implements Lifecycle, Checkpointable { /** - * Suffix to tag onto files. If value is '${HOSTNAME}', will use hostname - * for suffix. If empty, no suffix will be added. + * Template from which a filename is interpolated. Expressions of the + * form ${key} will be replaced by values from a local map of useful + * values (including 'prefix', 'timestamp17', and 'serialno') or + * global system properties (which includes the local hostname/port/pid). + * + * The default pattern will generate unique names under reasonable + * assumptions; be sure you know what you're doing before customizing, + * as you could easily create filename collisions with a poorly-designed + * filename template. + * */ - String suffix = WriterPoolMember.DEFAULT_SUFFIX; - public String getSuffix() { - return suffix; + String template = WriterPoolMember.DEFAULT_TEMPLATE; + public String getTemplate() { + return template; } - public void setSuffix(String suffix) { - this.suffix = suffix; + public void setTemplate(String template) { + this.template = template; } /** @@ -224,7 +231,6 @@ implements Lifecycle, Checkpointable { */ private long totalBytesWritten = 0; - private WriterPoolSettings settings; private AtomicInteger serial = new AtomicInteger(); @@ -242,7 +248,6 @@ implements Lifecycle, Checkpointable { return; } super.start(); - this.settings = makeWriterPoolSettings(); setupPool(serial); } @@ -252,7 +257,6 @@ implements Lifecycle, Checkpointable { } super.stop(); this.pool.close(); - this.settings = null; } @@ -391,9 +395,9 @@ implements Lifecycle, Checkpointable { this.totalBytesWritten = totalBytesWritten; } - protected abstract List getMetadata(); + public abstract List getMetadata(); - private List getOutputDirs() { + public List getOutputDirs() { List list = getStorePaths(); ArrayList results = new ArrayList(); for (String path: list) { @@ -412,35 +416,6 @@ implements Lifecycle, Checkpointable { } return results; } - - - protected WriterPoolSettings getWriterPoolSettings() { - return settings; - } - - private WriterPoolSettings makeWriterPoolSettings() { - DefaultWriterPoolSettings result = new DefaultWriterPoolSettings(); - result.setMaxSize(getMaxFileSizeBytes()); - result.setMetadata(getMetadata()); - result.setOutputDirs(getOutputDirs()); - result.setPrefix(getPrefix()); - - String sfx = getSuffix(); - sfx = sfx.trim(); - if (sfx.contains(WriterPoolMember.HOSTNAME_VARIABLE)) { - String str = "localhost.localdomain"; - try { - str = InetAddress.getLocalHost().getCanonicalHostName(); - } catch (UnknownHostException ue) { - logger.severe("Failed getHostAddress for this host: " + ue); - } - sfx = sfx.replace(WriterPoolMember.HOSTNAME_VARIABLE, str); - } - - result.setSuffix(sfx); - result.setCompressed(getCompress()); - return result; - } @Override protected void innerProcess(CrawlURI puri) {