diff --git a/commons/src/main/java/org/archive/io/DefaultWriterPoolSettings.java b/commons/src/main/java/org/archive/io/DefaultWriterPoolSettings.java deleted file mode 100644 index ebcf49d3..00000000 --- a/commons/src/main/java/org/archive/io/DefaultWriterPoolSettings.java +++ /dev/null @@ -1,105 +0,0 @@ -/* - * This file is part of the Heritrix web crawler (crawler.archive.org). - * - * Licensed to the Internet Archive (IA) by one or more individual - * contributors. - * - * The IA licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.archive.io; - -import java.io.File; -import java.io.Serializable; -import java.util.ArrayList; -import java.util.List; - -/** - * @author pjack - */ -public class DefaultWriterPoolSettings -implements WriterPoolSettings, Serializable { - - private static final long serialVersionUID = 1L; - - private long maxSize; - private List metadata = new ArrayList();; - transient private List outputDirs = new ArrayList(); - private String prefix; - private String suffix; - private boolean compressed; - - - public DefaultWriterPoolSettings() { - } - - - public boolean isCompressed() { - return compressed; - } - - - public void setCompressed(boolean compressed) { - this.compressed = compressed; - } - - - public long getMaxSize() { - return maxSize; - } - - - public void setMaxSize(long maxSize) { - this.maxSize = maxSize; - } - - - public List getMetadata() { - return metadata; - } - - - public void setMetadata(List metadata) { - this.metadata = metadata; - } - - - public List getOutputDirs() { - return outputDirs; - } - - - public void setOutputDirs(List outputDirs) { - this.outputDirs = outputDirs; - } - - - public String getPrefix() { - return prefix; - } - - - public void setPrefix(String prefix) { - this.prefix = prefix; - } - - - public String getSuffix() { - return suffix; - } - - - public void setSuffix(String suffix) { - this.suffix = suffix; - } -} diff --git a/commons/src/main/java/org/archive/io/WriterPool.java b/commons/src/main/java/org/archive/io/WriterPool.java index 0c555fa4..f1f8c954 100644 --- a/commons/src/main/java/org/archive/io/WriterPool.java +++ b/commons/src/main/java/org/archive/io/WriterPool.java @@ -102,9 +102,9 @@ public abstract class WriterPool { final int poolMaximumActive, final int poolMaximumWait) { logger.info("Initial configuration:" + " prefix=" + settings.getPrefix() + - ", suffix=" + settings.getSuffix() + - ", compress=" + settings.isCompressed() + - ", maxSize=" + settings.getMaxSize() + + ", suffix=" + settings.getTemplate() + + ", compress=" + settings.getCompress() + + ", maxSize=" + settings.getMaxFileSizeBytes() + ", maxActive=" + poolMaximumActive + ", maxWait=" + poolMaximumWait); this.settings = settings; diff --git a/commons/src/main/java/org/archive/io/WriterPoolMember.java b/commons/src/main/java/org/archive/io/WriterPoolMember.java index d32a2ff8..f609aab0 100644 --- a/commons/src/main/java/org/archive/io/WriterPoolMember.java +++ b/commons/src/main/java/org/archive/io/WriterPoolMember.java @@ -30,13 +30,14 @@ import java.text.DecimalFormat; import java.text.NumberFormat; import java.util.Iterator; import java.util.List; +import java.util.Properties; import java.util.concurrent.atomic.AtomicInteger; import java.util.logging.Logger; import java.util.zip.GZIPOutputStream; import org.archive.util.ArchiveUtils; import org.archive.util.FileUtils; -import org.archive.util.TimestampSerialno; +import org.archive.util.PropertyUtils; /** @@ -53,21 +54,22 @@ public abstract class WriterPoolMember implements ArchiveFileConstants { public static final String UTF8 = "UTF-8"; /** - * Default file prefix. + * Default archival-aggregate filename template. + * + * Under usual assumptions -- hostnames aren't shared among crawling hosts; + * processes have unique PIDs and admin ports; timestamps inside one process + * don't repeat (see UniqueTimestampService); clocks are generally + * accurate -- will generate a unique name. * * Stands for Internet Archive Heritrix. */ - public static final String DEFAULT_PREFIX = "IAH"; + public static final String DEFAULT_TEMPLATE = + "${prefix}-${timestamp17}-${serialno}-${heritrix.pid}@${heritrix.hostname}#${heritrix.port}"; /** - * Value to interpolate with actual hostname. + * Default for file prefix. */ - public static final String HOSTNAME_VARIABLE = "${HOSTNAME}"; - - /** - * Default for file suffix. - */ - public static final String DEFAULT_SUFFIX = HOSTNAME_VARIABLE; + public static final String DEFAULT_PREFIX = "WEB"; /** * Reference to file we're currently writing. @@ -87,8 +89,8 @@ public abstract class WriterPoolMember implements ArchiveFileConstants { private final boolean compressed; private List writeDirs = null; + private String template = DEFAULT_TEMPLATE; private String prefix = DEFAULT_PREFIX; - private String suffix = DEFAULT_SUFFIX; private final long maxSize; private final String extension; @@ -96,8 +98,10 @@ public abstract class WriterPoolMember implements ArchiveFileConstants { * Creation date for the current file. * Set by {@link #createFile()}. */ - private String createTimestamp = "UNSET!!!"; + protected String currentTimestamp = "UNSET!!!"; + protected String currentBasename; + /** * A running sequence used making unique file names. */ @@ -172,9 +176,9 @@ public abstract class WriterPoolMember implements ArchiveFileConstants { */ public WriterPoolMember(AtomicInteger serialNo, final List dirs, final String prefix, - final String suffix, final boolean cmprs, + final String template, final boolean cmprs, final long maxSize, final String extension) { - this.suffix = suffix; + this.template = template; this.prefix = prefix; this.maxSize = maxSize; this.writeDirs = dirs; @@ -211,13 +215,10 @@ public abstract class WriterPoolMember implements ArchiveFileConstants { * @throws IOException */ protected String createFile() throws IOException { - TimestampSerialno tsn = getTimestampSerialNo(); - String name = this.prefix + '-' + getUniqueBasename(tsn) + - ((this.suffix == null || this.suffix.length() <= 0)? - "": "-" + this.suffix) + '.' + this.extension + + generateNewBasename(); + String name = currentBasename + '.' + this.extension + ((this.compressed)? DOT_COMPRESSED_FILE_EXTENSION: "") + OCCUPIED_SUFFIX; - this.createTimestamp = tsn.getTimestamp(); File dir = getNextDirectory(this.writeDirs); return createFile(new File(dir, name)); } @@ -277,39 +278,28 @@ public abstract class WriterPoolMember implements ArchiveFileConstants { } return d; } - - protected synchronized TimestampSerialno getTimestampSerialNo() { - return getTimestampSerialNo(null); - } - + /** - * Do static synchronization around getting of counter and timestamp so - * no chance of a thread getting in between the getting of timestamp and - * allocation of serial number throwing the two out of alignment. - * - * @param timestamp If non-null, use passed timestamp (must be 14 digit - * ARC format), else if null, timestamp with now. - * @return Instance of data structure that has timestamp and serial no. + * Generate a new basename by interpolating values in the configured + * template. Values come from local state, other configured values, and + * global system properties. The recommended default template will + * generate a unique basename under reasonable assumptions. */ - protected synchronized TimestampSerialno - getTimestampSerialNo(final String timestamp) { - return new TimestampSerialno((timestamp != null)? - timestamp: ArchiveUtils.get14DigitDate(), - serialNo.getAndIncrement()); - } - - /** - * Return a unique basename. - * - * Name is timestamp + an every increasing sequence number. - * - * @param tsn Structure with timestamp and serial number. - * - * @return Unique basename. - */ - private String getUniqueBasename(TimestampSerialno tsn) { - return tsn.getTimestamp() + "-" + - WriterPoolMember.serialNoFormatter.format(tsn.getSerialNumber()); + protected void generateNewBasename() { + Properties localProps = new Properties(); + localProps.setProperty("prefix", prefix); + synchronized(this.getClass()) { + // ensure that serialNo and timestamp are minted together (never inverted sort order) + String paddedSerialNumber = WriterPoolMember.serialNoFormatter.format(serialNo.getAndIncrement()); + String timestamp17 = ArchiveUtils.getUnique17DigitDate(); + String timestamp14 = ArchiveUtils.getUnique14DigitDate(); + currentTimestamp = timestamp17; + localProps.setProperty("serialno", paddedSerialNumber); + localProps.setProperty("timestamp17", timestamp17); + localProps.setProperty("timestamp14", timestamp14); + } + currentBasename = PropertyUtils.interpolateWithProperties(template, + localProps, System.getProperties()); } @@ -384,7 +374,7 @@ public abstract class WriterPoolMember implements ArchiveFileConstants { } /** - * Postion in current physical file. + * Position in current physical file. * Used making accounting of bytes written. * @return Position in underlying file. Call before or after writing * records *only* to be safe. @@ -482,12 +472,7 @@ public abstract class WriterPoolMember implements ArchiveFileConstants { protected OutputStream getOutputStream() { return this.out; } - - protected String getCreateTimestamp() { - return createTimestamp; - } - - + /** * An override so we get access to underlying output stream. * and offer an end() that does not accompany closing underlying diff --git a/commons/src/main/java/org/archive/io/WriterPoolSettings.java b/commons/src/main/java/org/archive/io/WriterPoolSettings.java index c9812175..baa36ce7 100644 --- a/commons/src/main/java/org/archive/io/WriterPoolSettings.java +++ b/commons/src/main/java/org/archive/io/WriterPoolSettings.java @@ -28,10 +28,10 @@ import java.util.List; * @version $Date$, $Revision$ */ public interface WriterPoolSettings { - public long getMaxSize(); + public long getMaxFileSizeBytes(); public String getPrefix(); - public String getSuffix(); + public String getTemplate(); public List getOutputDirs(); - public boolean isCompressed(); + public boolean getCompress(); public List getMetadata(); } \ No newline at end of file diff --git a/commons/src/main/java/org/archive/io/arc/ARCWriter.java b/commons/src/main/java/org/archive/io/arc/ARCWriter.java index b58b74a4..10acfe80 100644 --- a/commons/src/main/java/org/archive/io/arc/ARCWriter.java +++ b/commons/src/main/java/org/archive/io/arc/ARCWriter.java @@ -186,7 +186,7 @@ public class ARCWriter extends WriterPoolMember implements ARCConstants { protected String createFile() throws IOException { String name = super.createFile(); - writeFirstRecord(getCreateTimestamp()); + writeFirstRecord(currentTimestamp); return name; } @@ -223,13 +223,17 @@ public class ARCWriter extends WriterPoolMember implements ARCConstants { *

We do things in this roundabout manner because the java * GZIPOutputStream does not give access to GZIP header fields. * - * @param date Date to put into the ARC metadata. + * @param date Date to put into the ARC metadata; if 17-digit will be + * truncated to traditional 14-digits * * @return Byte array filled w/ the arc header. * @throws IOException */ private byte [] generateARCFileMetaData(String date) throws IOException { + if(date.length()>14) { + date = date.substring(0,14); + } int metadataBodyLength = getMetadataLength(); // If metadata body, then the minor part of the version is '1' rather // than '0'. diff --git a/commons/src/main/java/org/archive/io/arc/ARCWriterPool.java b/commons/src/main/java/org/archive/io/arc/ARCWriterPool.java index e06dc086..8e05ddb2 100644 --- a/commons/src/main/java/org/archive/io/arc/ARCWriterPool.java +++ b/commons/src/main/java/org/archive/io/arc/ARCWriterPool.java @@ -58,8 +58,8 @@ public class ARCWriterPool extends WriterPool { super(serial, new BasePoolableObjectFactory() { public Object makeObject() throws Exception { return new ARCWriter(serial, settings.getOutputDirs(), - settings.getPrefix(), settings.getSuffix(), - settings.isCompressed(), settings.getMaxSize(), + settings.getPrefix(), settings.getTemplate(), + settings.getCompress(), settings.getMaxFileSizeBytes(), settings.getMetadata()); } diff --git a/commons/src/main/java/org/archive/io/warc/WARCWriterPool.java b/commons/src/main/java/org/archive/io/warc/WARCWriterPool.java index e9adfb02..414e9da4 100644 --- a/commons/src/main/java/org/archive/io/warc/WARCWriterPool.java +++ b/commons/src/main/java/org/archive/io/warc/WARCWriterPool.java @@ -57,8 +57,8 @@ public class WARCWriterPool extends WriterPool { public Object makeObject() throws Exception { return new WARCWriter(serial, settings.getOutputDirs(), - settings.getPrefix(), settings.getSuffix(), - settings.isCompressed(), settings.getMaxSize(), + settings.getPrefix(), settings.getTemplate(), + settings.getCompress(), settings.getMaxFileSizeBytes(), settings.getMetadata()); } diff --git a/commons/src/main/java/org/archive/util/ArchiveUtils.java b/commons/src/main/java/org/archive/util/ArchiveUtils.java index 2e71fd92..45e5a466 100644 --- a/commons/src/main/java/org/archive/util/ArchiveUtils.java +++ b/commons/src/main/java/org/archive/util/ArchiveUtils.java @@ -123,6 +123,31 @@ public class ArchiveUtils { public static String get17DigitDate(){ return TIMESTAMP17.get().format(new Date()); } + + static long LAST_UNIQUE_NOW17 = 0; + static String LAST_TIMESTAMP17 = ""; + /** + * Utility function for creating UNIQUE-from-this-class + * arc-style date stamps in the format yyyMMddHHmmssSSS. + * Rather than giving a duplicate datestamp on a + * subsequent call, will increment the milliseconds until a + * unique value is returned. + * + * Date stamps are in the UTC time zone + * @return the date stamp + */ + public synchronized static String getUnique17DigitDate(){ + long effectiveNow = System.currentTimeMillis(); + effectiveNow = Math.max(effectiveNow, LAST_UNIQUE_NOW17+1); + String candidate = get17DigitDate(effectiveNow); + while(candidate.equals(LAST_TIMESTAMP17)) { + effectiveNow++; + candidate = get17DigitDate(effectiveNow); + } + LAST_UNIQUE_NOW17 = effectiveNow; + LAST_TIMESTAMP17 = candidate; + return candidate; + } /** * Utility function for creating arc-style date stamps @@ -133,6 +158,31 @@ public class ArchiveUtils { public static String get14DigitDate(){ return TIMESTAMP14.get().format(new Date()); } + + static long LAST_UNIQUE_NOW14 = 0; + static String LAST_TIMESTAMP14 = ""; + /** + * Utility function for creating UNIQUE-from-this-class + * arc-style date stamps in the format yyyMMddHHmmss. + * Rather than giving a duplicate datestamp on a + * subsequent call, will increment the seconds until a + * unique value is returned. + * + * Date stamps are in the UTC time zone + * @return the date stamp + */ + public synchronized static String getUnique14DigitDate(){ + long effectiveNow = System.currentTimeMillis(); + effectiveNow = Math.max(effectiveNow, LAST_UNIQUE_NOW14+1); + String candidate = get14DigitDate(effectiveNow); + while(candidate.equals(LAST_TIMESTAMP14)) { + effectiveNow += 1000; + candidate = get14DigitDate(effectiveNow); + } + LAST_UNIQUE_NOW14 = effectiveNow; + LAST_TIMESTAMP14 = candidate; + return candidate; + } /** * Utility function for creating arc-style date stamps diff --git a/commons/src/main/java/org/archive/util/PropertyUtils.java b/commons/src/main/java/org/archive/util/PropertyUtils.java index ad60a131..0760cea5 100644 --- a/commons/src/main/java/org/archive/util/PropertyUtils.java +++ b/commons/src/main/java/org/archive/util/PropertyUtils.java @@ -18,8 +18,16 @@ */ package org.archive.util; +import java.util.Properties; +import java.util.regex.Matcher; + +import org.apache.commons.lang.StringUtils; + /** - * @author stack + * Utilities for dealing with Java Properties (incl. System Properties) + * + * @contributor stack + * @contributor gojomo * @version $Date$ $Revision$ */ public class PropertyUtils { @@ -51,4 +59,56 @@ public class PropertyUtils { return getPropertyOrNull(key) == null? fallback: Integer.parseInt(getPropertyOrNull(key)); } + + /** + * Given a string which may contain expressions of the form + * ${key}, replace each expression with the value corresponding to the + * given key in System Properties. If no value is present, + * the expression is replaced with the empty-string. + * + * @param original String + * @param properties Properties to try in order; first value found (if any) is used + * @return modified String + */ + public static String interpolateWithProperties(String original) { + return interpolateWithProperties(original,System.getProperties()); + } + + static String propRefPattern = "\\$\\{([^{}]+)\\}"; + + /** + * Given a string which may contain expressions of the form + * ${key}, replace each expression with the value corresponding to the + * given key in the supplied Properties instance. If no value is present, + * the expression is replaced with the empty-string. + * + * @param original String + * @param properties Properties to try in order; first value found (if any) is used + * @return modified String + */ + public static String interpolateWithProperties(String original, + Properties... props) { + String result = original; + // cap number of interpolations as guard against unending loop + inter: for(int i =0; i < original.length()*2; i++) { + Matcher m = TextUtils.getMatcher(propRefPattern, result); + while(m.find()) { + String key = m.group(1); + String value = ""; + for(Properties properties : props) { + value = properties.getProperty(key, ""); + if(StringUtils.isNotEmpty(value)) { + break; + } + } + result = result.substring(0,m.start()) + + value + + result.substring(m.end()); + continue inter; + } + // we only hit here if there were no interpolations last while loop + break; + } + return result; + } } diff --git a/commons/src/test/java/org/archive/io/arc/ARCWriterPoolTest.java b/commons/src/test/java/org/archive/io/arc/ARCWriterPoolTest.java index b0425fc8..c833c078 100644 --- a/commons/src/test/java/org/archive/io/arc/ARCWriterPoolTest.java +++ b/commons/src/test/java/org/archive/io/arc/ARCWriterPoolTest.java @@ -129,7 +129,7 @@ public class ARCWriterPoolTest extends TmpDirTestCase { private WriterPoolSettings getSettings(final boolean isCompressed) { return new WriterPoolSettings() { - public long getMaxSize() { + public long getMaxFileSizeBytes() { return ARCConstants.DEFAULT_MAX_ARC_FILE_SIZE; } @@ -137,8 +137,8 @@ public class ARCWriterPoolTest extends TmpDirTestCase { return PREFIX; } - public String getSuffix() { - return ""; + public String getTemplate() { + return "${prefix}-${timestamp17}-${serialno}-${heritrix.hostname}"; } public List getOutputDirs() { @@ -146,7 +146,7 @@ public class ARCWriterPoolTest extends TmpDirTestCase { return Arrays.asList(files); } - public boolean isCompressed() { + public boolean getCompress() { return isCompressed; } diff --git a/commons/src/main/java/org/archive/util/TimestampSerialno.java b/commons/src/test/java/org/archive/util/PropertyUtilsTest.java similarity index 55% rename from commons/src/main/java/org/archive/util/TimestampSerialno.java rename to commons/src/test/java/org/archive/util/PropertyUtilsTest.java index ed4550d6..0666efb7 100644 --- a/commons/src/main/java/org/archive/util/TimestampSerialno.java +++ b/commons/src/test/java/org/archive/util/PropertyUtilsTest.java @@ -16,41 +16,30 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.archive.util; + +import java.io.IOException; +import java.util.Properties; + +import junit.framework.TestCase; + + /** - * Immutable data structure that holds a timestamp and an accompanying - * serial number. + * PropertyUtils tests. * - * For Igor! - * - * @author stack + * @contributor gojomo + * @version $Date: 2009-11-19 14:39:53 -0800 (Thu, 19 Nov 2009) $, $Revision: 6674 $ */ -public class TimestampSerialno { - private final String ts; - private final int serialNumber; - - public TimestampSerialno(String ts, int serialNo) { - this.ts = ts; - this.serialNumber = serialNo; - } +public class PropertyUtilsTest extends TestCase { - public TimestampSerialno(int serialNo) { - this.ts = ArchiveUtils.get14DigitDate(); - this.serialNumber = serialNo; + public void testSimpleInterpolate() throws IOException { + Properties props = new Properties(); + props.put("foo", "OOF"); + props.put("bar","RAB"); + String original = "FOO|${foo} BAR|${bar}"; + String expected = "FOO|OOF BAR|RAB"; + assertEquals("interpalation problem",expected,PropertyUtils.interpolateWithProperties(original,props)); } - - /** - * @return Returns the now. - */ - public String getTimestamp() { - return this.ts; - } - - /** - * @return Returns the serialNumber. - */ - public int getSerialNumber() { - return this.serialNumber; - } -} \ No newline at end of file +} diff --git a/engine/src/main/java/org/archive/crawler/Heritrix.java b/engine/src/main/java/org/archive/crawler/Heritrix.java index 54341129..5173e349 100644 --- a/engine/src/main/java/org/archive/crawler/Heritrix.java +++ b/engine/src/main/java/org/archive/crawler/Heritrix.java @@ -27,6 +27,9 @@ import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.PrintStream; +import java.lang.management.ManagementFactory; +import java.net.InetAddress; +import java.net.UnknownHostException; import java.security.KeyStore; import java.security.MessageDigest; import java.security.cert.Certificate; @@ -90,7 +93,6 @@ public class Heritrix { private static final String ADHOC_KEYSTORE = "adhoc.keystore"; - @SuppressWarnings("unused") private static final Logger logger = Logger.getLogger(Heritrix.class.getName()); /** Name of configuration directory */ @@ -322,6 +324,8 @@ public class Heritrix { // inside in a container. TimeZone.setDefault(TimeZone.getTimeZone("GMT")); + setupGlobalProperties(port); + // Start Heritrix. try { engine = new Engine(jobsDir); @@ -372,6 +376,39 @@ public class Heritrix { } } + /** + * Setup global system properties that may be of use elsewhere. + * + * @param port + */ + protected void setupGlobalProperties(int port) { + if (System.getProperty("heritrix.port") == null) { + System.setProperty("heritrix.port", port + ""); + } + + String hostname = "localhost.localdomain"; + if (System.getProperty("heritrix.hostname") == null) { + try { + hostname = InetAddress.getLocalHost().getCanonicalHostName(); + } catch (UnknownHostException ue) { + logger.warning("Failed getHostAddress for this host: " + ue); + } + System.setProperty("heritrix.hostname", hostname); + } + + // while not guaranteed, on our platforms of interest this name + // always seems to be PID@HOSTNAME + String runtimeName = ManagementFactory.getRuntimeMXBean().getName(); + if(System.getProperty("heritrix.runtimeName") == null) { + System.setProperty("heritrix.runtimeName", runtimeName); + } + if (System.getProperty("heritrix.pid") == null + && runtimeName.matches("\\d+@\\S+")) { + System.setProperty("heritrix.pid", runtimeName.substring(0,runtimeName.indexOf("@"))); + } + } + + /** * Perform preparation to use an ad-hoc, created-as-necessary * certificate/keystore for HTTPS access. A keystore with new diff --git a/modules/src/main/java/org/archive/modules/writer/ARCWriterProcessor.java b/modules/src/main/java/org/archive/modules/writer/ARCWriterProcessor.java index cb0b9685..24b770f3 100644 --- a/modules/src/main/java/org/archive/modules/writer/ARCWriterProcessor.java +++ b/modules/src/main/java/org/archive/modules/writer/ARCWriterProcessor.java @@ -35,7 +35,6 @@ import org.apache.commons.lang.StringEscapeUtils; import org.apache.commons.lang.StringUtils; import org.archive.io.ReplayInputStream; import org.archive.io.WriterPoolMember; -import org.archive.io.WriterPoolSettings; import org.archive.io.arc.ARCWriter; import org.archive.io.arc.ARCWriterPool; import org.archive.modules.ProcessResult; @@ -54,7 +53,7 @@ import org.archive.util.ArchiveUtils; */ public class ARCWriterProcessor extends WriterPoolProcessor { - final static private String TEMPLATE = readTemplate(); + final static private String METADATA_TEMPLATE = readMetadataTemplate(); private static final long serialVersionUID = 3L; @@ -77,8 +76,7 @@ public class ARCWriterProcessor extends WriterPoolProcessor { @Override protected void setupPool(AtomicInteger serialNo) { - WriterPoolSettings wps = getWriterPoolSettings(); - setPool(new ARCWriterPool(serialNo, wps, getPoolMaxActive(), getPoolMaxWaitMs())); + setPool(new ARCWriterPool(serialNo, this, getPoolMaxActive(), getPoolMaxWaitMs())); } /** @@ -160,8 +158,8 @@ public class ARCWriterProcessor extends WriterPoolProcessor { return checkBytesWritten(); } - protected List getMetadata() { - if (TEMPLATE == null) { + public List getMetadata() { + if (METADATA_TEMPLATE == null) { return null; } @@ -169,7 +167,7 @@ public class ARCWriterProcessor extends WriterPoolProcessor { return cachedMetadata; } - String meta = TEMPLATE; + String meta = METADATA_TEMPLATE; meta = replace(meta, "${VERSION}", ArchiveUtils.VERSION); meta = replace(meta, "${HOST}", getHostName()); meta = replace(meta, "${IP}", getHostAddress()); @@ -225,7 +223,7 @@ public class ARCWriterProcessor extends WriterPoolProcessor { } } - private static String readTemplate() { + private static String readMetadataTemplate() { InputStream input = ARCWriterProcessor.class.getResourceAsStream( "arc_metadata_template.xml"); if (input == null) { diff --git a/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java b/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java index 8a574ea7..33cf4e69 100644 --- a/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java +++ b/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java @@ -70,7 +70,6 @@ import org.apache.commons.io.IOUtils; import org.apache.commons.lang.StringUtils; import org.archive.io.ReplayInputStream; import org.archive.io.WriterPoolMember; -import org.archive.io.WriterPoolSettings; import org.archive.io.warc.WARCWriter; import org.archive.io.warc.WARCWriterPool; import org.archive.modules.CrawlMetadata; @@ -166,8 +165,7 @@ public class WARCWriterProcessor extends WriterPoolProcessor { @Override protected void setupPool(final AtomicInteger serialNo) { - WriterPoolSettings wps = getWriterPoolSettings(); - setPool(new WARCWriterPool(serialNo, wps, getPoolMaxActive(), getPoolMaxWaitMs())); + setPool(new WARCWriterPool(serialNo, this, getPoolMaxActive(), getPoolMaxWaitMs())); } /** diff --git a/modules/src/main/java/org/archive/modules/writer/WriterPoolProcessor.java b/modules/src/main/java/org/archive/modules/writer/WriterPoolProcessor.java index ae5e7027..17099f7d 100644 --- a/modules/src/main/java/org/archive/modules/writer/WriterPoolProcessor.java +++ b/modules/src/main/java/org/archive/modules/writer/WriterPoolProcessor.java @@ -25,7 +25,6 @@ import static org.archive.modules.fetcher.FetchStatusCodes.S_DNS_SUCCESS; import java.io.File; import java.io.IOException; import java.net.InetAddress; -import java.net.UnknownHostException; import java.util.ArrayList; import java.util.List; import java.util.concurrent.atomic.AtomicInteger; @@ -33,7 +32,6 @@ import java.util.logging.Logger; import org.archive.checkpointing.Checkpoint; import org.archive.checkpointing.Checkpointable; -import org.archive.io.DefaultWriterPoolSettings; import org.archive.io.WriterPool; import org.archive.io.WriterPoolMember; import org.archive.io.WriterPoolSettings; @@ -57,8 +55,9 @@ import org.springframework.context.Lifecycle; * @author stack */ public abstract class WriterPoolProcessor extends Processor -implements Lifecycle, Checkpointable { +implements Lifecycle, Checkpointable, WriterPoolSettings { private static final long serialVersionUID = 1L; + @SuppressWarnings("unused") private static final Logger logger = Logger.getLogger(WriterPoolProcessor.class.getName()); @@ -76,9 +75,9 @@ implements Lifecycle, Checkpointable { /** * File prefix. The text supplied here will be used as a prefix naming - * writer files. For example if the prefix is 'IAH', then file names will - * look like IAH-20040808101010-0001-HOSTNAME.arc.gz ...if writing ARCs (The - * prefix will be separated from the date by a hyphen). + * writer files. For example if the prefix is 'WEB', then file names will + * look like WEB-20040808101010-0001-PID@HOSTNAME#PORT.arc.gz ...if + * writing ARCs (The prefix will be separated from the date by a hyphen). */ String prefix = WriterPoolMember.DEFAULT_PREFIX; public String getPrefix() { @@ -90,15 +89,23 @@ implements Lifecycle, Checkpointable { /** - * Suffix to tag onto files. If value is '${HOSTNAME}', will use hostname - * for suffix. If empty, no suffix will be added. + * Template from which a filename is interpolated. Expressions of the + * form ${key} will be replaced by values from a local map of useful + * values (including 'prefix', 'timestamp17', and 'serialno') or + * global system properties (which includes the local hostname/port/pid). + * + * The default pattern will generate unique names under reasonable + * assumptions; be sure you know what you're doing before customizing, + * as you could easily create filename collisions with a poorly-designed + * filename template. + * */ - String suffix = WriterPoolMember.DEFAULT_SUFFIX; - public String getSuffix() { - return suffix; + String template = WriterPoolMember.DEFAULT_TEMPLATE; + public String getTemplate() { + return template; } - public void setSuffix(String suffix) { - this.suffix = suffix; + public void setTemplate(String template) { + this.template = template; } /** @@ -224,7 +231,6 @@ implements Lifecycle, Checkpointable { */ private long totalBytesWritten = 0; - private WriterPoolSettings settings; private AtomicInteger serial = new AtomicInteger(); @@ -242,7 +248,6 @@ implements Lifecycle, Checkpointable { return; } super.start(); - this.settings = makeWriterPoolSettings(); setupPool(serial); } @@ -252,7 +257,6 @@ implements Lifecycle, Checkpointable { } super.stop(); this.pool.close(); - this.settings = null; } @@ -391,9 +395,9 @@ implements Lifecycle, Checkpointable { this.totalBytesWritten = totalBytesWritten; } - protected abstract List getMetadata(); + public abstract List getMetadata(); - private List getOutputDirs() { + public List getOutputDirs() { List list = getStorePaths(); ArrayList results = new ArrayList(); for (String path: list) { @@ -412,35 +416,6 @@ implements Lifecycle, Checkpointable { } return results; } - - - protected WriterPoolSettings getWriterPoolSettings() { - return settings; - } - - private WriterPoolSettings makeWriterPoolSettings() { - DefaultWriterPoolSettings result = new DefaultWriterPoolSettings(); - result.setMaxSize(getMaxFileSizeBytes()); - result.setMetadata(getMetadata()); - result.setOutputDirs(getOutputDirs()); - result.setPrefix(getPrefix()); - - String sfx = getSuffix(); - sfx = sfx.trim(); - if (sfx.contains(WriterPoolMember.HOSTNAME_VARIABLE)) { - String str = "localhost.localdomain"; - try { - str = InetAddress.getLocalHost().getCanonicalHostName(); - } catch (UnknownHostException ue) { - logger.severe("Failed getHostAddress for this host: " + ue); - } - sfx = sfx.replace(WriterPoolMember.HOSTNAME_VARIABLE, str); - } - - result.setSuffix(sfx); - result.setCompressed(getCompress()); - return result; - } @Override protected void innerProcess(CrawlURI puri) {