mirror of
https://github.com/internetarchive/heritrix3.git
synced 2026-09-25 07:05:49 +00:00
[HER-1727] (W)ARC filename uniqueness guarantees: adminport option, process_id option(?), JVM-global-repeat-suppression
* Heritrix.java
make heritrix.hostname, heritrix.pid, heritrix.port available as global properties
* PropertyUtils.java, PropertyUtilsTest.java
utility methods to interpolate string values from one or several supplied Properties instances
* ArchiveUtils.java
utility methods to give timestamps guaranteed larger/different than any previously-issued timestamp
* WriterPoolSettings.java, WriterPool.java, ARCWriterPool.java, WARCWriterPool.java, ARCWriterPoolTest.java
improve field names, change 'suffix' to more general 'template'
* WriterPoolMember.java
replace 'suffix' with 'template' which is interpolated when specific name is needed
set default template to pattern extremely unlikely to generate duplicate filenames
change default prefix to 'WEB'
centralize creation of new basenames into generateNewBasename() method, which internalizes timetamp/serialNo minting and interpolation
* WriterPoolProcessor.java, ARCWriterProcessor.java, WARCWriterProcessor.java
serve as own WriterPoolSettings instance
replace 'suffix' with 'template'
* ARCWriter.java
accept (but truncate) 17-digit timestamps at creation
* DefaultWriterPoolSettings.java, TimestampSerialNo.java
delete as superfluous
This commit is contained in:
gojomo
@@ -1,105 +0,0 @@
|
||||
/*
|
||||
* This file is part of the Heritrix web crawler (crawler.archive.org).
|
||||
*
|
||||
* Licensed to the Internet Archive (IA) by one or more individual
|
||||
* contributors.
|
||||
*
|
||||
* The IA licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.archive.io;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.Serializable;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* @author pjack
|
||||
*/
|
||||
public class DefaultWriterPoolSettings
|
||||
implements WriterPoolSettings, Serializable {
|
||||
|
||||
private static final long serialVersionUID = 1L;
|
||||
|
||||
private long maxSize;
|
||||
private List<String> metadata = new ArrayList<String>();;
|
||||
transient private List<File> outputDirs = new ArrayList<File>();
|
||||
private String prefix;
|
||||
private String suffix;
|
||||
private boolean compressed;
|
||||
|
||||
|
||||
public DefaultWriterPoolSettings() {
|
||||
}
|
||||
|
||||
|
||||
public boolean isCompressed() {
|
||||
return compressed;
|
||||
}
|
||||
|
||||
|
||||
public void setCompressed(boolean compressed) {
|
||||
this.compressed = compressed;
|
||||
}
|
||||
|
||||
|
||||
public long getMaxSize() {
|
||||
return maxSize;
|
||||
}
|
||||
|
||||
|
||||
public void setMaxSize(long maxSize) {
|
||||
this.maxSize = maxSize;
|
||||
}
|
||||
|
||||
|
||||
public List<String> getMetadata() {
|
||||
return metadata;
|
||||
}
|
||||
|
||||
|
||||
public void setMetadata(List<String> metadata) {
|
||||
this.metadata = metadata;
|
||||
}
|
||||
|
||||
|
||||
public List<File> getOutputDirs() {
|
||||
return outputDirs;
|
||||
}
|
||||
|
||||
|
||||
public void setOutputDirs(List<File> outputDirs) {
|
||||
this.outputDirs = outputDirs;
|
||||
}
|
||||
|
||||
|
||||
public String getPrefix() {
|
||||
return prefix;
|
||||
}
|
||||
|
||||
|
||||
public void setPrefix(String prefix) {
|
||||
this.prefix = prefix;
|
||||
}
|
||||
|
||||
|
||||
public String getSuffix() {
|
||||
return suffix;
|
||||
}
|
||||
|
||||
|
||||
public void setSuffix(String suffix) {
|
||||
this.suffix = suffix;
|
||||
}
|
||||
}
|
||||
@@ -102,9 +102,9 @@ public abstract class WriterPool {
|
||||
final int poolMaximumActive, final int poolMaximumWait) {
|
||||
logger.info("Initial configuration:" +
|
||||
" prefix=" + settings.getPrefix() +
|
||||
", suffix=" + settings.getSuffix() +
|
||||
", compress=" + settings.isCompressed() +
|
||||
", maxSize=" + settings.getMaxSize() +
|
||||
", suffix=" + settings.getTemplate() +
|
||||
", compress=" + settings.getCompress() +
|
||||
", maxSize=" + settings.getMaxFileSizeBytes() +
|
||||
", maxActive=" + poolMaximumActive +
|
||||
", maxWait=" + poolMaximumWait);
|
||||
this.settings = settings;
|
||||
|
||||
@@ -30,13 +30,14 @@ import java.text.DecimalFormat;
|
||||
import java.text.NumberFormat;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.Properties;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import java.util.logging.Logger;
|
||||
import java.util.zip.GZIPOutputStream;
|
||||
|
||||
import org.archive.util.ArchiveUtils;
|
||||
import org.archive.util.FileUtils;
|
||||
import org.archive.util.TimestampSerialno;
|
||||
import org.archive.util.PropertyUtils;
|
||||
|
||||
|
||||
/**
|
||||
@@ -53,21 +54,22 @@ public abstract class WriterPoolMember implements ArchiveFileConstants {
|
||||
public static final String UTF8 = "UTF-8";
|
||||
|
||||
/**
|
||||
* Default file prefix.
|
||||
* Default archival-aggregate filename template.
|
||||
*
|
||||
* Under usual assumptions -- hostnames aren't shared among crawling hosts;
|
||||
* processes have unique PIDs and admin ports; timestamps inside one process
|
||||
* don't repeat (see UniqueTimestampService); clocks are generally
|
||||
* accurate -- will generate a unique name.
|
||||
*
|
||||
* Stands for Internet Archive Heritrix.
|
||||
*/
|
||||
public static final String DEFAULT_PREFIX = "IAH";
|
||||
public static final String DEFAULT_TEMPLATE =
|
||||
"${prefix}-${timestamp17}-${serialno}-${heritrix.pid}@${heritrix.hostname}#${heritrix.port}";
|
||||
|
||||
/**
|
||||
* Value to interpolate with actual hostname.
|
||||
* Default for file prefix.
|
||||
*/
|
||||
public static final String HOSTNAME_VARIABLE = "${HOSTNAME}";
|
||||
|
||||
/**
|
||||
* Default for file suffix.
|
||||
*/
|
||||
public static final String DEFAULT_SUFFIX = HOSTNAME_VARIABLE;
|
||||
public static final String DEFAULT_PREFIX = "WEB";
|
||||
|
||||
/**
|
||||
* Reference to file we're currently writing.
|
||||
@@ -87,8 +89,8 @@ public abstract class WriterPoolMember implements ArchiveFileConstants {
|
||||
|
||||
private final boolean compressed;
|
||||
private List<File> writeDirs = null;
|
||||
private String template = DEFAULT_TEMPLATE;
|
||||
private String prefix = DEFAULT_PREFIX;
|
||||
private String suffix = DEFAULT_SUFFIX;
|
||||
private final long maxSize;
|
||||
private final String extension;
|
||||
|
||||
@@ -96,8 +98,10 @@ public abstract class WriterPoolMember implements ArchiveFileConstants {
|
||||
* Creation date for the current file.
|
||||
* Set by {@link #createFile()}.
|
||||
*/
|
||||
private String createTimestamp = "UNSET!!!";
|
||||
protected String currentTimestamp = "UNSET!!!";
|
||||
|
||||
protected String currentBasename;
|
||||
|
||||
/**
|
||||
* A running sequence used making unique file names.
|
||||
*/
|
||||
@@ -172,9 +176,9 @@ public abstract class WriterPoolMember implements ArchiveFileConstants {
|
||||
*/
|
||||
public WriterPoolMember(AtomicInteger serialNo,
|
||||
final List<File> dirs, final String prefix,
|
||||
final String suffix, final boolean cmprs,
|
||||
final String template, final boolean cmprs,
|
||||
final long maxSize, final String extension) {
|
||||
this.suffix = suffix;
|
||||
this.template = template;
|
||||
this.prefix = prefix;
|
||||
this.maxSize = maxSize;
|
||||
this.writeDirs = dirs;
|
||||
@@ -211,13 +215,10 @@ public abstract class WriterPoolMember implements ArchiveFileConstants {
|
||||
* @throws IOException
|
||||
*/
|
||||
protected String createFile() throws IOException {
|
||||
TimestampSerialno tsn = getTimestampSerialNo();
|
||||
String name = this.prefix + '-' + getUniqueBasename(tsn) +
|
||||
((this.suffix == null || this.suffix.length() <= 0)?
|
||||
"": "-" + this.suffix) + '.' + this.extension +
|
||||
generateNewBasename();
|
||||
String name = currentBasename + '.' + this.extension +
|
||||
((this.compressed)? DOT_COMPRESSED_FILE_EXTENSION: "") +
|
||||
OCCUPIED_SUFFIX;
|
||||
this.createTimestamp = tsn.getTimestamp();
|
||||
File dir = getNextDirectory(this.writeDirs);
|
||||
return createFile(new File(dir, name));
|
||||
}
|
||||
@@ -277,39 +278,28 @@ public abstract class WriterPoolMember implements ArchiveFileConstants {
|
||||
}
|
||||
return d;
|
||||
}
|
||||
|
||||
protected synchronized TimestampSerialno getTimestampSerialNo() {
|
||||
return getTimestampSerialNo(null);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Do static synchronization around getting of counter and timestamp so
|
||||
* no chance of a thread getting in between the getting of timestamp and
|
||||
* allocation of serial number throwing the two out of alignment.
|
||||
*
|
||||
* @param timestamp If non-null, use passed timestamp (must be 14 digit
|
||||
* ARC format), else if null, timestamp with now.
|
||||
* @return Instance of data structure that has timestamp and serial no.
|
||||
* Generate a new basename by interpolating values in the configured
|
||||
* template. Values come from local state, other configured values, and
|
||||
* global system properties. The recommended default template will
|
||||
* generate a unique basename under reasonable assumptions.
|
||||
*/
|
||||
protected synchronized TimestampSerialno
|
||||
getTimestampSerialNo(final String timestamp) {
|
||||
return new TimestampSerialno((timestamp != null)?
|
||||
timestamp: ArchiveUtils.get14DigitDate(),
|
||||
serialNo.getAndIncrement());
|
||||
}
|
||||
|
||||
/**
|
||||
* Return a unique basename.
|
||||
*
|
||||
* Name is timestamp + an every increasing sequence number.
|
||||
*
|
||||
* @param tsn Structure with timestamp and serial number.
|
||||
*
|
||||
* @return Unique basename.
|
||||
*/
|
||||
private String getUniqueBasename(TimestampSerialno tsn) {
|
||||
return tsn.getTimestamp() + "-" +
|
||||
WriterPoolMember.serialNoFormatter.format(tsn.getSerialNumber());
|
||||
protected void generateNewBasename() {
|
||||
Properties localProps = new Properties();
|
||||
localProps.setProperty("prefix", prefix);
|
||||
synchronized(this.getClass()) {
|
||||
// ensure that serialNo and timestamp are minted together (never inverted sort order)
|
||||
String paddedSerialNumber = WriterPoolMember.serialNoFormatter.format(serialNo.getAndIncrement());
|
||||
String timestamp17 = ArchiveUtils.getUnique17DigitDate();
|
||||
String timestamp14 = ArchiveUtils.getUnique14DigitDate();
|
||||
currentTimestamp = timestamp17;
|
||||
localProps.setProperty("serialno", paddedSerialNumber);
|
||||
localProps.setProperty("timestamp17", timestamp17);
|
||||
localProps.setProperty("timestamp14", timestamp14);
|
||||
}
|
||||
currentBasename = PropertyUtils.interpolateWithProperties(template,
|
||||
localProps, System.getProperties());
|
||||
}
|
||||
|
||||
|
||||
@@ -384,7 +374,7 @@ public abstract class WriterPoolMember implements ArchiveFileConstants {
|
||||
}
|
||||
|
||||
/**
|
||||
* Postion in current physical file.
|
||||
* Position in current physical file.
|
||||
* Used making accounting of bytes written.
|
||||
* @return Position in underlying file. Call before or after writing
|
||||
* records *only* to be safe.
|
||||
@@ -482,12 +472,7 @@ public abstract class WriterPoolMember implements ArchiveFileConstants {
|
||||
protected OutputStream getOutputStream() {
|
||||
return this.out;
|
||||
}
|
||||
|
||||
protected String getCreateTimestamp() {
|
||||
return createTimestamp;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* An override so we get access to underlying output stream.
|
||||
* and offer an end() that does not accompany closing underlying
|
||||
|
||||
@@ -28,10 +28,10 @@ import java.util.List;
|
||||
* @version $Date$, $Revision$
|
||||
*/
|
||||
public interface WriterPoolSettings {
|
||||
public long getMaxSize();
|
||||
public long getMaxFileSizeBytes();
|
||||
public String getPrefix();
|
||||
public String getSuffix();
|
||||
public String getTemplate();
|
||||
public List<File> getOutputDirs();
|
||||
public boolean isCompressed();
|
||||
public boolean getCompress();
|
||||
public List<String> getMetadata();
|
||||
}
|
||||
@@ -186,7 +186,7 @@ public class ARCWriter extends WriterPoolMember implements ARCConstants {
|
||||
protected String createFile()
|
||||
throws IOException {
|
||||
String name = super.createFile();
|
||||
writeFirstRecord(getCreateTimestamp());
|
||||
writeFirstRecord(currentTimestamp);
|
||||
return name;
|
||||
}
|
||||
|
||||
@@ -223,13 +223,17 @@ public class ARCWriter extends WriterPoolMember implements ARCConstants {
|
||||
* <p>We do things in this roundabout manner because the java
|
||||
* GZIPOutputStream does not give access to GZIP header fields.
|
||||
*
|
||||
* @param date Date to put into the ARC metadata.
|
||||
* @param date Date to put into the ARC metadata; if 17-digit will be
|
||||
* truncated to traditional 14-digits
|
||||
*
|
||||
* @return Byte array filled w/ the arc header.
|
||||
* @throws IOException
|
||||
*/
|
||||
private byte [] generateARCFileMetaData(String date)
|
||||
throws IOException {
|
||||
if(date.length()>14) {
|
||||
date = date.substring(0,14);
|
||||
}
|
||||
int metadataBodyLength = getMetadataLength();
|
||||
// If metadata body, then the minor part of the version is '1' rather
|
||||
// than '0'.
|
||||
|
||||
@@ -58,8 +58,8 @@ public class ARCWriterPool extends WriterPool {
|
||||
super(serial, new BasePoolableObjectFactory() {
|
||||
public Object makeObject() throws Exception {
|
||||
return new ARCWriter(serial, settings.getOutputDirs(),
|
||||
settings.getPrefix(), settings.getSuffix(),
|
||||
settings.isCompressed(), settings.getMaxSize(),
|
||||
settings.getPrefix(), settings.getTemplate(),
|
||||
settings.getCompress(), settings.getMaxFileSizeBytes(),
|
||||
settings.getMetadata());
|
||||
}
|
||||
|
||||
|
||||
@@ -57,8 +57,8 @@ public class WARCWriterPool extends WriterPool {
|
||||
public Object makeObject() throws Exception {
|
||||
return new WARCWriter(serial,
|
||||
settings.getOutputDirs(),
|
||||
settings.getPrefix(), settings.getSuffix(),
|
||||
settings.isCompressed(), settings.getMaxSize(),
|
||||
settings.getPrefix(), settings.getTemplate(),
|
||||
settings.getCompress(), settings.getMaxFileSizeBytes(),
|
||||
settings.getMetadata());
|
||||
}
|
||||
|
||||
|
||||
@@ -123,6 +123,31 @@ public class ArchiveUtils {
|
||||
public static String get17DigitDate(){
|
||||
return TIMESTAMP17.get().format(new Date());
|
||||
}
|
||||
|
||||
static long LAST_UNIQUE_NOW17 = 0;
|
||||
static String LAST_TIMESTAMP17 = "";
|
||||
/**
|
||||
* Utility function for creating UNIQUE-from-this-class
|
||||
* arc-style date stamps in the format yyyMMddHHmmssSSS.
|
||||
* Rather than giving a duplicate datestamp on a
|
||||
* subsequent call, will increment the milliseconds until a
|
||||
* unique value is returned.
|
||||
*
|
||||
* Date stamps are in the UTC time zone
|
||||
* @return the date stamp
|
||||
*/
|
||||
public synchronized static String getUnique17DigitDate(){
|
||||
long effectiveNow = System.currentTimeMillis();
|
||||
effectiveNow = Math.max(effectiveNow, LAST_UNIQUE_NOW17+1);
|
||||
String candidate = get17DigitDate(effectiveNow);
|
||||
while(candidate.equals(LAST_TIMESTAMP17)) {
|
||||
effectiveNow++;
|
||||
candidate = get17DigitDate(effectiveNow);
|
||||
}
|
||||
LAST_UNIQUE_NOW17 = effectiveNow;
|
||||
LAST_TIMESTAMP17 = candidate;
|
||||
return candidate;
|
||||
}
|
||||
|
||||
/**
|
||||
* Utility function for creating arc-style date stamps
|
||||
@@ -133,6 +158,31 @@ public class ArchiveUtils {
|
||||
public static String get14DigitDate(){
|
||||
return TIMESTAMP14.get().format(new Date());
|
||||
}
|
||||
|
||||
static long LAST_UNIQUE_NOW14 = 0;
|
||||
static String LAST_TIMESTAMP14 = "";
|
||||
/**
|
||||
* Utility function for creating UNIQUE-from-this-class
|
||||
* arc-style date stamps in the format yyyMMddHHmmss.
|
||||
* Rather than giving a duplicate datestamp on a
|
||||
* subsequent call, will increment the seconds until a
|
||||
* unique value is returned.
|
||||
*
|
||||
* Date stamps are in the UTC time zone
|
||||
* @return the date stamp
|
||||
*/
|
||||
public synchronized static String getUnique14DigitDate(){
|
||||
long effectiveNow = System.currentTimeMillis();
|
||||
effectiveNow = Math.max(effectiveNow, LAST_UNIQUE_NOW14+1);
|
||||
String candidate = get14DigitDate(effectiveNow);
|
||||
while(candidate.equals(LAST_TIMESTAMP14)) {
|
||||
effectiveNow += 1000;
|
||||
candidate = get14DigitDate(effectiveNow);
|
||||
}
|
||||
LAST_UNIQUE_NOW14 = effectiveNow;
|
||||
LAST_TIMESTAMP14 = candidate;
|
||||
return candidate;
|
||||
}
|
||||
|
||||
/**
|
||||
* Utility function for creating arc-style date stamps
|
||||
|
||||
@@ -18,8 +18,16 @@
|
||||
*/
|
||||
package org.archive.util;
|
||||
|
||||
import java.util.Properties;
|
||||
import java.util.regex.Matcher;
|
||||
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
|
||||
/**
|
||||
* @author stack
|
||||
* Utilities for dealing with Java Properties (incl. System Properties)
|
||||
*
|
||||
* @contributor stack
|
||||
* @contributor gojomo
|
||||
* @version $Date$ $Revision$
|
||||
*/
|
||||
public class PropertyUtils {
|
||||
@@ -51,4 +59,56 @@ public class PropertyUtils {
|
||||
return getPropertyOrNull(key) == null?
|
||||
fallback: Integer.parseInt(getPropertyOrNull(key));
|
||||
}
|
||||
|
||||
/**
|
||||
* Given a string which may contain expressions of the form
|
||||
* ${key}, replace each expression with the value corresponding to the
|
||||
* given key in System Properties. If no value is present,
|
||||
* the expression is replaced with the empty-string.
|
||||
*
|
||||
* @param original String
|
||||
* @param properties Properties to try in order; first value found (if any) is used
|
||||
* @return modified String
|
||||
*/
|
||||
public static String interpolateWithProperties(String original) {
|
||||
return interpolateWithProperties(original,System.getProperties());
|
||||
}
|
||||
|
||||
static String propRefPattern = "\\$\\{([^{}]+)\\}";
|
||||
|
||||
/**
|
||||
* Given a string which may contain expressions of the form
|
||||
* ${key}, replace each expression with the value corresponding to the
|
||||
* given key in the supplied Properties instance. If no value is present,
|
||||
* the expression is replaced with the empty-string.
|
||||
*
|
||||
* @param original String
|
||||
* @param properties Properties to try in order; first value found (if any) is used
|
||||
* @return modified String
|
||||
*/
|
||||
public static String interpolateWithProperties(String original,
|
||||
Properties... props) {
|
||||
String result = original;
|
||||
// cap number of interpolations as guard against unending loop
|
||||
inter: for(int i =0; i < original.length()*2; i++) {
|
||||
Matcher m = TextUtils.getMatcher(propRefPattern, result);
|
||||
while(m.find()) {
|
||||
String key = m.group(1);
|
||||
String value = "";
|
||||
for(Properties properties : props) {
|
||||
value = properties.getProperty(key, "");
|
||||
if(StringUtils.isNotEmpty(value)) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
result = result.substring(0,m.start())
|
||||
+ value
|
||||
+ result.substring(m.end());
|
||||
continue inter;
|
||||
}
|
||||
// we only hit here if there were no interpolations last while loop
|
||||
break;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -129,7 +129,7 @@ public class ARCWriterPoolTest extends TmpDirTestCase {
|
||||
|
||||
private WriterPoolSettings getSettings(final boolean isCompressed) {
|
||||
return new WriterPoolSettings() {
|
||||
public long getMaxSize() {
|
||||
public long getMaxFileSizeBytes() {
|
||||
return ARCConstants.DEFAULT_MAX_ARC_FILE_SIZE;
|
||||
}
|
||||
|
||||
@@ -137,8 +137,8 @@ public class ARCWriterPoolTest extends TmpDirTestCase {
|
||||
return PREFIX;
|
||||
}
|
||||
|
||||
public String getSuffix() {
|
||||
return "";
|
||||
public String getTemplate() {
|
||||
return "${prefix}-${timestamp17}-${serialno}-${heritrix.hostname}";
|
||||
}
|
||||
|
||||
public List<File> getOutputDirs() {
|
||||
@@ -146,7 +146,7 @@ public class ARCWriterPoolTest extends TmpDirTestCase {
|
||||
return Arrays.asList(files);
|
||||
}
|
||||
|
||||
public boolean isCompressed() {
|
||||
public boolean getCompress() {
|
||||
return isCompressed;
|
||||
}
|
||||
|
||||
|
||||
+20
-31
@@ -16,41 +16,30 @@
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.archive.util;
|
||||
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Properties;
|
||||
|
||||
import junit.framework.TestCase;
|
||||
|
||||
|
||||
/**
|
||||
* Immutable data structure that holds a timestamp and an accompanying
|
||||
* serial number.
|
||||
* PropertyUtils tests.
|
||||
*
|
||||
* For Igor!
|
||||
*
|
||||
* @author stack
|
||||
* @contributor gojomo
|
||||
* @version $Date: 2009-11-19 14:39:53 -0800 (Thu, 19 Nov 2009) $, $Revision: 6674 $
|
||||
*/
|
||||
public class TimestampSerialno {
|
||||
private final String ts;
|
||||
private final int serialNumber;
|
||||
|
||||
public TimestampSerialno(String ts, int serialNo) {
|
||||
this.ts = ts;
|
||||
this.serialNumber = serialNo;
|
||||
}
|
||||
public class PropertyUtilsTest extends TestCase {
|
||||
|
||||
public TimestampSerialno(int serialNo) {
|
||||
this.ts = ArchiveUtils.get14DigitDate();
|
||||
this.serialNumber = serialNo;
|
||||
public void testSimpleInterpolate() throws IOException {
|
||||
Properties props = new Properties();
|
||||
props.put("foo", "OOF");
|
||||
props.put("bar","RAB");
|
||||
String original = "FOO|${foo} BAR|${bar}";
|
||||
String expected = "FOO|OOF BAR|RAB";
|
||||
assertEquals("interpalation problem",expected,PropertyUtils.interpolateWithProperties(original,props));
|
||||
}
|
||||
|
||||
/**
|
||||
* @return Returns the now.
|
||||
*/
|
||||
public String getTimestamp() {
|
||||
return this.ts;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return Returns the serialNumber.
|
||||
*/
|
||||
public int getSerialNumber() {
|
||||
return this.serialNumber;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -27,6 +27,9 @@ import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.PrintStream;
|
||||
import java.lang.management.ManagementFactory;
|
||||
import java.net.InetAddress;
|
||||
import java.net.UnknownHostException;
|
||||
import java.security.KeyStore;
|
||||
import java.security.MessageDigest;
|
||||
import java.security.cert.Certificate;
|
||||
@@ -90,7 +93,6 @@ public class Heritrix {
|
||||
|
||||
private static final String ADHOC_KEYSTORE = "adhoc.keystore";
|
||||
|
||||
@SuppressWarnings("unused")
|
||||
private static final Logger logger = Logger.getLogger(Heritrix.class.getName());
|
||||
|
||||
/** Name of configuration directory */
|
||||
@@ -322,6 +324,8 @@ public class Heritrix {
|
||||
// inside in a container.
|
||||
TimeZone.setDefault(TimeZone.getTimeZone("GMT"));
|
||||
|
||||
setupGlobalProperties(port);
|
||||
|
||||
// Start Heritrix.
|
||||
try {
|
||||
engine = new Engine(jobsDir);
|
||||
@@ -372,6 +376,39 @@ public class Heritrix {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Setup global system properties that may be of use elsewhere.
|
||||
*
|
||||
* @param port
|
||||
*/
|
||||
protected void setupGlobalProperties(int port) {
|
||||
if (System.getProperty("heritrix.port") == null) {
|
||||
System.setProperty("heritrix.port", port + "");
|
||||
}
|
||||
|
||||
String hostname = "localhost.localdomain";
|
||||
if (System.getProperty("heritrix.hostname") == null) {
|
||||
try {
|
||||
hostname = InetAddress.getLocalHost().getCanonicalHostName();
|
||||
} catch (UnknownHostException ue) {
|
||||
logger.warning("Failed getHostAddress for this host: " + ue);
|
||||
}
|
||||
System.setProperty("heritrix.hostname", hostname);
|
||||
}
|
||||
|
||||
// while not guaranteed, on our platforms of interest this name
|
||||
// always seems to be PID@HOSTNAME
|
||||
String runtimeName = ManagementFactory.getRuntimeMXBean().getName();
|
||||
if(System.getProperty("heritrix.runtimeName") == null) {
|
||||
System.setProperty("heritrix.runtimeName", runtimeName);
|
||||
}
|
||||
if (System.getProperty("heritrix.pid") == null
|
||||
&& runtimeName.matches("\\d+@\\S+")) {
|
||||
System.setProperty("heritrix.pid", runtimeName.substring(0,runtimeName.indexOf("@")));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Perform preparation to use an ad-hoc, created-as-necessary
|
||||
* certificate/keystore for HTTPS access. A keystore with new
|
||||
|
||||
@@ -35,7 +35,6 @@ import org.apache.commons.lang.StringEscapeUtils;
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.archive.io.ReplayInputStream;
|
||||
import org.archive.io.WriterPoolMember;
|
||||
import org.archive.io.WriterPoolSettings;
|
||||
import org.archive.io.arc.ARCWriter;
|
||||
import org.archive.io.arc.ARCWriterPool;
|
||||
import org.archive.modules.ProcessResult;
|
||||
@@ -54,7 +53,7 @@ import org.archive.util.ArchiveUtils;
|
||||
*/
|
||||
public class ARCWriterProcessor extends WriterPoolProcessor {
|
||||
|
||||
final static private String TEMPLATE = readTemplate();
|
||||
final static private String METADATA_TEMPLATE = readMetadataTemplate();
|
||||
|
||||
private static final long serialVersionUID = 3L;
|
||||
|
||||
@@ -77,8 +76,7 @@ public class ARCWriterProcessor extends WriterPoolProcessor {
|
||||
|
||||
@Override
|
||||
protected void setupPool(AtomicInteger serialNo) {
|
||||
WriterPoolSettings wps = getWriterPoolSettings();
|
||||
setPool(new ARCWriterPool(serialNo, wps, getPoolMaxActive(), getPoolMaxWaitMs()));
|
||||
setPool(new ARCWriterPool(serialNo, this, getPoolMaxActive(), getPoolMaxWaitMs()));
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -160,8 +158,8 @@ public class ARCWriterProcessor extends WriterPoolProcessor {
|
||||
return checkBytesWritten();
|
||||
}
|
||||
|
||||
protected List<String> getMetadata() {
|
||||
if (TEMPLATE == null) {
|
||||
public List<String> getMetadata() {
|
||||
if (METADATA_TEMPLATE == null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
@@ -169,7 +167,7 @@ public class ARCWriterProcessor extends WriterPoolProcessor {
|
||||
return cachedMetadata;
|
||||
}
|
||||
|
||||
String meta = TEMPLATE;
|
||||
String meta = METADATA_TEMPLATE;
|
||||
meta = replace(meta, "${VERSION}", ArchiveUtils.VERSION);
|
||||
meta = replace(meta, "${HOST}", getHostName());
|
||||
meta = replace(meta, "${IP}", getHostAddress());
|
||||
@@ -225,7 +223,7 @@ public class ARCWriterProcessor extends WriterPoolProcessor {
|
||||
}
|
||||
}
|
||||
|
||||
private static String readTemplate() {
|
||||
private static String readMetadataTemplate() {
|
||||
InputStream input = ARCWriterProcessor.class.getResourceAsStream(
|
||||
"arc_metadata_template.xml");
|
||||
if (input == null) {
|
||||
|
||||
@@ -70,7 +70,6 @@ import org.apache.commons.io.IOUtils;
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.archive.io.ReplayInputStream;
|
||||
import org.archive.io.WriterPoolMember;
|
||||
import org.archive.io.WriterPoolSettings;
|
||||
import org.archive.io.warc.WARCWriter;
|
||||
import org.archive.io.warc.WARCWriterPool;
|
||||
import org.archive.modules.CrawlMetadata;
|
||||
@@ -166,8 +165,7 @@ public class WARCWriterProcessor extends WriterPoolProcessor {
|
||||
|
||||
@Override
|
||||
protected void setupPool(final AtomicInteger serialNo) {
|
||||
WriterPoolSettings wps = getWriterPoolSettings();
|
||||
setPool(new WARCWriterPool(serialNo, wps, getPoolMaxActive(), getPoolMaxWaitMs()));
|
||||
setPool(new WARCWriterPool(serialNo, this, getPoolMaxActive(), getPoolMaxWaitMs()));
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -25,7 +25,6 @@ import static org.archive.modules.fetcher.FetchStatusCodes.S_DNS_SUCCESS;
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.net.InetAddress;
|
||||
import java.net.UnknownHostException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
@@ -33,7 +32,6 @@ import java.util.logging.Logger;
|
||||
|
||||
import org.archive.checkpointing.Checkpoint;
|
||||
import org.archive.checkpointing.Checkpointable;
|
||||
import org.archive.io.DefaultWriterPoolSettings;
|
||||
import org.archive.io.WriterPool;
|
||||
import org.archive.io.WriterPoolMember;
|
||||
import org.archive.io.WriterPoolSettings;
|
||||
@@ -57,8 +55,9 @@ import org.springframework.context.Lifecycle;
|
||||
* @author stack
|
||||
*/
|
||||
public abstract class WriterPoolProcessor extends Processor
|
||||
implements Lifecycle, Checkpointable {
|
||||
implements Lifecycle, Checkpointable, WriterPoolSettings {
|
||||
private static final long serialVersionUID = 1L;
|
||||
@SuppressWarnings("unused")
|
||||
private static final Logger logger =
|
||||
Logger.getLogger(WriterPoolProcessor.class.getName());
|
||||
|
||||
@@ -76,9 +75,9 @@ implements Lifecycle, Checkpointable {
|
||||
|
||||
/**
|
||||
* File prefix. The text supplied here will be used as a prefix naming
|
||||
* writer files. For example if the prefix is 'IAH', then file names will
|
||||
* look like IAH-20040808101010-0001-HOSTNAME.arc.gz ...if writing ARCs (The
|
||||
* prefix will be separated from the date by a hyphen).
|
||||
* writer files. For example if the prefix is 'WEB', then file names will
|
||||
* look like WEB-20040808101010-0001-PID@HOSTNAME#PORT.arc.gz ...if
|
||||
* writing ARCs (The prefix will be separated from the date by a hyphen).
|
||||
*/
|
||||
String prefix = WriterPoolMember.DEFAULT_PREFIX;
|
||||
public String getPrefix() {
|
||||
@@ -90,15 +89,23 @@ implements Lifecycle, Checkpointable {
|
||||
|
||||
|
||||
/**
|
||||
* Suffix to tag onto files. If value is '${HOSTNAME}', will use hostname
|
||||
* for suffix. If empty, no suffix will be added.
|
||||
* Template from which a filename is interpolated. Expressions of the
|
||||
* form ${key} will be replaced by values from a local map of useful
|
||||
* values (including 'prefix', 'timestamp17', and 'serialno') or
|
||||
* global system properties (which includes the local hostname/port/pid).
|
||||
*
|
||||
* The default pattern will generate unique names under reasonable
|
||||
* assumptions; be sure you know what you're doing before customizing,
|
||||
* as you could easily create filename collisions with a poorly-designed
|
||||
* filename template.
|
||||
*
|
||||
*/
|
||||
String suffix = WriterPoolMember.DEFAULT_SUFFIX;
|
||||
public String getSuffix() {
|
||||
return suffix;
|
||||
String template = WriterPoolMember.DEFAULT_TEMPLATE;
|
||||
public String getTemplate() {
|
||||
return template;
|
||||
}
|
||||
public void setSuffix(String suffix) {
|
||||
this.suffix = suffix;
|
||||
public void setTemplate(String template) {
|
||||
this.template = template;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -224,7 +231,6 @@ implements Lifecycle, Checkpointable {
|
||||
*/
|
||||
private long totalBytesWritten = 0;
|
||||
|
||||
private WriterPoolSettings settings;
|
||||
private AtomicInteger serial = new AtomicInteger();
|
||||
|
||||
|
||||
@@ -242,7 +248,6 @@ implements Lifecycle, Checkpointable {
|
||||
return;
|
||||
}
|
||||
super.start();
|
||||
this.settings = makeWriterPoolSettings();
|
||||
setupPool(serial);
|
||||
}
|
||||
|
||||
@@ -252,7 +257,6 @@ implements Lifecycle, Checkpointable {
|
||||
}
|
||||
super.stop();
|
||||
this.pool.close();
|
||||
this.settings = null;
|
||||
}
|
||||
|
||||
|
||||
@@ -391,9 +395,9 @@ implements Lifecycle, Checkpointable {
|
||||
this.totalBytesWritten = totalBytesWritten;
|
||||
}
|
||||
|
||||
protected abstract List<String> getMetadata();
|
||||
public abstract List<String> getMetadata();
|
||||
|
||||
private List<File> getOutputDirs() {
|
||||
public List<File> getOutputDirs() {
|
||||
List<String> list = getStorePaths();
|
||||
ArrayList<File> results = new ArrayList<File>();
|
||||
for (String path: list) {
|
||||
@@ -412,35 +416,6 @@ implements Lifecycle, Checkpointable {
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
|
||||
protected WriterPoolSettings getWriterPoolSettings() {
|
||||
return settings;
|
||||
}
|
||||
|
||||
private WriterPoolSettings makeWriterPoolSettings() {
|
||||
DefaultWriterPoolSettings result = new DefaultWriterPoolSettings();
|
||||
result.setMaxSize(getMaxFileSizeBytes());
|
||||
result.setMetadata(getMetadata());
|
||||
result.setOutputDirs(getOutputDirs());
|
||||
result.setPrefix(getPrefix());
|
||||
|
||||
String sfx = getSuffix();
|
||||
sfx = sfx.trim();
|
||||
if (sfx.contains(WriterPoolMember.HOSTNAME_VARIABLE)) {
|
||||
String str = "localhost.localdomain";
|
||||
try {
|
||||
str = InetAddress.getLocalHost().getCanonicalHostName();
|
||||
} catch (UnknownHostException ue) {
|
||||
logger.severe("Failed getHostAddress for this host: " + ue);
|
||||
}
|
||||
sfx = sfx.replace(WriterPoolMember.HOSTNAME_VARIABLE, str);
|
||||
}
|
||||
|
||||
result.setSuffix(sfx);
|
||||
result.setCompressed(getCompress());
|
||||
return result;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void innerProcess(CrawlURI puri) {
|
||||
|
||||
Reference in New Issue
Block a user