mirror of
https://github.com/internetarchive/heritrix3.git
synced 2026-11-09 07:24:41 +00:00
[HER-1727] (W)ARC filename uniqueness guarantees: adminport option, process_id option(?), JVM-global-repeat-suppression
* Heritrix.java
make heritrix.hostname, heritrix.pid, heritrix.port available as global properties
* PropertyUtils.java, PropertyUtilsTest.java
utility methods to interpolate string values from one or several supplied Properties instances
* ArchiveUtils.java
utility methods to give timestamps guaranteed larger/different than any previously-issued timestamp
* WriterPoolSettings.java, WriterPool.java, ARCWriterPool.java, WARCWriterPool.java, ARCWriterPoolTest.java
improve field names, change 'suffix' to more general 'template'
* WriterPoolMember.java
replace 'suffix' with 'template' which is interpolated when specific name is needed
set default template to pattern extremely unlikely to generate duplicate filenames
change default prefix to 'WEB'
centralize creation of new basenames into generateNewBasename() method, which internalizes timetamp/serialNo minting and interpolation
* WriterPoolProcessor.java, ARCWriterProcessor.java, WARCWriterProcessor.java
serve as own WriterPoolSettings instance
replace 'suffix' with 'template'
* ARCWriter.java
accept (but truncate) 17-digit timestamps at creation
* DefaultWriterPoolSettings.java, TimestampSerialNo.java
delete as superfluous
This commit is contained in:
1 parent
c410c90b76
commit
b388479d32
15 files changed
+260
-269
No files matched your search
@@ -27,6 +27,9 @@ import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.PrintStream;
|
||||
import java.lang.management.ManagementFactory;
|
||||
import java.net.InetAddress;
|
||||
import java.net.UnknownHostException;
|
||||
import java.security.KeyStore;
|
||||
import java.security.MessageDigest;
|
||||
import java.security.cert.Certificate;
|
||||
@@ -90,7 +93,6 @@ public class Heritrix {
|
||||
|
||||
private static final String ADHOC_KEYSTORE = "adhoc.keystore";
|
||||
|
||||
@SuppressWarnings("unused")
|
||||
private static final Logger logger = Logger.getLogger(Heritrix.class.getName());
|
||||
|
||||
/** Name of configuration directory */
|
||||
@@ -322,6 +324,8 @@ public class Heritrix {
|
||||
// inside in a container.
|
||||
TimeZone.setDefault(TimeZone.getTimeZone("GMT"));
|
||||
|
||||
setupGlobalProperties(port);
|
||||
|
||||
// Start Heritrix.
|
||||
try {
|
||||
engine = new Engine(jobsDir);
|
||||
@@ -372,6 +376,39 @@ public class Heritrix {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Setup global system properties that may be of use elsewhere.
|
||||
*
|
||||
* @param port
|
||||
*/
|
||||
protected void setupGlobalProperties(int port) {
|
||||
if (System.getProperty("heritrix.port") == null) {
|
||||
System.setProperty("heritrix.port", port + "");
|
||||
}
|
||||
|
||||
String hostname = "localhost.localdomain";
|
||||
if (System.getProperty("heritrix.hostname") == null) {
|
||||
try {
|
||||
hostname = InetAddress.getLocalHost().getCanonicalHostName();
|
||||
} catch (UnknownHostException ue) {
|
||||
logger.warning("Failed getHostAddress for this host: " + ue);
|
||||
}
|
||||
System.setProperty("heritrix.hostname", hostname);
|
||||
}
|
||||
|
||||
// while not guaranteed, on our platforms of interest this name
|
||||
// always seems to be PID@HOSTNAME
|
||||
String runtimeName = ManagementFactory.getRuntimeMXBean().getName();
|
||||
if(System.getProperty("heritrix.runtimeName") == null) {
|
||||
System.setProperty("heritrix.runtimeName", runtimeName);
|
||||
}
|
||||
if (System.getProperty("heritrix.pid") == null
|
||||
&& runtimeName.matches("\\d+@\\S+")) {
|
||||
System.setProperty("heritrix.pid", runtimeName.substring(0,runtimeName.indexOf("@")));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Perform preparation to use an ad-hoc, created-as-necessary
|
||||
* certificate/keystore for HTTPS access. A keystore with new
|
||||
|
||||
Reference in new issue
Block a user