/* * This file is part of the Heritrix web crawler (crawler.archive.org). * * Licensed to the Internet Archive (IA) by one or more individual * contributors. * * The IA licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.archive.crawler; import java.io.BufferedOutputStream; import java.io.ByteArrayInputStream; import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.PrintStream; import java.lang.management.ManagementFactory; import java.net.InetAddress; import java.net.UnknownHostException; import java.security.KeyStore; import java.security.MessageDigest; import java.security.cert.Certificate; import java.util.*; import java.util.logging.LogManager; import java.util.logging.Logger; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.CommandLineParser; import org.apache.commons.cli.GnuParser; import org.apache.commons.cli.HelpFormatter; import org.apache.commons.cli.Options; import org.apache.commons.cli.ParseException; import org.apache.commons.io.FileUtils; import org.apache.commons.io.output.TeeOutputStream; import org.apache.commons.lang.StringUtils; import org.archive.crawler.framework.Engine; import org.archive.crawler.restlet.EngineApplication; import org.archive.crawler.restlet.RateLimitGuard; import org.archive.util.ArchiveUtils; import org.archive.util.KeyTool; import org.restlet.Component; import org.restlet.Server; import org.restlet.data.ChallengeScheme; import org.restlet.data.Protocol; import org.restlet.security.ChallengeAuthenticator; import org.restlet.security.MapVerifier; /** * Main class for Heritrix crawler. * * Heritrix is usually launched by a shell script that backgrounds heritrix * that redirects all stdout and stderr emitted by heritrix to a log file. So * that startup messages emitted subsequent to the redirection of stdout and * stderr show on the console, this class prints usage or startup output * such as where the web UI can be found, etc., to a STARTLOG that the shell * script is waiting on. As soon as the shell script sees output in this file, * it prints its content and breaks out of its wait. * See ${HERITRIX_HOME}/bin/heritrix. * *
Heritrix can also be embedded or launched by webapp initialization or * by JMX bootstrapping. So far I count 4 methods of instantiation: *
bin/heritrix shell script. This is a DEPENDENCY the shell
* wrapper has on this here java heritrix.
*/
private static final String STARTLOG = "heritrix_dmesg.log";
private static void usage(PrintStream out, String[] args) {
HelpFormatter hf = new HelpFormatter();
hf.printHelp("Heritrix", options());
out.println("Your arguments were: "+StringUtils.join(args, ' '));
}
private static Options options() {
Options options = new Options();
options.addOption("h", "help", true, "Usage information." );
options.addOption("a", "web-admin", true, "REQUIRED. Specifies the " +
"authorization username and password which must be supplied to " +
"access the web interface. This may be of the form " +
"\"password\" (which leaves username as the default 'admin'), " +
"\"username:password\", or \"@filename\" for a file that " +
"includes the single line \"username:password\". ");
options.addOption("j", "jobs-dir", true, "The jobs directory. " +
"Defaults to ./jobs");
options.addOption("l", "logging-properties", true,
"The full path to the logging properties file " +
"(eg, conf/logging.properties). If present, this file " +
"will be used to configure Java logging. Defaults to " +
"${heritrix.home}/conf/logging.properties or if no " +
"heritrix.home property set, ./conf/logging.properties");
options.addOption("b", "web-bind-hosts", true,
"A comma-separated list of addresses/hostnames for the " +
"web interface to bind to.");
options.addOption("p", "web-port", true, "The port the web interface " +
"should listen on.");
options.addOption("s", "ssl-params", true, "Specify a keystore " +
"path, keystore password, and key password for HTTPS use. " +
"Separate with commas, no whitespace.");
return options;
}
private static File getDefaultPropertiesFile() {
File confDir = new File(getHeritrixHome(),CONF);
File props = new File(confDir, PROPERTIES);
return props;
}
private static CommandLine getCommandLine(PrintStream out, String[] args) {
CommandLineParser clp = new GnuParser();
CommandLine cl;
try {
cl = clp.parse(options(), args);
} catch (ParseException e) {
usage(out, args);
return null;
}
if (cl.getArgList().size() != 0) {
usage(out, args);
return null;
}
return cl;
}
/**
* Launches a local Engine and restfgul web interface given the
* command-line options or defaults.
*
* @param args Command line arguments.
* @throws Exception
*/
public static void main(String[] args)
throws Exception {
new Heritrix().instanceMain(args);
}
public void instanceMain(String[] args)
throws Exception {
System.out.println(System.getProperty("java.vendor")
+ ' ' + System.getProperty("java.runtime.name")
+ ' ' + System.getProperty("java.runtime.version"));
// ensure using java 1.6+ before hitting a later cryptic error
String version = System.getProperty("java.version");
float floatVersion = Float.valueOf(version.substring(0,version.indexOf('.',2)));
if(floatVersion<1.6) {
System.err.println("Heritrix (as of version 3) requires Java 1.6 or higher.");
System.err.println("You attempted to launch with: "+version);
System.err.println("Please try again with a later Java.");
System.exit(1);
}
// Set some system properties early.
// Can't use class names here without loading them.
String ignoredSchemes = "org.archive.net.UURIFactory.ignored-schemes";
if (System.getProperty(ignoredSchemes) == null) {
System.setProperty(ignoredSchemes,
"mailto, clsid, res, file, rtsp, about");
}
String maxFormSize = "org.eclipse.jetty.server.Request.maxFormContentSize";
if (System.getProperty(maxFormSize) == null) {
System.setProperty(maxFormSize, "52428800");
}
BufferedOutputStream startupOutStream =
new BufferedOutputStream(
new FileOutputStream(
new File(getHeritrixHome(), STARTLOG)),16384);
PrintStream startupOut =
new PrintStream(
new TeeOutputStream(
System.out,
startupOutStream));
CommandLine cl = getCommandLine(startupOut, args);
if (cl == null) return;
if (cl.hasOption('h')) {
usage(startupOut, args);
return ;
}
// DEFAULTS until changed by cmd-line options
int port = 8443;
Set-Dheritrix.home if available to us.
* Is current working dir if no heritrix.home property supplied.
* @return Heritrix home directory.
*/
protected static File getHeritrixHome() {
String home = System.getProperty("heritrix.home");
if (home != null && home.length() > 0) {
File candidate = new File(home);
if (candidate.exists()) {
return candidate;
}
logger.warning("HERITRIX_HOME <" + home +"> does not exist; " +
"using current working directory instead.");
}
return new File(System.getProperty("user.dir"));
}
public Engine getEngine() {
return engine;
}
public Component getComponent() {
return component;
}
}