mirror of
https://github.com/internetarchive/heritrix3.git
synced 2026-09-24 14:45:56 +00:00
a bit more error-handling for input args
This commit is contained in:
szznax
@@ -43,6 +43,8 @@ import org.apache.commons.lang.StringUtils;
|
||||
import org.w3c.dom.Document;
|
||||
import org.w3c.dom.Node;
|
||||
import org.w3c.dom.NodeList;
|
||||
import org.xbill.DNS.SRVRecord;
|
||||
import org.xml.sax.SAXException;
|
||||
|
||||
/**
|
||||
* Utility class which takes a H1 order.xml and creates a similar
|
||||
@@ -71,14 +73,21 @@ public class MigrateH1to3Tool {
|
||||
}
|
||||
|
||||
public void instanceMain(String[] args) throws Exception {
|
||||
if(args.length == 0) {
|
||||
if(args.length < 2) {
|
||||
printHelp();
|
||||
return;
|
||||
}
|
||||
|
||||
File sourceOrderXmlFile = new File(args[0]);
|
||||
File destinationH3JobDir = new File(args[1]);
|
||||
|
||||
|
||||
String sourceOrderXmlFileArg = args[0];
|
||||
String destinationH3JobDirArg = args[1];
|
||||
|
||||
File sourceOrderXmlFile = new File(sourceOrderXmlFileArg);
|
||||
if (!sourceOrderXmlFile.isFile()) {
|
||||
System.err.println("ERROR sourceOrderXmlFileArg is not a file: " + sourceOrderXmlFileArg);
|
||||
System.exit(1);
|
||||
}
|
||||
File destinationH3JobDir = new File(destinationH3JobDirArg);
|
||||
|
||||
destinationH3JobDir.mkdirs();
|
||||
|
||||
System.out.println("H1 source: "+sourceOrderXmlFile.getAbsolutePath());
|
||||
@@ -93,77 +102,91 @@ public class MigrateH1to3Tool {
|
||||
|
||||
Map<String,String> migrateH1toH3Map = getMigrateMap();
|
||||
|
||||
Document sourceOrderXmlDom = DOCUMENT_BUILDER.parse(sourceOrderXmlFile);
|
||||
Map<String,String> h1simpleSettings = flattenH1Order(sourceOrderXmlDom);
|
||||
|
||||
List<String> notApplicable = new ArrayList<String>();
|
||||
List<String> needsAttention = new ArrayList<String>();
|
||||
int migrated = 0;
|
||||
StringBuilder sb = new StringBuilder();
|
||||
for(String key : h1simpleSettings.keySet()) {
|
||||
String beanPath = migrateH1toH3Map.get(key);
|
||||
String value = h1simpleSettings.get(key);
|
||||
System.out.print(".");
|
||||
if(beanPath==null) {
|
||||
// no equivalence rule
|
||||
needsAttention.add(key+" "+value);
|
||||
continue;
|
||||
}
|
||||
if (beanPath.startsWith("$")) {
|
||||
// rule indicates not-available/not-applicable
|
||||
notApplicable.add(key+" "+value);
|
||||
continue;
|
||||
}
|
||||
if (beanPath.startsWith("*")) {
|
||||
// TODO: needs special handling
|
||||
if(beanPath.equals("*metadata.userAgentTemplate")) {
|
||||
splitH1userAgent(value,sb);
|
||||
migrated += 2;
|
||||
} else {
|
||||
try {
|
||||
Document sourceOrderXmlDom = DOCUMENT_BUILDER.parse(sourceOrderXmlFile);
|
||||
|
||||
Map<String,String> h1simpleSettings = flattenH1Order(sourceOrderXmlDom);
|
||||
|
||||
List<String> notApplicable = new ArrayList<String>();
|
||||
List<String> needsAttention = new ArrayList<String>();
|
||||
int migrated = 0;
|
||||
StringBuilder sb = new StringBuilder();
|
||||
for(String key : h1simpleSettings.keySet()) {
|
||||
String beanPath = migrateH1toH3Map.get(key);
|
||||
String value = h1simpleSettings.get(key);
|
||||
System.out.print(".");
|
||||
if(beanPath==null) {
|
||||
// no equivalence rule
|
||||
needsAttention.add(key+" "+value);
|
||||
continue;
|
||||
}
|
||||
if (beanPath.startsWith("$")) {
|
||||
// rule indicates not-available/not-applicable
|
||||
notApplicable.add(key+" "+value);
|
||||
continue;
|
||||
}
|
||||
if (beanPath.startsWith("*")) {
|
||||
// TODO: needs special handling
|
||||
if(beanPath.equals("*metadata.userAgentTemplate")) {
|
||||
splitH1userAgent(value,sb);
|
||||
migrated += 2;
|
||||
} else {
|
||||
needsAttention.add(key+" "+value);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (beanPath.startsWith("^")) {
|
||||
// uppercase to new enum-style
|
||||
value = value.toUpperCase();
|
||||
beanPath = beanPath.substring(1);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (beanPath.startsWith("^")) {
|
||||
// uppercase to new enum-style
|
||||
value = value.toUpperCase();
|
||||
beanPath = beanPath.substring(1);
|
||||
sb
|
||||
.append(beanPath)
|
||||
.append("=")
|
||||
.append(value)
|
||||
.append("\n");
|
||||
migrated++;
|
||||
}
|
||||
sb
|
||||
.append(beanPath)
|
||||
.append("=")
|
||||
.append(value)
|
||||
.append("\n");
|
||||
migrated++;
|
||||
System.out.println();
|
||||
System.out.println();
|
||||
|
||||
// patch all overrides derived from H1 into H3 template
|
||||
String beansCxml = template.replace("###MIGRATE_OVERRIDES###", sb.toString());
|
||||
|
||||
File targetBeansXmlFile = new File(destinationH3JobDir,"crawler-beans.cxml");
|
||||
FileUtils.writeStringToFile(targetBeansXmlFile, beansCxml);
|
||||
|
||||
File sourceSeedsTxtFile = new File(sourceOrderXmlFile.getParentFile(), "seeds.txt");
|
||||
File destinationSeedsTxtFile = new File(destinationH3JobDir, "seeds.txt");
|
||||
|
||||
if (!sourceSeedsTxtFile.isFile()) {
|
||||
System.err.println("ERROR sourceSeedsTxtFile not found: " + sourceSeedsTxtFile);
|
||||
System.exit(1);
|
||||
}
|
||||
|
||||
FileUtils.copyFile(sourceSeedsTxtFile,destinationSeedsTxtFile);
|
||||
|
||||
System.out.println(notApplicable.size()+" settings skipped as not-applicable");
|
||||
System.out.println("These are probably harmless, but if the following settings were");
|
||||
System.out.println("important to your crawl process, investigate other options.");
|
||||
listProblems(notApplicable);
|
||||
System.out.println();
|
||||
System.out.println(needsAttention.size()+" settings may need attention");
|
||||
System.out.println("Please review your original crawl and the created H3 job, for each");
|
||||
System.out.println("of the following, and manually update as needed.");
|
||||
listProblems(needsAttention);
|
||||
System.out.println();
|
||||
System.out.println(migrated +" H1 settings successfully migrated to H3 configuration");
|
||||
System.out.println();
|
||||
System.out.println("Review your converted crawler-beans.cxml at:");
|
||||
System.out.println(targetBeansXmlFile.getAbsolutePath());
|
||||
|
||||
} catch (SAXException e) {
|
||||
System.err.println("ERROR caught exception parsing input file: "
|
||||
+ e.getMessage() + "\n");
|
||||
e.printStackTrace();
|
||||
}
|
||||
System.out.println();
|
||||
System.out.println();
|
||||
|
||||
// patch all overrides derived from H1 into H3 template
|
||||
String beansCxml = template.replace("###MIGRATE_OVERRIDES###", sb.toString());
|
||||
|
||||
File targetBeansXmlFile = new File(destinationH3JobDir,"crawler-beans.cxml");
|
||||
FileUtils.writeStringToFile(targetBeansXmlFile, beansCxml);
|
||||
|
||||
FileUtils.copyFile(
|
||||
new File(sourceOrderXmlFile.getParentFile(), "seeds.txt"),
|
||||
new File(destinationH3JobDir, "seeds.txt"));
|
||||
|
||||
|
||||
System.out.println(notApplicable.size()+" settings skipped as not-applicable");
|
||||
System.out.println("These are probably harmless, but if the following settings were");
|
||||
System.out.println("important to your crawl process, investigate other options.");
|
||||
listProblems(notApplicable);
|
||||
System.out.println();
|
||||
System.out.println(needsAttention.size()+" settings may need attention");
|
||||
System.out.println("Please review your original crawl and the created H3 job, for each");
|
||||
System.out.println("of the following, and manually update as needed.");
|
||||
listProblems(needsAttention);
|
||||
System.out.println();
|
||||
System.out.println(migrated +" H1 settings successfully migrated to H3 configuration");
|
||||
System.out.println();
|
||||
System.out.println("Review your converted crawler-beans.cxml at:");
|
||||
System.out.println(targetBeansXmlFile.getAbsolutePath());
|
||||
|
||||
}
|
||||
|
||||
protected void listProblems(List<String> problems) {
|
||||
|
||||
Reference in New Issue
Block a user