lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-09-26 23:55:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

check for exactly 2 args, tightened up SAXException handling by making sourceOrderXmlDom

Browse Source 
a protected field, cleaned up unused import and whitespace.
This commit is contained in:
szznax
2009-11-16 23:45:04 +00:00
parent 9059395617
commit 92f3bfe0ec
1 changed files with 100 additions and 95 deletions
Download Patch File Download Diff File Expand all files Collapse all files
engine/src/main/java/org/archive/crawler/migrate/MigrateH1to3Tool.java
+100 -95
View File
@@ -43,7 +43,6 @@ import org.apache.commons.lang.StringUtils;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.xbill.DNS.SRVRecord;
import org.xml.sax.SAXException;
/**
@@ -59,7 +58,11 @@ import org.xml.sax.SAXException;
* @contributor gojomo
*/
public class MigrateH1to3Tool {
protected Document sourceOrderXmlDom;
static DocumentBuilder DOCUMENT_BUILDER;
static {
try {
DOCUMENT_BUILDER = DocumentBuilderFactory.newInstance().newDocumentBuilder();
@@ -71,25 +74,26 @@ public class MigrateH1to3Tool {
public static void main(String[] args) throws Exception {
new MigrateH1to3Tool().instanceMain(args);
}
public void instanceMain(String[] args) throws Exception {
if(args.length < 2) {
printHelp();
public void instanceMain(String[] args) throws Exception {
if(args.length != 2) {
printHelp();
return;
}
String sourceOrderXmlFileArg = args[0];
String destinationH3JobDirArg = args[1];
File sourceOrderXmlFile = new File(sourceOrderXmlFileArg);
if (!sourceOrderXmlFile.isFile()) {
System.err.println("ERROR sourceOrderXmlFileArg is not a file: " + sourceOrderXmlFileArg);
String sourceOrderXmlFileArg = args[0];
String destinationH3JobDirArg = args[1];
File sourceOrderXmlFile = new File(sourceOrderXmlFileArg);
if (!sourceOrderXmlFile.isFile()) {
System.err.println("ERROR sourceOrderXmlFileArg is not a file: " + sourceOrderXmlFileArg);
System.exit(1);
}
}
File destinationH3JobDir = new File(destinationH3JobDirArg);
destinationH3JobDir.mkdirs();
System.out.println("H1 source: "+sourceOrderXmlFile.getAbsolutePath());
System.out.println("H3 destination: "+destinationH3JobDir.getAbsolutePath());
@@ -99,95 +103,96 @@ public class MigrateH1to3Tool {
"/org/archive/crawler/migrate/migrate-template-crawler-beans.cxml");
String template = IOUtils.toString(inStream);
inStream.close();
Map<String,String> migrateH1toH3Map = getMigrateMap();
try {
Document sourceOrderXmlDom = DOCUMENT_BUILDER.parse(sourceOrderXmlFile);
Map<String,String> h1simpleSettings = flattenH1Order(sourceOrderXmlDom);
List<String> notApplicable = new ArrayList<String>();
List<String> needsAttention = new ArrayList<String>();
int migrated = 0;
StringBuilder sb = new StringBuilder();
for(String key : h1simpleSettings.keySet()) {
String beanPath = migrateH1toH3Map.get(key);
String value = h1simpleSettings.get(key);
System.out.print(".");
if(beanPath==null) {
// no equivalence rule
needsAttention.add(key+" "+value);
continue;
}
if (beanPath.startsWith("$")) {
// rule indicates not-available/not-applicable
notApplicable.add(key+" "+value);
continue;
}
if (beanPath.startsWith("*")) {
// TODO: needs special handling
if(beanPath.equals("*metadata.userAgentTemplate")) {
splitH1userAgent(value,sb);
migrated += 2;
} else {
needsAttention.add(key+" "+value);
}
continue;
}
if (beanPath.startsWith("^")) {
// uppercase to new enum-style
value = value.toUpperCase();
beanPath = beanPath.substring(1);
}
sb
.append(beanPath)
.append("=")
.append(value)
.append("\n");
migrated++;
}
System.out.println();
System.out.println();
// patch all overrides derived from H1 into H3 template
String beansCxml = template.replace("###MIGRATE_OVERRIDES###", sb.toString());
File targetBeansXmlFile = new File(destinationH3JobDir,"crawler-beans.cxml");
FileUtils.writeStringToFile(targetBeansXmlFile, beansCxml);
File sourceSeedsTxtFile = new File(sourceOrderXmlFile.getParentFile(), "seeds.txt");
File destinationSeedsTxtFile = new File(destinationH3JobDir, "seeds.txt");
if (!sourceSeedsTxtFile.isFile()) {
System.err.println("ERROR sourceSeedsTxtFile not found: " + sourceSeedsTxtFile);
System.exit(1);
}
FileUtils.copyFile(sourceSeedsTxtFile,destinationSeedsTxtFile);
System.out.println(notApplicable.size()+" settings skipped as not-applicable");
System.out.println("These are probably harmless, but if the following settings were");
System.out.println("important to your crawl process, investigate other options.");
listProblems(notApplicable);
System.out.println();
System.out.println(needsAttention.size()+" settings may need attention");
System.out.println("Please review your original crawl and the created H3 job, for each");
System.out.println("of the following, and manually update as needed.");
listProblems(needsAttention);
System.out.println();
System.out.println(migrated +" H1 settings successfully migrated to H3 configuration");
System.out.println();
System.out.println("Review your converted crawler-beans.cxml at:");
System.out.println(targetBeansXmlFile.getAbsolutePath());
sourceOrderXmlDom = DOCUMENT_BUILDER.parse(sourceOrderXmlFile);
} catch (SAXException e) {
System.err.println("ERROR caught exception parsing input file: "
+ e.getMessage() + "\n");
e.printStackTrace();
}
}
Map<String,String> h1simpleSettings = flattenH1Order(sourceOrderXmlDom);
List<String> notApplicable = new ArrayList<String>();
List<String> needsAttention = new ArrayList<String>();
int migrated = 0;
StringBuilder sb = new StringBuilder();
for(String key : h1simpleSettings.keySet()) {
String beanPath = migrateH1toH3Map.get(key);
String value = h1simpleSettings.get(key);
System.out.print(".");
if(beanPath==null) {
// no equivalence rule
needsAttention.add(key+" "+value);
continue;
}
if (beanPath.startsWith("$")) {
// rule indicates not-available/not-applicable
notApplicable.add(key+" "+value);
continue;
}
if (beanPath.startsWith("*")) {
// TODO: needs special handling
if(beanPath.equals("*metadata.userAgentTemplate")) {
splitH1userAgent(value,sb);
migrated += 2;
} else {
needsAttention.add(key+" "+value);
}
continue;
}
if (beanPath.startsWith("^")) {
// uppercase to new enum-style
value = value.toUpperCase();
beanPath = beanPath.substring(1);
}
sb
.append(beanPath)
.append("=")
.append(value)
.append("\n");
migrated++;
}
System.out.println();
System.out.println();
// patch all overrides derived from H1 into H3 template
String beansCxml = template.replace("###MIGRATE_OVERRIDES###", sb.toString());
File targetBeansXmlFile = new File(destinationH3JobDir,"crawler-beans.cxml");
FileUtils.writeStringToFile(targetBeansXmlFile, beansCxml);
File sourceSeedsTxtFile = new File(sourceOrderXmlFile.getParentFile(), "seeds.txt");
File destinationSeedsTxtFile = new File(destinationH3JobDir, "seeds.txt");
if (!sourceSeedsTxtFile.isFile()) {
System.err.println("ERROR sourceSeedsTxtFile not found: " + sourceSeedsTxtFile);
System.exit(1);
}
FileUtils.copyFile(sourceSeedsTxtFile,destinationSeedsTxtFile);
System.out.println(notApplicable.size()+" settings skipped as not-applicable");
System.out.println("These are probably harmless, but if the following settings were");
System.out.println("important to your crawl process, investigate other options.");
listProblems(notApplicable);
System.out.println();
System.out.println(needsAttention.size()+" settings may need attention");
System.out.println("Please review your original crawl and the created H3 job, for each");
System.out.println("of the following, and manually update as needed.");
listProblems(needsAttention);
System.out.println();
System.out.println(migrated +" H1 settings successfully migrated to H3 configuration");
System.out.println();
System.out.println("Review your converted crawler-beans.cxml at:");
System.out.println(targetBeansXmlFile.getAbsolutePath());
}
protected void listProblems(List<String> problems) {
for(String problem : problems) {
@@ -231,7 +236,7 @@ public class MigrateH1to3Tool {
inStream.close();
return map;
}
/**
* Given a Document, return a Map of all non-blank simple text
* nodes, keyed by the pseudo-XPath to their parent element.