diff --git a/engine/src/main/java/org/archive/crawler/migrate/MigrateH1to3Tool.java b/engine/src/main/java/org/archive/crawler/migrate/MigrateH1to3Tool.java index e257a4f1..115389a7 100644 --- a/engine/src/main/java/org/archive/crawler/migrate/MigrateH1to3Tool.java +++ b/engine/src/main/java/org/archive/crawler/migrate/MigrateH1to3Tool.java @@ -43,7 +43,6 @@ import org.apache.commons.lang.StringUtils; import org.w3c.dom.Document; import org.w3c.dom.Node; import org.w3c.dom.NodeList; -import org.xbill.DNS.SRVRecord; import org.xml.sax.SAXException; /** @@ -59,7 +58,11 @@ import org.xml.sax.SAXException; * @contributor gojomo */ public class MigrateH1to3Tool { + + protected Document sourceOrderXmlDom; + static DocumentBuilder DOCUMENT_BUILDER; + static { try { DOCUMENT_BUILDER = DocumentBuilderFactory.newInstance().newDocumentBuilder(); @@ -71,25 +74,26 @@ public class MigrateH1to3Tool { public static void main(String[] args) throws Exception { new MigrateH1to3Tool().instanceMain(args); } - - public void instanceMain(String[] args) throws Exception { - if(args.length < 2) { - printHelp(); + + public void instanceMain(String[] args) throws Exception { + + if(args.length != 2) { + printHelp(); return; } - String sourceOrderXmlFileArg = args[0]; - String destinationH3JobDirArg = args[1]; - - File sourceOrderXmlFile = new File(sourceOrderXmlFileArg); - if (!sourceOrderXmlFile.isFile()) { - System.err.println("ERROR sourceOrderXmlFileArg is not a file: " + sourceOrderXmlFileArg); + String sourceOrderXmlFileArg = args[0]; + String destinationH3JobDirArg = args[1]; + + File sourceOrderXmlFile = new File(sourceOrderXmlFileArg); + if (!sourceOrderXmlFile.isFile()) { + System.err.println("ERROR sourceOrderXmlFileArg is not a file: " + sourceOrderXmlFileArg); System.exit(1); - } + } File destinationH3JobDir = new File(destinationH3JobDirArg); destinationH3JobDir.mkdirs(); - + System.out.println("H1 source: "+sourceOrderXmlFile.getAbsolutePath()); System.out.println("H3 destination: "+destinationH3JobDir.getAbsolutePath()); @@ -99,95 +103,96 @@ public class MigrateH1to3Tool { "/org/archive/crawler/migrate/migrate-template-crawler-beans.cxml"); String template = IOUtils.toString(inStream); inStream.close(); - + Map migrateH1toH3Map = getMigrateMap(); - + try { - Document sourceOrderXmlDom = DOCUMENT_BUILDER.parse(sourceOrderXmlFile); - - Map h1simpleSettings = flattenH1Order(sourceOrderXmlDom); - - List notApplicable = new ArrayList(); - List needsAttention = new ArrayList(); - int migrated = 0; - StringBuilder sb = new StringBuilder(); - for(String key : h1simpleSettings.keySet()) { - String beanPath = migrateH1toH3Map.get(key); - String value = h1simpleSettings.get(key); - System.out.print("."); - if(beanPath==null) { - // no equivalence rule - needsAttention.add(key+" "+value); - continue; - } - if (beanPath.startsWith("$")) { - // rule indicates not-available/not-applicable - notApplicable.add(key+" "+value); - continue; - } - if (beanPath.startsWith("*")) { - // TODO: needs special handling - if(beanPath.equals("*metadata.userAgentTemplate")) { - splitH1userAgent(value,sb); - migrated += 2; - } else { - needsAttention.add(key+" "+value); - } - continue; - } - if (beanPath.startsWith("^")) { - // uppercase to new enum-style - value = value.toUpperCase(); - beanPath = beanPath.substring(1); - } - sb - .append(beanPath) - .append("=") - .append(value) - .append("\n"); - migrated++; - } - System.out.println(); - System.out.println(); - - // patch all overrides derived from H1 into H3 template - String beansCxml = template.replace("###MIGRATE_OVERRIDES###", sb.toString()); - - File targetBeansXmlFile = new File(destinationH3JobDir,"crawler-beans.cxml"); - FileUtils.writeStringToFile(targetBeansXmlFile, beansCxml); - - File sourceSeedsTxtFile = new File(sourceOrderXmlFile.getParentFile(), "seeds.txt"); - File destinationSeedsTxtFile = new File(destinationH3JobDir, "seeds.txt"); - - if (!sourceSeedsTxtFile.isFile()) { - System.err.println("ERROR sourceSeedsTxtFile not found: " + sourceSeedsTxtFile); - System.exit(1); - } - - FileUtils.copyFile(sourceSeedsTxtFile,destinationSeedsTxtFile); - - System.out.println(notApplicable.size()+" settings skipped as not-applicable"); - System.out.println("These are probably harmless, but if the following settings were"); - System.out.println("important to your crawl process, investigate other options."); - listProblems(notApplicable); - System.out.println(); - System.out.println(needsAttention.size()+" settings may need attention"); - System.out.println("Please review your original crawl and the created H3 job, for each"); - System.out.println("of the following, and manually update as needed."); - listProblems(needsAttention); - System.out.println(); - System.out.println(migrated +" H1 settings successfully migrated to H3 configuration"); - System.out.println(); - System.out.println("Review your converted crawler-beans.cxml at:"); - System.out.println(targetBeansXmlFile.getAbsolutePath()); - + sourceOrderXmlDom = DOCUMENT_BUILDER.parse(sourceOrderXmlFile); } catch (SAXException e) { System.err.println("ERROR caught exception parsing input file: " + e.getMessage() + "\n"); e.printStackTrace(); } - } + Map h1simpleSettings = flattenH1Order(sourceOrderXmlDom); + + List notApplicable = new ArrayList(); + List needsAttention = new ArrayList(); + int migrated = 0; + StringBuilder sb = new StringBuilder(); + for(String key : h1simpleSettings.keySet()) { + String beanPath = migrateH1toH3Map.get(key); + String value = h1simpleSettings.get(key); + System.out.print("."); + if(beanPath==null) { + // no equivalence rule + needsAttention.add(key+" "+value); + continue; + } + if (beanPath.startsWith("$")) { + // rule indicates not-available/not-applicable + notApplicable.add(key+" "+value); + continue; + } + if (beanPath.startsWith("*")) { + // TODO: needs special handling + if(beanPath.equals("*metadata.userAgentTemplate")) { + splitH1userAgent(value,sb); + migrated += 2; + } else { + needsAttention.add(key+" "+value); + } + continue; + } + if (beanPath.startsWith("^")) { + // uppercase to new enum-style + value = value.toUpperCase(); + beanPath = beanPath.substring(1); + } + sb + .append(beanPath) + .append("=") + .append(value) + .append("\n"); + migrated++; + } + + + System.out.println(); + System.out.println(); + + // patch all overrides derived from H1 into H3 template + String beansCxml = template.replace("###MIGRATE_OVERRIDES###", sb.toString()); + + File targetBeansXmlFile = new File(destinationH3JobDir,"crawler-beans.cxml"); + FileUtils.writeStringToFile(targetBeansXmlFile, beansCxml); + + File sourceSeedsTxtFile = new File(sourceOrderXmlFile.getParentFile(), "seeds.txt"); + File destinationSeedsTxtFile = new File(destinationH3JobDir, "seeds.txt"); + + if (!sourceSeedsTxtFile.isFile()) { + System.err.println("ERROR sourceSeedsTxtFile not found: " + sourceSeedsTxtFile); + System.exit(1); + } + + FileUtils.copyFile(sourceSeedsTxtFile,destinationSeedsTxtFile); + + System.out.println(notApplicable.size()+" settings skipped as not-applicable"); + System.out.println("These are probably harmless, but if the following settings were"); + System.out.println("important to your crawl process, investigate other options."); + listProblems(notApplicable); + System.out.println(); + System.out.println(needsAttention.size()+" settings may need attention"); + System.out.println("Please review your original crawl and the created H3 job, for each"); + System.out.println("of the following, and manually update as needed."); + listProblems(needsAttention); + System.out.println(); + System.out.println(migrated +" H1 settings successfully migrated to H3 configuration"); + System.out.println(); + System.out.println("Review your converted crawler-beans.cxml at:"); + System.out.println(targetBeansXmlFile.getAbsolutePath()); + + } protected void listProblems(List problems) { for(String problem : problems) { @@ -231,7 +236,7 @@ public class MigrateH1to3Tool { inStream.close(); return map; } - + /** * Given a Document, return a Map of all non-blank simple text * nodes, keyed by the pseudo-XPath to their parent element.