diff --git a/engine/src/main/java/org/archive/crawler/migrate/MigrateH1to3Tool.java b/engine/src/main/java/org/archive/crawler/migrate/MigrateH1to3Tool.java index 02a0cb83..e257a4f1 100644 --- a/engine/src/main/java/org/archive/crawler/migrate/MigrateH1to3Tool.java +++ b/engine/src/main/java/org/archive/crawler/migrate/MigrateH1to3Tool.java @@ -43,6 +43,8 @@ import org.apache.commons.lang.StringUtils; import org.w3c.dom.Document; import org.w3c.dom.Node; import org.w3c.dom.NodeList; +import org.xbill.DNS.SRVRecord; +import org.xml.sax.SAXException; /** * Utility class which takes a H1 order.xml and creates a similar @@ -71,14 +73,21 @@ public class MigrateH1to3Tool { } public void instanceMain(String[] args) throws Exception { - if(args.length == 0) { + if(args.length < 2) { printHelp(); return; } - - File sourceOrderXmlFile = new File(args[0]); - File destinationH3JobDir = new File(args[1]); - + + String sourceOrderXmlFileArg = args[0]; + String destinationH3JobDirArg = args[1]; + + File sourceOrderXmlFile = new File(sourceOrderXmlFileArg); + if (!sourceOrderXmlFile.isFile()) { + System.err.println("ERROR sourceOrderXmlFileArg is not a file: " + sourceOrderXmlFileArg); + System.exit(1); + } + File destinationH3JobDir = new File(destinationH3JobDirArg); + destinationH3JobDir.mkdirs(); System.out.println("H1 source: "+sourceOrderXmlFile.getAbsolutePath()); @@ -93,77 +102,91 @@ public class MigrateH1to3Tool { Map migrateH1toH3Map = getMigrateMap(); - Document sourceOrderXmlDom = DOCUMENT_BUILDER.parse(sourceOrderXmlFile); - Map h1simpleSettings = flattenH1Order(sourceOrderXmlDom); - - List notApplicable = new ArrayList(); - List needsAttention = new ArrayList(); - int migrated = 0; - StringBuilder sb = new StringBuilder(); - for(String key : h1simpleSettings.keySet()) { - String beanPath = migrateH1toH3Map.get(key); - String value = h1simpleSettings.get(key); - System.out.print("."); - if(beanPath==null) { - // no equivalence rule - needsAttention.add(key+" "+value); - continue; - } - if (beanPath.startsWith("$")) { - // rule indicates not-available/not-applicable - notApplicable.add(key+" "+value); - continue; - } - if (beanPath.startsWith("*")) { - // TODO: needs special handling - if(beanPath.equals("*metadata.userAgentTemplate")) { - splitH1userAgent(value,sb); - migrated += 2; - } else { + try { + Document sourceOrderXmlDom = DOCUMENT_BUILDER.parse(sourceOrderXmlFile); + + Map h1simpleSettings = flattenH1Order(sourceOrderXmlDom); + + List notApplicable = new ArrayList(); + List needsAttention = new ArrayList(); + int migrated = 0; + StringBuilder sb = new StringBuilder(); + for(String key : h1simpleSettings.keySet()) { + String beanPath = migrateH1toH3Map.get(key); + String value = h1simpleSettings.get(key); + System.out.print("."); + if(beanPath==null) { + // no equivalence rule needsAttention.add(key+" "+value); + continue; + } + if (beanPath.startsWith("$")) { + // rule indicates not-available/not-applicable + notApplicable.add(key+" "+value); + continue; + } + if (beanPath.startsWith("*")) { + // TODO: needs special handling + if(beanPath.equals("*metadata.userAgentTemplate")) { + splitH1userAgent(value,sb); + migrated += 2; + } else { + needsAttention.add(key+" "+value); + } + continue; + } + if (beanPath.startsWith("^")) { + // uppercase to new enum-style + value = value.toUpperCase(); + beanPath = beanPath.substring(1); } - continue; - } - if (beanPath.startsWith("^")) { - // uppercase to new enum-style - value = value.toUpperCase(); - beanPath = beanPath.substring(1); + sb + .append(beanPath) + .append("=") + .append(value) + .append("\n"); + migrated++; } - sb - .append(beanPath) - .append("=") - .append(value) - .append("\n"); - migrated++; + System.out.println(); + System.out.println(); + + // patch all overrides derived from H1 into H3 template + String beansCxml = template.replace("###MIGRATE_OVERRIDES###", sb.toString()); + + File targetBeansXmlFile = new File(destinationH3JobDir,"crawler-beans.cxml"); + FileUtils.writeStringToFile(targetBeansXmlFile, beansCxml); + + File sourceSeedsTxtFile = new File(sourceOrderXmlFile.getParentFile(), "seeds.txt"); + File destinationSeedsTxtFile = new File(destinationH3JobDir, "seeds.txt"); + + if (!sourceSeedsTxtFile.isFile()) { + System.err.println("ERROR sourceSeedsTxtFile not found: " + sourceSeedsTxtFile); + System.exit(1); + } + + FileUtils.copyFile(sourceSeedsTxtFile,destinationSeedsTxtFile); + + System.out.println(notApplicable.size()+" settings skipped as not-applicable"); + System.out.println("These are probably harmless, but if the following settings were"); + System.out.println("important to your crawl process, investigate other options."); + listProblems(notApplicable); + System.out.println(); + System.out.println(needsAttention.size()+" settings may need attention"); + System.out.println("Please review your original crawl and the created H3 job, for each"); + System.out.println("of the following, and manually update as needed."); + listProblems(needsAttention); + System.out.println(); + System.out.println(migrated +" H1 settings successfully migrated to H3 configuration"); + System.out.println(); + System.out.println("Review your converted crawler-beans.cxml at:"); + System.out.println(targetBeansXmlFile.getAbsolutePath()); + + } catch (SAXException e) { + System.err.println("ERROR caught exception parsing input file: " + + e.getMessage() + "\n"); + e.printStackTrace(); } - System.out.println(); - System.out.println(); - - // patch all overrides derived from H1 into H3 template - String beansCxml = template.replace("###MIGRATE_OVERRIDES###", sb.toString()); - - File targetBeansXmlFile = new File(destinationH3JobDir,"crawler-beans.cxml"); - FileUtils.writeStringToFile(targetBeansXmlFile, beansCxml); - - FileUtils.copyFile( - new File(sourceOrderXmlFile.getParentFile(), "seeds.txt"), - new File(destinationH3JobDir, "seeds.txt")); - - - System.out.println(notApplicable.size()+" settings skipped as not-applicable"); - System.out.println("These are probably harmless, but if the following settings were"); - System.out.println("important to your crawl process, investigate other options."); - listProblems(notApplicable); - System.out.println(); - System.out.println(needsAttention.size()+" settings may need attention"); - System.out.println("Please review your original crawl and the created H3 job, for each"); - System.out.println("of the following, and manually update as needed."); - listProblems(needsAttention); - System.out.println(); - System.out.println(migrated +" H1 settings successfully migrated to H3 configuration"); - System.out.println(); - System.out.println("Review your converted crawler-beans.cxml at:"); - System.out.println(targetBeansXmlFile.getAbsolutePath()); + } protected void listProblems(List problems) {