diff --git a/.classpath b/.classpath index e24be8d9..0ac0d222 100644 --- a/.classpath +++ b/.classpath @@ -1,15 +1,11 @@ - - - - @@ -40,7 +36,7 @@ - + @@ -60,5 +56,9 @@ + + + + diff --git a/engine/src/main/java/org/archive/crawler/migrate/MigrateH1to3Tool.java b/engine/src/main/java/org/archive/crawler/migrate/MigrateH1to3Tool.java index 53ffd5b4..02a0cb83 100644 --- a/engine/src/main/java/org/archive/crawler/migrate/MigrateH1to3Tool.java +++ b/engine/src/main/java/org/archive/crawler/migrate/MigrateH1to3Tool.java @@ -76,8 +76,6 @@ public class MigrateH1to3Tool { return; } - - File sourceOrderXmlFile = new File(args[0]); File destinationH3JobDir = new File(args[1]); @@ -89,7 +87,7 @@ public class MigrateH1to3Tool { System.out.print("Migrating settings..."); InputStream inStream = getClass().getResourceAsStream( - "migrate-template-crawler-beans.cxml"); + "/org/archive/crawler/migrate/migrate-template-crawler-beans.cxml"); String template = IOUtils.toString(inStream); inStream.close(); @@ -105,11 +103,18 @@ public class MigrateH1to3Tool { for(String key : h1simpleSettings.keySet()) { String beanPath = migrateH1toH3Map.get(key); String value = h1simpleSettings.get(key); + System.out.print("."); if(beanPath==null) { - System.err.println("no rule found for: "+key); - } else if (beanPath.startsWith("$")) { - notApplicable.add(key+" "+value); - } else if (beanPath.startsWith("*")) { + // no equivalence rule + needsAttention.add(key+" "+value); + continue; + } + if (beanPath.startsWith("$")) { + // rule indicates not-available/not-applicable + notApplicable.add(key+" "+value); + continue; + } + if (beanPath.startsWith("*")) { // TODO: needs special handling if(beanPath.equals("*metadata.userAgentTemplate")) { splitH1userAgent(value,sb); @@ -117,16 +122,19 @@ public class MigrateH1to3Tool { } else { needsAttention.add(key+" "+value); } - } else { - sb - .append(beanPath) - .append("=") - .append(value) - .append("\n"); - migrated++; - + continue; + } + if (beanPath.startsWith("^")) { + // uppercase to new enum-style + value = value.toUpperCase(); + beanPath = beanPath.substring(1); } - System.out.print("."); + sb + .append(beanPath) + .append("=") + .append(value) + .append("\n"); + migrated++; } System.out.println(); System.out.println(); @@ -144,12 +152,12 @@ public class MigrateH1to3Tool { System.out.println(notApplicable.size()+" settings skipped as not-applicable"); System.out.println("These are probably harmless, but if the following settings were"); - System.out.println("important to your crawl, investigate other options."); + System.out.println("important to your crawl process, investigate other options."); listProblems(notApplicable); System.out.println(); System.out.println(needsAttention.size()+" settings may need attention"); System.out.println("Please review your original crawl and the created H3 job, for each"); - System.out.println("of the following and update manually to reflect your intent "); + System.out.println("of the following, and manually update as needed."); listProblems(needsAttention); System.out.println(); System.out.println(migrated +" H1 settings successfully migrated to H3 configuration"); @@ -178,6 +186,7 @@ public class MigrateH1to3Tool { "^.*?\\+(http://[^)]*).*$", "$1"); String newTemplate = userAgent.replace(originalUrl,"@OPERATOR_CONTACT_URL@"); + // TODO: catch, change outdated version info? sb .append("metadata.operatorContactUrl=") .append(originalUrl) @@ -189,7 +198,8 @@ public class MigrateH1to3Tool { protected Map getMigrateMap() throws IOException { Map map = new HashMap(); - InputStream inStream = getClass().getResourceAsStream("H1toH3.map"); + InputStream inStream = getClass().getResourceAsStream( + "/org/archive/crawler/migrate/H1toH3.map"); LineIterator iter = IOUtils.lineIterator(inStream, "UTF-8"); while(iter.hasNext()) { String[] fields = iter.nextLine().split("\\|"); diff --git a/engine/src/main/java/org/archive/crawler/migrate/H1toH3.map b/engine/src/main/resources/org/archive/crawler/migrate/H1toH3.map similarity index 91% rename from engine/src/main/java/org/archive/crawler/migrate/H1toH3.map rename to engine/src/main/resources/org/archive/crawler/migrate/H1toH3.map index 12e622d0..9b91aa1d 100644 --- a/engine/src/main/java/org/archive/crawler/migrate/H1toH3.map +++ b/engine/src/main/resources/org/archive/crawler/migrate/H1toH3.map @@ -28,7 +28,7 @@ scope.rules[6].maxRepetitions|//controller/*[@name="scope"]/*[@name="decide-rule scope.rules[7].maxPathDepth|//controller/*[@name="scope"]/*[@name="decide-rules"]/*[@name="rules"]/*[@name="rejectIfTooManyPathSegs"]/*[@name="max-path-depth"]|20 *metadata.userAgentTemplate|//controller/*[@name="http-headers"]/*[@name="user-agent"]|Mozilla/5.0|(compatible;|heritrix/@VERSION@|+PROJECT_URL_HERE) metadata.operatorFrom|//controller/*[@name="http-headers"]/*[@name="from"]|CONTACT_EMAIL_ADDRESS_HERE -*metadata.robotsHonoringPolicy.type|//controller/*[@name="robots-honoring-policy"]/*[@name="type"]|classic +^metadata.robotsHonoringPolicy.type|//controller/*[@name="robots-honoring-policy"]/*[@name="type"]|classic metadata.robotsHonoringPolicy.masquerade|//controller/*[@name="robots-honoring-policy"]/*[@name="masquerade"]|false disposition.delayFactor|//controller/*[@name="frontier"]/*[@name="delay-factor"]|4.0 disposition.maxDelayMs|//controller/*[@name="frontier"]/*[@name="max-delay-ms"]|20000 @@ -53,12 +53,12 @@ frontier.snoozeLongMs|//controller/*[@name="frontier"]/*[@name="snooze-deactivat $|//controller/*[@name="frontier"]/*[@name="target-ready-backlog"]|50 *frontier.uriUniqFilter|//controller/*[@name="frontier"]/*[@name="uri-included-structure"]|org.archive.crawler.util.BdbUriUniqFilter frontier.dumpPendingAtClose|//controller/*[@name="frontier"]/*[@name="dump-pending-at-close"]|false -$|//controller/*[@name="uri-canonicalization-rules"]/*[@name="Lowercase"]/*[@name="enabled"]|true -$|//controller/*[@name="uri-canonicalization-rules"]/*[@name="Userinfo"]/*[@name="enabled"]|true -$|//controller/*[@name="uri-canonicalization-rules"]/*[@name="WWW[0-9]*"]/*[@name="enabled"]|true -$|//controller/*[@name="uri-canonicalization-rules"]/*[@name="SessionIDs"]/*[@name="enabled"]|true -$|//controller/*[@name="uri-canonicalization-rules"]/*[@name="SessionCFIDs"]/*[@name="enabled"]|true -$|//controller/*[@name="uri-canonicalization-rules"]/*[@name="QueryStrPrefix"]/*[@name="enabled"]|true +preparer.uriCanonicalizationPolicy.rules[0].enabled|//controller/*[@name="uri-canonicalization-rules"]/*[@name="Lowercase"]/*[@name="enabled"]|true +preparer.uriCanonicalizationPolicy.rules[1].enabled|//controller/*[@name="uri-canonicalization-rules"]/*[@name="Userinfo"]/*[@name="enabled"]|true +preparer.uriCanonicalizationPolicy.rules[2].enabled|//controller/*[@name="uri-canonicalization-rules"]/*[@name="WWW[0-9]*"]/*[@name="enabled"]|true +preparer.uriCanonicalizationPolicy.rules[3].enabled|//controller/*[@name="uri-canonicalization-rules"]/*[@name="SessionIDs"]/*[@name="enabled"]|true +preparer.uriCanonicalizationPolicy.rules[4].enabled|//controller/*[@name="uri-canonicalization-rules"]/*[@name="SessionCFIDs"]/*[@name="enabled"]|true +preparer.uriCanonicalizationPolicy.rules[5].enabled|//controller/*[@name="uri-canonicalization-rules"]/*[@name="QueryStrPrefix"]/*[@name="enabled"]|true preselector.enabled|//controller/*[@name="pre-fetch-processors"]/*[@name="Preselector"]/*[@name="enabled"]|true preselector.logToFile|//controller/*[@name="pre-fetch-processors"]/*[@name="Preselector"]/*[@name="override-logger"]|false preselector.recheckScope|//controller/*[@name="pre-fetch-processors"]/*[@name="Preselector"]/*[@name="recheck-scope"]|true @@ -78,7 +78,7 @@ fetchHttp.maxFetchKBSec|//controller/*[@name="fetch-processors"]/*[@name="HTTP"] fetchHttp.maxLengthBytes|//controller/*[@name="fetch-processors"]/*[@name="HTTP"]/*[@name="max-length-bytes"]|0 fetchHttp.ignoreCookies|//controller/*[@name="fetch-processors"]/*[@name="HTTP"]/*[@name="ignore-cookies"]|false $|//controller/*[@name="fetch-processors"]/*[@name="HTTP"]/*[@name="use-bdb-for-cookies"]|true -*fetchHttp.sslTrustLevel|//controller/*[@name="fetch-processors"]/*[@name="HTTP"]/*[@name="trust-level"]|open +^fetchHttp.sslTrustLevel|//controller/*[@name="fetch-processors"]/*[@name="HTTP"]/*[@name="trust-level"]|open fetchHttp.defaultEncoding|//controller/*[@name="fetch-processors"]/*[@name="HTTP"]/*[@name="default-encoding"]|ISO-8859-1 fetchHttp.digestContent|//controller/*[@name="fetch-processors"]/*[@name="HTTP"]/*[@name="digest-content"]|true fetchHttp.digestAlgorithm|//controller/*[@name="fetch-processors"]/*[@name="HTTP"]/*[@name="digest-algorithm"]|sha1 diff --git a/engine/src/main/java/org/archive/crawler/migrate/migrate-template-crawler-beans.cxml b/engine/src/main/resources/org/archive/crawler/migrate/migrate-template-crawler-beans.cxml similarity index 100% rename from engine/src/main/java/org/archive/crawler/migrate/migrate-template-crawler-beans.cxml rename to engine/src/main/resources/org/archive/crawler/migrate/migrate-template-crawler-beans.cxml