mirror of
https://github.com/internetarchive/heritrix3.git
synced 2026-11-09 15:36:40 +00:00
[HER-1641] migration tool to convert Heritrix 1.14.x order.xml to H3 crawler-beans.cxml
* MigrateSettings.java, MigrateH1toH3Tool.java
renamings, 'flatten' method for building map
* H1toH3.map
order-XPath-to-override-key map for translating simple H1 settings to H3 overrides
This commit is contained in:
gojomo committed
1 parent
eb659a9fad
commit
329204b9ff
3 files changed
+401
-215
No files matched your search
@@ -0,0 +1,121 @@
|
||||
metadata.jobName //meta/name default
|
||||
metadata.description //meta/description Default Profile
|
||||
metadata.operator //meta/operator Admin
|
||||
# //meta/date 20091006215050
|
||||
# //controller/*[@name="settings-directory"] settings
|
||||
loggerModule.path //controller/*[@name="logs-path"] logs
|
||||
crawlController.checkpointsDir //controller/*[@name="checkpoints-path"] checkpoints
|
||||
bdb.dir //controller/*[@name="state-path"] state
|
||||
crawlController.scratchDir //controller/*[@name="scratch-path"] scratch
|
||||
crawlLimiter.maxBytesDownload //controller/*[@name="max-bytes-download"] 0
|
||||
crawlLimiter.maxDocumentsDownload //controller/*[@name="max-document-download"] 0
|
||||
crawlLimiter.maxTimeSeconds //controller/*[@name="max-time-sec"] 0
|
||||
crawlController.maxToeThreads //controller/*[@name="max-toe-threads"] 50
|
||||
crawlController.recorderOutBufferBytes //controller/*[@name="recorder-out-buffer-bytes"] 4096
|
||||
crawlController.recorderInBufferBytes //controller/*[@name="recorder-in-buffer-bytes"] 65536
|
||||
bdb.cachePercent //controller/*[@name="bdb-cache-percent"] 0
|
||||
! //controller/*[@name="scope"]/*[@name="enabled"] true
|
||||
seeds.textSource.path //controller/*[@name="scope"]/*[@name="seedsfile"] seeds.txt
|
||||
# //controller/*[@name="scope"]/*[@name="reread-seeds-on-config"] true
|
||||
?scope.rules[1].decision //controller/*[@name="scope"]/*[@name="decide-rules"]/*[@name="rules"]/*[@name="acceptIfSurtPrefixed"]/*[@name="decision"] ACCEPT
|
||||
?scope.rules[1].seedsAsSurtPrefixes //controller/*[@name="scope"]/*[@name="decide-rules"]/*[@name="rules"]/*[@name="acceptIfSurtPrefixed"]/*[@name="seeds-as-surt-prefixes"] true
|
||||
//controller/*[@name="scope"]/*[@name="decide-rules"]/*[@name="rules"]/*[@name="acceptIfSurtPrefixed"]/*[@name="also-check-via"] false
|
||||
//controller/*[@name="scope"]/*[@name="decide-rules"]/*[@name="rules"]/*[@name="acceptIfSurtPrefixed"]/*[@name="rebuild-on-reconfig"] true
|
||||
//controller/*[@name="scope"]/*[@name="decide-rules"]/*[@name="rules"]/*[@name="rejectIfTooManyHops"]/*[@name="max-hops"] 20
|
||||
//controller/*[@name="scope"]/*[@name="decide-rules"]/*[@name="rules"]/*[@name="acceptIfTranscluded"]/*[@name="max-trans-hops"] 3
|
||||
//controller/*[@name="scope"]/*[@name="decide-rules"]/*[@name="rules"]/*[@name="acceptIfTranscluded"]/*[@name="max-speculative-hops"] 1
|
||||
//controller/*[@name="scope"]/*[@name="decide-rules"]/*[@name="rules"]/*[@name="rejectIfPathological"]/*[@name="max-repetitions"] 2
|
||||
//controller/*[@name="scope"]/*[@name="decide-rules"]/*[@name="rules"]/*[@name="rejectIfTooManyPathSegs"]/*[@name="max-path-depth"] 20
|
||||
//controller/*[@name="http-headers"]/*[@name="user-agent"] Mozilla/5.0 (compatible; heritrix/@VERSION@ +PROJECT_URL_HERE)
|
||||
//controller/*[@name="http-headers"]/*[@name="from"] CONTACT_EMAIL_ADDRESS_HERE
|
||||
//controller/*[@name="robots-honoring-policy"]/*[@name="type"] classic
|
||||
//controller/*[@name="robots-honoring-policy"]/*[@name="masquerade"] false
|
||||
//controller/*[@name="frontier"]/*[@name="delay-factor"] 4.0
|
||||
//controller/*[@name="frontier"]/*[@name="max-delay-ms"] 20000
|
||||
//controller/*[@name="frontier"]/*[@name="min-delay-ms"] 2000
|
||||
//controller/*[@name="frontier"]/*[@name="respect-crawl-delay-up-to-secs"] 300
|
||||
//controller/*[@name="frontier"]/*[@name="max-retries"] 30
|
||||
//controller/*[@name="frontier"]/*[@name="retry-delay-seconds"] 900
|
||||
//controller/*[@name="frontier"]/*[@name="preference-embed-hops"] 1
|
||||
//controller/*[@name="frontier"]/*[@name="total-bandwidth-usage-KB-sec"] 0
|
||||
//controller/*[@name="frontier"]/*[@name="max-per-host-bandwidth-usage-KB-sec"] 0
|
||||
//controller/*[@name="frontier"]/*[@name="queue-assignment-policy"] org.archive.crawler.frontier.HostnameQueueAssignmentPolicy
|
||||
//controller/*[@name="frontier"]/*[@name="pause-at-start"] false
|
||||
//controller/*[@name="frontier"]/*[@name="pause-at-finish"] false
|
||||
//controller/*[@name="frontier"]/*[@name="source-tag-seeds"] false
|
||||
//controller/*[@name="frontier"]/*[@name="recovery-log-enabled"] true
|
||||
//controller/*[@name="frontier"]/*[@name="hold-queues"] true
|
||||
//controller/*[@name="frontier"]/*[@name="balance-replenish-amount"] 3000
|
||||
//controller/*[@name="frontier"]/*[@name="error-penalty-amount"] 100
|
||||
//controller/*[@name="frontier"]/*[@name="queue-total-budget"] -1
|
||||
//controller/*[@name="frontier"]/*[@name="cost-policy"] org.archive.crawler.frontier.ZeroCostAssignmentPolicy
|
||||
//controller/*[@name="frontier"]/*[@name="snooze-deactivate-ms"] 300000
|
||||
//controller/*[@name="frontier"]/*[@name="target-ready-backlog"] 50
|
||||
//controller/*[@name="frontier"]/*[@name="uri-included-structure"] org.archive.crawler.util.BdbUriUniqFilter
|
||||
//controller/*[@name="frontier"]/*[@name="dump-pending-at-close"] false
|
||||
//controller/*[@name="uri-canonicalization-rules"]/*[@name="Lowercase"]/*[@name="enabled"] true
|
||||
//controller/*[@name="uri-canonicalization-rules"]/*[@name="Userinfo"]/*[@name="enabled"] true
|
||||
//controller/*[@name="uri-canonicalization-rules"]/*[@name="WWW[0-9]*"]/*[@name="enabled"] true
|
||||
//controller/*[@name="uri-canonicalization-rules"]/*[@name="SessionIDs"]/*[@name="enabled"] true
|
||||
//controller/*[@name="uri-canonicalization-rules"]/*[@name="SessionCFIDs"]/*[@name="enabled"] true
|
||||
//controller/*[@name="uri-canonicalization-rules"]/*[@name="QueryStrPrefix"]/*[@name="enabled"] true
|
||||
//controller/*[@name="pre-fetch-processors"]/*[@name="Preselector"]/*[@name="enabled"] true
|
||||
//controller/*[@name="pre-fetch-processors"]/*[@name="Preselector"]/*[@name="override-logger"] false
|
||||
//controller/*[@name="pre-fetch-processors"]/*[@name="Preselector"]/*[@name="recheck-scope"] true
|
||||
//controller/*[@name="pre-fetch-processors"]/*[@name="Preselector"]/*[@name="block-all"] false
|
||||
//controller/*[@name="pre-fetch-processors"]/*[@name="Preprocessor"]/*[@name="enabled"] true
|
||||
//controller/*[@name="pre-fetch-processors"]/*[@name="Preprocessor"]/*[@name="ip-validity-duration-seconds"] 21600
|
||||
//controller/*[@name="pre-fetch-processors"]/*[@name="Preprocessor"]/*[@name="robot-validity-duration-seconds"] 86400
|
||||
//controller/*[@name="pre-fetch-processors"]/*[@name="Preprocessor"]/*[@name="calculate-robots-only"] false
|
||||
//controller/*[@name="fetch-processors"]/*[@name="DNS"]/*[@name="enabled"] true
|
||||
//controller/*[@name="fetch-processors"]/*[@name="DNS"]/*[@name="accept-non-dns-resolves"] false
|
||||
//controller/*[@name="fetch-processors"]/*[@name="DNS"]/*[@name="digest-content"] true
|
||||
//controller/*[@name="fetch-processors"]/*[@name="DNS"]/*[@name="digest-algorithm"] sha1
|
||||
//controller/*[@name="fetch-processors"]/*[@name="HTTP"]/*[@name="enabled"] true
|
||||
//controller/*[@name="fetch-processors"]/*[@name="HTTP"]/*[@name="timeout-seconds"] 1200
|
||||
//controller/*[@name="fetch-processors"]/*[@name="HTTP"]/*[@name="sotimeout-ms"] 20000
|
||||
//controller/*[@name="fetch-processors"]/*[@name="HTTP"]/*[@name="fetch-bandwidth"] 0
|
||||
//controller/*[@name="fetch-processors"]/*[@name="HTTP"]/*[@name="max-length-bytes"] 0
|
||||
//controller/*[@name="fetch-processors"]/*[@name="HTTP"]/*[@name="ignore-cookies"] false
|
||||
//controller/*[@name="fetch-processors"]/*[@name="HTTP"]/*[@name="use-bdb-for-cookies"] true
|
||||
//controller/*[@name="fetch-processors"]/*[@name="HTTP"]/*[@name="trust-level"] open
|
||||
//controller/*[@name="fetch-processors"]/*[@name="HTTP"]/*[@name="default-encoding"] ISO-8859-1
|
||||
//controller/*[@name="fetch-processors"]/*[@name="HTTP"]/*[@name="digest-content"] true
|
||||
//controller/*[@name="fetch-processors"]/*[@name="HTTP"]/*[@name="digest-algorithm"] sha1
|
||||
//controller/*[@name="fetch-processors"]/*[@name="HTTP"]/*[@name="send-if-modified-since"] true
|
||||
//controller/*[@name="fetch-processors"]/*[@name="HTTP"]/*[@name="send-if-none-match"] true
|
||||
//controller/*[@name="fetch-processors"]/*[@name="HTTP"]/*[@name="send-connection-close"] true
|
||||
//controller/*[@name="fetch-processors"]/*[@name="HTTP"]/*[@name="send-referer"] true
|
||||
//controller/*[@name="fetch-processors"]/*[@name="HTTP"]/*[@name="send-range"] false
|
||||
//controller/*[@name="extract-processors"]/*[@name="ExtractorHTTP"]/*[@name="enabled"] true
|
||||
//controller/*[@name="extract-processors"]/*[@name="ExtractorHTML"]/*[@name="enabled"] true
|
||||
//controller/*[@name="extract-processors"]/*[@name="ExtractorHTML"]/*[@name="extract-javascript"] true
|
||||
//controller/*[@name="extract-processors"]/*[@name="ExtractorHTML"]/*[@name="treat-frames-as-embed-links"] true
|
||||
//controller/*[@name="extract-processors"]/*[@name="ExtractorHTML"]/*[@name="ignore-form-action-urls"] false
|
||||
//controller/*[@name="extract-processors"]/*[@name="ExtractorHTML"]/*[@name="extract-only-form-gets"] true
|
||||
//controller/*[@name="extract-processors"]/*[@name="ExtractorHTML"]/*[@name="extract-value-attributes"] true
|
||||
//controller/*[@name="extract-processors"]/*[@name="ExtractorHTML"]/*[@name="ignore-unexpected-html"] true
|
||||
//controller/*[@name="extract-processors"]/*[@name="ExtractorCSS"]/*[@name="enabled"] true
|
||||
//controller/*[@name="extract-processors"]/*[@name="ExtractorJS"]/*[@name="enabled"] true
|
||||
//controller/*[@name="extract-processors"]/*[@name="ExtractorSWF"]/*[@name="enabled"] true
|
||||
//controller/*[@name="write-processors"]/*[@name="Archiver"]/*[@name="enabled"] true
|
||||
//controller/*[@name="write-processors"]/*[@name="Archiver"]/*[@name="compress"] true
|
||||
//controller/*[@name="write-processors"]/*[@name="Archiver"]/*[@name="prefix"] IAH
|
||||
//controller/*[@name="write-processors"]/*[@name="Archiver"]/*[@name="suffix"] ${HOSTNAME}
|
||||
//controller/*[@name="write-processors"]/*[@name="Archiver"]/*[@name="max-size-bytes"] 100000000
|
||||
//controller/*[@name="write-processors"]/*[@name="Archiver"]/*[@name="path"]/string arcs
|
||||
//controller/*[@name="write-processors"]/*[@name="Archiver"]/*[@name="pool-max-active"] 5
|
||||
//controller/*[@name="write-processors"]/*[@name="Archiver"]/*[@name="pool-max-wait"] 300000
|
||||
//controller/*[@name="write-processors"]/*[@name="Archiver"]/*[@name="total-bytes-to-write"] 0
|
||||
//controller/*[@name="write-processors"]/*[@name="Archiver"]/*[@name="skip-identical-digests"] false
|
||||
//controller/*[@name="post-processors"]/*[@name="Updater"]/*[@name="enabled"] true
|
||||
//controller/*[@name="post-processors"]/*[@name="LinksScoper"]/*[@name="enabled"] true
|
||||
//controller/*[@name="post-processors"]/*[@name="LinksScoper"]/*[@name="override-logger"] false
|
||||
//controller/*[@name="post-processors"]/*[@name="LinksScoper"]/*[@name="seed-redirects-new-seed"] true
|
||||
//controller/*[@name="post-processors"]/*[@name="LinksScoper"]/*[@name="preference-depth-hops"] -1
|
||||
//controller/*[@name="post-processors"]/*[@name="Scheduler"]/*[@name="enabled"] true
|
||||
//controller/*[@name="loggers"]/*[@name="crawl-statistics"]/*[@name="interval-seconds"] 20
|
||||
//controller/*[@name="checkpoint-copy-bdbje-logs"] true
|
||||
//controller/*[@name="recover-retain-failures"] false
|
||||
//controller/*[@name="recover-scope-includes"] true
|
||||
//controller/*[@name="recover-scope-enqueues"] true
|
||||
@@ -0,0 +1,280 @@
|
||||
package org.archive.crawler.migrate;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
import javax.xml.parsers.DocumentBuilder;
|
||||
import javax.xml.parsers.DocumentBuilderFactory;
|
||||
import javax.xml.parsers.ParserConfigurationException;
|
||||
import javax.xml.xpath.XPath;
|
||||
import javax.xml.xpath.XPathConstants;
|
||||
import javax.xml.xpath.XPathExpression;
|
||||
import javax.xml.xpath.XPathExpressionException;
|
||||
import javax.xml.xpath.XPathFactory;
|
||||
|
||||
import org.apache.commons.io.FileUtils;
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.w3c.dom.Document;
|
||||
import org.w3c.dom.Element;
|
||||
import org.w3c.dom.Node;
|
||||
import org.w3c.dom.NodeList;
|
||||
import org.xml.sax.SAXException;
|
||||
|
||||
public class MigrateH1to3Tool {
|
||||
static DocumentBuilder DOCUMENT_BUILDER;
|
||||
static {
|
||||
try {
|
||||
DOCUMENT_BUILDER = DocumentBuilderFactory.newInstance().newDocumentBuilder();
|
||||
} catch (ParserConfigurationException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
|
||||
// TODO: fixxx
|
||||
protected File templateH3job = new File("/Users/steve/Documents/workspace-3.4.2/h3/dist/src/main/conf/jobs/profile-defaults");
|
||||
protected File defaultBeansXmlFile = new File(templateH3job,"profile-crawler-beans.cxml");
|
||||
|
||||
protected File sourceOrderXmlFile;
|
||||
protected File destinationH3JobDir;
|
||||
protected File targetBeansXmlFile;
|
||||
|
||||
protected Document sourceOrderXmlDom;
|
||||
protected Document targetBeansXmlDom;
|
||||
|
||||
public void instanceMain(String[] args) throws Exception {
|
||||
|
||||
sourceOrderXmlFile = new File(args[0]);
|
||||
destinationH3JobDir = new File(args[1]);
|
||||
if(args.length>2) {
|
||||
templateH3job = new File(args[2]);
|
||||
}
|
||||
|
||||
targetBeansXmlFile = new File(destinationH3JobDir,"crawler-beans.cxml");
|
||||
|
||||
FileUtils.copyFile(defaultBeansXmlFile, targetBeansXmlFile);
|
||||
|
||||
sourceOrderXmlDom = DOCUMENT_BUILDER.parse(sourceOrderXmlFile);
|
||||
|
||||
domFindOrderXmlControllerStrings(sourceOrderXmlDom);
|
||||
domFindOrderXmlScopeElements(sourceOrderXmlDom);
|
||||
xpathFindOrderXmlScopeNode(sourceOrderXmlDom);
|
||||
|
||||
targetBeansXmlDom = DOCUMENT_BUILDER.parse(targetBeansXmlFile);
|
||||
System.out.println("targetBeansXmlDom = " + targetBeansXmlDom.getLocalName());
|
||||
unRollNode(targetBeansXmlDom.getElementsByTagName("property"));
|
||||
|
||||
migrateMetadata();
|
||||
|
||||
}
|
||||
|
||||
protected void unRollNode(NodeList node) {
|
||||
System.out.println("unRollNode " + node.toString()); // + node.getLength());
|
||||
for (int i=0; i<node.getLength(); i++) {
|
||||
System.out.println(node.item(i).getNodeName()
|
||||
+ " " + node.item(i).getNodeValue()
|
||||
+ " " + node.item(i).getTextContent());
|
||||
for ( int j=0; j<node.item(i).getChildNodes().getLength(); j++) {
|
||||
Node beanChild = node.item(i).getChildNodes().item(j);
|
||||
System.out.println(" > " + beanChild.getNodeName()
|
||||
+ " " + beanChild.getNodeValue()
|
||||
+ " " + beanChild.getTextContent());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
protected void setBeanProperty(String id, String property, String value)
|
||||
throws Exception {
|
||||
System.out.println("setBeanProperty("+id+","+property+","+value+")");
|
||||
|
||||
// String xpathStr = "//bean[@id=\""+id+"\"]//property[@name=\""+property+"\"]";
|
||||
String xpathStr = "//bean[@id=\"" + id + "\"]";
|
||||
System.out.println(" xpathStr = " + xpathStr);
|
||||
NodeList node = getBeanNode(xpathStr);
|
||||
unRollNode(node);
|
||||
|
||||
}
|
||||
|
||||
protected NodeList getBeanNode(String xpath) throws Exception {
|
||||
return getDocNode(targetBeansXmlDom,xpath);
|
||||
}
|
||||
|
||||
protected NodeList getDocNode(Document doc, String str) throws Exception {
|
||||
System.out.println("getDocNode(" + doc.getLocalName() + "," + str + " )");
|
||||
XPathFactory factory = XPathFactory.newInstance();
|
||||
XPath xpath = factory.newXPath();
|
||||
try {
|
||||
XPathExpression expr = xpath.compile(str);
|
||||
NodeList list = (NodeList) expr.evaluate(doc,XPathConstants.NODE);
|
||||
System.out.println("xpath \"" + str + "\" found " + list.getLength() + " members");
|
||||
return list;
|
||||
} catch (XPathExpressionException e) {
|
||||
System.out.println("XPathExpressionException!");
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
protected String getDocString(Document doc, String str) {
|
||||
XPathFactory factory = XPathFactory.newInstance();
|
||||
XPath xpath = factory.newXPath();
|
||||
try {
|
||||
XPathExpression expr = xpath.compile(str);
|
||||
return expr.evaluate(doc);
|
||||
} catch (XPathExpressionException e) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// JOB_NAME => order_meta_name = bean_metadata_jobName
|
||||
// DESCRIPTION => order_meta_description = bean_metadata_description
|
||||
protected void migrateMetadata() throws Exception {
|
||||
|
||||
String order_meta_name = getDocString(sourceOrderXmlDom,"//meta/name");
|
||||
String order_meta_description = getDocString(sourceOrderXmlDom,"//meta/description");
|
||||
System.out.println("order_meta/name = " + order_meta_name);
|
||||
System.out.println("order_meta/description = " + order_meta_description);
|
||||
|
||||
// NodeList bean_simpleOverrides = getBeanNode("//bean[@=\"simpleOverrides\"]");
|
||||
// unRollNode(bean_simpleOverrides);
|
||||
|
||||
setBeanProperty("simpleOverrides","jobName",order_meta_name);
|
||||
// setBeanProperty("metadata","description",order_meta_description);
|
||||
}
|
||||
|
||||
// CONTACT_URL => map_http-headers_from = bean_metadata_operatorContactUrl
|
||||
// HVERSION => map_http-headers_user-agent=" heritrix/(.*) " =>
|
||||
protected void migrateHttpHeaders() throws Exception {
|
||||
}
|
||||
|
||||
// protected void migrateControllerStrings()
|
||||
// protected void migrateScope()
|
||||
|
||||
protected void xpathFindOrderXmlScopeNode(Document orderXmlDom) throws Exception {
|
||||
// use Xpath to extract scope node
|
||||
XPathFactory factory = XPathFactory.newInstance();
|
||||
XPath xpath = factory.newXPath();
|
||||
String xp = "//controller/newObject[@name=\"scope\"]";
|
||||
System.out.print("==== XPATH {" + xp + "} ====\n");
|
||||
XPathExpression expr = xpath.compile(xp);
|
||||
NodeList scope = (NodeList) expr.evaluate(orderXmlDom,XPathConstants.NODE);
|
||||
for (int j=0; j< scope.getLength(); j++) {
|
||||
if (scope.item(j).getNodeType() == org.w3c.dom.Node.ELEMENT_NODE) {
|
||||
System.out.println(" " + scope.item(j).getNodeName());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
protected void domFindOrderXmlScopeElements(Document orderXmlDom) {
|
||||
Element crawlOrder = orderXmlDom.getDocumentElement();
|
||||
NodeList newObjects = crawlOrder.getElementsByTagName("newObject");
|
||||
for (int i=0; i<newObjects.getLength(); i++) {
|
||||
if (newObjects.item(i).hasAttributes()) {
|
||||
Node name = newObjects.item(i).getAttributes().getNamedItem("name");
|
||||
|
||||
// scope
|
||||
if (name.getNodeValue().equalsIgnoreCase("scope")) {
|
||||
String npath = newObjects.item(i).getNodeName()
|
||||
+ "[" + i + "][@" + name.getNodeName()
|
||||
+ "=\"" + name.getNodeValue() + "\"]";
|
||||
System.out.println("==== DOM search {" + npath + "} ====");
|
||||
NodeList scope = newObjects.item(i).getChildNodes();
|
||||
for (int j=0; j<scope.getLength(); j++) {
|
||||
if (scope.item(j).getNodeType() == org.w3c.dom.Node.ELEMENT_NODE) {
|
||||
System.out.println(" " + scope.item(j).getNodeName());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
protected void domFindOrderXmlControllerStrings(Document orderXmlDom) {
|
||||
Element crawlOrder = orderXmlDom.getDocumentElement();
|
||||
NodeList strings = crawlOrder.getElementsByTagName("string");
|
||||
for (int i=0; i<strings.getLength(); i++) {
|
||||
if (strings.item(i).getParentNode().getNodeName() == "controller") {
|
||||
if (strings.item(i).hasAttributes()) {
|
||||
Node s = strings.item(i);
|
||||
System.out.println(s.getParentNode().getNodeName()
|
||||
+ "/"
|
||||
+ s.getNodeName()
|
||||
+ "[" + i + "]"
|
||||
+ "@"
|
||||
+ s.getAttributes().item(0).getNodeName()
|
||||
+ "["
|
||||
+ s.getAttributes().item(0).getNodeValue()
|
||||
+ "]"
|
||||
+ " = "
|
||||
+ s.getTextContent());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public static void main(String[] args) throws Exception {
|
||||
new MigrateH1to3Tool().instanceMain(args);
|
||||
}
|
||||
|
||||
public static Map<String,String> flattenH1Order(String path) throws Exception {
|
||||
return flattenH1Order(DOCUMENT_BUILDER.parse(new File(path)));
|
||||
}
|
||||
|
||||
/**
|
||||
* Given a Document, return a Map of all non-blank simple text
|
||||
* nodes, keyed by the pseudo-XPath to their parent element.
|
||||
*
|
||||
* @param h1order Document to extract Map
|
||||
* @return Map<String,String> Xpath-like-String -> non-blank text content
|
||||
* @throws XPathExpressionException
|
||||
*/
|
||||
public static Map<String,String> flattenH1Order(Document h1order) throws XPathExpressionException {
|
||||
Map<String,String> flattened = new HashMap<String,String>();
|
||||
XPathExpression xpath = XPathFactory.newInstance().newXPath().compile("//text()");
|
||||
NodeList nodes = (NodeList) xpath.evaluate(h1order,XPathConstants.NODESET);
|
||||
for(int i = 0; i< nodes.getLength(); i++) {
|
||||
Node node = nodes.item(i);
|
||||
if(StringUtils.isNotBlank(node.getTextContent())) {
|
||||
String pseudoXPath = getPseudoXpath(node.getParentNode());
|
||||
pseudoXPath = pseudoXPath.replaceFirst("/crawl-order", "/");
|
||||
|
||||
System.out.println(
|
||||
pseudoXPath
|
||||
+" "+node.getTextContent());
|
||||
|
||||
flattened.put(getPseudoXpath(node.getParentNode()), node.getTextContent());
|
||||
}
|
||||
}
|
||||
System.err.println(flattened.size());
|
||||
System.err.println(flattened);
|
||||
|
||||
return flattened;
|
||||
}
|
||||
|
||||
/**
|
||||
* Given a node, give back an XPath-like string that addresses it.
|
||||
* (For our constrained order.xml files, it is a valid and unique
|
||||
* XPath, but the simple approach used here might not generate
|
||||
* unique XPaths on all XML.
|
||||
*
|
||||
* @param node node to get pseudo-XPath
|
||||
* @return String pseudo-XPath
|
||||
*/
|
||||
protected static String getPseudoXpath(Node node) {
|
||||
String pseudoXpath = "";
|
||||
Node currentNode = node;
|
||||
while(currentNode.getParentNode()!=null) {
|
||||
String thisSegment = currentNode.getNodeName();
|
||||
if(currentNode.getAttributes().getNamedItem("name")!=null) {
|
||||
thisSegment =
|
||||
"*[@"
|
||||
+currentNode.getAttributes().getNamedItem("name")
|
||||
+"]";
|
||||
}
|
||||
pseudoXpath = "/" + thisSegment + pseudoXpath;
|
||||
currentNode = currentNode.getParentNode();
|
||||
}
|
||||
return pseudoXpath;
|
||||
}
|
||||
}
|
||||
@@ -1,215 +0,0 @@
|
||||
package org.archive.crawler.migrate;
|
||||
|
||||
import java.io.File;
|
||||
import javax.xml.parsers.DocumentBuilderFactory;
|
||||
import javax.xml.xpath.XPath;
|
||||
import javax.xml.xpath.XPathConstants;
|
||||
import javax.xml.xpath.XPathExpressionException;
|
||||
import javax.xml.xpath.XPathFactory;
|
||||
import javax.xml.xpath.XPathExpression;
|
||||
|
||||
import org.archive.util.FileUtils;
|
||||
import org.w3c.dom.Document;
|
||||
import org.w3c.dom.Element;
|
||||
import org.w3c.dom.Node;
|
||||
import org.w3c.dom.NodeList;
|
||||
|
||||
public class MigrateSettings {
|
||||
|
||||
private File defaultConfigDir = new File("/Users/steve/Documents/workspace-3.4.2/h3/dist/src/main/conf/jobs/profile-defaults"); // TODO: fixxx
|
||||
private File defaultBeansXmlFile = new File(defaultConfigDir,"profile-crawler-beans.cxml");
|
||||
|
||||
private File srcOrderXmlFile;
|
||||
private File destConfigDir;
|
||||
private File targetBeansXmlFile;
|
||||
|
||||
private Document srcOrderXmlDom;
|
||||
private Document targetBeansXmlDom;
|
||||
|
||||
public void instanceMain(String[] args) throws Exception {
|
||||
|
||||
srcOrderXmlFile = new File(args[0]);
|
||||
destConfigDir = new File(args[1]);
|
||||
targetBeansXmlFile = new File(destConfigDir,"crawler-beans.cxml");
|
||||
|
||||
try {
|
||||
|
||||
FileUtils.copyFile(defaultBeansXmlFile, targetBeansXmlFile);
|
||||
|
||||
setSrcXmlDom(srcOrderXmlFile);
|
||||
domFindOrderXmlControllerStrings(srcOrderXmlDom);
|
||||
domFindOrderXmlScopeElements(srcOrderXmlDom);
|
||||
xpathFindOrderXmlScopeNode(srcOrderXmlDom);
|
||||
|
||||
setTargetXmlDom(targetBeansXmlFile);
|
||||
System.out.println("targetBeansXmlDom = " + targetBeansXmlDom.getLocalName());
|
||||
unRollNode(targetBeansXmlDom.getElementsByTagName("property"));
|
||||
|
||||
migrateMetadata();
|
||||
|
||||
} catch (Exception e) {
|
||||
System.out.println("Caught Exception: " + e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private void unRollNode(NodeList node) {
|
||||
System.out.println("unRollNode " + node.toString()); // + node.getLength());
|
||||
for (int i=0; i<node.getLength(); i++) {
|
||||
System.out.println(node.item(i).getNodeName()
|
||||
+ " " + node.item(i).getNodeValue()
|
||||
+ " " + node.item(i).getTextContent());
|
||||
for ( int j=0; j<node.item(i).getChildNodes().getLength(); j++) {
|
||||
Node beanChild = node.item(i).getChildNodes().item(j);
|
||||
System.out.println(" > " + beanChild.getNodeName()
|
||||
+ " " + beanChild.getNodeValue()
|
||||
+ " " + beanChild.getTextContent());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void setBeanProperty(String id, String property, String value)
|
||||
throws Exception {
|
||||
System.out.println("setBeanProperty("+id+","+property+","+value+")");
|
||||
|
||||
// String xpathStr = "//bean[@id=\""+id+"\"]//property[@name=\""+property+"\"]";
|
||||
String xpathStr = "//bean[@id=\"" + id + "\"]";
|
||||
System.out.println(" xpathStr = " + xpathStr);
|
||||
NodeList node = getBeanNode(xpathStr);
|
||||
unRollNode(node);
|
||||
|
||||
}
|
||||
|
||||
private NodeList getBeanNode(String xpath) throws Exception {
|
||||
return getDocNode(targetBeansXmlDom,xpath);
|
||||
}
|
||||
|
||||
private NodeList getDocNode(Document doc, String str) throws Exception {
|
||||
System.out.println("getDocNode(" + doc.getLocalName() + "," + str + " )");
|
||||
XPathFactory factory = XPathFactory.newInstance();
|
||||
XPath xpath = factory.newXPath();
|
||||
try {
|
||||
XPathExpression expr = xpath.compile(str);
|
||||
NodeList list = (NodeList) expr.evaluate(doc,XPathConstants.NODE);
|
||||
System.out.println("xpath \"" + str + "\" found " + list.getLength() + " members");
|
||||
return list;
|
||||
} catch (XPathExpressionException e) {
|
||||
System.out.println("XPathExpressionException!");
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private String getDocString(Document doc, String str) {
|
||||
XPathFactory factory = XPathFactory.newInstance();
|
||||
XPath xpath = factory.newXPath();
|
||||
try {
|
||||
XPathExpression expr = xpath.compile(str);
|
||||
return expr.evaluate(doc);
|
||||
} catch (XPathExpressionException e) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// JOB_NAME => order_meta_name = bean_metadata_jobName
|
||||
// DESCRIPTION => order_meta_description = bean_metadata_description
|
||||
private void migrateMetadata() throws Exception {
|
||||
|
||||
String order_meta_name = getDocString(srcOrderXmlDom,"//meta/name");
|
||||
String order_meta_description = getDocString(srcOrderXmlDom,"//meta/description");
|
||||
System.out.println("order_meta/name = " + order_meta_name);
|
||||
System.out.println("order_meta/description = " + order_meta_description);
|
||||
|
||||
// NodeList bean_simpleOverrides = getBeanNode("//bean[@=\"simpleOverrides\"]");
|
||||
// unRollNode(bean_simpleOverrides);
|
||||
|
||||
setBeanProperty("simpleOverrides","jobName",order_meta_name);
|
||||
// setBeanProperty("metadata","description",order_meta_description);
|
||||
}
|
||||
|
||||
// CONTACT_URL => map_http-headers_from = bean_metadata_operatorContactUrl
|
||||
// HVERSION => map_http-headers_user-agent=" heritrix/(.*) " =>
|
||||
private void migrateHttpHeaders() throws Exception {
|
||||
}
|
||||
|
||||
// private void migrateControllerStrings()
|
||||
// private void migrateScope()
|
||||
|
||||
private void setTargetXmlDom(File file) throws Exception {
|
||||
DocumentBuilderFactory beansXmlFactory = DocumentBuilderFactory.newInstance();
|
||||
beansXmlFactory.setNamespaceAware(true); // never forget this!
|
||||
targetBeansXmlDom = beansXmlFactory.newDocumentBuilder().parse(file);
|
||||
}
|
||||
|
||||
private void setSrcXmlDom(File file) throws Exception {
|
||||
DocumentBuilderFactory orderXmlFactory = DocumentBuilderFactory.newInstance();
|
||||
orderXmlFactory.setNamespaceAware(true); // never forget this!
|
||||
srcOrderXmlDom = orderXmlFactory.newDocumentBuilder().parse(file);
|
||||
}
|
||||
|
||||
private void xpathFindOrderXmlScopeNode(Document orderXmlDom) throws Exception {
|
||||
// use Xpath to extract scope node
|
||||
XPathFactory factory = XPathFactory.newInstance();
|
||||
XPath xpath = factory.newXPath();
|
||||
String xp = "//controller/newObject[@name=\"scope\"]";
|
||||
System.out.print("==== XPATH {" + xp + "} ====\n");
|
||||
XPathExpression expr = xpath.compile(xp);
|
||||
NodeList scope = (NodeList) expr.evaluate(orderXmlDom,XPathConstants.NODE);
|
||||
for (int j=0; j< scope.getLength(); j++) {
|
||||
if (scope.item(j).getNodeType() == org.w3c.dom.Node.ELEMENT_NODE) {
|
||||
System.out.println(" " + scope.item(j).getNodeName());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void domFindOrderXmlScopeElements(Document orderXmlDom) {
|
||||
Element crawlOrder = orderXmlDom.getDocumentElement();
|
||||
NodeList newObjects = crawlOrder.getElementsByTagName("newObject");
|
||||
for (int i=0; i<newObjects.getLength(); i++) {
|
||||
if (newObjects.item(i).hasAttributes()) {
|
||||
Node name = newObjects.item(i).getAttributes().getNamedItem("name");
|
||||
|
||||
// scope
|
||||
if (name.getNodeValue().equalsIgnoreCase("scope")) {
|
||||
String npath = newObjects.item(i).getNodeName()
|
||||
+ "[" + i + "][@" + name.getNodeName()
|
||||
+ "=\"" + name.getNodeValue() + "\"]";
|
||||
System.out.println("==== DOM search {" + npath + "} ====");
|
||||
NodeList scope = newObjects.item(i).getChildNodes();
|
||||
for (int j=0; j<scope.getLength(); j++) {
|
||||
if (scope.item(j).getNodeType() == org.w3c.dom.Node.ELEMENT_NODE) {
|
||||
System.out.println(" " + scope.item(j).getNodeName());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void domFindOrderXmlControllerStrings(Document orderXmlDom) {
|
||||
Element crawlOrder = orderXmlDom.getDocumentElement();
|
||||
NodeList strings = crawlOrder.getElementsByTagName("string");
|
||||
for (int i=0; i<strings.getLength(); i++) {
|
||||
if (strings.item(i).getParentNode().getNodeName() == "controller") {
|
||||
if (strings.item(i).hasAttributes()) {
|
||||
Node s = strings.item(i);
|
||||
System.out.println(s.getParentNode().getNodeName()
|
||||
+ "/"
|
||||
+ s.getNodeName()
|
||||
+ "[" + i + "]"
|
||||
+ "@"
|
||||
+ s.getAttributes().item(0).getNodeName()
|
||||
+ "["
|
||||
+ s.getAttributes().item(0).getNodeValue()
|
||||
+ "]"
|
||||
+ " = "
|
||||
+ s.getTextContent());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public static void main(String[] args) throws Exception {
|
||||
new MigrateSettings().instanceMain(args);
|
||||
}
|
||||
|
||||
}
|
||||
Reference in new issue
Block a user