From e2ee037786c1ba3fb9ef3dd7e3e9460fd75f7d3a Mon Sep 17 00:00:00 2001 From: szznax Date: Tue, 6 Oct 2009 22:24:19 +0000 Subject: [PATCH] Working commit... --- .../crawler/migrate/MigrateSettings.java | 139 ++++++++++++++---- 1 file changed, 108 insertions(+), 31 deletions(-) diff --git a/engine/src/main/java/org/archive/crawler/migrate/MigrateSettings.java b/engine/src/main/java/org/archive/crawler/migrate/MigrateSettings.java index 05633f5e..6d93192c 100644 --- a/engine/src/main/java/org/archive/crawler/migrate/MigrateSettings.java +++ b/engine/src/main/java/org/archive/crawler/migrate/MigrateSettings.java @@ -4,6 +4,7 @@ import java.io.File; import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.xpath.XPath; import javax.xml.xpath.XPathConstants; +import javax.xml.xpath.XPathExpressionException; import javax.xml.xpath.XPathFactory; import javax.xml.xpath.XPathExpression; @@ -15,57 +16,133 @@ import org.w3c.dom.NodeList; public class MigrateSettings { - private File orderXmlFile; + private File defaultConfigDir = new File("/Users/steve/Documents/workspace-3.4.2/h3/dist/src/main/conf/jobs/profile-defaults"); // TODO: fixxx + private File defaultBeansXmlFile = new File(defaultConfigDir,"profile-crawler-beans.cxml"); + + private File srcOrderXmlFile; private File destConfigDir; - private File defaultConfigDir = new File("/Users/steve/Documents/workspace-3.4.2/h3/jobs/profile-defaults/"); // TODO: fixxx - private File destDefaultBeansXmlFile; - private File destCrawlerBeansXmlFile; - private Document orderXmlDom; - private Document defaultBeansXmlDom; + private File targetBeansXmlFile; + + private Document srcOrderXmlDom; + private Document targetBeansXmlDom; public void instanceMain(String[] args) throws Exception { - orderXmlFile = new File(args[0]); + srcOrderXmlFile = new File(args[0]); destConfigDir = new File(args[1]); - destDefaultBeansXmlFile = new File(destConfigDir,"defaults.xml"); - destCrawlerBeansXmlFile = new File(destConfigDir,"crawler-beans.cxml"); + targetBeansXmlFile = new File(destConfigDir,"crawler-beans.cxml"); try { - FileUtils.copyFile(new File(defaultConfigDir,"defaults.xml"), destDefaultBeansXmlFile); - FileUtils.copyFile(new File(defaultConfigDir,"profile-crawler-beans.cxml"), destCrawlerBeansXmlFile); + FileUtils.copyFile(defaultBeansXmlFile, targetBeansXmlFile); - // parse order XML - if (!orderXmlFile.canRead()) { - throw new Exception("can not read file: " - + orderXmlFile.getAbsolutePath()); - } else { + setSrcXmlDom(srcOrderXmlFile); + domFindOrderXmlControllerStrings(srcOrderXmlDom); + domFindOrderXmlScopeElements(srcOrderXmlDom); + xpathFindOrderXmlScopeNode(srcOrderXmlDom); - setOrderXmlDom(orderXmlFile); + setTargetXmlDom(targetBeansXmlFile); + System.out.println("targetBeansXmlDom = " + targetBeansXmlDom.getLocalName()); + unRollNode(targetBeansXmlDom.getElementsByTagName("property")); - domFindOrderXmlControllerStrings(orderXmlDom); - domFindOrderXmlScopeElements(orderXmlDom); - xpathFindOrderXmlScopeNode(orderXmlDom); + migrateMetadata(); - setDefaultBeansXmlDom(destDefaultBeansXmlFile); - - } - } catch (Exception e) { System.out.println("Caught Exception: " + e.getMessage()); } } - - private void setDefaultBeansXmlDom(File destDefaultBeansXmlFile) throws Exception { - DocumentBuilderFactory defaultBeansXmlFactory = DocumentBuilderFactory.newInstance(); - defaultBeansXmlFactory.setNamespaceAware(true); // never forget this! - this.defaultBeansXmlDom = defaultBeansXmlFactory.newDocumentBuilder().parse(destDefaultBeansXmlFile); + + private void unRollNode(NodeList node) { + System.out.println("unRollNode " + node.toString()); // + node.getLength()); + for (int i=0; i " + beanChild.getNodeName() + + " " + beanChild.getNodeValue() + + " " + beanChild.getTextContent()); + } + } } - private void setOrderXmlDom(File orderXmlFile) throws Exception { + private void setBeanProperty(String id, String property, String value) + throws Exception { + System.out.println("setBeanProperty("+id+","+property+","+value+")"); + + // String xpathStr = "//bean[@id=\""+id+"\"]//property[@name=\""+property+"\"]"; + String xpathStr = "//bean[@id=\"" + id + "\"]"; + System.out.println(" xpathStr = " + xpathStr); + NodeList node = getBeanNode(xpathStr); + unRollNode(node); + + } + + private NodeList getBeanNode(String xpath) throws Exception { + return getDocNode(targetBeansXmlDom,xpath); + } + + private NodeList getDocNode(Document doc, String str) throws Exception { + System.out.println("getDocNode(" + doc.getLocalName() + "," + str + " )"); + XPathFactory factory = XPathFactory.newInstance(); + XPath xpath = factory.newXPath(); + try { + XPathExpression expr = xpath.compile(str); + NodeList list = (NodeList) expr.evaluate(doc,XPathConstants.NODE); + System.out.println("xpath \"" + str + "\" found " + list.getLength() + " members"); + return list; + } catch (XPathExpressionException e) { + System.out.println("XPathExpressionException!"); + return null; + } + } + + private String getDocString(Document doc, String str) { + XPathFactory factory = XPathFactory.newInstance(); + XPath xpath = factory.newXPath(); + try { + XPathExpression expr = xpath.compile(str); + return expr.evaluate(doc); + } catch (XPathExpressionException e) { + return null; + } + } + + // JOB_NAME => order_meta_name = bean_metadata_jobName + // DESCRIPTION => order_meta_description = bean_metadata_description + private void migrateMetadata() throws Exception { + + String order_meta_name = getDocString(srcOrderXmlDom,"//meta/name"); + String order_meta_description = getDocString(srcOrderXmlDom,"//meta/description"); + System.out.println("order_meta/name = " + order_meta_name); + System.out.println("order_meta/description = " + order_meta_description); + +// NodeList bean_simpleOverrides = getBeanNode("//bean[@=\"simpleOverrides\"]"); +// unRollNode(bean_simpleOverrides); + + setBeanProperty("simpleOverrides","jobName",order_meta_name); + // setBeanProperty("metadata","description",order_meta_description); + } + + // CONTACT_URL => map_http-headers_from = bean_metadata_operatorContactUrl + // HVERSION => map_http-headers_user-agent=" heritrix/(.*) " => + private void migrateHttpHeaders() throws Exception { + } + +// private void migrateControllerStrings() +// private void migrateScope() + + private void setTargetXmlDom(File file) throws Exception { + DocumentBuilderFactory beansXmlFactory = DocumentBuilderFactory.newInstance(); + beansXmlFactory.setNamespaceAware(true); // never forget this! + targetBeansXmlDom = beansXmlFactory.newDocumentBuilder().parse(file); + } + + private void setSrcXmlDom(File file) throws Exception { DocumentBuilderFactory orderXmlFactory = DocumentBuilderFactory.newInstance(); orderXmlFactory.setNamespaceAware(true); // never forget this! - this.orderXmlDom = orderXmlFactory.newDocumentBuilder().parse(orderXmlFile); + srcOrderXmlDom = orderXmlFactory.newDocumentBuilder().parse(file); } private void xpathFindOrderXmlScopeNode(Document orderXmlDom) throws Exception {