mirror of
https://github.com/internetarchive/heritrix3.git
synced 2026-11-09 15:36:40 +00:00
Working commit...
This commit is contained in:
szznax committed
1 parent
258da53d57
commit
e2ee037786
1 file changed
+108
-31
@@ -4,6 +4,7 @@ import java.io.File;
|
||||
import javax.xml.parsers.DocumentBuilderFactory;
|
||||
import javax.xml.xpath.XPath;
|
||||
import javax.xml.xpath.XPathConstants;
|
||||
import javax.xml.xpath.XPathExpressionException;
|
||||
import javax.xml.xpath.XPathFactory;
|
||||
import javax.xml.xpath.XPathExpression;
|
||||
|
||||
@@ -15,57 +16,133 @@ import org.w3c.dom.NodeList;
|
||||
|
||||
public class MigrateSettings {
|
||||
|
||||
private File orderXmlFile;
|
||||
private File defaultConfigDir = new File("/Users/steve/Documents/workspace-3.4.2/h3/dist/src/main/conf/jobs/profile-defaults"); // TODO: fixxx
|
||||
private File defaultBeansXmlFile = new File(defaultConfigDir,"profile-crawler-beans.cxml");
|
||||
|
||||
private File srcOrderXmlFile;
|
||||
private File destConfigDir;
|
||||
private File defaultConfigDir = new File("/Users/steve/Documents/workspace-3.4.2/h3/jobs/profile-defaults/"); // TODO: fixxx
|
||||
private File destDefaultBeansXmlFile;
|
||||
private File destCrawlerBeansXmlFile;
|
||||
private Document orderXmlDom;
|
||||
private Document defaultBeansXmlDom;
|
||||
private File targetBeansXmlFile;
|
||||
|
||||
private Document srcOrderXmlDom;
|
||||
private Document targetBeansXmlDom;
|
||||
|
||||
public void instanceMain(String[] args) throws Exception {
|
||||
|
||||
orderXmlFile = new File(args[0]);
|
||||
srcOrderXmlFile = new File(args[0]);
|
||||
destConfigDir = new File(args[1]);
|
||||
destDefaultBeansXmlFile = new File(destConfigDir,"defaults.xml");
|
||||
destCrawlerBeansXmlFile = new File(destConfigDir,"crawler-beans.cxml");
|
||||
targetBeansXmlFile = new File(destConfigDir,"crawler-beans.cxml");
|
||||
|
||||
try {
|
||||
|
||||
FileUtils.copyFile(new File(defaultConfigDir,"defaults.xml"), destDefaultBeansXmlFile);
|
||||
FileUtils.copyFile(new File(defaultConfigDir,"profile-crawler-beans.cxml"), destCrawlerBeansXmlFile);
|
||||
FileUtils.copyFile(defaultBeansXmlFile, targetBeansXmlFile);
|
||||
|
||||
// parse order XML
|
||||
if (!orderXmlFile.canRead()) {
|
||||
throw new Exception("can not read file: "
|
||||
+ orderXmlFile.getAbsolutePath());
|
||||
} else {
|
||||
setSrcXmlDom(srcOrderXmlFile);
|
||||
domFindOrderXmlControllerStrings(srcOrderXmlDom);
|
||||
domFindOrderXmlScopeElements(srcOrderXmlDom);
|
||||
xpathFindOrderXmlScopeNode(srcOrderXmlDom);
|
||||
|
||||
setOrderXmlDom(orderXmlFile);
|
||||
setTargetXmlDom(targetBeansXmlFile);
|
||||
System.out.println("targetBeansXmlDom = " + targetBeansXmlDom.getLocalName());
|
||||
unRollNode(targetBeansXmlDom.getElementsByTagName("property"));
|
||||
|
||||
domFindOrderXmlControllerStrings(orderXmlDom);
|
||||
domFindOrderXmlScopeElements(orderXmlDom);
|
||||
xpathFindOrderXmlScopeNode(orderXmlDom);
|
||||
migrateMetadata();
|
||||
|
||||
setDefaultBeansXmlDom(destDefaultBeansXmlFile);
|
||||
|
||||
}
|
||||
|
||||
} catch (Exception e) {
|
||||
System.out.println("Caught Exception: " + e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private void setDefaultBeansXmlDom(File destDefaultBeansXmlFile) throws Exception {
|
||||
DocumentBuilderFactory defaultBeansXmlFactory = DocumentBuilderFactory.newInstance();
|
||||
defaultBeansXmlFactory.setNamespaceAware(true); // never forget this!
|
||||
this.defaultBeansXmlDom = defaultBeansXmlFactory.newDocumentBuilder().parse(destDefaultBeansXmlFile);
|
||||
|
||||
private void unRollNode(NodeList node) {
|
||||
System.out.println("unRollNode " + node.toString()); // + node.getLength());
|
||||
for (int i=0; i<node.getLength(); i++) {
|
||||
System.out.println(node.item(i).getNodeName()
|
||||
+ " " + node.item(i).getNodeValue()
|
||||
+ " " + node.item(i).getTextContent());
|
||||
for ( int j=0; j<node.item(i).getChildNodes().getLength(); j++) {
|
||||
Node beanChild = node.item(i).getChildNodes().item(j);
|
||||
System.out.println(" > " + beanChild.getNodeName()
|
||||
+ " " + beanChild.getNodeValue()
|
||||
+ " " + beanChild.getTextContent());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void setOrderXmlDom(File orderXmlFile) throws Exception {
|
||||
private void setBeanProperty(String id, String property, String value)
|
||||
throws Exception {
|
||||
System.out.println("setBeanProperty("+id+","+property+","+value+")");
|
||||
|
||||
// String xpathStr = "//bean[@id=\""+id+"\"]//property[@name=\""+property+"\"]";
|
||||
String xpathStr = "//bean[@id=\"" + id + "\"]";
|
||||
System.out.println(" xpathStr = " + xpathStr);
|
||||
NodeList node = getBeanNode(xpathStr);
|
||||
unRollNode(node);
|
||||
|
||||
}
|
||||
|
||||
private NodeList getBeanNode(String xpath) throws Exception {
|
||||
return getDocNode(targetBeansXmlDom,xpath);
|
||||
}
|
||||
|
||||
private NodeList getDocNode(Document doc, String str) throws Exception {
|
||||
System.out.println("getDocNode(" + doc.getLocalName() + "," + str + " )");
|
||||
XPathFactory factory = XPathFactory.newInstance();
|
||||
XPath xpath = factory.newXPath();
|
||||
try {
|
||||
XPathExpression expr = xpath.compile(str);
|
||||
NodeList list = (NodeList) expr.evaluate(doc,XPathConstants.NODE);
|
||||
System.out.println("xpath \"" + str + "\" found " + list.getLength() + " members");
|
||||
return list;
|
||||
} catch (XPathExpressionException e) {
|
||||
System.out.println("XPathExpressionException!");
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private String getDocString(Document doc, String str) {
|
||||
XPathFactory factory = XPathFactory.newInstance();
|
||||
XPath xpath = factory.newXPath();
|
||||
try {
|
||||
XPathExpression expr = xpath.compile(str);
|
||||
return expr.evaluate(doc);
|
||||
} catch (XPathExpressionException e) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// JOB_NAME => order_meta_name = bean_metadata_jobName
|
||||
// DESCRIPTION => order_meta_description = bean_metadata_description
|
||||
private void migrateMetadata() throws Exception {
|
||||
|
||||
String order_meta_name = getDocString(srcOrderXmlDom,"//meta/name");
|
||||
String order_meta_description = getDocString(srcOrderXmlDom,"//meta/description");
|
||||
System.out.println("order_meta/name = " + order_meta_name);
|
||||
System.out.println("order_meta/description = " + order_meta_description);
|
||||
|
||||
// NodeList bean_simpleOverrides = getBeanNode("//bean[@=\"simpleOverrides\"]");
|
||||
// unRollNode(bean_simpleOverrides);
|
||||
|
||||
setBeanProperty("simpleOverrides","jobName",order_meta_name);
|
||||
// setBeanProperty("metadata","description",order_meta_description);
|
||||
}
|
||||
|
||||
// CONTACT_URL => map_http-headers_from = bean_metadata_operatorContactUrl
|
||||
// HVERSION => map_http-headers_user-agent=" heritrix/(.*) " =>
|
||||
private void migrateHttpHeaders() throws Exception {
|
||||
}
|
||||
|
||||
// private void migrateControllerStrings()
|
||||
// private void migrateScope()
|
||||
|
||||
private void setTargetXmlDom(File file) throws Exception {
|
||||
DocumentBuilderFactory beansXmlFactory = DocumentBuilderFactory.newInstance();
|
||||
beansXmlFactory.setNamespaceAware(true); // never forget this!
|
||||
targetBeansXmlDom = beansXmlFactory.newDocumentBuilder().parse(file);
|
||||
}
|
||||
|
||||
private void setSrcXmlDom(File file) throws Exception {
|
||||
DocumentBuilderFactory orderXmlFactory = DocumentBuilderFactory.newInstance();
|
||||
orderXmlFactory.setNamespaceAware(true); // never forget this!
|
||||
this.orderXmlDom = orderXmlFactory.newDocumentBuilder().parse(orderXmlFile);
|
||||
srcOrderXmlDom = orderXmlFactory.newDocumentBuilder().parse(file);
|
||||
}
|
||||
|
||||
private void xpathFindOrderXmlScopeNode(Document orderXmlDom) throws Exception {
|
||||
|
||||
Reference in new issue
Block a user