Update of
/var/cvs/contributions/CMSContainer_Modules/dataconversion/src/java/com/finalist/cmsc/dataconversion/service
In directory james.mmbase.org:/tmp/cvs-serv19613/service
Modified Files:
Constants.java Conversion.java Data.java NodeService.java
XMLUtil.java
Log Message:
CMSC-1166, KNMI migration to the CMS Container,make some improvements to get
the data imported properly
See also:
http://cvs.mmbase.org/viewcvs/contributions/CMSContainer_Modules/dataconversion/src/java/com/finalist/cmsc/dataconversion/service
See also: http://www.mmbase.org/jira/browse/CMSC-1166
Index: Constants.java
===================================================================
RCS file:
/var/cvs/contributions/CMSContainer_Modules/dataconversion/src/java/com/finalist/cmsc/dataconversion/service/Constants.java,v
retrieving revision 1.1
retrieving revision 1.2
diff -u -b -r1.1 -r1.2
--- Constants.java 6 May 2008 11:05:49 -0000 1.1
+++ Constants.java 28 Nov 2008 07:30:44 -0000 1.2
@@ -10,4 +10,6 @@
public static final Byte ROOT_CATEGORY_TYPE = 3;
+ public static final Byte RELATION_DATA_TYPE = 4;
+
}
Index: Conversion.java
===================================================================
RCS file:
/var/cvs/contributions/CMSContainer_Modules/dataconversion/src/java/com/finalist/cmsc/dataconversion/service/Conversion.java,v
retrieving revision 1.4
retrieving revision 1.5
diff -u -b -r1.4 -r1.5
--- Conversion.java 7 Oct 2008 13:05:59 -0000 1.4
+++ Conversion.java 28 Nov 2008 07:30:44 -0000 1.5
@@ -4,6 +4,7 @@
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
+import java.util.Map;
import java.util.Properties;
import javax.servlet.ServletContext;
@@ -16,6 +17,7 @@
import org.w3c.dom.NodeList;
import com.finalist.cmsc.dataconversion.dataaccess.DataAccessDelegate;
+import com.finalist.cmsc.dataconversion.dataaccess.DataAccessor;
import com.finalist.cmsc.dataconversion.dataaccess.DataHolder;
import com.finalist.cmsc.dataconversion.dataaccess.DataSourceFactory;
@@ -60,125 +62,199 @@
Document dom = null;
long beginTime = System.currentTimeMillis();
try {
-
-
log.info(" ############################### begin import
######################");
dom =
XMLParser.parseXMLToDOM(context.getResourceAsStream(IMPORT_FILE));
// XMLParser.DTDValidator(dom);
- Element element = (Element)dom.getElementsByTagName("import").item(0);
+ Element element = (Element)
dom.getElementsByTagName("import").item(0);
NodeList list = element.getChildNodes();
List<Data> sources = new ArrayList<Data>();
- for(int i = 0 ; i < list.getLength(); i++) {
- //datatype node
- if(list.item(i).getNodeName().equals("datatype")) {
- //deal with data type node
-
log.info(String.format(table_log,XMLUtil.getSourceType((Element)list.item(i))));
- Data sorData = getDataOfDataType((Element)list.item(i));
- //load data number from
- HashMap<Integer,Integer> keys =
DataAccessDelegate.getNumbersOfDataType((Element)list.item(i), dataSource);
-
log.info(String.format(result_count,keys.size(),XMLUtil.getSourceType((Element)list.item(i))));
+ List<Data> clondSources = new ArrayList<Data>();
+ int j= list.getLength();
+ for (int i = 0; i < j; i++) {
+ // datatype node
+ if (list.item(i).getNodeName().equals("datatype")) {
+ // deal with data type node
+ log.info(String.format(table_log,
XMLUtil.getSourceType((Element) list.item(i))));
+ Data sorData = getDataOfDataType((Element) list.item(i));
+ // load data number from
+ HashMap<Integer, Integer> keys =
DataAccessDelegate.getNumbersOfDataType((Element) list.item(i),
+ dataSource);
+ log.info(String.format(result_count, keys.size(), XMLUtil
+ .getSourceType((Element) list.item(i))));
sorData.setIdentifiers(keys);
sources.add(sorData);
-
- if(XMLUtil.hasSelfRelation((Element)list.item(i))) {
-
log.info(String.format(sefRelation_log,XMLUtil.getSourceType((Element)list.item(i))));
- Data relData = getDataOfSelfRelation((Element)list.item(i));
- HashMap<Integer,Integer> keys1 =
DataAccessDelegate.getNumbersOfSelfRelation((Element)list.item(i), dataSource);
-
log.info(String.format(result_count,keys1.size(),XMLUtil.getSourceType((Element)list.item(i))));
+ //log.info("did I chage? "+Utils.getXML(list.item(i)));
+ if (XMLUtil.hasSelfRelation((Element) list.item(i))) {
+ log.info(String.format(sefRelation_log,
XMLUtil.getSourceType((Element) list.item(i))));
+ Data relData = getDataOfSelfRelation((Element) list.item(i));
+ HashMap<Integer, Integer> keys1 =
DataAccessDelegate.getNumbersOfSelfRelation((Element) list.item(i),
+ dataSource);
+ log.info(String.format(result_count, keys1.size(),
XMLUtil.getSourceType((Element) list
+ .item(i))));
if (keys1 == null || keys1.isEmpty()) {
continue;
}
relData.setIdentifiers(keys1);
sources.add(relData);
}
- Data rootData = getRootData((Element)list.item(i));
- HashMap<Integer,Integer> keysd =
DataAccessDelegate.addChildRelation((Element)list.item(i), dataSource);
- log.info(String.format(sefRelation_log,keysd.size()));
+ Data rootData = getRootData((Element) list.item(i));
+ HashMap<Integer, Integer> keysd = DataAccessDelegate
+ .addChildRelation((Element) list.item(i), dataSource);
+ log.info(String.format(sefRelation_log, keysd.size()));
rootData.setIdentifiers(keysd);
rootData.setRelateId(Integer.parseInt(node));
-
- // sources.add(rootData);
log.info("----> before recur");
- recur((Element)list.item(i),sources);
+ recur((Element) list.item(i), sources);
log.info("----> after recur");
- for(Data data:sources) {
- if(data.getType() == Constants.ENTITY_TYPE) {
- log.info(String.format(" ----> begin to save type
[%s]",data.getDestinationType()));
+ relDatatype((Element) list.item(i), sources);
+ saveType(list, sources, i);
+ log.info("entity type and it relation finished.........");
+ NodeService.insertData(null, rootData, sources, null);
+ }
+ //end if
+ clondSources.addAll(sources);
+ sources.clear();
+ }
+ NodeService.insertMigrationMappings(clondSources);
+ createRelationDataType(clondSources);
+ linkRoot(list, j);
+ log.info(String.format(
+ " ---->#######################finished importing [time:%s
mins] #############################",
+ (System.currentTimeMillis() - beginTime) / (1000 * 60)));
+ NodeService.insertProperties(properties);
+ } catch (Exception e) {
+ log.error(e.getMessage());
+ e.printStackTrace();
}
- else if(data.getType() == Constants.RELATION_TYPE) {
- log.info(String.format(" ----> begin to save type
[%s]",data.getDestinationRelationType()));
}
- else if(data.getType() == Constants.SELF_RELATION_TYPE) {
- log.info(String.format(" ----> begin to save type
[%s]",data.getSourceRelationType()));
+
+ private void linkRoot(NodeList list, int j) {
+ Map<String, ArrayList<Integer>> rootDatas=new HashMap<String,
ArrayList<Integer>>();
+ for (int i = 0; i < j; i++) {
+ if (list.item(i).getNodeName().equals("datatype")) {
+ String destinationtype=XMLUtil.getDestinationType((Element)
list.item(i));
+ String sourcetype=XMLUtil.getSourceType((Element) list.item(i));
+ DataAccessor da=new DataAccessor(dataSource);
+ ArrayList<Integer> primerKeys=da.getPrimerKeyList(sourcetype);
+ rootDatas.put(destinationtype, primerKeys);
+ }
+ }
+ NodeService.linkRootDatas(rootDatas,node);
+ }
+
+ private void createRelationDataType(List<Data> sources) throws Exception {
+ List<String> reskeys = new ArrayList<String>();
+ for (Data reldata : sources) {
+ if (reldata.getType() == Constants.RELATION_DATA_TYPE) {
+ reskeys = DataAccessDelegate.getResOfRelation(sources, reldata,
dataSource);
+ NodeService.createRelationData(reskeys, reldata);
+ }
+ }
+ }
+
+ private void saveType(NodeList list, List<Data> sources, int i) throws
Exception {
+ for (Data data : sources) {
+ if (data.getType() != Constants.RELATION_DATA_TYPE) {
+ if (data.getType() == Constants.ENTITY_TYPE) {
+ log.info(String.format(" ----> begin to save type [%s]",
data.getDestinationType()));
+ } else if (data.getType() == Constants.RELATION_TYPE) {
+ log.info(String.format(" ----> begin to save type [%s]",
data.getDestinationRelationType()));
+ } else if (data.getType() == Constants.SELF_RELATION_TYPE) {
+ log.info(String.format(" ----> begin to save type [%s]",
data.getSourceRelationType()));
}
Iterator<Integer> iterator =
data.getIdentifiers().keySet().iterator();
- while(iterator.hasNext()) {
+ while (iterator.hasNext()) {
Integer key = iterator.next();
- Element elment =
XMLUtil.getElementByTableName(data.getTableName(),(Element)list.item(i));
- DataHolder holder =
DataAccessDelegate.getElementByPrimaryKey(elment, dataSource,
key,data.getType());
- Integer number =
NodeService.insertData(holder,data,sources,key);
+ Element elment =
XMLUtil.getElementByTableName(data.getTableName(), (Element) list.item(i));
+ DataHolder holder =
DataAccessDelegate.getElementByPrimaryKey(elment, dataSource, key,
data.getType());
+ Integer number = NodeService.insertData(holder, data, sources,
key);
+ if (null!=number) {
data.getIdentifiers().put(key, number);
}
- log.info("----> end saving ");
- }
- System.out.println("finished.........");
- NodeService.insertData(null,rootData,sources,null);
- }
}
- log.info(String.format(" ---->#######################finished
importing [time:%s mins]
#############################",(System.currentTimeMillis()-beginTime)/(1000*60)));
- NodeService.insertProperties(properties);
+ log.info("----> end saving ");
}
- catch (Exception e) {
- log.error(e.getMessage());
- e.printStackTrace();
}
}
+
/**
* recur to get elements related
- * @param element dom element
- * @param sources a collection object which hold Data Objects
+ *
+ * @param element
+ * dom element
+ * @param sources
+ * a collection object which hold Data Objects
* @throws Exception
*/
- private void recur(Element element,List<Data> sources) throws Exception {
+ private void recur(Element element, List<Data> sources) throws Exception {
Element[] relates = XMLUtil.getDirectRelateChildNodes(element);
- if(relates.length < 1) {
- return ;
+ if (relates.length < 1) {
+ return;
}
- for(int i = 0 ; i < relates.length ; i++) {
- log.info("###################### begin Import table
["+XMLUtil.getSourceType(relates[i])+"] ####################");
- log.info(String.format(table_log,XMLUtil.getSourceType(relates[i])));
+ for (int i = 0; i < relates.length; i++) {
+ log.info("###################### begin Import table [" +
XMLUtil.getSourceType(relates[i])
+ + "] ####################");
+ log.info(String.format(table_log, XMLUtil.getSourceType(relates[i])));
Data sorData = getDataOfDataType(relates[i]);
- HashMap<Integer,Integer> keys =
DataAccessDelegate.getNumbersOfDataType(relates[i], dataSource);
-
log.info(String.format(result_count,keys.size(),XMLUtil.getSourceType(relates[i])));
+ HashMap<Integer, Integer> keys =
DataAccessDelegate.getNumbersOfDataType(relates[i], dataSource);
+ log.info(String.format(result_count, keys.size(),
XMLUtil.getSourceType(relates[i])));
if (keys == null || keys.isEmpty()) {
continue;
}
sorData.setIdentifiers(keys);
sources.add(sorData);
-
log.info(String.format(relation_log,XMLUtil.getSourceType(element),XMLUtil.getSourceType(relates[i]),XMLUtil.getSourceRelationType(relates[i])));
+ log.info(String.format(relation_log, XMLUtil.getSourceType(element),
XMLUtil
+ .getSourceType(relates[i]),
XMLUtil.getSourceRelationType(relates[i])));
Data relData = getDataOfRelation(relates[i]);
- HashMap<Integer,Integer> keys1 =
DataAccessDelegate.getNumbersOfRelation(relates[i], dataSource);
-
log.info(String.format(result_rel_count,keys1.size(),XMLUtil.getSourceRelationType(relates[i])));
+ HashMap<Integer, Integer> keys1 =
DataAccessDelegate.getNumbersOfRelation(relates[i], dataSource);
+ log.info(String.format(result_rel_count, keys1.size(),
XMLUtil.getSourceRelationType(relates[i])));
if (keys1 != null && !keys1.isEmpty()) {
relData.setIdentifiers(keys1);
sources.add(relData);
}
- log.info("###################### end Import table
["+XMLUtil.getSourceType(relates[i])+"] ####################");
+ log.info("###################### end Import table [" +
XMLUtil.getSourceType(relates[i])
+ + "] ####################");
recur(relates[i], sources);
}
}
- private void setData(Data data,Element element) {
+ private void relDatatype(Element element, List<Data> sources) throws
Exception {
+ Element[] relates = XMLUtil.getRelateChildNodes(element);
+ for (int i = 0; i < relates.length; i++) {
+ log.info("###################### begin import relateddatatype Type ["
+ + XMLUtil.getSourceType(relates[i]) + "]
####################");
+ Data relData = getDataOfRelationType(relates[i]);
+ sources.add(relData);
+ log.info("###################### end import relateddatatype Type ["
+ + XMLUtil.getSourceType(relates[i]) + "]
####################");
+ }
+ }
+
+ // add getDataOfRelationType method
+ private Data getDataOfRelationType(Element element) {
+ Data relData = new Data(Constants.RELATION_DATA_TYPE);
+ setData(relData, element);
+ String destion = XMLUtil.getDestinationRelationType(element);
+ relData.setDestinationRelationType(destion);
+ String sourceRelationType = XMLUtil.getSourceRelationType(element);
+ relData.setSourceRelationType(sourceRelationType);
+ String reverse = XMLUtil.getReverseRelationType(element);
+ relData.setReverse(reverse);
+ return relData;
+ }
+
+ private void setData(Data data, Element element) {
data.setTableName(XMLUtil.getSourceType(element));
data.setDestinationType(XMLUtil.getDestinationType(element));
data.setRelationType(XMLUtil.getRelateType(element));
data.setRelateTable(XMLUtil.getRelateSourceType(element));
}
+
private Data getDataOfDataType(Element element) {
Data sorData = new Data(Constants.ENTITY_TYPE);
- setData(sorData,element);
+ setData(sorData, element);
sorData.setDestinationRelationType(XMLUtil.getDestinationRelationType(element));
return sorData;
}
@@ -195,8 +271,12 @@
Data relData = new Data(Constants.RELATION_TYPE);
setData(relData,element);
relData.setDestinationRelationType(XMLUtil.getDestinationRelationType(element));
+ //add #############
+ String reverse=XMLUtil.getReverseRelationType(element);
+ relData.setReverse(reverse);
return relData;
}
+
private Data getRootData(Element element) {
Data rootData = new Data(Constants.ROOT_CATEGORY_TYPE);
setData(rootData,element);
Index: Data.java
===================================================================
RCS file:
/var/cvs/contributions/CMSContainer_Modules/dataconversion/src/java/com/finalist/cmsc/dataconversion/service/Data.java,v
retrieving revision 1.2
retrieving revision 1.3
diff -u -b -r1.2 -r1.3
--- Data.java 7 Oct 2008 13:05:59 -0000 1.2
+++ Data.java 28 Nov 2008 07:30:44 -0000 1.3
@@ -24,6 +24,8 @@
private String relationType;
+ private String reverse ;
+
public Data() {
}
@@ -110,4 +112,12 @@
public void setRelationType(String relationType) {
this.relationType = relationType;
}
+
+ public String getReverse() {
+ return reverse;
+ }
+
+ public void setReverse(String reverse) {
+ this.reverse = reverse;
+ }
}
Index: NodeService.java
===================================================================
RCS file:
/var/cvs/contributions/CMSContainer_Modules/dataconversion/src/java/com/finalist/cmsc/dataconversion/service/NodeService.java,v
retrieving revision 1.4
retrieving revision 1.5
diff -u -b -r1.4 -r1.5
--- NodeService.java 16 May 2008 11:45:45 -0000 1.4
+++ NodeService.java 28 Nov 2008 07:30:44 -0000 1.5
@@ -1,17 +1,26 @@
package com.finalist.cmsc.dataconversion.service;
+import java.util.ArrayList;
import java.util.Collection;
+import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Properties;
+
import net.sf.mmapps.commons.bridge.RelationUtil;
+
import org.mmbase.bridge.Cloud;
import org.mmbase.bridge.CloudContext;
import org.mmbase.bridge.ContextProvider;
+import org.mmbase.bridge.Field;
import org.mmbase.bridge.Node;
+import org.mmbase.bridge.NodeList;
import org.mmbase.bridge.NodeManager;
+import org.mmbase.bridge.NodeQuery;
import org.mmbase.bridge.Relation;
+import org.mmbase.bridge.util.SearchUtil;
+import org.mmbase.datatypes.DataType;
import org.mmbase.module.core.MMBase;
import org.mmbase.module.core.MMObjectNode;
import org.mmbase.util.logging.Logger;
@@ -87,7 +96,12 @@
Node sourceNode = cloud.getNode(snumber.intValue());
Node desNode = cloud.getNode(dnumber.intValue());
type = data.getDestinationRelationType();
- Relation relate = RelationUtil.createRelation(sourceNode, desNode,
data.getDestinationRelationType());
+ Relation relate;
+ if ("true".equals(data.getReverse())) {
+ relate = RelationUtil.createRelation(desNode, sourceNode,
data.getDestinationRelationType());
+ }else{
+ relate = RelationUtil.createRelation(sourceNode, desNode,
data.getDestinationRelationType());
+ }
if(sourceNode.getNodeManager().getName().equals("contentchannel")
&& desNode.getNodeManager().getName().equals("article")) {
Relation creationRelation =
RelationUtil.createRelation(desNode, sourceNode, "creationrel");
creationRelation.commit();
@@ -131,16 +145,57 @@
Integer id = iterator.next();
Integer dId = getDnumber( data, id,sources);
Node desNode = cloud.getNode(dId.intValue());
- Relation relate = RelationUtil.createRelation(sourceNode,
desNode, data.getDestinationRelationType());
+ Relation relate = RelationUtil.createRelation(sourceNode,
desNode, type);
relate.commit();
}
}
} catch (Exception e) {
- // TODO Auto-generated catch block
log.info(String.format("[type %s] [old Node %s] [Node %s ]
+"+e.getMessage(),type,key,number));
}
return number;
}
+ //createRelationData for the element relateddatatype
+ public static void createRelationData(List<String> reskeys, Data reldata) {
+ Cloud cloud = initCloud();
+ String type = "";
+ try {
+ if (reldata.getType() == Constants.RELATION_DATA_TYPE) {
+ for (String sd : reskeys) {
+ String[] spd = sd.split(",");
+ Integer snumber = getNewkey(cloud, Integer.parseInt(spd[0]));
+ Integer dnumber = getNewkey(cloud, Integer.parseInt(spd[1]));
+
+ if (snumber > 0 && dnumber > 0) {
+ Node sourceNode = cloud.getNode(snumber.intValue());
+ Node desNode = cloud.getNode(dnumber.intValue());
+ type = reldata.getDestinationRelationType();
+ Relation relate;
+ if ("true".equals(reldata.getReverse())) {
+ relate = RelationUtil.createRelation(desNode, sourceNode,
type);
+ } else {
+ relate = RelationUtil.createRelation(sourceNode, desNode,
type);
+ }
+ relate.commit();
+ }
+ }
+ }
+ } catch (Exception e) {
+ log.info(String.format("[type %s] [old Node %s] [Node %s ] +" +
e.getMessage(), type));
+ }
+ }
+ // search newNumber from mapping
+ private static Integer getNewkey(Cloud cloud,int parseInt) {
+ NodeManager manager = cloud.getNodeManager("migration_mappings");
+ NodeQuery query = manager.createQuery();
+ SearchUtil.addEqualConstraint(query, manager.getField("old_number"),
parseInt);
+ NodeList list = query.getList();
+ if(list != null && list.size() >0) {
+ Node node = (Node)list.get(0);
+ return node.getIntValue("new_number");
+ }else{
+ return -1;
+ }
+ }
/**
* get the dnumber from data collection
* @param data
@@ -210,9 +265,50 @@
node.commit();
}
+ // insertMigrationMappings to produce a mapping between old and new node
numbers.
+ public static void insertMigrationMappings(List<Data> clonedSources) throws
Exception {
+ Cloud cloud = initCloud();
+ NodeManager manager = cloud.getNodeManager("migration_mappings");
+ for (Data data : clonedSources) {
+ if (data.getType() == Constants.ENTITY_TYPE || data.getType() ==
Constants.ROOT_CATEGORY_TYPE) {
+ HashMap<Integer, Integer> mappingNumber = data.getIdentifiers();
+ for (Iterator iter = mappingNumber.entrySet().iterator();
iter.hasNext();) {
+ Map.Entry entry = (Map.Entry) iter.next();
+ Node node = manager.createNode();
+ Object key = entry.getKey();
+ Object val = entry.getValue();
+ node.setObjectValue("old_number", key);
+ node.setObjectValue("new_number", val);
+ node.commit();
+ }
+ }
+ }
+ log.info("----> end insertMigrationMappings ");
+ }
+
private static Cloud initCloud() {
CloudContext context = ContextProvider.getDefaultCloudContext();
return context.getCloud("mmbase");
}
+
+ public static void linkRootDatas(Map<String, ArrayList<Integer>> rootDatas,
String node) {
+ Cloud cloud = initCloud();
+ Node desNode = cloud.getNode(Integer.parseInt(node));
+ for (Iterator iter = rootDatas.entrySet().iterator(); iter.hasNext();) {
+ Map.Entry entry = (Map.Entry) iter.next();
+ String key = (String) entry.getKey();
+ ArrayList<Integer> val = (ArrayList<Integer>) entry.getValue();
+ NodeManager manager = cloud.getNodeManager(key);
+ for (Integer i : val) {
+ int sn=getNewkey(cloud,i);
+ if (sn>0) {
+ Node sourceNode = cloud.getNode(sn);
+ Relation relate = RelationUtil.createRelation(sourceNode,
desNode, "creationrel");
+ Relation rel = RelationUtil.createRelation(desNode, sourceNode,
"contentrel");
+ }
+ }
+ }
+ }
+
}
Index: XMLUtil.java
===================================================================
RCS file:
/var/cvs/contributions/CMSContainer_Modules/dataconversion/src/java/com/finalist/cmsc/dataconversion/service/XMLUtil.java,v
retrieving revision 1.3
retrieving revision 1.4
diff -u -b -r1.3 -r1.4
--- XMLUtil.java 7 Oct 2008 13:05:59 -0000 1.3
+++ XMLUtil.java 28 Nov 2008 07:30:44 -0000 1.4
@@ -145,5 +145,21 @@
return ele;
}
+ public static String getReverseRelationType(Element element) {
+ return element.getAttribute("reverse");
+ }
+
+ public static Element[] getRelateChildNodes(Element element) {
+ Element[] elements = new Element[0];
+ List<Element> list = new ArrayList<Element>();
+ NodeList relationList = element.getElementsByTagName("relateddatatype");
+ for(int i = 0 ; i < relationList.getLength() ; i++) {
+ if(((Element)relationList.item(i)).getParentNode() == element) {
+ list.add((Element)relationList.item(i));
+ }
+ }
+ return list.toArray(elements);
+ }
+
}
_______________________________________________
Cvs mailing list
[email protected]
http://lists.mmbase.org/mailman/listinfo/cvs
