This is an automated email from the ASF dual-hosted git repository. east pushed a commit to branch reformat_mmanager_tostring in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit e7c6ca82173578644627a6b00960333b4920bc85 Author: mdf369 <[email protected]> AuthorDate: Thu Apr 4 16:39:17 2019 +0800 reimplement MTree.toString() by JSON add MManager.combineMetadataInStrings(String[] metadatas) for cluster module --- .../java/org/apache/iotdb/db/metadata/MGraph.java | 30 +++++- .../org/apache/iotdb/db/metadata/MManager.java | 16 +++- .../java/org/apache/iotdb/db/metadata/MTree.java | 103 ++++++++++++--------- .../org/apache/iotdb/db/monitor/StatMonitor.java | 2 +- .../iotdb/db/qp/executor/OverflowQPExecutor.java | 2 +- .../org/apache/iotdb/db/metadata/MGraphTest.java | 79 ++++++++++++++++ .../iotdb/db/metadata/MManagerBasicTest.java | 22 ++--- .../org/apache/iotdb/db/metadata/MTreeTest.java | 43 +++++++++ 8 files changed, 237 insertions(+), 60 deletions(-) diff --git a/iotdb/src/main/java/org/apache/iotdb/db/metadata/MGraph.java b/iotdb/src/main/java/org/apache/iotdb/db/metadata/MGraph.java index 5cf2847..afe0cdc 100644 --- a/iotdb/src/main/java/org/apache/iotdb/db/metadata/MGraph.java +++ b/iotdb/src/main/java/org/apache/iotdb/db/metadata/MGraph.java @@ -20,12 +20,14 @@ package org.apache.iotdb.db.metadata; import java.io.Serializable; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import org.apache.iotdb.db.exception.MetadataArgsErrorException; import org.apache.iotdb.db.exception.PathErrorException; +import org.apache.iotdb.tsfile.common.conf.TSFileConfig; import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; @@ -39,6 +41,7 @@ public class MGraph implements Serializable { private static final long serialVersionUID = 8214849219614352834L; private static final String DOUB_SEPARATOR = "\\."; private static final String TIME_SERIES_INCORRECT = "Timeseries's root is not Correct. RootName: "; + public static final String TIME_SERIES_TREE_HEADER = "=== Timeseries Tree ===\n\n"; private MTree mtree; private HashMap<String, PTree> ptreeMap; @@ -59,12 +62,24 @@ public class MGraph implements Serializable { } /** + * this is just for compatibility + */ + public void addPathToMTree(String path, String dataType, String encoding) + throws PathErrorException { + TSDataType tsDataType = TSDataType.valueOf(dataType); + TSEncoding tsEncoding = TSEncoding.valueOf(encoding); + CompressionType compressionType = CompressionType.valueOf(TSFileConfig.compressor); + addPathToMTree(path, tsDataType, tsEncoding, compressionType, + Collections.emptyMap()); + } + + /** * Add a seriesPath to Metadata Tree. * * @param path Format: root.node.(node)* */ public void addPathToMTree(String path, TSDataType dataType, TSEncoding encoding, - CompressionType compressor, Map<String, String> props) throws PathErrorException, MetadataArgsErrorException { + CompressionType compressor, Map<String, String> props) throws PathErrorException { String[] nodes = path.trim().split(DOUB_SEPARATOR); if (nodes.length == 0) { throw new PathErrorException("Timeseries is null"); @@ -366,8 +381,19 @@ public class MGraph implements Serializable { @Override public String toString() { StringBuilder sb = new StringBuilder(); - sb.append("=== Timeseries Tree ===\n\n"); + sb.append(TIME_SERIES_TREE_HEADER); sb.append(mtree.toString()); return sb.toString(); } + + /** + * combine multiple metadata in string format + */ + public static String combineMetadataInStrings(String[] metadatas) { + for (int i = 0; i < metadatas.length; i++) { + metadatas[i] = metadatas[i].replace(TIME_SERIES_TREE_HEADER, ""); + } + String res = MTree.combineMetadataInStrings(metadatas); + return TIME_SERIES_TREE_HEADER + res; + } } diff --git a/iotdb/src/main/java/org/apache/iotdb/db/metadata/MManager.java b/iotdb/src/main/java/org/apache/iotdb/db/metadata/MManager.java index eef21ca..9edb93f 100644 --- a/iotdb/src/main/java/org/apache/iotdb/db/metadata/MManager.java +++ b/iotdb/src/main/java/org/apache/iotdb/db/metadata/MManager.java @@ -259,7 +259,7 @@ public class MManager { */ public void addPathToMTree(String path, TSDataType dataType, TSEncoding encoding, CompressionType compressor, Map<String, String> props) - throws PathErrorException, IOException, MetadataArgsErrorException { + throws PathErrorException, IOException { lock.writeLock().lock(); try { @@ -292,7 +292,7 @@ public class MManager { * @param encoding the encoding function {@code Encoding} for the timeseries */ public void addPathToMTree(String path, String dataType, String encoding) - throws PathErrorException, IOException, MetadataArgsErrorException { + throws PathErrorException, IOException { TSDataType tsDataType = TSDataType.valueOf(dataType); TSEncoding tsEncoding = TSEncoding.valueOf(encoding); CompressionType type = CompressionType.valueOf(TSFileConfig.compressor); @@ -992,6 +992,18 @@ public class MManager { } /** + * combine multiple metadata in string format + */ + private String combineMetadataInStrings(String[] metadatas) { + lock.readLock().lock(); + try { + return MGraph.combineMetadataInStrings(metadatas); + } finally { + lock.readLock().unlock(); + } + } + + /** * Check whether {@code seriesPath} exists and whether {@code seriesPath} has been set storage * level. * diff --git a/iotdb/src/main/java/org/apache/iotdb/db/metadata/MTree.java b/iotdb/src/main/java/org/apache/iotdb/db/metadata/MTree.java index 6444b4a..a000de6 100644 --- a/iotdb/src/main/java/org/apache/iotdb/db/metadata/MTree.java +++ b/iotdb/src/main/java/org/apache/iotdb/db/metadata/MTree.java @@ -18,6 +18,9 @@ */ package org.apache.iotdb.db.metadata; +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONObject; +import com.alibaba.fastjson.serializer.SerializerFeature; import java.io.Serializable; import java.util.ArrayList; import java.util.Collections; @@ -25,6 +28,7 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import org.apache.iotdb.db.exception.PathErrorException; import org.apache.iotdb.tsfile.common.conf.TSFileConfig; import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType; @@ -67,6 +71,7 @@ public class MTree implements Serializable { addTimeseriesPath(timeseriesPath, tsDataType, tsEncoding, compressionType, Collections.emptyMap()); } + /** * function for adding timeseries.It should check whether seriesPath exists. */ @@ -1012,59 +1017,71 @@ public class MTree implements Serializable { @Override public String toString() { - return mnodeToString(getRoot(), 0); + return jsonToString(toJson()); } - private String mnodeToString(MNode node, int tab) { - StringBuilder builder = new StringBuilder(); - for (int i = 0; i < tab; i++) { - builder.append(QUAD_SPACE); - } - builder.append(node.getName()); + private static String jsonToString(JSONObject jsonObject) { + return JSON.toJSONString(jsonObject, SerializerFeature.PrettyFormat); + } + + private JSONObject toJson() { + return mnodeToJSON(getRoot()); + } + + private JSONObject mnodeToJSON(MNode node) { + JSONObject jsonObject = new JSONObject(); if (!node.isLeaf() && node.getChildren().size() > 0) { - builder.append(":{\n"); - int first = 0; for (MNode child : node.getChildren().values()) { - if (first == 0) { - first = 1; - } else { - builder.append(",\n"); - } - builder.append(mnodeToString(child, tab + 1)); - } - builder.append("\n"); - for (int i = 0; i < tab; i++) { - builder.append(QUAD_SPACE); + jsonObject.put(child.getName(), mnodeToJSON(child)); } - builder.append("}"); } else if (node.isLeaf()) { - builder.append(":{\n"); - builder - .append(String.format("%s DataType: %s,\n", getTabs(tab + 1), node.getSchema().getType())); - builder - .append(String.format("%s Encoding: %s,\n", getTabs(tab + 1), node.getSchema().getEncodingType())); - - builder - .append(String.format("%s Compressor: %s,\n", getTabs(tab + 1), node.getSchema().getCompressor())); - builder - .append(String.format("%s args: %s,\n", getTabs(tab + 1), node.getSchema().getProps())); - builder.append( - String.format("%s StorageGroup: %s\n", getTabs(tab + 1), node.getDataFileName())); - builder.append(getTabs(tab)); - builder.append("}"); - } - return builder.toString(); - } - - private String getTabs(int count) { - StringBuilder sb = new StringBuilder(); - for (int i = 0; i < count; i++) { - sb.append(QUAD_SPACE); + jsonObject.put("DataType", node.getSchema().getType()); + jsonObject.put("Encoding", node.getSchema().getEncodingType()); + jsonObject.put("Compressor", node.getSchema().getCompressor()); + jsonObject.put("args", node.getSchema().getProps().toString()); + jsonObject.put("StorageGroup", node.getDataFileName()); } - return sb.toString(); + return jsonObject; } public MNode getRoot() { return root; } + + /** + * combine multiple metadata in string format + */ + public static String combineMetadataInStrings(String[] metadatas) { + JSONObject[] jsonObjects = new JSONObject[metadatas.length]; + for (int i = 0; i < jsonObjects.length; i++) { + jsonObjects[i] = JSONObject.parseObject(metadatas[i]); + } + + JSONObject root = jsonObjects[0]; + for (int i = 1; i < jsonObjects.length; i++) { + root = combineJSONObjects(root, jsonObjects[i]); + } + return jsonToString(root); + } + + private static JSONObject combineJSONObjects(JSONObject a, JSONObject b) { + JSONObject res = new JSONObject(); + + Set<String> retainSet = new HashSet<>(a.keySet()); + retainSet.retainAll(b.keySet()); + Set<String> aCha = new HashSet<>(a.keySet()); + Set<String> bCha = new HashSet<>(b.keySet()); + aCha.removeAll(retainSet); + bCha.removeAll(retainSet); + for (String key : aCha) { + res.put(key, a.getJSONObject(key)); + } + for (String key : bCha) { + res.put(key, b.get(key)); + } + for (String key : retainSet) { + res.put(key, combineJSONObjects(a.getJSONObject(key), b.getJSONObject(key))); + } + return res; + } } diff --git a/iotdb/src/main/java/org/apache/iotdb/db/monitor/StatMonitor.java b/iotdb/src/main/java/org/apache/iotdb/db/monitor/StatMonitor.java index a9a5355..052c324 100644 --- a/iotdb/src/main/java/org/apache/iotdb/db/monitor/StatMonitor.java +++ b/iotdb/src/main/java/org/apache/iotdb/db/monitor/StatMonitor.java @@ -169,7 +169,7 @@ public class StatMonitor implements IService { Collections.emptyMap()); } } - } catch (MetadataArgsErrorException | IOException | PathErrorException e) { + } catch (IOException | PathErrorException e) { LOGGER.error("Initialize the metadata error.", e); } } diff --git a/iotdb/src/main/java/org/apache/iotdb/db/qp/executor/OverflowQPExecutor.java b/iotdb/src/main/java/org/apache/iotdb/db/qp/executor/OverflowQPExecutor.java index 061bb47..d2cad93 100644 --- a/iotdb/src/main/java/org/apache/iotdb/db/qp/executor/OverflowQPExecutor.java +++ b/iotdb/src/main/java/org/apache/iotdb/db/qp/executor/OverflowQPExecutor.java @@ -618,7 +618,7 @@ public class OverflowQPExecutor extends QueryProcessExecutor { default: throw new ProcessorException("unknown namespace type:" + namespaceType); } - } catch (PathErrorException | IOException | ArgsErrorException | FileNodeManagerException e) { + } catch (PathErrorException | IOException | FileNodeManagerException e) { throw new ProcessorException(e.getMessage()); } return true; diff --git a/iotdb/src/test/java/org/apache/iotdb/db/metadata/MGraphTest.java b/iotdb/src/test/java/org/apache/iotdb/db/metadata/MGraphTest.java new file mode 100644 index 0000000..2d7dd3e --- /dev/null +++ b/iotdb/src/test/java/org/apache/iotdb/db/metadata/MGraphTest.java @@ -0,0 +1,79 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata; + +import static org.junit.Assert.*; + +import org.apache.iotdb.db.exception.PathErrorException; +import org.apache.iotdb.db.utils.EnvironmentUtils; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +public class MGraphTest { + + @Before + public void setUp() throws Exception { + } + + @After + public void tearDown() throws Exception { + EnvironmentUtils.cleanEnv(); + } + + @Test + public void testCombineMetadataInStrings() { + MGraph root = new MGraph("root"); + MGraph root1 = new MGraph("root"); + MGraph root2 = new MGraph("root"); + MGraph root3 = new MGraph("root"); + try { + root.setStorageLevel("root.a.d0"); + root.addPathToMTree("root.a.d0.s0", "INT32", "RLE"); + root.addPathToMTree("root.a.d0.s1", "INT32", "RLE"); + + root.setStorageLevel("root.a.d1"); + root.addPathToMTree("root.a.d1.s0", "INT32", "RLE"); + root.addPathToMTree("root.a.d1.s1", "INT32", "RLE"); + + root.setStorageLevel("root.a.b.d0"); + root.addPathToMTree("root.a.b.d0.s0", "INT32", "RLE"); + + root1.setStorageLevel("root.a.d0"); + root1.addPathToMTree("root.a.d0.s0", "INT32", "RLE"); + root1.addPathToMTree("root.a.d0.s1", "INT32", "RLE"); + + root2.setStorageLevel("root.a.d1"); + root2.addPathToMTree("root.a.d1.s0", "INT32", "RLE"); + root2.addPathToMTree("root.a.d1.s1", "INT32", "RLE"); + + root3.setStorageLevel("root.a.b.d0"); + root3.addPathToMTree("root.a.b.d0.s0", "INT32", "RLE"); + + String[] metadatas = new String[3]; + metadatas[0] = root1.toString(); + metadatas[1] = root2.toString(); + metadatas[2] = root3.toString(); + assertEquals(MGraph.combineMetadataInStrings(metadatas), root.toString()); + } catch (PathErrorException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } +} \ No newline at end of file diff --git a/iotdb/src/test/java/org/apache/iotdb/db/metadata/MManagerBasicTest.java b/iotdb/src/test/java/org/apache/iotdb/db/metadata/MManagerBasicTest.java index 2133cc0..36dba77 100644 --- a/iotdb/src/test/java/org/apache/iotdb/db/metadata/MManagerBasicTest.java +++ b/iotdb/src/test/java/org/apache/iotdb/db/metadata/MManagerBasicTest.java @@ -79,7 +79,7 @@ public class MManagerBasicTest { manager.addPathToMTree("root.laptop.d1.s0", TSDataType.valueOf("INT32"), TSEncoding.valueOf("RLE"), compressionType, Collections .emptyMap()); - } catch (PathErrorException | MetadataArgsErrorException | IOException e) { + } catch (PathErrorException | IOException e) { e.printStackTrace(); fail(e.getMessage()); } @@ -90,7 +90,7 @@ public class MManagerBasicTest { try { manager.addPathToMTree("root.laptop.d1.s1", TSDataType.valueOf("INT32"), TSEncoding.valueOf("RLE"), compressionType, Collections.emptyMap()); - } catch (PathErrorException | MetadataArgsErrorException | IOException e1) { + } catch (PathErrorException | IOException e1) { e1.printStackTrace(); fail(e1.getMessage()); } @@ -125,7 +125,7 @@ public class MManagerBasicTest { try { manager.addPathToMTree("root.laptop.d1.s1", TSDataType.valueOf("INT32"), TSEncoding.valueOf("RLE"), compressionType, Collections.emptyMap()); - } catch (PathErrorException | MetadataArgsErrorException | IOException e1) { + } catch (PathErrorException | IOException e1) { e1.printStackTrace(); fail(e1.getMessage()); } @@ -133,7 +133,7 @@ public class MManagerBasicTest { try { manager.addPathToMTree("root.laptop.d1.s0", TSDataType.valueOf("INT32"), TSEncoding.valueOf("RLE"), compressionType, Collections.emptyMap()); - } catch (PathErrorException | MetadataArgsErrorException | IOException e1) { + } catch (PathErrorException | IOException e1) { e1.printStackTrace(); fail(e1.getMessage()); } @@ -178,7 +178,7 @@ public class MManagerBasicTest { try { manager.addPathToMTree("root.laptop.d2.s1", TSDataType.valueOf("INT32"), TSEncoding.valueOf("RLE"), compressionType, Collections.emptyMap()); - } catch (PathErrorException | MetadataArgsErrorException | IOException e1) { + } catch (PathErrorException | IOException e1) { e1.printStackTrace(); fail(e1.getMessage()); } @@ -186,7 +186,7 @@ public class MManagerBasicTest { try { manager.addPathToMTree("root.laptop.d2.s0", TSDataType.valueOf("INT32"), TSEncoding.valueOf("RLE"), compressionType, Collections.emptyMap()); - } catch (PathErrorException | MetadataArgsErrorException | IOException e1) { + } catch (PathErrorException | IOException e1) { e1.printStackTrace(); fail(e1.getMessage()); } @@ -207,7 +207,7 @@ public class MManagerBasicTest { try { manager.addPathToMTree("root.laptop.d1.s0", TSDataType.valueOf("INT32"), TSEncoding.valueOf("RLE"), compressionType, Collections.emptyMap()); - } catch (PathErrorException | MetadataArgsErrorException | IOException e1) { + } catch (PathErrorException | IOException e1) { e1.printStackTrace(); fail(e1.getMessage()); } @@ -215,7 +215,7 @@ public class MManagerBasicTest { try { manager.addPathToMTree("root.laptop.d1.s1", TSDataType.valueOf("INT32"), TSEncoding.valueOf("RLE"), compressionType, Collections.emptyMap()); - } catch (PathErrorException | MetadataArgsErrorException | IOException e1) { + } catch (PathErrorException | IOException e1) { e1.printStackTrace(); fail(e1.getMessage()); } @@ -232,7 +232,7 @@ public class MManagerBasicTest { try { manager.addPathToMTree("root.laptop.d1.s2", TSDataType.valueOf("INT32"), TSEncoding.valueOf("RLE"), compressionType, Collections.emptyMap()); - } catch (PathErrorException | MetadataArgsErrorException | IOException e1) { + } catch (PathErrorException | IOException e1) { e1.printStackTrace(); fail(e1.getMessage()); } @@ -248,7 +248,7 @@ public class MManagerBasicTest { try { manager.addPathToMTree("root.laptop.d1.s3", TSDataType.valueOf("INT32"), TSEncoding.valueOf("RLE"), compressionType, Collections.emptyMap()); - } catch (PathErrorException | MetadataArgsErrorException | IOException e1) { + } catch (PathErrorException | IOException e1) { e1.printStackTrace(); fail(e1.getMessage()); } @@ -309,7 +309,7 @@ public class MManagerBasicTest { list.add("root.laptop.d2"); assertEquals(list, manager.getAllFileNamesByPath("root.laptop")); assertEquals(list, manager.getAllFileNamesByPath("root")); - } catch (PathErrorException | IOException | MetadataArgsErrorException e) { + } catch (PathErrorException | IOException e) { e.printStackTrace(); fail(e.getMessage()); } diff --git a/iotdb/src/test/java/org/apache/iotdb/db/metadata/MTreeTest.java b/iotdb/src/test/java/org/apache/iotdb/db/metadata/MTreeTest.java index 3e45d4b..4f838e1 100644 --- a/iotdb/src/test/java/org/apache/iotdb/db/metadata/MTreeTest.java +++ b/iotdb/src/test/java/org/apache/iotdb/db/metadata/MTreeTest.java @@ -22,6 +22,9 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONObject; +import com.alibaba.fastjson.serializer.SerializerFeature; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -128,6 +131,46 @@ public class MTreeTest { } @Test + public void testCombineMetadataInStrings() { + MTree root = new MTree("root"); + MTree root1 = new MTree("root"); + MTree root2 = new MTree("root"); + MTree root3 = new MTree("root"); + try { + root.setStorageGroup("root.a.d0"); + root.addTimeseriesPath("root.a.d0.s0", "INT32", "RLE"); + root.addTimeseriesPath("root.a.d0.s1", "INT32", "RLE"); + + root.setStorageGroup("root.a.d1"); + root.addTimeseriesPath("root.a.d1.s0", "INT32", "RLE"); + root.addTimeseriesPath("root.a.d1.s1", "INT32", "RLE"); + + root.setStorageGroup("root.a.b.d0"); + root.addTimeseriesPath("root.a.b.d0.s0", "INT32", "RLE"); + + root1.setStorageGroup("root.a.d0"); + root1.addTimeseriesPath("root.a.d0.s0", "INT32", "RLE"); + root1.addTimeseriesPath("root.a.d0.s1", "INT32", "RLE"); + + root2.setStorageGroup("root.a.d1"); + root2.addTimeseriesPath("root.a.d1.s0", "INT32", "RLE"); + root2.addTimeseriesPath("root.a.d1.s1", "INT32", "RLE"); + + root3.setStorageGroup("root.a.b.d0"); + root3.addTimeseriesPath("root.a.b.d0.s0", "INT32", "RLE"); + + String[] metadatas = new String[3]; + metadatas[0] = root1.toString(); + metadatas[1] = root2.toString(); + metadatas[2] = root3.toString(); + assertEquals(MTree.combineMetadataInStrings(metadatas), root.toString()); + } catch (PathErrorException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test public void testSetStorageGroup() { // set storage group first MTree root = new MTree("root");
