This is an automated email from the ASF dual-hosted git repository. haonan pushed a commit to branch new_TsFile in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit 544453d77af0303f40f634c21c73e591bcd8a6d1 Merge: 3c7ea7b d6ae1c3 Author: HTHou <[email protected]> AuthorDate: Fri Feb 28 14:51:07 2020 +0800 new TsFile README.md | 2 +- client-py/src/client_example.py | 33 +++++------ .../SystemDesign/5-DataQuery/2-SeriesReader.md | 2 +- .../UserGuide/0-Get Started/1-QuickStart.md | 2 +- .../UserGuide/4-Client/6-Status Codes.md | 65 +++++++++++++++++++++ .../2-DML (Data Manipulation Language).md | 23 +++++++- .../5-Operation Manual/4-SQL Reference.md | 25 ++++---- docs/Documentation/UserGuide/0-Content.md | 1 + .../UserGuide/0-Get Started/1-QuickStart.md | 9 ++- .../UserGuide/4-Client/6-Status Codes.md | 66 ++++++++++++++++++++++ .../2-DML (Data Manipulation Language).md | 19 ++++++- .../5-Operation Manual/4-SQL Reference.md | 12 ++-- .../main/java/org/apache/iotdb/JDBCExample.java | 2 +- .../org/apache/iotdb/jdbc/IoTDBConnection.java | 27 +++++---- .../org/apache/iotdb/jdbc/IoTDBResultMetadata.java | 7 +-- .../apache/iotdb/jdbc/IoTDBResultMetadataTest.java | 30 +++++----- .../org/apache/iotdb/db/qp/strategy/SqlBase.g4 | 33 ++--------- .../org/apache/iotdb/db/conf/IoTDBConfigCheck.java | 33 +++++++---- .../apache/iotdb/db/engine/cache/ChunkCache.java | 37 +++++++----- .../iotdb/db/engine/memtable/AbstractMemTable.java | 2 +- .../iotdb/db/engine/merge/task/MergeTask.java | 3 +- .../engine/storagegroup/StorageGroupProcessor.java | 37 +++++++----- .../org/apache/iotdb/db/metadata/MManager.java | 2 +- .../java/org/apache/iotdb/db/metadata/MTree.java | 64 +++++++++++++++++---- .../apache/iotdb/db/qp/executor/PlanExecutor.java | 9 ++- .../iotdb/db/qp/strategy/PhysicalGenerator.java | 13 +---- .../db/query/aggregation/impl/AvgAggrResult.java | 6 ++ .../db/query/aggregation/impl/CountAggrResult.java | 2 +- .../iotdb/db/query/control/FileReaderManager.java | 34 +++++++---- .../iotdb/db/query/control/QueryFileManager.java | 17 +++--- .../dataset/RawQueryDataSetWithoutValueFilter.java | 13 +++-- .../iotdb/db/query/reader/series/SeriesReader.java | 8 +-- .../reader/universal/PriorityMergeReader.java | 26 +++++++-- .../org/apache/iotdb/db/service/TSServiceImpl.java | 19 ++++++- .../java/org/apache/iotdb/db/utils/MemUtils.java | 21 ++++--- .../org/apache/iotdb/db/utils/SchemaUtils.java | 3 +- .../integration/IOTDBGroupByInnerIntervalIT.java | 4 +- .../iotdb/db/integration/IoTDBAggregationIT.java | 8 +-- .../integration/IoTDBAggregationSmallDataIT.java | 2 +- .../iotdb/db/integration/IoTDBAlignByDeviceIT.java | 4 +- .../integration/IoTDBLoadExternalTsfileTest.java | 2 +- .../iotdb/db/integration/IoTDBMultiSeriesIT.java | 4 +- .../iotdb/db/integration/IoTDBSeriesReaderIT.java | 4 ++ .../iotdb/db/integration/IoTDBSimpleQueryTest.java | 59 +++++++++++++++++++ .../iotdb/db/metadata/MManagerAdvancedTest.java | 14 +++-- .../iotdb/db/metadata/MManagerBasicTest.java | 57 ++++++++++++++++++- .../java/org/apache/iotdb/db/qp/PlannerTest.java | 16 ++++-- .../db/query/aggregation/AggregateResultTest.java | 2 + .../main/java/org/apache/iotdb/rpc/RpcUtils.java | 2 +- .../java/org/apache/iotdb/rpc/TSStatusCode.java | 4 +- service-rpc/src/main/thrift/rpc.thrift | 3 +- .../java/org/apache/iotdb/session/Session.java | 14 +++-- .../org/apache/iotdb/spark/db/Converter.scala | 37 ++++++------ .../org/apache/iotdb/spark/db/DefaultSource.scala | 37 ++++++------ .../org/apache/iotdb/spark/db/IoTDBOptions.scala | 37 ++++++------ .../scala/org/apache/iotdb/spark/db/IoTDBRDD.scala | 37 ++++++------ .../org/apache/iotdb/spark/db/IoTDBRelation.scala | 37 ++++++------ .../org/apache/iotdb/spark/db/Transformer.scala | 37 ++++++------ .../scala/org/apache/iotdb/spark/db/package.scala | 37 ++++++------ .../org/apache/iotdb/spark/db/IoTDBTest.scala | 37 ++++++------ .../org/apache/iotdb/spark/tsfile/Converter.scala | 37 ++++++------ .../apache/iotdb/spark/tsfile/DefaultSource.scala | 36 ++++++------ .../iotdb/spark/tsfile/NarrowConverter.scala | 37 ++++++------ .../spark/tsfile/NarrowTsFileOutputWriter.scala | 37 ++++++------ .../apache/iotdb/spark/tsfile/Transformer.scala | 37 ++++++------ .../iotdb/spark/tsfile/TsFileWriterFactory.scala | 36 ++++++------ .../apache/iotdb/spark/tsfile/WideConverter.scala | 37 ++++++------ .../spark/tsfile/WideTsFileOutputWriter.scala | 37 ++++++------ .../org/apache/iotdb/spark/tsfile/package.scala | 36 ++++++------ .../apache/iotdb/spark/tsfile/ConverterTest.scala | 37 ++++++------ .../org/apache/iotdb/spark/tsfile/TSFileSuit.scala | 37 ++++++------ .../iotdb/tsfile/read/TsFileSequenceReader.java | 27 ++++----- .../tsfile/read/query/dataset/EmptyDataSet.java | 39 +++++++++++++ .../read/query/timegenerator/node/AndNode.java | 14 ++++- .../read/query/timegenerator/node/LeafNode.java | 11 ++-- .../tsfile/read/reader/DefaultTsFileInput.java | 2 +- .../tsfile/read/query/timegenerator/NodeTest.java | 66 +++++----------------- .../iotdb/tsfile/read/reader/FakedBatchReader.java | 55 ++++++++++++++++++ 78 files changed, 1172 insertions(+), 648 deletions(-) diff --cc server/src/main/java/org/apache/iotdb/db/engine/merge/task/MergeTask.java index b84b393,533bd9f..c17783f --- a/server/src/main/java/org/apache/iotdb/db/engine/merge/task/MergeTask.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/merge/task/MergeTask.java @@@ -23,10 -23,7 +23,11 @@@ import java.io.File import java.io.IOException; import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.Map.Entry; ++import java.util.Set; import java.util.concurrent.Callable; import org.apache.iotdb.db.engine.merge.manage.MergeContext; import org.apache.iotdb.db.engine.merge.manage.MergeResource; @@@ -113,15 -110,9 +114,15 @@@ public class MergeTask implements Calla mergeLogger.logFiles(resource); - List<String> devices = MManager.getInstance().getDevices(storageGroupName); - List<MeasurementSchema> measurementSchemas = MManager.getInstance() - .getStorageGroupSchema(storageGroupName); - resource.addMeasurements(measurementSchemas); ++ Set<String> devices = MManager.getInstance().getDevices(storageGroupName); + Map<Path, MeasurementSchema> measurementSchemaMap = new HashMap<>(); + for (String device : devices) { + Map<String, MeasurementSchema> schema = MManager.getInstance().getDeviceSchemaMap(device); + for (Entry<String, MeasurementSchema> entry : schema.entrySet()) { + measurementSchemaMap.put(new Path(device, entry.getKey()), entry.getValue()); + } + } + resource.setMeasurementSchemaMap(measurementSchemaMap); List<String> storageGroupPaths = MManager.getInstance().getAllTimeseriesName(storageGroupName + ".*"); List<Path> unmergedSeries = new ArrayList<>(); diff --cc server/src/main/java/org/apache/iotdb/db/metadata/MTree.java index 090590d,9a07c20..b8f8fd7 --- a/server/src/main/java/org/apache/iotdb/db/metadata/MTree.java +++ b/server/src/main/java/org/apache/iotdb/db/metadata/MTree.java @@@ -538,15 -505,18 +540,18 @@@ public class MTree implements Serializa return; } String nodeReg = MetaUtils.getNodeRegByIdx(idx, nodes); - if (!(PATH_WILDCARD).equals(nodeReg)) { + if (!nodeReg.contains(PATH_WILDCARD)) { if (node.hasChild(nodeReg)) { findPath(node.getChild(nodeReg), nodes, idx + 1, parent + node.getName() + PATH_SEPARATOR, - timeseriesSchemaList); + MeasurementSchemaList); } } else { for (MNode child : node.getChildren().values()) { + if (!Pattern.matches(nodeReg.replace("*", ".*"), child.getName())) { + continue; + } findPath(child, nodes, idx + 1, parent + node.getName() + PATH_SEPARATOR, - timeseriesSchemaList); + MeasurementSchemaList); } } } diff --cc server/src/main/java/org/apache/iotdb/db/utils/SchemaUtils.java index 388dc13,c18ce58..21c99f6 --- a/server/src/main/java/org/apache/iotdb/db/utils/SchemaUtils.java +++ b/server/src/main/java/org/apache/iotdb/db/utils/SchemaUtils.java @@@ -18,14 -18,9 +18,15 @@@ */ package org.apache.iotdb.db.utils; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.Map.Entry; ++import java.util.Set; + import org.apache.iotdb.db.exception.metadata.MetadataException; import org.apache.iotdb.db.metadata.MManager; +import org.apache.iotdb.tsfile.read.common.Path; import org.apache.iotdb.tsfile.write.schema.MeasurementSchema; import org.apache.iotdb.tsfile.write.schema.Schema; @@@ -42,15 -37,9 +43,15 @@@ public class SchemaUtils * @return the schema of the FileNode named processorName. */ public static Schema constructSchema(String processorName) throws MetadataException { - List<String> devices = MManager.getInstance().getDevices(processorName); - List<MeasurementSchema> columnSchemaList; - columnSchemaList = MManager.getInstance().getStorageGroupSchema(processorName); - return getSchemaFromColumnSchema(columnSchemaList); ++ Set<String> devices = MManager.getInstance().getDevices(processorName); + Map<Path, MeasurementSchema> measurementSchemaMap = new HashMap<>(); + for (String device : devices) { + Map<String, MeasurementSchema> schema = MManager.getInstance().getDeviceSchemaMap(device); + for (Entry<String, MeasurementSchema> entry : schema.entrySet()) { + measurementSchemaMap.put(new Path(device, entry.getKey()), entry.getValue()); + } + } + return getSchemaFromColumnSchema(measurementSchemaMap); } /** diff --cc tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java index a81a0b9,6fd2e43..8a3e96d --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java @@@ -53,6 -40,17 +41,19 @@@ import org.apache.iotdb.tsfile.write.sc import org.slf4j.Logger; import org.slf4j.LoggerFactory; + import java.io.File; + import java.io.IOException; + import java.nio.ByteBuffer; + import java.util.ArrayList; + import java.util.Comparator; ++import java.util.HashMap; ++import java.util.LinkedHashMap; + import java.util.List; + import java.util.Map; + import java.util.concurrent.ConcurrentHashMap; + + import static org.apache.iotdb.tsfile.write.writer.TsFileIOWriter.magicStringBytes; + public class TsFileSequenceReader implements AutoCloseable { private static final Logger logger = LoggerFactory.getLogger(TsFileSequenceReader.class);
