This is an automated email from the ASF dual-hosted git repository. qiaojialin pushed a commit to branch fix_conflict_for_merge in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit befe5f805ec0cd560afc30f391ba595134cea238 Merge: 7422734 c626197 Author: qiaojialin <[email protected]> AuthorDate: Thu Sep 3 19:35:43 2020 +0800 resolve conflict with bug in IoTDBRemovePartitionIT .mvn/wrapper/MavenWrapperDownloader.java | 10 +- .../org/apache/iotdb/db/qp/strategy/SqlBase.g4 | 16 +- .../java/org/apache/iotdb/cli/AbstractCli.java | 3 + .../main/java/org/apache/iotdb/tool/ExportCsv.java | 1 + .../main/java/org/apache/iotdb/tool/ImportCsv.java | 51 +- .../DML Data Manipulation Language.md | 18 +- docs/UserGuide/Operation Manual/SQL Reference.md | 47 + .../DML Data Manipulation Language.md | 18 +- .../zh/UserGuide/Operation Manual/SQL Reference.md | 47 + .../org/apache/iotdb/flink/FlinkIoTDBSink.java | 4 +- .../iotdb/hadoop/tsfile/TSFMRReadExample.java | 3 +- .../iotdb/hadoop/tsfile/TSMRWriteExample.java | 3 +- .../apache/iotdb/hadoop/tsfile/TsFileHelper.java | 44 +- .../java/org/apache/iotdb/tsfile/TsFileRead.java | 49 +- .../apache/iotdb/tsfile/TsFileSequenceRead.java | 137 +-- .../iotdb/tsfile/TsFileWriteWithTSRecord.java | 46 +- .../iotdb/flink/tsfile/TsFileInputFormat.java | 19 +- .../iotdb/hadoop/tsfile/TSFRecordReader.java | 21 +- .../org/apache/iotdb/hive/TSFHiveRecordReader.java | 1 + .../org/apache/iotdb/hive/TsFileDeserializer.java | 1 + .../org/apache/iotdb/jdbc/IoTDBConnection.java | 1 + .../apache/iotdb/jdbc/IoTDBDatabaseMetadata.java | 2 +- .../resources/conf/iotdb-engine.properties | 6 +- .../iotdb/db/auth/role/LocalFileRoleAccessor.java | 4 + .../iotdb/db/auth/user/LocalFileUserAccessor.java | 4 + .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 8 +- .../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 37 +- .../iotdb/db/conf/adapter/CompressionRatio.java | 4 +- .../strategy/RandomOnDiskUsableSpaceStrategy.java | 4 +- .../org/apache/iotdb/db/engine/StorageEngine.java | 7 +- .../db/engine/flush/pool/AbstractPoolManager.java | 1 + .../iotdb/db/engine/merge/recover/LogAnalyzer.java | 5 + .../db/engine/merge/task/MergeMultiChunkTask.java | 1 + .../engine/storagegroup/StorageGroupProcessor.java | 4 + .../db/engine/storagegroup/TsFileProcessor.java | 2 + .../version/SimpleFileVersionController.java | 1 + .../org/apache/iotdb/db/metadata/MManager.java | 10 +- .../java/org/apache/iotdb/db/metadata/MTree.java | 5 +- .../org/apache/iotdb/db/metadata/PartialPath.java | 43 +- .../apache/iotdb/db/metrics/server/JettyUtil.java | 4 +- .../iotdb/db/metrics/server/QueryServlet.java | 2 +- .../iotdb/db/metrics/server/ServerArgument.java | 37 +- .../apache/iotdb/db/metrics/ui/MetricsPage.java | 10 +- .../apache/iotdb/db/qp/constant/DatetimeUtils.java | 14 +- .../apache/iotdb/db/qp/executor/PlanExecutor.java | 1 + .../iotdb/db/qp/logical/crud/InOperator.java | 1 + .../db/qp/physical/crud/AlignByDevicePlan.java | 10 + .../iotdb/db/qp/physical/crud/InsertRowPlan.java | 1 + .../iotdb/db/qp/strategy/LogicalGenerator.java | 59 +- .../iotdb/db/qp/strategy/PhysicalGenerator.java | 43 +- .../qp/strategy/optimizer/ConcatPathOptimizer.java | 45 +- .../optimizer/MergeSingleFilterOptimizer.java | 1 + .../db/query/control/QueryResourceManager.java | 1 + .../db/query/dataset/AlignByDeviceDataSet.java | 1 + .../db/query/dataset/NonAlignEngineDataSet.java | 1 + .../dataset/RawQueryDataSetWithoutValueFilter.java | 2 + .../query/dataset/groupby/GroupByFillDataSet.java | 10 +- .../groupby/GroupByWithValueFilterDataSet.java | 1 + .../dataset/groupby/LocalGroupByExecutor.java | 1 + .../db/query/executor/AggregationExecutor.java | 2 + .../iotdb/db/query/executor/LastQueryExecutor.java | 21 +- .../db/query/executor/fill/LastPointReader.java | 1 + .../FixLengthIExternalSortFileDeserializer.java | 75 +- .../iotdb/db/query/reader/series/SeriesReader.java | 3 + .../org/apache/iotdb/db/rescon/MemTablePool.java | 2 +- .../java/org/apache/iotdb/db/service/IoTDB.java | 1 + .../org/apache/iotdb/db/service/TSServiceImpl.java | 47 +- .../iotdb/db/sync/receiver/load/FileLoader.java | 1 + .../db/sync/receiver/load/FileLoaderManager.java | 1 + .../receiver/recover/SyncReceiverLogAnalyzer.java | 1 + .../db/sync/receiver/transfer/SyncServiceImpl.java | 26 +- .../db/sync/sender/manage/SyncFileManager.java | 2 + .../iotdb/db/sync/sender/transfer/SyncClient.java | 8 +- .../apache/iotdb/db/tools/TsFileSketchTool.java | 254 ++-- .../db/tools/upgrade/TsFileOnlineUpgradeTool.java | 1 + .../db/tools/watermark/WatermarkDetector.java | 94 +- .../org/apache/iotdb/db/utils/CommonUtils.java | 7 +- .../org/apache/iotdb/db/utils/FilePathUtils.java | 1 + .../org/apache/iotdb/db/utils/OpenFileNumUtil.java | 10 +- .../apache/iotdb/db/utils/QueryDataSetUtils.java | 2 + .../java/org/apache/iotdb/db/utils/QueryUtils.java | 1 + .../org/apache/iotdb/db/utils/SchemaUtils.java | 13 +- .../org/apache/iotdb/db/utils/UpgradeUtils.java | 1 + .../writelog/manager/MultiFileLogNodeManager.java | 36 +- .../iotdb/db/writelog/recover/LogReplayer.java | 1 + .../writelog/recover/TsFileRecoverPerformer.java | 3 +- .../iotdb/db/engine/memtable/MemTablePoolTest.java | 1 + .../version/SysTimeVersionControllerTest.java | 1 + .../iotdb/db/integration/IoTDBAggregationIT.java | 35 +- .../integration/IoTDBAggregationLargeDataIT.java | 285 +++-- .../apache/iotdb/db/integration/IoTDBAliasIT.java | 49 +- .../org/apache/iotdb/db/integration/IoTDBAsIT.java | 603 ++++++++++ .../iotdb/db/integration/IoTDBAuthorizationIT.java | 1234 ++++++++++---------- .../apache/iotdb/db/integration/IoTDBCloseIT.java | 112 +- .../db/integration/IoTDBDeleteStorageGroupIT.java | 14 +- .../db/integration/IoTDBDeleteTimeseriesIT.java | 3 +- .../apache/iotdb/db/integration/IoTDBFillIT.java | 283 ++--- .../db/integration/IoTDBFlushQueryMergeIT.java | 8 +- .../IoTDBMultiOverlappedChunkInUnseqIT.java | 9 +- .../db/integration/IoTDBMultiOverlappedPageIT.java | 20 +- .../db/integration/IoTDBMultiStatementsIT.java | 92 +- .../iotdb/db/integration/IoTDBNumberPathIT.java | 13 +- .../db/integration/IoTDBOverlappedPageIT.java | 12 +- .../iotdb/db/integration/IoTDBQuotedPathIT.java | 57 +- .../iotdb/db/integration/IoTDBRecoverIT.java | 3 + .../iotdb/db/integration/IoTDBRestartIT.java | 84 +- .../iotdb/db/integration/IoTDBSeriesReaderIT.java | 7 +- .../iotdb/db/integration/IoTDBSimpleQueryIT.java | 164 +-- .../iotdb/db/integration/IoTDBTagAlterIT.java | 410 ++++--- .../apache/iotdb/db/integration/IoTDBTagIT.java | 623 +++++----- .../iotdb/db/integration/IoTDBTimeZoneIT.java | 12 +- .../org/apache/iotdb/db/metadata/MTreeTest.java | 4 +- .../db/query/dataset/GroupByLevelDataSetTest.java | 1 - .../db/query/dataset/GroupByTimeDataSetTest.java | 1 - .../db/sync/receiver/load/FileLoaderTest.java | 8 +- .../recover/SyncReceiverLogAnalyzerTest.java | 2 + .../apache/iotdb/db/tools/IoTDBWatermarkTest.java | 73 +- .../org/apache/iotdb/db/tools/WalCheckerTest.java | 7 +- .../apache/iotdb/db/utils/EnvironmentUtils.java | 5 +- .../db/utils/datastructure/PrecisionTest.java | 2 +- .../java/org/apache/iotdb/rpc/IoTDBRpcDataSet.java | 1 + .../org/apache/iotdb/session/SessionUtils.java | 1 + .../org/apache/iotdb/session/pool/SessionPool.java | 1 + .../org/apache/iotdb/session/IoTDBSessionIT.java | 188 +-- .../iotdb/spark/tsfile/qp/QueryProcessor.java | 2 + .../qp/optimizer/MergeSingleFilterOptimizer.java | 1 + .../tsfile/qp/optimizer/PhysicalOptimizer.java | 2 + .../iotdb/tsfile/encoding/encoder/RleEncoder.java | 1 + .../file/metadata/MetadataIndexConstructor.java | 1 + .../file/metadata/statistics/Statistics.java | 5 + .../iotdb/tsfile/read/TsFileSequenceReader.java | 2 +- .../org/apache/iotdb/tsfile/read/common/Path.java | 12 +- .../apache/iotdb/tsfile/read/common/TimeRange.java | 11 +- .../read/controller/MetadataQuerierByFileImpl.java | 2 + .../query/dataset/DataSetWithoutTimeGenerator.java | 1 + .../read/query/timegenerator/node/AndNode.java | 1 + .../iotdb/tsfile/read/reader/page/PageReader.java | 1 + .../reader/series/FileSeriesReaderByTimestamp.java | 2 + .../iotdb/tsfile/utils/RamUsageEstimator.java | 1 + .../iotdb/tsfile/read/TimePlainEncodeReadTest.java | 2 +- .../iotdb/tsfile/utils/TsFileGeneratorForTest.java | 79 +- .../org/apache/iotdb/tsfile/write/PerfTest.java | 62 +- .../org/apache/iotdb/tsfile/write/WriteTest.java | 51 +- .../write/writer/RestorableTsFileIOWriterTest.java | 7 +- 144 files changed, 3764 insertions(+), 2458 deletions(-) diff --cc antlr/src/main/antlr4/org/apache/iotdb/db/qp/strategy/SqlBase.g4 index f34e32e,8cea342..8cea342 mode 100755,100644..100755 --- a/antlr/src/main/antlr4/org/apache/iotdb/db/qp/strategy/SqlBase.g4 +++ b/antlr/src/main/antlr4/org/apache/iotdb/db/qp/strategy/SqlBase.g4 diff --cc server/src/assembly/resources/conf/iotdb-engine.properties index cfd1182,c2b16fd..988e6d7 --- a/server/src/assembly/resources/conf/iotdb-engine.properties +++ b/server/src/assembly/resources/conf/iotdb-engine.properties @@@ -251,21 -251,17 +251,21 @@@ default_fill_interval=- #################### ### Merge Configurations #################### -# Is vm merge enable. -enable_vm=false +# TsFile manage strategy, define use which hot compaction strategy +# now we have normal_strategy, level_strategy +tsfile_manage_strategy=NORMAL_STRATEGY -# The max vm num of each memtable. When vm num exceeds this, the vm files will merge to one. -max_vm_num=10 +# Work when tsfile_manage_strategy is level_strategy. +# The max file num of each level. When file num exceeds this, the files in one level will merge to one. - max_file_num_in_each_level=10 ++max_file_num_in_each_level=100 -# When merge point number reaches this, merge the vmfile to the tsfile. -merge_chunk_point_number=100000 +# Work when tsfile_manage_strategy is level_strategy. +# The max num of level. - max_level_num=3 ++max_level_num=2 -# When vmfiles merge times exceeds this, merge the vmfile to the tsfile. -max_merge_chunk_num_in_tsfile=100 +# Work when tsfile_manage_strategy is level_strategy. +# When merge point number reaches this, merge the files to the last level. +merge_chunk_point_number=100000 # How many thread will be set up to perform merge main tasks, 1 by default. # Set to 1 when less than or equal to 0. diff --cc server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java index 37dca75,b6bcb3f..e914cfc --- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java +++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java @@@ -259,20 -257,19 +259,20 @@@ public class IoTDBConfig private int mergeChunkPointNumberThreshold = 100000; /** - * Is vm merge enable + * TsFile manage strategy, define use which hot compaction strategy */ - private boolean enableVm = false; + private TsFileManagementStrategy tsFileManagementStrategy = TsFileManagementStrategy.NORMAL_STRATEGY; /** - * The max vm num of each memtable. When vm num exceeds this, the vm files will merge to one. + * Work when tsfile_manage_strategy is level_strategy. The max file num of each level. When file + * num exceeds this, the files in one level will merge to one. */ - private int maxFileNumInEachLevel = 10; - private int maxVmNum = 10; ++ private int maxFileNumInEachLevel = 100; /** - * When vmfiles merge times exceeds this, merge the vmfile to the tsfile. + * Work when tsfile_manage_strategy is level_strategy. The max num of level. */ - private int maxLevelNum = 3; - private int maxMergeChunkNumInTsFile = 100; ++ private int maxLevelNum = 2; /** * whether to cache meta data(ChunkMetaData and TsFileMetaData) or not. diff --cc server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java index 8650e44,6c6a637..39efcf0 --- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java @@@ -710,11 -1012,12 +711,12 @@@ public class TsFileProcessor * memtables and then compact them into one TimeValuePairSorter). Then get the related * ChunkMetadata of data on disk. * - * @param deviceId device id + * @param deviceId device id * @param measurementId measurements id - * @param dataType data type - * @param encoding encoding + * @param dataType data type + * @param encoding encoding */ + @SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity warning public void query(String deviceId, String measurementId, TSDataType dataType, TSEncoding encoding, Map<String, String> props, QueryContext context, List<TsFileResource> tsfileResourcesForQuery) throws IOException { diff --cc server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java index 3c8e4d0,3b1bd73..e5f7157 --- a/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java +++ b/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java @@@ -1608,13 -1614,13 +1614,12 @@@ public class TSServiceImpl implements T return QueryResourceManager.getInstance().assignQueryId(isDataQuery); } - protected List<TSDataType> getSeriesTypesByPath(List<PartialPath> paths, List<String> aggregations) + protected List<TSDataType> getSeriesTypesByPaths(List<PartialPath> paths, List<String> aggregations) throws MetadataException { - return SchemaUtils.getSeriesTypesByPath(paths, aggregations); + return SchemaUtils.getSeriesTypesByPaths(paths, aggregations); } - protected List<TSDataType> getSeriesTypesByString(List<PartialPath> paths, String aggregation) - throws MetadataException { - return SchemaUtils.getSeriesTypesByString(paths, aggregation); - + protected TSDataType getSeriesTypeByPath(PartialPath path) throws MetadataException { + return SchemaUtils.getSeriesTypeByPath(path); } } diff --cc server/src/main/java/org/apache/iotdb/db/writelog/recover/TsFileRecoverPerformer.java index 2ea32ef,6c3d25c..537e9fd --- a/server/src/main/java/org/apache/iotdb/db/writelog/recover/TsFileRecoverPerformer.java +++ b/server/src/main/java/org/apache/iotdb/db/writelog/recover/TsFileRecoverPerformer.java @@@ -84,10 -95,19 +84,9 @@@ public class TsFileRecoverPerformer * file and the vmfiles are not closed before crash, so these writers can be used to continue * writing */ - public RestorableTsFileIOWriter recover() - @SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity warning - public Pair<RestorableTsFileIOWriter, List<List<RestorableTsFileIOWriter>>> recover() -- throws StorageGroupProcessorException { ++ public RestorableTsFileIOWriter recover() throws StorageGroupProcessorException { File file = FSFactoryProducer.getFSFactory().getFile(filePath); - List<List<File>> vmFileList = new ArrayList<>(); - for (List<TsFileResource> subTsFileResource : vmTsFileResources) { - List<File> subVmFileList = new ArrayList<>(); - for (TsFileResource tsFileResource : subTsFileResource) { - subVmFileList.add(FSFactoryProducer.getFSFactory().getFile(tsFileResource.getTsFilePath())); - } - vmFileList.add(subVmFileList); - } if (!file.exists()) { logger.error("TsFile {} is missing, will skip its recovery.", filePath); return null; diff --cc server/src/test/java/org/apache/iotdb/db/integration/IoTDBDeleteTimeseriesIT.java index de6e438,264f340..66a9038 --- a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBDeleteTimeseriesIT.java +++ b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBDeleteTimeseriesIT.java @@@ -53,8 -50,7 +53,7 @@@ public class IoTDBDeleteTimeseriesIT @After public void tearDown() throws Exception { IoTDBDescriptor.getInstance().getConfig().setMemtableSizeThreshold(memtableSizeThreshold); - IoTDBDescriptor.getInstance().getConfig() - .setTsFileManagementStrategy(tsFileManagementStrategy); - IoTDBDescriptor.getInstance().getConfig().setEnableVm(enableVm); ++ IoTDBDescriptor.getInstance().getConfig().setTsFileManagementStrategy(tsFileManagementStrategy); EnvironmentUtils.cleanEnv(); } diff --cc server/src/test/java/org/apache/iotdb/db/integration/IoTDBQuotedPathIT.java index db1fe94,276bd73..098dfae --- a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBQuotedPathIT.java +++ b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBQuotedPathIT.java @@@ -78,36 -78,40 +78,39 @@@ public class IoTDBQuotedPathIT ArrayList<String> ans = new ArrayList<>(); ResultSet resultSet = statement.getResultSet(); cnt = 0; - while (resultSet.next()) { - String result = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(2); - ans.add(result); - cnt++; - } - - for (int i = 0; i < ans.size(); i++) { - assertEquals(exp[i], ans.get(i)); - } + try { + while (resultSet.next()) { + String result = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(2); + ans.add(result); + cnt++; + } - hasResultSet = statement.execute("SELECT * FROM root.ln.\"wf.01\".wt01 WHERE \"status.2.3\" = false"); - assertTrue(hasResultSet); - exp = new String[]{ - "1509465600002,false", - "1509465600003,false" - }; - ans = new ArrayList<>(); - resultSet = statement.getResultSet(); - cnt = 0; - while (resultSet.next()) { - String result = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(2); - ans.add(result); - cnt++; - } + for (int i = 0; i < ans.size(); i++) { + assertEquals(exp[i], ans.get(i)); + } - + hasResultSet = statement.execute("SELECT * FROM root.ln.\"wf.01\".wt01 WHERE \"status.2.3\" = false"); + assertTrue(hasResultSet); + exp = new String[]{ + "1509465600002,false", + "1509465600003,false" + }; + ans = new ArrayList<>(); + resultSet = statement.getResultSet(); + cnt = 0; + while (resultSet.next()) { + String result = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(2); + ans.add(result); + cnt++; + } - for (int i = 0; i < exp.length; i++) { - assertEquals(exp[i], ans.get(i)); + for (int i = 0; i < exp.length; i++) { + assertEquals(exp[i], ans.get(i)); + } + statement.execute("DELETE FROM root.ln.\"wf.01\".wt01.\"status.2.3\" WHERE time < 1509465600001"); + statement.execute("DELETE TIMESERIES root.ln.\"wf.01\".wt01.\"status.2.3\""); + } finally { + resultSet.close(); } - statement.execute("DELETE FROM root.ln.\"wf.01\".wt01.\"status.2.3\" WHERE time < 1509465600001"); - statement.execute("DELETE TIMESERIES root.ln.\"wf.01\".wt01.\"status.2.3\""); } catch (Exception e) { e.printStackTrace(); Assert.fail();
