This is an automated email from the ASF dual-hosted git repository. lta pushed a commit to branch cluster_scalability in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit e5e4411bd57076245943a315835f789ada699840 Merge: dd64f0e 6f632e2 Author: lta <[email protected]> AuthorDate: Thu Jan 21 17:50:44 2021 +0800 merge master .../client-cpp-example/src/SessionExample.cpp | 13 + client-cpp/src/main/Session.cpp | 18 +- client-cpp/src/main/Session.h | 4 + client-cpp/src/test/cpp/sessionIT.cpp | 22 + cluster/pom.xml | 16 + .../cluster/client/async/AsyncClientFactory.java | 6 +- .../cluster/client/async/AsyncDataClient.java | 11 +- .../cluster/query/ClusterPhysicalGenerator.java | 15 +- .../iotdb/cluster/query/LoadConfigurationTest.java | 122 +++ docs/Download/README.md | 17 +- docs/zh/Download/README.md | 14 +- .../main/java/org/apache/iotdb/JDBCExample.java | 25 +- .../apache/iotdb/tsfile/TsFileSequenceRead.java | 62 +- .../apache/iotdb/hadoop/fileSystem/HDFSInput.java | 11 + .../apache/iotdb/hadoop/fileSystem/HDFSOutput.java | 5 + .../iotdb/hadoop/tsfile/record/HDFSTSRecord.java | 4 +- .../main/java/org/apache/iotdb/jdbc/Config.java | 4 +- .../org/apache/iotdb/jdbc/IoTDBConnection.java | 4 +- .../java/org/apache/iotdb/jdbc/IoTDBStatement.java | 6 +- server/file-changelists/TsFileMods-changelist.md | 11 + .../system.properties-changelist.md | 9 + .../resources/conf/iotdb-engine.properties | 5 +- .../resources/tools/upgrade/config.properties | 27 - .../resources/tools/upgrade/offline-upgrade.bat | 67 -- .../resources/tools/upgrade/offline-upgrade.sh | 47 - .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 15 +- .../org/apache/iotdb/db/conf/IoTDBConfigCheck.java | 199 +++- .../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 8 + .../apache/iotdb/db/engine/cache/ChunkCache.java | 5 +- .../level/LevelCompactionTsFileManagement.java | 9 +- .../iotdb/db/engine/flush/MemTableFlushTask.java | 169 ++-- .../merge/selector/MaxFileMergeFileSelector.java | 4 +- .../io/LocalTextModificationAccessor.java | 8 +- .../engine/storagegroup/StorageGroupProcessor.java | 181 ++-- .../db/engine/storagegroup/TsFileResource.java | 50 +- .../iotdb/db/engine/upgrade/UpgradeTask.java | 108 +-- .../iotdb/db/metadata/logfile/MLogWriter.java | 6 +- .../iotdb/db/qp/physical/crud/InsertRowPlan.java | 4 +- .../db/qp/physical/crud/InsertTabletPlan.java | 4 +- .../db/query/aggregation/AggregateResult.java | 2 +- .../db/query/aggregation/impl/AvgAggrResult.java | 12 +- .../db/query/aggregation/impl/SumAggrResult.java | 10 +- .../iotdb/db/query/control/FileReaderManager.java | 16 +- .../iotdb/db/query/control/QueryTimeManager.java | 16 +- .../dataset/RawQueryDataSetWithValueFilter.java | 68 +- .../dataset/RawQueryDataSetWithoutValueFilter.java | 107 +- .../iotdb/db/query/dataset/UDFInputDataSet.java | 19 +- .../db/query/dataset/UDTFAlignByTimeDataSet.java | 6 +- .../FixLengthIExternalSortFileDeserializer.java | 2 +- .../chunk/metadata/DiskChunkMetadataLoader.java | 5 +- .../iotdb/db/query/udf/core/access/RowImpl.java | 23 +- .../iotdb/db/query/udf/core/input/InputLayer.java | 120 ++- .../iotdb/db/query/udf/core/input/SafetyLine.java | 40 +- .../iotdb/db/query/udf/datastructure/Cache.java | 99 ++ .../primitive/ElasticSerializableIntList.java | 25 +- .../row/ElasticSerializableRowRecordList.java | 69 +- .../row/SerializableRowRecordList.java | 98 +- .../tv/ElasticSerializableTVList.java | 28 +- .../org/apache/iotdb/db/rescon/SystemInfo.java | 28 +- .../org/apache/iotdb/db/service/TSServiceImpl.java | 12 +- .../org/apache/iotdb/db/service/UpgradeSevice.java | 3 - .../apache/iotdb/db/tools/TsFileSketchTool.java | 36 +- .../db/tools/upgrade/TsFileOnlineUpgradeTool.java | 451 +++------ .../org/apache/iotdb/db/utils/FileLoaderUtils.java | 12 +- .../java/org/apache/iotdb/db/utils/QueryUtils.java | 9 + .../org/apache/iotdb/db/utils/UpgradeUtils.java | 120 ++- .../iotdb/db/utils/datastructure/TimeSelector.java | 155 +++ .../engine/merge/MaxFileMergeFileSelectorTest.java | 4 +- .../merge/MaxSeriesMergeFileSelectorTest.java | 8 +- .../iotdb/db/integration/IoTDBDisableAlignIT.java | 6 +- .../db/integration/IoTDBNewTsFileCompactionIT.java | 1018 ++++++++++++++++++++ .../db/integration/IoTDBUDTFNonAlignQueryIT.java | 2 +- .../db/qp/physical/InsertTabletMultiPlanTest.java | 2 +- .../iotdb/db/qp/physical/InsertTabletPlanTest.java | 2 +- .../ElasticSerializableRowRecordListTest.java | 61 +- .../iotdb/db/query/udf/datastructure/LRUCache.java | 59 ++ .../db/query/udf/datastructure/LRUCacheTest.java | 115 +++ .../udf/datastructure/SerializableListTest.java | 2 + .../SerializableRowRecordListTest.java | 49 +- .../db/utils/datastructure/TimeSelectorTest.java | 217 +++++ .../db/writelog/recover/SeqTsFileRecoverTest.java | 2 +- .../main/java/org/apache/iotdb/session/Config.java | 2 +- .../org/apache/iotdb/session/pool/SessionPool.java | 22 + .../iotdb/session/IoTDBSessionIteratorIT.java | 2 +- .../apache/iotdb/session/pool/SessionPoolTest.java | 33 +- .../apache/iotdb/spark/tsfile/HDFSInputTest.java | 4 +- tsfile/format-changelist.md | 4 +- .../iotdb/tsfile/common/conf/TSFileConfig.java | 6 +- .../iotdb/tsfile/encoding/decoder/Decoder.java | 8 +- .../tsfile/encoding/decoder/FloatDecoder.java | 11 +- .../tsfile/encoding/decoder/IntRleDecoder.java | 11 +- .../tsfile/encoding/decoder/LongRleDecoder.java | 11 +- .../tsfile/encoding/decoder/PlainDecoder.java | 21 +- .../iotdb/tsfile/encoding/decoder/RleDecoder.java | 15 +- .../tsfile/encoding/encoder/FloatEncoder.java | 6 +- .../tsfile/encoding/encoder/IntRleEncoder.java | 8 +- .../tsfile/encoding/encoder/LongRleEncoder.java | 6 +- .../tsfile/encoding/encoder/PlainEncoder.java | 57 +- .../iotdb/tsfile/encoding/encoder/RleEncoder.java | 20 +- .../tsfile/encoding/encoder/TSEncodingBuilder.java | 13 +- .../org/apache/iotdb/tsfile/file/MetaMarker.java | 17 +- .../iotdb/tsfile/file/footer/ChunkGroupFooter.java | 156 --- .../iotdb/tsfile/file/header/ChunkGroupHeader.java | 117 +++ .../iotdb/tsfile/file/header/ChunkHeader.java | 151 +-- .../iotdb/tsfile/file/header/PageHeader.java | 54 +- .../iotdb/tsfile/file/metadata/ChunkMetadata.java | 35 +- .../file/metadata/MetadataIndexConstructor.java | 5 +- .../tsfile/file/metadata/MetadataIndexEntry.java | 4 +- .../tsfile/file/metadata/MetadataIndexNode.java | 12 +- .../tsfile/file/metadata/TimeseriesMetadata.java | 36 +- .../iotdb/tsfile/file/metadata/TsFileMetadata.java | 45 +- .../file/metadata/enums/CompressionType.java | 64 +- .../file/metadata/enums/MetadataIndexNodeType.java | 57 +- .../tsfile/file/metadata/enums/TSDataType.java | 113 +-- .../tsfile/file/metadata/enums/TSEncoding.java | 94 +- .../file/metadata/statistics/BinaryStatistics.java | 13 +- .../metadata/statistics/BooleanStatistics.java | 48 +- .../file/metadata/statistics/DoubleStatistics.java | 23 +- .../file/metadata/statistics/FloatStatistics.java | 23 +- .../metadata/statistics/IntegerStatistics.java | 48 +- .../file/metadata/statistics/LongStatistics.java | 31 +- .../file/metadata/statistics/Statistics.java | 22 +- .../iotdb/tsfile/read/TsFileSequenceReader.java | 404 ++++---- .../apache/iotdb/tsfile/read/common/BatchData.java | 2 +- .../org/apache/iotdb/tsfile/read/common/Chunk.java | 99 +- .../read/controller/CachedChunkLoaderImpl.java | 3 +- .../iotdb/tsfile/read/reader/LocalTsFileInput.java | 20 + .../iotdb/tsfile/read/reader/TsFileInput.java | 5 + .../tsfile/read/reader/chunk/ChunkReader.java | 105 +- .../iotdb/tsfile/read/reader/page/PageReader.java | 14 +- .../tsfile/utils/ReadWriteForEncodingUtils.java | 96 +- .../iotdb/tsfile/utils/ReadWriteIOUtils.java | 103 +- .../v1/file/metadata/ChunkGroupMetaDataV1.java | 118 --- .../tsfile/v1/file/metadata/ChunkMetadataV1.java | 131 --- .../v1/file/metadata/TimeseriesMetadataForV1.java | 42 - .../v1/file/metadata/TsDeviceMetadataIndexV1.java | 78 -- .../v1/file/metadata/TsDeviceMetadataV1.java | 87 -- .../iotdb/tsfile/v1/file/metadata/TsDigestV1.java | 75 -- .../tsfile/v1/file/metadata/TsFileMetadataV1.java | 106 -- .../metadata/statistics/BinaryStatisticsV1.java | 84 -- .../metadata/statistics/BooleanStatisticsV1.java | 80 -- .../metadata/statistics/DoubleStatisticsV1.java | 79 -- .../metadata/statistics/FloatStatisticsV1.java | 79 -- .../metadata/statistics/IntegerStatisticsV1.java | 79 -- .../file/metadata/statistics/LongStatisticsV1.java | 80 -- .../v1/file/metadata/statistics/StatisticsV1.java | 225 ----- .../iotdb/tsfile/v1/file/utils/HeaderUtils.java | 141 --- .../tsfile/v1/read/TsFileSequenceReaderForV1.java | 409 -------- .../tsfile/v2/file/footer/ChunkGroupFooterV2.java | 86 ++ .../iotdb/tsfile/v2/file/header/ChunkHeaderV2.java | 108 +++ .../iotdb/tsfile/v2/file/header/PageHeaderV2.java | 51 + .../tsfile/v2/file/metadata/ChunkMetadataV2.java | 52 + .../file/metadata/MetadataIndexEntryV2.java} | 28 +- .../v2/file/metadata/MetadataIndexNodeV2.java | 47 + .../v2/file/metadata/TimeseriesMetadataV2.java | 42 + .../tsfile/v2/file/metadata/TsFileMetadataV2.java | 75 ++ .../v2/file/metadata/statistics/StatisticsV2.java | 104 ++ .../tsfile/v2/read/TsFileSequenceReaderForV2.java | 581 +++++++++++ .../tsfile/v2/read/reader/page/PageReaderV2.java | 85 ++ .../apache/iotdb/tsfile/write/TsFileWriter.java | 2 +- .../iotdb/tsfile/write/chunk/ChunkWriterImpl.java | 84 +- .../iotdb/tsfile/write/chunk/IChunkWriter.java | 11 +- .../apache/iotdb/tsfile/write/page/PageWriter.java | 30 +- .../tsfile/write/schema/MeasurementSchema.java | 38 +- .../write/writer/ForceAppendTsFileWriter.java | 3 - .../tsfile/write/writer/LocalTsFileOutput.java | 5 + .../write/writer/RestorableTsFileIOWriter.java | 1 - .../iotdb/tsfile/write/writer/TsFileIOWriter.java | 54 +- .../iotdb/tsfile/write/writer/TsFileOutput.java | 8 + .../tsfile/encoding/decoder/IntRleDecoderTest.java | 83 +- .../encoding/decoder/LongRleDecoderTest.java | 17 +- .../iotdb/tsfile/file/header/PageHeaderTest.java | 2 +- .../metadata/statistics/DoubleStatisticsTest.java | 10 +- .../metadata/statistics/FloatStatisticsTest.java | 11 +- .../metadata/statistics/IntegerStatisticsTest.java | 10 +- .../metadata/statistics/LongStatisticsTest.java | 15 +- .../iotdb/tsfile/file/metadata/utils/Utils.java | 36 +- .../iotdb/tsfile/read/GetAllDevicesTest.java | 14 +- .../tsfile/read/TsFileSequenceReaderTest.java | 66 +- .../iotdb/tsfile/read/reader/PageReaderTest.java | 54 +- .../utils/ReadWriteForEncodingUtilsTest.java | 54 ++ .../iotdb/tsfile/write/TsFileIOWriterTest.java | 20 +- .../iotdb/tsfile/write/writer/PageWriterTest.java | 54 +- .../write/writer/RestorableTsFileIOWriterTest.java | 15 +- 184 files changed, 6107 insertions(+), 4474 deletions(-) diff --cc server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java index dd9ef15,7d3f114..42575d2 --- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java @@@ -183,9 -186,11 +186,11 @@@ public class StorageGroupProcessor private CopyOnReadLinkedList<TsFileProcessor> closingSequenceTsFileProcessor = new CopyOnReadLinkedList<>(); // upgrading unsequence TsFile resource list -- private List<TsFileResource> upgradeUnseqFileList = new LinkedList<>(); ++ private final List<TsFileResource> upgradeUnseqFileList = new LinkedList<>(); private CopyOnReadLinkedList<TsFileProcessor> closingUnSequenceTsFileProcessor = new CopyOnReadLinkedList<>(); + + private AtomicInteger upgradeFileCount = new AtomicInteger(); /* * time partition id -> map, which contains * device -> global latest timestamp of each device latestTimeForEachDevice caches non-flushed @@@ -2312,23 -2336,7 +2336,24 @@@ e.getMessage())); } + File modFileToLoad = fsFactory.getFile( + tsFileToLoad.getAbsolutePath() + TsFileResource.RESOURCE_SUFFIX); + if (modFileToLoad.exists()) { + File targetModFile = fsFactory.getFile( + targetFile.getAbsolutePath() + TsFileResource.RESOURCE_SUFFIX); + try { + FileUtils.moveFile(modFileToLoad, targetModFile); + } catch (IOException e) { + logger.error("File renaming failed when loading .mod file. Origin: {}, Target: {}", + resourceFileToLoad.getAbsolutePath(), targetModFile.getAbsolutePath(), e); + throw new LoadFileException(String.format( + "File renaming failed when loading .mod file. Origin: %s, Target: %s, because %s", + resourceFileToLoad.getAbsolutePath(), targetModFile.getAbsolutePath(), + e.getMessage())); + } + } + + updatePartitionFileVersion(filePartitionId, tsFileResource.getVersion()); return true; }
