This is an automated email from the ASF dual-hosted git repository. haonan pushed a commit to branch mv_storage_groups_databases in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 345c71f2d2b149453556cdb4899450e0240c7a40 Author: HTHou <[email protected]> AuthorDate: Tue Nov 15 14:24:20 2022 +0800 Rename storage_groups dir name to databases --- docs/UserGuide/Data-Concept/Compression.md | 2 +- docs/UserGuide/QuickStart/ServerFileList.md | 49 ++-------------------- docs/zh/UserGuide/Data-Concept/Compression.md | 2 +- docs/zh/UserGuide/QuickStart/ServerFileList.md | 46 ++------------------ .../apache/iotdb/db/engine/StorageEngineV2.java | 6 +-- .../iotdb/db/engine/snapshot/SnapshotLoader.java | 2 +- .../iotdb/db/engine/storagegroup/DataRegion.java | 2 +- .../iotdb/db/metadata/idtable/IDTableManager.java | 2 +- .../apache/iotdb/db/utils/EnvironmentUtils.java | 2 +- 9 files changed, 15 insertions(+), 98 deletions(-) diff --git a/docs/UserGuide/Data-Concept/Compression.md b/docs/UserGuide/Data-Concept/Compression.md index adc808f3e0..01b0271833 100644 --- a/docs/UserGuide/Data-Concept/Compression.md +++ b/docs/UserGuide/Data-Concept/Compression.md @@ -130,7 +130,7 @@ point and updates the upper and lower boundaries. The last point at time 2017-11 ## Compression Ratio Statistics -Compression ratio statistics file: data/system/storage_groups/compression_ratio/Ratio-{ratio_sum}-{memtable_flush_time} +Compression ratio statistics file: data/system/compression_ratio/Ratio-{ratio_sum}-{memtable_flush_time} * ratio_sum: sum of memtable compression ratios * memtable_flush_time: memtable flush times diff --git a/docs/UserGuide/QuickStart/ServerFileList.md b/docs/UserGuide/QuickStart/ServerFileList.md index 6e7070e9d3..839239d71d 100644 --- a/docs/UserGuide/QuickStart/ServerFileList.md +++ b/docs/UserGuide/QuickStart/ServerFileList.md @@ -67,7 +67,7 @@ + every database has several wal files, and every memtable has one associated wal file before it is flushed into a TsFile #### TsFile -> under directory data/sequence or unsequence/{StorageGroupName}/{TimePartitionId}/ +> under directory data/sequence or unsequence/{DatabaseName}/{DataRegionId}/{TimePartitionId}/ 1. {time}-{version}-{mergeCnt}.tsfile + normal data file @@ -85,7 +85,7 @@ + close flag file, to mark a tsfile closing so during restarts we can continue to close it or reopen it #### Version -> under directory basedir/system/storage_groups/{StorageGroupName}/{TimePartitionId} or upgrade +> under directory basedir/system/databases/{DatabaseName}/{DataRegionId}/{TimePartitionId} or upgrade 1. Version-{version} + version file, record the max version in fileName of a database @@ -97,7 +97,7 @@ + record which files have been upgraded #### Merge -> under directory basedir/system/storage_groups/{StorageGroup}/ +> under directory basedir/system/databases/{StorageGroup}/ 1. merge.mods + modification file generated during a merge @@ -115,46 +115,3 @@ 1. Ration-{compressionRatioSum}-{calTimes} + record compression ratio of each tsfile ---- - -# Cluster-Mode -> Attention: the following files are newly added - -## Configuration Files -1. iotdb-cluster.properties - -## State Related Files -> under directory basedir/ - -1. node_identifier - + the identifier of the local node in a cluster -2. partitions - + partition table file, records the distribution of data -3. {time}_{random}.task - + pullSnapshotTask file, record the slots and owners. When a node joins a cluster, - it will create pullSnapshotTask file to track which data to be pulled - + under directory basedir/raft/{nodeIdentifier}/snapshot_task/ - -## Raft Related Files -> under directory basedir/system/raftLog/{nodeIdentifier}/ - -### Raft Log -1. .data-{version} - + raft committed logs, only save the latest 1000(configurable) committed logs - -### Raft Meta -1. logMeta - + raft meta, like hardState and Meta - + hardState: voteFor, term - + Meta: commitLogTerm, commitLogIndex, lastLogTerm, lastLogIndex - + ... -2. logMeta.tmp - + temp file, to avoid damaging the logMeta when updating it - -### Raft Catch Up -> under directory basedir/remote/{nodeIdentifier}/{storageGroupName}/{partitionNum}/ - -1. {fileName}.tsfile - + remote TsFile, will be loaded during snapshot installation -2. {fileName}.tsfile.mod - + remote TsFile modification file, will be loaded during snapshot installation \ No newline at end of file diff --git a/docs/zh/UserGuide/Data-Concept/Compression.md b/docs/zh/UserGuide/Data-Concept/Compression.md index 82ebffd09f..848a1f2c16 100644 --- a/docs/zh/UserGuide/Data-Concept/Compression.md +++ b/docs/zh/UserGuide/Data-Concept/Compression.md @@ -122,7 +122,7 @@ SDT 在刷新到磁盘时进行压缩。 SDT 算法始终存储第一个点， ## 压缩比统计信息 -压缩比统计信息文件：data/system/storage_groups/compression_ratio/Ratio-{ratio_sum}-{memtable_flush_time} +压缩比统计信息文件：data/system/compression_ratio/Ratio-{ratio_sum}-{memtable_flush_time} * ratio_sum: memtable压缩比的总和 * memtable_flush_time: memtable刷盘的总次数 diff --git a/docs/zh/UserGuide/QuickStart/ServerFileList.md b/docs/zh/UserGuide/QuickStart/ServerFileList.md index c49f74116f..1bf3f97e00 100644 --- a/docs/zh/UserGuide/QuickStart/ServerFileList.md +++ b/docs/zh/UserGuide/QuickStart/ServerFileList.md @@ -66,7 +66,7 @@ + 每个 memtable 会对应一个 wal 文件 #### TsFile -> 在 basedir/data/sequence or unsequence/{StorageGroupName}/{TimePartitionId}/目录下 +> 在 basedir/data/sequence or unsequence/{DatabaseName}/{DataRegionId}/{TimePartitionId}/目录下 1. {time}-{version}-{mergeCnt}.tsfile + 数据文件 2. {TsFileName}.tsfile.mod @@ -81,7 +81,7 @@ + 关闭标记文件，用于标记 TsFile 处于关闭状态，重启后可以据此选择是关闭或继续写入该文件 #### Version -> 在 basedir/system/storage_groups/{StorageGroupName}/{TimePartitionId} or upgrade 目录下 +> 在 basedir/system/databases/{DatabaseName}/{DataRegionId}/{TimePartitionId} or upgrade 目录下 1. Version-{version} + 版本号文件，使用文件名来记录当前最大的版本号 @@ -91,7 +91,7 @@ + 记录升级进度 #### Merge -> 在 basedir/system/storage_groups/{StorageGroup}/目录下 +> 在 basedir/system/databases/{DatabaseName}/目录下 1. merge.mods + 记录合并过程中发生的删除等操作 2. merge.log @@ -108,43 +108,3 @@ 1. Ration-{compressionRatioSum}-{calTimes} + 记录每个文件的压缩率 ---- - -# 集群模式 -> 注意：下面文件是相对单机新增的文件列表 - -## 配置文件 -1. iotdb-cluster.properties - -## 状态相关文件 -> 在 basedir 目录下 -1. node_identifier - + 本地节点在集群中的唯一标识 -2. partitions - + 分区表文件，记录数据分布信息 -3. {time}_{random}.task - + pullSnapshotTask 文件，记录的是 slot 和 owner，当一个节点加入集群时，会创建这个文件来跟踪数据拉取情况 - + 在 basedir/raft/{nodeIdentifier}/snapshot_task/目录下 - -## Raft 相关文件 -> 在 basedir/system/raftLog/{nodeIdentifier}/目录下 - -### Raft Log -1. .data-{version} - + raft committed log, 默认只保留最新的 1000 条日志 - -### Raft Meta -1. logMeta - + raft 一些相关元数据，例如选举相关信息和日志提交信息 - + hardState: voteFor, term - + Meta: commitLogTerm, commitLogIndex, lastLogTerm, lastLogIndex - + ... -2. logMeta.tmp - + 临时文件，用于更新 logMeta 时防止损坏旧的 logMeta 文件 - -### Raft Catch Up -> 在 basedir/remote/{nodeIdentifier}/{storageGroupName}/{partitionNum}/目录下 -1. {fileName}.tsfile - + 远程 TsFile，拉取完毕会 load 进对应的 database 数据目录下 -2. {fileName}.tsfile.mod - + 远程更新文件 TsFile.mod，拉取完毕会 load 进对应的 database 数据目录下 diff --git a/server/src/main/java/org/apache/iotdb/db/engine/StorageEngineV2.java b/server/src/main/java/org/apache/iotdb/db/engine/StorageEngineV2.java index 4c03ee5a67..94ae35c0cd 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/StorageEngineV2.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/StorageEngineV2.java @@ -102,11 +102,11 @@ public class StorageEngineV2 implements IService { @ServerConfigConsistent private static boolean enablePartition = config.isEnablePartition(); /** - * a folder (system/storage_groups/ by default) that persist system info. Each Storage Processor - * will have a subfolder under the systemDir. + * a folder (system/databases/ by default) that persist system info. Each database will have a + * subfolder under the systemDir. */ private final String systemDir = - FilePathUtils.regularizePath(config.getSystemDir()) + "storage_groups"; + FilePathUtils.regularizePath(config.getSystemDir()) + "databases"; /** DataRegionId -> DataRegion */ private final ConcurrentHashMap<DataRegionId, DataRegion> dataRegionMap = diff --git a/server/src/main/java/org/apache/iotdb/db/engine/snapshot/SnapshotLoader.java b/server/src/main/java/org/apache/iotdb/db/engine/snapshot/SnapshotLoader.java index 41cb58d580..c70fa8844e 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/snapshot/SnapshotLoader.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/snapshot/SnapshotLoader.java @@ -62,7 +62,7 @@ public class SnapshotLoader { return new DataRegion( IoTDBDescriptor.getInstance().getConfig().getSystemDir() + File.separator - + "storage_groups" + + "databases" + File.separator + storageGroupName, dataRegionId, diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/DataRegion.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/DataRegion.java index 3bcf0a916e..39de6413ce 100755 --- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/DataRegion.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/DataRegion.java @@ -1404,7 +1404,7 @@ public class DataRegion { } /** - * delete the storageGroup's own folder in folder data/system/storage_groups + * delete the database's own folder in folder data/system/databases * * @param systemDir system dir */ diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/idtable/IDTableManager.java b/server/src/main/java/org/apache/iotdb/db/metadata/idtable/IDTableManager.java index 675552e618..ee52952fe8 100644 --- a/server/src/main/java/org/apache/iotdb/db/metadata/idtable/IDTableManager.java +++ b/server/src/main/java/org/apache/iotdb/db/metadata/idtable/IDTableManager.java @@ -46,7 +46,7 @@ public class IDTableManager { /** system dir */ private final String systemDir = FilePathUtils.regularizePath(IoTDBDescriptor.getInstance().getConfig().getSystemDir()) - + "storage_groups"; + + "databases"; // region IDManager Singleton private static class IDManagerHolder { diff --git a/server/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java b/server/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java index f840265c2f..b681205060 100644 --- a/server/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java +++ b/server/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java @@ -366,7 +366,7 @@ public class EnvironmentUtils { // create database createDir(config.getSystemDir()); // create sg dir - String sgDir = FilePathUtils.regularizePath(config.getSystemDir()) + "storage_groups"; + String sgDir = FilePathUtils.regularizePath(config.getSystemDir()) + "databases"; createDir(sgDir); // create sync createDir(commonConfig.getSyncFolder());
