This is an automated email from the ASF dual-hosted git repository.
shuwenwei pushed a commit to branch table_disk_usage_statistics_with_cache
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to
refs/heads/table_disk_usage_statistics_with_cache by this push:
new 2b87ce713a7 refactor
2b87ce713a7 is described below
commit 2b87ce713a7fc7a8bf1079fee0c982c3bdbf1b49
Author: shuwenwei <[email protected]>
AuthorDate: Tue Jan 20 18:45:27 2026 +0800
refactor
---
.../AbstractTableSizeCacheWriter.java | 80 +++++++++++++++
.../TsFileTableDiskUsageCacheWriter.java | 111 ++++++++-------------
...er.java => TsFileTableSizeIndexFileWriter.java} | 6 +-
3 files changed, 122 insertions(+), 75 deletions(-)
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/AbstractTableSizeCacheWriter.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/AbstractTableSizeCacheWriter.java
new file mode 100644
index 00000000000..8e10932ebd6
--- /dev/null
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/AbstractTableSizeCacheWriter.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.storageengine.dataregion.utils.tableDiskUsageCache;
+
+import org.apache.iotdb.db.storageengine.StorageEngine;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+public abstract class AbstractTableSizeCacheWriter {
+ protected final int regionId;
+ protected int activeReaderNum = 0;
+ protected long previousCompactionTimestamp = System.currentTimeMillis();
+ protected long lastWriteTimestamp = System.currentTimeMillis();
+ protected int currentIndexFileVersion = 0;
+ protected final File dir;
+
+ public AbstractTableSizeCacheWriter(String database, int regionId) {
+ this.regionId = regionId;
+ this.dir = StorageEngine.getDataRegionSystemDir(database, regionId + "");
+ }
+
+ protected void deleteOldVersionFiles(int maxVersion, String prefix,
List<File> files) {
+ for (File file : files) {
+ try {
+ int version =
Integer.parseInt(file.getName().substring(prefix.length()));
+ if (version != maxVersion) {
+ Files.deleteIfExists(file.toPath());
+ }
+ } catch (Exception e) {
+ }
+ }
+ }
+
+ public void closeIfIdle() {
+ if (System.currentTimeMillis() - lastWriteTimestamp >=
TimeUnit.MINUTES.toMillis(1)) {
+ close();
+ }
+ }
+
+ public abstract boolean needCompact();
+
+ public abstract void compact();
+
+ public abstract void flush() throws IOException;
+
+ public abstract void sync() throws IOException;
+
+ public void increaseActiveReaderNum() {
+ activeReaderNum++;
+ }
+
+ public void decreaseActiveReaderNum() {
+ if (activeReaderNum > 0) {
+ activeReaderNum--;
+ }
+ }
+
+ public abstract void close();
+}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/TsFileTableDiskUsageCacheWriter.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/TsFileTableDiskUsageCacheWriter.java
index fd1c79b06ed..1dc81056b21 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/TsFileTableDiskUsageCacheWriter.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/TsFileTableDiskUsageCacheWriter.java
@@ -42,7 +42,7 @@ import java.util.Map;
import java.util.concurrent.TimeUnit;
import java.util.stream.Stream;
-public class TsFileTableDiskUsageCacheWriter {
+public class TsFileTableDiskUsageCacheWriter extends
AbstractTableSizeCacheWriter {
private static final Logger logger =
LoggerFactory.getLogger(TsFileTableDiskUsageCacheWriter.class);
private static final String TSFILE_CACHE_KEY_FILENAME_PREFIX =
"TableSizeKeyFile_";
@@ -53,24 +53,17 @@ public class TsFileTableDiskUsageCacheWriter {
public static final byte KEY_FILE_RECORD_TYPE_OFFSET = 1;
public static final byte KEY_FILE_RECORD_TYPE_REDIRECT = 2;
- private final int regionId;
- private int activeReaderNum = 0;
- private long previousCompactionTimestamp = System.currentTimeMillis();
- private long lastWriteTimestamp = System.currentTimeMillis();
- private int currentTsFileIndexFileVersion = 0;
- private final File dir;
- private TsFileTableSizeCacheWriter tsFileTableSizeCacheWriter;
+ private TsFileTableSizeIndexFileWriter tsFileTableSizeIndexFileWriter;
public TsFileTableDiskUsageCacheWriter(String database, int regionId) {
- this.regionId = regionId;
- this.dir = StorageEngine.getDataRegionSystemDir(database, regionId + "");
+ super(database, regionId);
recoverTsFileTableSizeIndexFile(true);
}
private void recoverTsFileTableSizeIndexFile(boolean needRecover) {
dir.mkdirs();
File[] files = dir.listFiles();
- currentTsFileIndexFileVersion = 0;
+ currentIndexFileVersion = 0;
List<File> keyFiles = new ArrayList<>();
List<File> valueFiles = new ArrayList<>();
if (files != null) {
@@ -114,54 +107,36 @@ public class TsFileTableDiskUsageCacheWriter {
continue;
}
}
- currentTsFileIndexFileVersion =
Math.max(currentTsFileIndexFileVersion, version);
+ currentIndexFileVersion = Math.max(currentIndexFileVersion, version);
keyFiles.add(file);
}
if (keyFiles.size() > 1) {
- deleteOldVersionFiles(
- currentTsFileIndexFileVersion, TSFILE_CACHE_KEY_FILENAME_PREFIX,
keyFiles);
+ deleteOldVersionFiles(currentIndexFileVersion,
TSFILE_CACHE_KEY_FILENAME_PREFIX, keyFiles);
}
if (valueFiles.size() > 1) {
deleteOldVersionFiles(
- currentTsFileIndexFileVersion, TSFILE_CACHE_VALUE_FILENAME_PREFIX,
valueFiles);
+ currentIndexFileVersion, TSFILE_CACHE_VALUE_FILENAME_PREFIX,
valueFiles);
}
}
- File currentKeyIndexFile = generateKeyFile(currentTsFileIndexFileVersion,
false);
- File currentValueIndexFile =
generateValueFile(currentTsFileIndexFileVersion, false);
+ File currentKeyIndexFile = generateKeyFile(currentIndexFileVersion, false);
+ File currentValueIndexFile = generateValueFile(currentIndexFileVersion,
false);
try {
- this.tsFileTableSizeCacheWriter =
- new TsFileTableSizeCacheWriter(
+ this.tsFileTableSizeIndexFileWriter =
+ new TsFileTableSizeIndexFileWriter(
regionId, currentKeyIndexFile, currentValueIndexFile,
needRecover);
} catch (IOException ignored) {
}
}
- private void deleteOldVersionFiles(int maxVersion, String prefix, List<File>
files) {
- for (File file : files) {
- try {
- int version =
Integer.parseInt(file.getName().substring(prefix.length()));
- if (version != maxVersion) {
- Files.deleteIfExists(file.toPath());
- }
- } catch (Exception e) {
- }
- }
- }
-
public void write(TsFileID tsFileID, Map<String, Long> tableSizeMap) throws
IOException {
- tsFileTableSizeCacheWriter.write(tsFileID, tableSizeMap);
+ tsFileTableSizeIndexFileWriter.write(tsFileID, tableSizeMap);
}
public void write(TsFileID originTsFileID, TsFileID newTsFileID) throws
IOException {
- tsFileTableSizeCacheWriter.write(originTsFileID, newTsFileID);
- }
-
- public void closeIfIdle() {
- if (System.currentTimeMillis() - lastWriteTimestamp >=
TimeUnit.MINUTES.toMillis(1)) {
- close();
- }
+ tsFileTableSizeIndexFileWriter.write(originTsFileID, newTsFileID);
}
+ @Override
public boolean needCompact() {
if (activeReaderNum > 0) {
return false;
@@ -180,15 +155,16 @@ public class TsFileTableDiskUsageCacheWriter {
return delta >= 1000;
}
+ @Override
public void compact() {
previousCompactionTimestamp = System.currentTimeMillis();
- this.tsFileTableSizeCacheWriter.close();
+ this.tsFileTableSizeIndexFileWriter.close();
TsFileTableSizeCacheReader cacheFileReader =
new TsFileTableSizeCacheReader(
- tsFileTableSizeCacheWriter.getKeyFile().length(),
- tsFileTableSizeCacheWriter.getKeyFile(),
- tsFileTableSizeCacheWriter.getValueFile().length(),
- tsFileTableSizeCacheWriter.getValueFile(),
+ tsFileTableSizeIndexFileWriter.getKeyFile().length(),
+ tsFileTableSizeIndexFileWriter.getKeyFile(),
+ tsFileTableSizeIndexFileWriter.getValueFile().length(),
+ tsFileTableSizeIndexFileWriter.getValueFile(),
regionId);
Map<Long, TimePartitionTableSizeQueryContext> contextMap = new HashMap<>();
try {
@@ -236,13 +212,13 @@ public class TsFileTableDiskUsageCacheWriter {
}
validFilesOrderByOffset.sort(Comparator.comparingLong(Pair::getRight));
- TsFileTableSizeCacheWriter targetFileWriter = null;
+ TsFileTableSizeIndexFileWriter targetFileWriter = null;
try {
targetFileWriter =
- new TsFileTableSizeCacheWriter(
+ new TsFileTableSizeIndexFileWriter(
regionId,
- generateKeyFile(currentTsFileIndexFileVersion + 1, true),
- generateValueFile(currentTsFileIndexFileVersion + 1, true));
+ generateKeyFile(currentIndexFileVersion + 1, true),
+ generateValueFile(currentIndexFileVersion + 1, true));
cacheFileReader.openValueFile();
for (Pair<TsFileID, Long> pair : validFilesOrderByOffset) {
TsFileID tsFileID = pair.getLeft();
@@ -253,16 +229,16 @@ public class TsFileTableDiskUsageCacheWriter {
targetFileWriter.close();
// replace
- File targetKeyFile = generateKeyFile(currentTsFileIndexFileVersion + 1,
false);
- File targetValueFile = generateValueFile(currentTsFileIndexFileVersion +
1, false);
+ File targetKeyFile = generateKeyFile(currentIndexFileVersion + 1, false);
+ File targetValueFile = generateValueFile(currentIndexFileVersion + 1,
false);
targetFileWriter.getKeyFile().renameTo(targetKeyFile);
targetFileWriter.getValueFile().renameTo(targetValueFile);
- this.tsFileTableSizeCacheWriter.close();
+ this.tsFileTableSizeIndexFileWriter.close();
} catch (Exception e) {
logger.error("Failed to execute compaction for tsfile table size cache
file", e);
} finally {
- if (tsFileTableSizeCacheWriter != null) {
- tsFileTableSizeCacheWriter.close();
+ if (tsFileTableSizeIndexFileWriter != null) {
+ tsFileTableSizeIndexFileWriter.close();
}
if (targetFileWriter != null) {
targetFileWriter.close();
@@ -290,43 +266,34 @@ public class TsFileTableDiskUsageCacheWriter {
+ (isTempFile ? TEMP_CACHE_FILE_SUBFIX : ""));
}
+ @Override
public void flush() throws IOException {
- tsFileTableSizeCacheWriter.flush();
+ tsFileTableSizeIndexFileWriter.flush();
}
public File getKeyFile() {
- return tsFileTableSizeCacheWriter.getKeyFile();
+ return tsFileTableSizeIndexFileWriter.getKeyFile();
}
public File getValueFile() {
- return tsFileTableSizeCacheWriter.getValueFile();
+ return tsFileTableSizeIndexFileWriter.getValueFile();
}
public long keyFileLength() {
- return tsFileTableSizeCacheWriter.keyFileLength();
+ return tsFileTableSizeIndexFileWriter.keyFileLength();
}
public long valueFileLength() {
- return tsFileTableSizeCacheWriter.valueFileLength();
+ return tsFileTableSizeIndexFileWriter.valueFileLength();
}
+ @Override
public void sync() throws IOException {
- tsFileTableSizeCacheWriter.sync();
- }
-
- public void increaseActiveReaderNum() {
- activeReaderNum++;
- }
-
- public void decreaseActiveReaderNum() {
- if (activeReaderNum > 0) {
- activeReaderNum--;
- }
+ tsFileTableSizeIndexFileWriter.sync();
}
- public void removeFiles() {}
-
+ @Override
public void close() {
- this.tsFileTableSizeCacheWriter.close();
+ this.tsFileTableSizeIndexFileWriter.close();
}
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/TsFileTableSizeCacheWriter.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/TsFileTableSizeIndexFileWriter.java
similarity index 98%
rename from
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/TsFileTableSizeCacheWriter.java
rename to
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/TsFileTableSizeIndexFileWriter.java
index b073a5f2204..a5f6aba555d 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/TsFileTableSizeCacheWriter.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/TsFileTableSizeIndexFileWriter.java
@@ -36,7 +36,7 @@ import java.util.Map;
import static
org.apache.iotdb.db.storageengine.dataregion.utils.tableDiskUsageCache.TsFileTableDiskUsageCacheWriter.KEY_FILE_RECORD_TYPE_OFFSET;
import static
org.apache.iotdb.db.storageengine.dataregion.utils.tableDiskUsageCache.TsFileTableDiskUsageCacheWriter.KEY_FILE_RECORD_TYPE_REDIRECT;
-public class TsFileTableSizeCacheWriter {
+public class TsFileTableSizeIndexFileWriter {
private final int regionId;
private final File currentKeyIndexFile;
private final File currentValueIndexFile;
@@ -47,12 +47,12 @@ public class TsFileTableSizeCacheWriter {
private long keyFileSize;
private long valueFileSize;
- public TsFileTableSizeCacheWriter(
+ public TsFileTableSizeIndexFileWriter(
int regionId, File currentKeyIndexFile, File currentValueIndexFile)
throws IOException {
this(regionId, currentKeyIndexFile, currentValueIndexFile, true);
}
- public TsFileTableSizeCacheWriter(
+ public TsFileTableSizeIndexFileWriter(
int regionId, File currentKeyIndexFile, File currentValueIndexFile,
boolean recover)
throws IOException {
this.regionId = regionId;
