This is an automated email from the ASF dual-hosted git repository.
shuwenwei pushed a commit to branch table_disk_usage_statistics_with_cache
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to
refs/heads/table_disk_usage_statistics_with_cache by this push:
new 3a91c805a56 refactor
3a91c805a56 is described below
commit 3a91c805a56c186d3c6e0f5d6103b9d40aad34a8
Author: shuwenwei <[email protected]>
AuthorDate: Thu Jan 22 18:57:32 2026 +0800
refactor
---
.../AbstractTableSizeCacheWriter.java | 4 ++++
.../tableDiskUsageCache/TableDiskUsageCache.java | 27 +++++++++++++++++-----
.../TableDiskUsageCacheReader.java | 1 +
.../object/IObjectTableSizeCacheReader.java | 2 +-
.../TsFileTableDiskUsageCacheWriter.java | 11 ++++-----
.../{ => tsfile}/TsFileTableSizeCacheReader.java | 3 ++-
.../TsFileTableSizeIndexFileWriter.java | 6 ++---
.../utils/TsFileTableSizeCacheWriterTest.java | 4 ++--
8 files changed, 39 insertions(+), 19 deletions(-)
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/AbstractTableSizeCacheWriter.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/AbstractTableSizeCacheWriter.java
index 37137ba1959..1ffcdeb7c68 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/AbstractTableSizeCacheWriter.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/AbstractTableSizeCacheWriter.java
@@ -40,6 +40,10 @@ public abstract class AbstractTableSizeCacheWriter {
this.dir = StorageEngine.getDataRegionSystemDir(database, regionId + "");
}
+ protected void failedToRecover(Exception e) {
+ TableDiskUsageCache.getInstance().failedToRecover(e);
+ }
+
protected void deleteOldVersionFiles(int maxVersion, String prefix,
List<File> files) {
for (File file : files) {
try {
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/TableDiskUsageCache.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/TableDiskUsageCache.java
index 081d5d116e7..132d39a3a74 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/TableDiskUsageCache.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/TableDiskUsageCache.java
@@ -24,6 +24,8 @@ import org.apache.iotdb.commons.concurrent.ThreadName;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileID;
import
org.apache.iotdb.db.storageengine.dataregion.utils.tableDiskUsageCache.object.EmptyObjectTableSizeCacheReader;
import
org.apache.iotdb.db.storageengine.dataregion.utils.tableDiskUsageCache.object.IObjectTableSizeCacheReader;
+import
org.apache.iotdb.db.storageengine.dataregion.utils.tableDiskUsageCache.tsfile.TsFileTableDiskUsageCacheWriter;
+import
org.apache.iotdb.db.storageengine.dataregion.utils.tableDiskUsageCache.tsfile.TsFileTableSizeCacheReader;
import org.apache.tsfile.utils.Pair;
import org.slf4j.Logger;
@@ -44,6 +46,7 @@ public class TableDiskUsageCache {
protected final BlockingQueue<Operation> queue = new LinkedBlockingQueue<>();
protected final Map<Integer, DataRegionTableSizeCacheWriter> writerMap = new
HashMap<>();
protected final ScheduledExecutorService scheduledExecutorService;
+ protected volatile boolean failedToRecover = false;
protected TableDiskUsageCache() {
scheduledExecutorService =
@@ -75,6 +78,11 @@ public class TableDiskUsageCache {
}
}
+ protected void failedToRecover(Exception e) {
+ failedToRecover = true;
+ LOGGER.error("Failed to recover TableDiskUsageCache", e);
+ }
+
protected void checkAndMaySyncObjectDeltaToFile() {}
protected void checkAndMayCompact(long maxRunTime) {
@@ -102,11 +110,11 @@ public class TableDiskUsageCache {
if (tableSizeMap == null || tableSizeMap.isEmpty()) {
return;
}
- queue.add(new WriteOperation(database, tsFileID, tableSizeMap));
+ addOperationToQueue(new WriteOperation(database, tsFileID, tableSizeMap));
}
public void write(String database, TsFileID originTsFileID, TsFileID
newTsFileID) {
- queue.add(new ReplaceTsFileOperation(database, originTsFileID,
newTsFileID));
+ addOperationToQueue(new ReplaceTsFileOperation(database, originTsFileID,
newTsFileID));
}
public void writeObjectDelta(
@@ -118,23 +126,23 @@ public class TableDiskUsageCache {
String database, int regionId, boolean readTsFileCache, boolean
readObjectFileCache) {
StartReadOperation operation =
new StartReadOperation(database, regionId, readTsFileCache,
readObjectFileCache);
- queue.add(operation);
+ addOperationToQueue(operation);
return operation.future;
}
public void endRead(String database, int regionId) {
EndReadOperation operation = new EndReadOperation(database, regionId);
- queue.add(operation);
+ addOperationToQueue(operation);
}
public void registerRegion(String database, int regionId) {
RegisterRegionOperation operation = new RegisterRegionOperation(database,
regionId);
- queue.add(operation);
+ addOperationToQueue(operation);
}
public void remove(String database, int regionId) {
RemoveRegionOperation operation = new RemoveRegionOperation(database,
regionId);
- queue.add(operation);
+ addOperationToQueue(operation);
try {
operation.future.get(5, TimeUnit.SECONDS);
} catch (InterruptedException e) {
@@ -143,6 +151,13 @@ public class TableDiskUsageCache {
}
}
+ protected void addOperationToQueue(Operation operation) {
+ if (failedToRecover) {
+ return;
+ }
+ queue.add(operation);
+ }
+
public void close() {
if (scheduledExecutorService != null) {
scheduledExecutorService.shutdownNow();
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/TableDiskUsageCacheReader.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/TableDiskUsageCacheReader.java
index 4c591f27266..7c6b8b713b1 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/TableDiskUsageCacheReader.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/TableDiskUsageCacheReader.java
@@ -25,6 +25,7 @@ import
org.apache.iotdb.db.storageengine.dataregion.DataRegion;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileID;
import
org.apache.iotdb.db.storageengine.dataregion.utils.TableDiskUsageStatisticUtil;
import
org.apache.iotdb.db.storageengine.dataregion.utils.tableDiskUsageCache.object.IObjectTableSizeCacheReader;
+import
org.apache.iotdb.db.storageengine.dataregion.utils.tableDiskUsageCache.tsfile.TsFileTableSizeCacheReader;
import org.apache.tsfile.utils.Pair;
import org.apache.tsfile.utils.RamUsageEstimator;
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/object/IObjectTableSizeCacheReader.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/object/IObjectTableSizeCacheReader.java
index cc38307f093..2953f37fc1b 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/object/IObjectTableSizeCacheReader.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/object/IObjectTableSizeCacheReader.java
@@ -23,7 +23,7 @@ import
org.apache.iotdb.db.storageengine.dataregion.utils.tableDiskUsageCache.Da
import java.io.IOException;
-public interface IObjectTableSizeCacheReader {
+public interface IObjectTableSizeCacheReader extends AutoCloseable {
boolean loadObjectFileTableSize(
DataRegionTableSizeQueryContext dataRegionContext, long startTime, long
maxRunTime)
throws IOException;
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/TsFileTableDiskUsageCacheWriter.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/tsfile/TsFileTableDiskUsageCacheWriter.java
similarity index 97%
rename from
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/TsFileTableDiskUsageCacheWriter.java
rename to
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/tsfile/TsFileTableDiskUsageCacheWriter.java
index 90c54526404..58011497a66 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/TsFileTableDiskUsageCacheWriter.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/tsfile/TsFileTableDiskUsageCacheWriter.java
@@ -17,7 +17,7 @@
* under the License.
*/
-package org.apache.iotdb.db.storageengine.dataregion.utils.tableDiskUsageCache;
+package
org.apache.iotdb.db.storageengine.dataregion.utils.tableDiskUsageCache.tsfile;
import org.apache.iotdb.commons.consensus.DataRegionId;
import org.apache.iotdb.db.storageengine.StorageEngine;
@@ -25,6 +25,8 @@ import
org.apache.iotdb.db.storageengine.dataregion.DataRegion;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileID;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileManager;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
+import
org.apache.iotdb.db.storageengine.dataregion.utils.tableDiskUsageCache.AbstractTableSizeCacheWriter;
+import
org.apache.iotdb.db.storageengine.dataregion.utils.tableDiskUsageCache.TimePartitionTableSizeQueryContext;
import org.apache.tsfile.utils.Pair;
import org.slf4j.Logger;
@@ -123,7 +125,8 @@ public class TsFileTableDiskUsageCacheWriter extends
AbstractTableSizeCacheWrite
this.tsFileTableSizeIndexFileWriter =
new TsFileTableSizeIndexFileWriter(
regionId, currentKeyIndexFile, currentValueIndexFile,
needRecover);
- } catch (IOException ignored) {
+ } catch (IOException e) {
+ failedToRecover(e);
}
}
@@ -229,13 +232,9 @@ public class TsFileTableDiskUsageCacheWriter extends
AbstractTableSizeCacheWrite
File targetValueFile = generateValueFile(currentIndexFileVersion + 1,
false);
targetFileWriter.getKeyFile().renameTo(targetKeyFile);
targetFileWriter.getValueFile().renameTo(targetValueFile);
- this.tsFileTableSizeIndexFileWriter.close();
} catch (Exception e) {
logger.error("Failed to execute compaction for tsfile table size cache
file", e);
} finally {
- if (tsFileTableSizeIndexFileWriter != null) {
- tsFileTableSizeIndexFileWriter.close();
- }
if (targetFileWriter != null) {
targetFileWriter.close();
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/TsFileTableSizeCacheReader.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/tsfile/TsFileTableSizeCacheReader.java
similarity index 98%
rename from
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/TsFileTableSizeCacheReader.java
rename to
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/tsfile/TsFileTableSizeCacheReader.java
index 351c2b10e9f..d387032da18 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/TsFileTableSizeCacheReader.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/tsfile/TsFileTableSizeCacheReader.java
@@ -17,10 +17,11 @@
* under the License.
*/
-package org.apache.iotdb.db.storageengine.dataregion.utils.tableDiskUsageCache;
+package
org.apache.iotdb.db.storageengine.dataregion.utils.tableDiskUsageCache.tsfile;
import org.apache.iotdb.commons.exception.IoTDBRuntimeException;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileID;
+import
org.apache.iotdb.db.storageengine.dataregion.utils.tableDiskUsageCache.DataRegionTableSizeQueryContext;
import org.apache.iotdb.db.utils.MmapUtil;
import org.apache.iotdb.rpc.TSStatusCode;
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/TsFileTableSizeIndexFileWriter.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/tsfile/TsFileTableSizeIndexFileWriter.java
similarity index 97%
rename from
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/TsFileTableSizeIndexFileWriter.java
rename to
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/tsfile/TsFileTableSizeIndexFileWriter.java
index a5f6aba555d..99fcc9308b9 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/TsFileTableSizeIndexFileWriter.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/tsfile/TsFileTableSizeIndexFileWriter.java
@@ -17,7 +17,7 @@
* under the License.
*/
-package org.apache.iotdb.db.storageengine.dataregion.utils.tableDiskUsageCache;
+package
org.apache.iotdb.db.storageengine.dataregion.utils.tableDiskUsageCache.tsfile;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileID;
@@ -33,8 +33,8 @@ import java.nio.channels.FileChannel;
import java.nio.file.StandardOpenOption;
import java.util.Map;
-import static
org.apache.iotdb.db.storageengine.dataregion.utils.tableDiskUsageCache.TsFileTableDiskUsageCacheWriter.KEY_FILE_RECORD_TYPE_OFFSET;
-import static
org.apache.iotdb.db.storageengine.dataregion.utils.tableDiskUsageCache.TsFileTableDiskUsageCacheWriter.KEY_FILE_RECORD_TYPE_REDIRECT;
+import static
org.apache.iotdb.db.storageengine.dataregion.utils.tableDiskUsageCache.tsfile.TsFileTableDiskUsageCacheWriter.KEY_FILE_RECORD_TYPE_OFFSET;
+import static
org.apache.iotdb.db.storageengine.dataregion.utils.tableDiskUsageCache.tsfile.TsFileTableDiskUsageCacheWriter.KEY_FILE_RECORD_TYPE_REDIRECT;
public class TsFileTableSizeIndexFileWriter {
private final int regionId;
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/utils/TsFileTableSizeCacheWriterTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/utils/TsFileTableSizeCacheWriterTest.java
index 6517ff466d8..e65be56db80 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/utils/TsFileTableSizeCacheWriterTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/utils/TsFileTableSizeCacheWriterTest.java
@@ -27,8 +27,8 @@ import
org.apache.iotdb.db.storageengine.dataregion.DataRegion;
import
org.apache.iotdb.db.storageengine.dataregion.compaction.AbstractCompactionTest;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileManager;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
-import
org.apache.iotdb.db.storageengine.dataregion.utils.tableDiskUsageCache.TsFileTableDiskUsageCacheWriter;
-import
org.apache.iotdb.db.storageengine.dataregion.utils.tableDiskUsageCache.TsFileTableSizeCacheReader;
+import
org.apache.iotdb.db.storageengine.dataregion.utils.tableDiskUsageCache.tsfile.TsFileTableDiskUsageCacheWriter;
+import
org.apache.iotdb.db.storageengine.dataregion.utils.tableDiskUsageCache.tsfile.TsFileTableSizeCacheReader;
import org.apache.tsfile.exception.write.WriteProcessException;
import org.junit.After;
