This is an automated email from the ASF dual-hosted git repository.
sarvekshayr pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 0e1b03cef7f HDDS-15466. Make RocksDB bottommost level compaction
options configurable for CLI tools (#10428)
0e1b03cef7f is described below
commit 0e1b03cef7f03aa652910c47561177c0975cc5aa
Author: Rishabh Patel <[email protected]>
AuthorDate: Tue Jun 16 19:28:59 2026 -0700
HDDS-15466. Make RocksDB bottommost level compaction options configurable
for CLI tools (#10428)
Co-authored-by: Sarveksha Yeshavantha Raju
<[email protected]>
---
.../ozone/repair/ldb/RocksDBManualCompaction.java | 15 ++++++++--
.../apache/hadoop/ozone/repair/om/CompactOMDB.java | 16 ++++++++--
.../hadoop/ozone/repair/ldb/TestLdbRepair.java | 3 +-
.../hadoop/ozone/om/protocol/OMAdminProtocol.java | 10 +++++--
.../protocolPB/OMAdminProtocolClientSideImpl.java | 3 +-
.../src/main/smoketest/repair/om-compact.robot | 2 +-
.../src/main/proto/OMAdminProtocol.proto | 4 +++
.../org/apache/hadoop/ozone/om/OzoneManager.java | 6 ++--
.../hadoop/ozone/om/service/CompactDBUtil.java | 34 +++++++++++++++++-----
.../hadoop/ozone/om/service/CompactionService.java | 7 +++--
.../protocolPB/OMAdminProtocolServerSideImpl.java | 6 +++-
11 files changed, 84 insertions(+), 22 deletions(-)
diff --git
a/hadoop-ozone/cli-repair/src/main/java/org/apache/hadoop/ozone/repair/ldb/RocksDBManualCompaction.java
b/hadoop-ozone/cli-repair/src/main/java/org/apache/hadoop/ozone/repair/ldb/RocksDBManualCompaction.java
index 5b6662805af..ffefe93b25d 100644
---
a/hadoop-ozone/cli-repair/src/main/java/org/apache/hadoop/ozone/repair/ldb/RocksDBManualCompaction.java
+++
b/hadoop-ozone/cli-repair/src/main/java/org/apache/hadoop/ozone/repair/ldb/RocksDBManualCompaction.java
@@ -27,6 +27,7 @@
import org.apache.hadoop.hdds.utils.db.managed.ManagedDBOptions;
import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksDB;
import org.apache.hadoop.ozone.debug.RocksDBUtils;
+import org.apache.hadoop.ozone.om.service.CompactDBUtil;
import org.apache.hadoop.ozone.repair.RepairTool;
import org.apache.hadoop.util.Time;
import org.rocksdb.ColumnFamilyDescriptor;
@@ -62,6 +63,13 @@ public class RocksDBManualCompaction extends RepairTool {
description = "Column family name")
private String columnFamilyName;
+ @CommandLine.Option(names = {"--bottommost-level-compaction", "--blc"},
+ description = "BottommostLevelCompaction option for RocksDB compaction."
+
+ " Valid values: 0 (kSkip), 1 (kIfHaveCompactionFilter), 2 (kForce),
3 (kForceOptimized).",
+ defaultValue = "0",
+ showDefaultValue = CommandLine.Help.Visibility.ALWAYS)
+ private int bottommostLevelCompaction;
+
private String getConsoleReadLineWithFormat() {
err().printf(WARNING_TO_STOP_SERVICE);
return getScanner().nextLine().trim();
@@ -96,11 +104,14 @@ public void execute() throws Exception {
" is not in a column family in DB for the given path.");
}
- info("Running compaction on " + columnFamilyName);
+ ManagedCompactRangeOptions.BottommostLevelCompaction blcOption =
+
CompactDBUtil.getBottommostLevelCompaction(bottommostLevelCompaction);
+ info("Running compaction on " + columnFamilyName +
+ " with bottommost level compaction: " + blcOption.name());
long startTime = Time.monotonicNow();
if (!isDryRun()) {
ManagedCompactRangeOptions compactOptions = new
ManagedCompactRangeOptions();
-
compactOptions.setBottommostLevelCompaction(ManagedCompactRangeOptions.BottommostLevelCompaction.kForce);
+ compactOptions.setBottommostLevelCompaction(blcOption);
db.get().compactRange(cfh, null, null, compactOptions);
}
long duration = Time.monotonicNow() - startTime;
diff --git
a/hadoop-ozone/cli-repair/src/main/java/org/apache/hadoop/ozone/repair/om/CompactOMDB.java
b/hadoop-ozone/cli-repair/src/main/java/org/apache/hadoop/ozone/repair/om/CompactOMDB.java
index ae3f421c1a1..2522c9cf8be 100644
---
a/hadoop-ozone/cli-repair/src/main/java/org/apache/hadoop/ozone/repair/om/CompactOMDB.java
+++
b/hadoop-ozone/cli-repair/src/main/java/org/apache/hadoop/ozone/repair/om/CompactOMDB.java
@@ -20,8 +20,10 @@
import java.io.IOException;
import org.apache.hadoop.hdds.cli.HddsVersionProvider;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedCompactRangeOptions;
import org.apache.hadoop.ozone.om.helpers.OMNodeDetails;
import org.apache.hadoop.ozone.om.protocolPB.OMAdminProtocolClientSideImpl;
+import org.apache.hadoop.ozone.om.service.CompactDBUtil;
import org.apache.hadoop.ozone.repair.RepairTool;
import org.apache.hadoop.security.UserGroupInformation;
import picocli.CommandLine;
@@ -58,18 +60,28 @@ public class CompactOMDB extends RepairTool {
)
private String nodeId;
+ @CommandLine.Option(names = {"--bottommost-level-compaction", "--blc"},
+ description = "BottommostLevelCompaction option for RocksDB compaction."
+
+ " Valid values: 0 (kSkip), 1 (kIfHaveCompactionFilter), 2 (kForce),
3 (kForceOptimized).",
+ defaultValue = "0",
+ showDefaultValue = CommandLine.Help.Visibility.ALWAYS)
+ private int bottommostLevelCompaction;
+
@Override
public void execute() throws Exception {
OzoneConfiguration conf = getOzoneConf();
OMNodeDetails omNodeDetails = OMNodeDetails.getOMNodeDetailsFromConf(
conf, omServiceId, nodeId);
+ ManagedCompactRangeOptions.BottommostLevelCompaction blcOption =
+ CompactDBUtil.getBottommostLevelCompaction(bottommostLevelCompaction);
if (!isDryRun()) {
try (OMAdminProtocolClientSideImpl omAdminProtocolClient =
OMAdminProtocolClientSideImpl.createProxyForSingleOM(conf,
UserGroupInformation.getCurrentUser(), omNodeDetails)) {
- omAdminProtocolClient.compactOMDB(columnFamilyName);
- info("Compaction request issued for om.db of om node: %s,
column-family: %s.", nodeId, columnFamilyName);
+ omAdminProtocolClient.compactOMDB(columnFamilyName,
blcOption.getValue());
+ info("Compaction request issued for om.db of om node: %s,
column-family: %s" +
+ " with bottommost level compaction: %s.", nodeId,
columnFamilyName, blcOption.name());
info("Please check role logs of %s for completion status.", nodeId);
} catch (IOException ex) {
error("Couldn't compact column %s. \nException: %s", columnFamilyName,
ex);
diff --git
a/hadoop-ozone/cli-repair/src/test/java/org/apache/hadoop/ozone/repair/ldb/TestLdbRepair.java
b/hadoop-ozone/cli-repair/src/test/java/org/apache/hadoop/ozone/repair/ldb/TestLdbRepair.java
index 2553af6d974..af5001ed174 100644
---
a/hadoop-ozone/cli-repair/src/test/java/org/apache/hadoop/ozone/repair/ldb/TestLdbRepair.java
+++
b/hadoop-ozone/cli-repair/src/test/java/org/apache/hadoop/ozone/repair/ldb/TestLdbRepair.java
@@ -125,7 +125,8 @@ public void testRocksDBManualCompaction() throws Exception {
CommandLine cmd = new CommandLine(compactionTool);
String[] args = {
"--db", dbPath.toString(),
- "--column-family", TEST_CF_NAME
+ "--column-family", TEST_CF_NAME,
+ "--blc", "2"
};
// Pass two "y" inputs - one for user confirmation and the other for
warning to stop service
int exitCode = withTextFromSystemIn("y", "y")
diff --git
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OMAdminProtocol.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OMAdminProtocol.java
index cb6baf79fe7..d6f9395dd37 100644
---
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OMAdminProtocol.java
+++
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OMAdminProtocol.java
@@ -41,10 +41,14 @@ public interface OMAdminProtocol extends Closeable {
void decommission(OMNodeDetails removeOMNode) throws IOException;
/**
- * Requests compaction of a column family of om.db.
- * @param columnFamily
+ * Requests compaction of a column family of om.db with the specified
+ * BottommostLevelCompaction option.
+ *
+ * @param columnFamily column family name
+ * @param bottommostLevelCompaction rocksId of BottommostLevelCompaction
+ * (0=kSkip, 1=kIfHaveCompactionFilter,
2=kForce, 3=kForceOptimized)
*/
- void compactOMDB(String columnFamily) throws IOException;
+ void compactOMDB(String columnFamily, int bottommostLevelCompaction) throws
IOException;
/**
* Triggers the Snapshot Defragmentation Service to run immediately.
diff --git
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OMAdminProtocolClientSideImpl.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OMAdminProtocolClientSideImpl.java
index db82b37245a..d248e03b1bd 100644
---
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OMAdminProtocolClientSideImpl.java
+++
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OMAdminProtocolClientSideImpl.java
@@ -216,9 +216,10 @@ public void decommission(OMNodeDetails removeOMNode)
throws IOException {
}
@Override
- public void compactOMDB(String columnFamily) throws IOException {
+ public void compactOMDB(String columnFamily, int bottommostLevelCompaction)
throws IOException {
CompactRequest compactRequest = CompactRequest.newBuilder()
.setColumnFamily(columnFamily)
+ .setBottommostLevelCompaction(bottommostLevelCompaction)
.build();
CompactResponse response;
try {
diff --git a/hadoop-ozone/dist/src/main/smoketest/repair/om-compact.robot
b/hadoop-ozone/dist/src/main/smoketest/repair/om-compact.robot
index 164cf410dcb..95f92ad3ce4 100644
--- a/hadoop-ozone/dist/src/main/smoketest/repair/om-compact.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/repair/om-compact.robot
@@ -34,7 +34,7 @@ Get OM DB SST Files Size
Compact OM DB Column Family
[Arguments] ${column_family}
- Execute ozone repair om compact --cf=${column_family} --service-id
${OM_SERVICE_ID} --node-id om1
+ Execute ozone repair om compact --cf=${column_family} --service-id
${OM_SERVICE_ID} --node-id om1 --blc 2
*** Test Cases ***
Testing OM DB Size Reduction After Compaction
diff --git a/hadoop-ozone/interface-client/src/main/proto/OMAdminProtocol.proto
b/hadoop-ozone/interface-client/src/main/proto/OMAdminProtocol.proto
index 5e726b400e8..4c9a73635bd 100644
--- a/hadoop-ozone/interface-client/src/main/proto/OMAdminProtocol.proto
+++ b/hadoop-ozone/interface-client/src/main/proto/OMAdminProtocol.proto
@@ -72,6 +72,10 @@ message DecommissionOMResponse {
message CompactRequest {
required string columnFamily = 1;
+ // BottommostLevelCompaction option:
+ // 0=kSkip, 1=kIfHaveCompactionFilter, 2=kForce, 3=kForceOptimized.
+ // Defaults to kSkip (0) if not set.
+ optional int32 bottommostLevelCompaction = 2 [default = 0];
}
message CompactResponse {
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
index 2bc8d501fc1..db7717c81ea 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
@@ -221,6 +221,7 @@
import org.apache.hadoop.hdds.utils.db.TableIterator;
import org.apache.hadoop.hdds.utils.db.cache.CacheKey;
import org.apache.hadoop.hdds.utils.db.cache.CacheValue;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedCompactRangeOptions;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.ipc_.ProtobufRpcEngine;
import org.apache.hadoop.ipc_.RPC;
@@ -5663,10 +5664,11 @@ public void checkFeatureEnabled(OzoneManagerVersion
feature) throws OMException
}
}
- public void compactOMDB(String columnFamily) throws IOException {
+ public void compactOMDB(String columnFamily,
+ ManagedCompactRangeOptions.BottommostLevelCompaction
bottommostLevelCompaction) throws IOException {
checkAdminUserPrivilege("compact column family " + columnFamily);
CompletableFuture<Void> compactFuture =
- CompactDBUtil.compactTableAsync(metadataManager, columnFamily);
+ CompactDBUtil.compactTableAsync(metadataManager, columnFamily,
bottommostLevelCompaction);
compactFuture.whenComplete((result, throwable) -> {
if (throwable == null) {
LOG.info("Compaction request for column family \"{}\" completed
successfully.",
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/CompactDBUtil.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/CompactDBUtil.java
index 6eb52abbcfa..73c54f2586d 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/CompactDBUtil.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/CompactDBUtil.java
@@ -38,13 +38,13 @@ public final class CompactDBUtil {
private CompactDBUtil() {
}
- public static void compactTable(OMMetadataManager omMetadataManager,
- String tableName) throws IOException {
+ public static void compactTable(OMMetadataManager omMetadataManager, String
tableName,
+ ManagedCompactRangeOptions.BottommostLevelCompaction compactionType)
throws IOException {
long startTime = Time.monotonicNow();
- LOG.info("Compacting column family: {}", tableName);
try (ManagedCompactRangeOptions options = new
ManagedCompactRangeOptions()) {
- options.setBottommostLevelCompaction(
- ManagedCompactRangeOptions.BottommostLevelCompaction.kForce);
+ options.setBottommostLevelCompaction(compactionType);
+ LOG.info("Compacting column family: {} with {} bottommost level
compaction",
+ tableName, options.bottommostLevelCompaction());
options.setExclusiveManualCompaction(true);
RocksDatabase rocksDatabase =
((RDBStore) omMetadataManager.getStore()).getDb();
@@ -62,14 +62,34 @@ public static void compactTable(OMMetadataManager
omMetadataManager,
}
}
- public static CompletableFuture<Void> compactTableAsync(OMMetadataManager
metadataManager, String tableName) {
+ public static CompletableFuture<Void> compactTableAsync(OMMetadataManager
metadataManager, String tableName,
+ ManagedCompactRangeOptions.BottommostLevelCompaction compactionType) {
return CompletableFuture.runAsync(() -> {
try {
- compactTable(metadataManager, tableName);
+ compactTable(metadataManager, tableName, compactionType);
} catch (Exception e) {
LOG.warn("Failed to compact column family: {}", tableName, e);
throw new CompletionException("Compaction failed for column family: "
+ tableName, e);
}
});
}
+
+ /**
+ * Converts the given RocksDB id to a
+ * {@link ManagedCompactRangeOptions.BottommostLevelCompaction} enum value.
+ * Defaults to {@code kSkip} if the id is invalid.
+ *
+ * @param bottommostLevelCompaction RocksDB id
+ * (0=kSkip, 1=kIfHaveCompactionFilter,
2=kForce, 3=kForceOptimized).
+ */
+ public static ManagedCompactRangeOptions.BottommostLevelCompaction
getBottommostLevelCompaction(
+ int bottommostLevelCompaction) {
+ ManagedCompactRangeOptions.BottommostLevelCompaction level =
+
ManagedCompactRangeOptions.BottommostLevelCompaction.fromRocksId(bottommostLevelCompaction);
+ if (level == null) {
+ LOG.warn("Invalid bottommost level compaction id: {}. Using default:
kSkip.", bottommostLevelCompaction);
+ return ManagedCompactRangeOptions.BottommostLevelCompaction.kSkip;
+ }
+ return level;
+ }
}
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/CompactionService.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/CompactionService.java
index ec24fe117bb..34c7a8ff867 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/CompactionService.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/CompactionService.java
@@ -32,6 +32,7 @@
import org.apache.hadoop.hdds.utils.BackgroundTask;
import org.apache.hadoop.hdds.utils.BackgroundTaskQueue;
import org.apache.hadoop.hdds.utils.BackgroundTaskResult;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedCompactRangeOptions;
import org.apache.hadoop.ozone.om.OMMetadataManager;
import org.apache.hadoop.ozone.om.OzoneManager;
import org.slf4j.Logger;
@@ -141,11 +142,13 @@ private boolean shouldRun() {
* @return CompletableFuture that completes when compaction finishes
*/
public CompletableFuture<Void> compactTableAsync(String tableName) {
- return CompactDBUtil.compactTableAsync(omMetadataManager, tableName);
+ return CompactDBUtil.compactTableAsync(omMetadataManager, tableName,
+ ManagedCompactRangeOptions.BottommostLevelCompaction.kForce);
}
protected void compactFully(String tableName) throws IOException {
- CompactDBUtil.compactTable(omMetadataManager, tableName);
+ CompactDBUtil.compactTable(omMetadataManager, tableName,
+ ManagedCompactRangeOptions.BottommostLevelCompaction.kForce);
}
private class CompactTask implements BackgroundTask {
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OMAdminProtocolServerSideImpl.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OMAdminProtocolServerSideImpl.java
index 8b76f6c0fe4..ba96368cfd5 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OMAdminProtocolServerSideImpl.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OMAdminProtocolServerSideImpl.java
@@ -25,12 +25,14 @@
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedCompactRangeOptions;
import org.apache.hadoop.ozone.om.OzoneManager;
import org.apache.hadoop.ozone.om.exceptions.OMException;
import org.apache.hadoop.ozone.om.helpers.OMNodeDetails;
import org.apache.hadoop.ozone.om.protocolPB.OMAdminProtocolPB;
import org.apache.hadoop.ozone.om.ratis.OzoneManagerRatisServer;
import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerRatisUtils;
+import org.apache.hadoop.ozone.om.service.CompactDBUtil;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerAdminProtocolProtos.CompactRequest;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerAdminProtocolProtos.CompactResponse;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerAdminProtocolProtos.DecommissionOMRequest;
@@ -121,7 +123,9 @@ public CompactResponse compactDB(RpcController controller,
CompactRequest compac
try {
// check if table exists. IOException is thrown if table is not found.
ozoneManager.getMetadataManager().getStore().getTable(compactRequest.getColumnFamily());
- ozoneManager.compactOMDB(compactRequest.getColumnFamily());
+ ManagedCompactRangeOptions.BottommostLevelCompaction
bottommostLevelCompaction =
+
CompactDBUtil.getBottommostLevelCompaction(compactRequest.getBottommostLevelCompaction());
+ ozoneManager.compactOMDB(compactRequest.getColumnFamily(),
bottommostLevelCompaction);
} catch (IOException ex) {
return CompactResponse.newBuilder()
.setSuccess(false)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
