This is an automated email from the ASF dual-hosted git repository.
swamirishi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 404a036348 HDDS-11191. Add a config to set max_open_files for OM
RocksDB. (#6954)
404a036348 is described below
commit 404a036348ef1cbddf24db8db914e5149f03d886
Author: Sadanand Shenoy <[email protected]>
AuthorDate: Tue Jul 16 21:19:17 2024 +0530
HDDS-11191. Add a config to set max_open_files for OM RocksDB. (#6954)
---
hadoop-hdds/common/src/main/resources/ozone-default.xml | 14 ++++++++++++++
.../main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java | 5 +++++
.../org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java | 14 ++++++++++++--
3 files changed, 31 insertions(+), 2 deletions(-)
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml
b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index fe6ef0d5e6..f34886400f 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -4216,6 +4216,20 @@
</description>
</property>
+ <property>
+ <name>ozone.om.db.max.open.files</name>
+ <value>-1</value>
+ <tag>OZONE, OM</tag>
+ <description>
+ Max number of open files that OM RocksDB will open simultaneously
+ Essentially sets `max_open_files` config for the active OM RocksDB
instance.
+ This will limit the total number of files opened by OM db.
+ Default is -1 which is unlimited and gives max performance.
+ If you are certain that your ulimit will always be bigger than number of
files in the database,
+ set max_open_files to -1, or else set it to a value lesser than or equal
to ulimit.
+ </description>
+ </property>
+
<property>
<name>ozone.om.snapshot.db.max.open.files</name>
<value>100</value>
diff --git
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
index 7b7556b5e3..d992f43877 100644
---
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
+++
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
@@ -53,6 +53,11 @@ public final class OMConfigKeys {
"ozone.om.handler.count.key";
public static final int OZONE_OM_HANDLER_COUNT_DEFAULT = 100;
+ public static final String OZONE_OM_DB_MAX_OPEN_FILES
+ = "ozone.om.db.max.open.files";
+ public static final int OZONE_OM_DB_MAX_OPEN_FILES_DEFAULT
+ = -1;
+
public static final String OZONE_OM_INTERNAL_SERVICE_ID =
"ozone.om.internal.service.id";
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
index f8f4c40bab..8275661161 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
@@ -106,6 +106,8 @@ import static
org.apache.hadoop.ozone.OzoneConsts.OM_DB_NAME;
import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX;
import static
org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_FS_SNAPSHOT_MAX_LIMIT;
import static
org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_FS_SNAPSHOT_MAX_LIMIT_DEFAULT;
+import static
org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_DB_MAX_OPEN_FILES;
+import static
org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_DB_MAX_OPEN_FILES_DEFAULT;
import static
org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_SNAPSHOT_CHECKPOINT_DIR_CREATION_POLL_TIMEOUT;
import static
org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_SNAPSHOT_CHECKPOINT_DIR_CREATION_POLL_TIMEOUT_DEFAULT;
import static
org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.BUCKET_NOT_FOUND;
@@ -559,7 +561,10 @@ public class OmMetadataManagerImpl implements
OMMetadataManager,
rocksDBConfiguration.setSyncOption(true);
}
- this.store = loadDB(configuration, metaDir);
+ int maxOpenFiles = configuration.getInt(OZONE_OM_DB_MAX_OPEN_FILES,
+ OZONE_OM_DB_MAX_OPEN_FILES_DEFAULT);
+
+ this.store = loadDB(configuration, metaDir, Optional.of(maxOpenFiles));
initializeOmTables(CacheType.FULL_CACHE, true);
}
@@ -569,8 +574,13 @@ public class OmMetadataManagerImpl implements
OMMetadataManager,
public static DBStore loadDB(OzoneConfiguration configuration, File metaDir)
throws IOException {
+ return loadDB(configuration, metaDir, Optional.empty());
+ }
+
+ public static DBStore loadDB(OzoneConfiguration configuration, File metaDir,
Optional<Integer> maxOpenFiles)
+ throws IOException {
return loadDB(configuration, metaDir, OM_DB_NAME, false,
- java.util.Optional.empty(), Optional.empty(), true, true);
+ java.util.Optional.empty(), maxOpenFiles, true, true);
}
public static DBStore loadDB(OzoneConfiguration configuration, File metaDir,
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
