This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 802fe5e215 HDDS-9818. Ensure valid Raft log write buffer size (#5713)
802fe5e215 is described below
commit 802fe5e21514405ce3ee4c5a2a737d409fe85ed7
Author: Doroszlai, Attila <[email protected]>
AuthorDate: Sat Dec 2 10:22:39 2023 +0100
HDDS-9818. Ensure valid Raft log write buffer size (#5713)
---
.../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 4 --
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 4 --
.../common/src/main/resources/ozone-default.xml | 8 ----
.../transport/server/ratis/XceiverServerRatis.java | 54 ++++++++++++----------
.../org/apache/hadoop/hdds/scm/ha/RatisUtil.java | 14 +++---
.../src/test/resources/ozone-site.xml | 12 +++++
.../ozone/om/ratis/OzoneManagerRatisServer.java | 2 +
7 files changed, 52 insertions(+), 46 deletions(-)
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
index 60d4ac6ae1..4b586b796d 100644
---
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
@@ -67,10 +67,6 @@ public final class ScmConfigKeys {
"dfs.container.ratis.segment.size";
public static final String DFS_CONTAINER_RATIS_SEGMENT_SIZE_DEFAULT =
"64MB";
- public static final String DFS_CONTAINER_RATIS_SEGMENT_BUFFER_SIZE_KEY =
- "dfs.container.ratis.segment.buffer.size";
- public static final String DFS_CONTAINER_RATIS_SEGMENT_BUFFER_SIZE_DEFAULT =
- "2MB";
public static final String DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_KEY
=
"dfs.container.ratis.segment.preallocated.size";
public static final String
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
index 338fd778b0..0b62b887a3 100644
---
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
@@ -352,10 +352,6 @@ public final class OzoneConfigKeys {
= ScmConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_SIZE_KEY;
public static final String DFS_CONTAINER_RATIS_SEGMENT_SIZE_DEFAULT
= ScmConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_SIZE_DEFAULT;
- public static final String DFS_CONTAINER_RATIS_SEGMENT_BUFFER_SIZE_KEY
- = ScmConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_BUFFER_SIZE_KEY;
- public static final String DFS_CONTAINER_RATIS_SEGMENT_BUFFER_SIZE_DEFAULT
- = ScmConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_BUFFER_SIZE_DEFAULT;
public static final String DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_KEY
= ScmConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_KEY;
public static final String
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml
b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index 3eaba5a5c6..65019796b8 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -320,14 +320,6 @@
by Apache Ratis on datanodes. (64 MB by default)
</description>
</property>
- <property>
- <name>dfs.container.ratis.segment.buffer.size</name>
- <value>2MB</value>
- <tag>OZONE, RATIS, PERFORMANCE</tag>
- <description>The size of the raft segment buffer used
- by Apache Ratis on datanodes. (2 MB by default)
- </description>
- </property>
<property>
<name>dfs.container.ratis.segment.preallocated.size</name>
<value>4MB</value>
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
index 0f8f7d4ecc..c9ac85414a 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
@@ -103,7 +103,6 @@ import org.apache.ratis.server.RaftServerConfigKeys;
import org.apache.ratis.server.RaftServerRpc;
import org.apache.ratis.server.protocol.TermIndex;
import org.apache.ratis.server.storage.RaftStorage;
-import org.apache.ratis.util.Preconditions;
import org.apache.ratis.util.SizeInBytes;
import org.apache.ratis.util.TimeDuration;
import org.apache.ratis.util.TraditionalBinaryPrefix;
@@ -111,6 +110,13 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static
org.apache.hadoop.hdds.DatanodeVersion.SEPARATE_RATIS_PORTS_AVAILABLE;
+import static
org.apache.hadoop.ozone.OzoneConfigKeys.DFS_CONTAINER_RATIS_LOG_APPENDER_QUEUE_BYTE_LIMIT;
+import static
org.apache.hadoop.ozone.OzoneConfigKeys.DFS_CONTAINER_RATIS_LOG_APPENDER_QUEUE_BYTE_LIMIT_DEFAULT;
+import static
org.apache.hadoop.ozone.OzoneConfigKeys.DFS_CONTAINER_RATIS_LOG_APPENDER_QUEUE_NUM_ELEMENTS;
+import static
org.apache.hadoop.ozone.OzoneConfigKeys.DFS_CONTAINER_RATIS_LOG_APPENDER_QUEUE_NUM_ELEMENTS_DEFAULT;
+import static
org.apache.hadoop.ozone.OzoneConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_SIZE_DEFAULT;
+import static
org.apache.hadoop.ozone.OzoneConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_SIZE_KEY;
+import static org.apache.ratis.util.Preconditions.assertTrue;
/**
* Creates a ratis server endpoint that acts as the communication layer for
@@ -428,40 +434,40 @@ public final class XceiverServerRatis implements
XceiverServerSpi {
OzoneConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_KEY,
OzoneConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_DEFAULT,
StorageUnit.BYTES);
- int logAppenderQueueNumElements = conf.getInt(
- OzoneConfigKeys.DFS_CONTAINER_RATIS_LOG_APPENDER_QUEUE_NUM_ELEMENTS,
- OzoneConfigKeys
- .DFS_CONTAINER_RATIS_LOG_APPENDER_QUEUE_NUM_ELEMENTS_DEFAULT);
- final int logAppenderQueueByteLimit = (int) conf.getStorageSize(
- OzoneConfigKeys.DFS_CONTAINER_RATIS_LOG_APPENDER_QUEUE_BYTE_LIMIT,
- OzoneConfigKeys
- .DFS_CONTAINER_RATIS_LOG_APPENDER_QUEUE_BYTE_LIMIT_DEFAULT,
- StorageUnit.BYTES);
- RaftServerConfigKeys.Log.Appender
- .setBufferElementLimit(properties, logAppenderQueueNumElements);
- RaftServerConfigKeys.Log.Appender.setBufferByteLimit(properties,
- SizeInBytes.valueOf(logAppenderQueueByteLimit));
RaftServerConfigKeys.Log.setPreallocatedSize(properties,
SizeInBytes.valueOf(raftSegmentPreallocatedSize));
return raftSegmentPreallocatedSize;
}
private void setRaftSegmentAndWriteBufferSize(RaftProperties properties) {
+ final int logAppenderQueueNumElements = conf.getInt(
+ DFS_CONTAINER_RATIS_LOG_APPENDER_QUEUE_NUM_ELEMENTS,
+ DFS_CONTAINER_RATIS_LOG_APPENDER_QUEUE_NUM_ELEMENTS_DEFAULT);
+ final int logAppenderQueueByteLimit = (int) conf.getStorageSize(
+ DFS_CONTAINER_RATIS_LOG_APPENDER_QUEUE_BYTE_LIMIT,
+ DFS_CONTAINER_RATIS_LOG_APPENDER_QUEUE_BYTE_LIMIT_DEFAULT,
+ StorageUnit.BYTES);
+
final long raftSegmentSize = (long) conf.getStorageSize(
- OzoneConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_SIZE_KEY,
- OzoneConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_SIZE_DEFAULT,
+ DFS_CONTAINER_RATIS_SEGMENT_SIZE_KEY,
+ DFS_CONTAINER_RATIS_SEGMENT_SIZE_DEFAULT,
StorageUnit.BYTES);
+ final long raftSegmentBufferSize = logAppenderQueueByteLimit + 8;
+
+ assertTrue(raftSegmentBufferSize <= raftSegmentSize,
+ () -> DFS_CONTAINER_RATIS_LOG_APPENDER_QUEUE_BYTE_LIMIT + " = "
+ + logAppenderQueueByteLimit
+ + " must be <= (" + DFS_CONTAINER_RATIS_SEGMENT_SIZE_KEY + " - 8"
+ + " = " + (raftSegmentSize - 8) + ")");
+
+ RaftServerConfigKeys.Log.Appender.setBufferElementLimit(properties,
+ logAppenderQueueNumElements);
+ RaftServerConfigKeys.Log.Appender.setBufferByteLimit(properties,
+ SizeInBytes.valueOf(logAppenderQueueByteLimit));
RaftServerConfigKeys.Log.setSegmentSizeMax(properties,
SizeInBytes.valueOf(raftSegmentSize));
- final long raftSegmentBufferSize = (long) conf.getStorageSize(
- OzoneConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_BUFFER_SIZE_KEY,
- OzoneConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_BUFFER_SIZE_DEFAULT,
- StorageUnit.BYTES);
RaftServerConfigKeys.Log.setWriteBufferSize(properties,
- SizeInBytes.valueOf(raftSegmentBufferSize));
- Preconditions.assertTrue(raftSegmentBufferSize <= raftSegmentSize,
- () -> "raftSegmentBufferSize = " + raftSegmentBufferSize
- + " > raftSegmentSize = " + raftSegmentSize);
+ SizeInBytes.valueOf(raftSegmentBufferSize));
}
private RpcType setRpcType(RaftProperties properties) {
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/RatisUtil.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/RatisUtil.java
index 4ae82d89bb..ccef5aab24 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/RatisUtil.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/RatisUtil.java
@@ -173,12 +173,14 @@ public final class RatisUtil {
ScmConfigKeys.OZONE_SCM_HA_RAFT_LOG_APPENDER_QUEUE_NUM,
ScmConfigKeys.
OZONE_SCM_HA_RAFT_LOG_APPENDER_QUEUE_NUM_DEFAULT));
- Log.Appender.setBufferByteLimit(properties, SizeInBytes.valueOf(
- (long) ozoneConf.getStorageSize(
- ScmConfigKeys.OZONE_SCM_HA_RAFT_LOG_APPENDER_QUEUE_BYTE_LIMIT,
- ScmConfigKeys.
- OZONE_SCM_HA_RAFT_LOG_APPENDER_QUEUE_BYTE_LIMIT_DEFAULT,
- StorageUnit.BYTES)));
+ final int logAppenderQueueByteLimit = (int) ozoneConf.getStorageSize(
+ ScmConfigKeys.OZONE_SCM_HA_RAFT_LOG_APPENDER_QUEUE_BYTE_LIMIT,
+ ScmConfigKeys.OZONE_SCM_HA_RAFT_LOG_APPENDER_QUEUE_BYTE_LIMIT_DEFAULT,
+ StorageUnit.BYTES);
+ Log.Appender.setBufferByteLimit(properties,
+ SizeInBytes.valueOf(logAppenderQueueByteLimit));
+ Log.setWriteBufferSize(properties,
+ SizeInBytes.valueOf(logAppenderQueueByteLimit + 8));
Log.setPreallocatedSize(properties, SizeInBytes.valueOf(
(long) ozoneConf.getStorageSize(
ScmConfigKeys.OZONE_SCM_HA_RAFT_SEGMENT_PRE_ALLOCATED_SIZE,
diff --git a/hadoop-ozone/integration-test/src/test/resources/ozone-site.xml
b/hadoop-ozone/integration-test/src/test/resources/ozone-site.xml
index a27b7210e9..4c415c0c7e 100644
--- a/hadoop-ozone/integration-test/src/test/resources/ozone-site.xml
+++ b/hadoop-ozone/integration-test/src/test/resources/ozone-site.xml
@@ -51,4 +51,16 @@
<value>1s</value>
</property>
+ <property>
+ <name>dfs.container.ratis.log.appender.queue.byte-limit</name>
+ <value>8MB</value>
+ </property>
+ <property>
+ <name>ozone.om.ratis.log.appender.queue.byte-limit</name>
+ <value>4MB</value>
+ </property>
+ <property>
+ <name>ozone.scm.ha.ratis.log.appender.queue.byte-limit</name>
+ <value>4MB</value>
+ </property>
</configuration>
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerRatisServer.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerRatisServer.java
index b5f460a85f..f26a5e300a 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerRatisServer.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerRatisServer.java
@@ -661,6 +661,8 @@ public final class OzoneManagerRatisServer {
logAppenderQueueNumElements);
RaftServerConfigKeys.Log.Appender.setBufferByteLimit(properties,
SizeInBytes.valueOf(logAppenderQueueByteLimit));
+ RaftServerConfigKeys.Log.setWriteBufferSize(properties,
+ SizeInBytes.valueOf(logAppenderQueueByteLimit + 8));
RaftServerConfigKeys.Log.setPreallocatedSize(properties,
SizeInBytes.valueOf(raftSegmentPreallocatedSize));
RaftServerConfigKeys.Log.Appender.setInstallSnapshotEnabled(properties,
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
