This is an automated email from the ASF dual-hosted git repository.
duong pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new b6f953bf70 HDDS-10671. Wrong size of response data in om-echo. (#6504)
b6f953bf70 is described below
commit b6f953bf707a63766ca3d3c4ff81d84f7825e607
Author: Duong Nguyen <[email protected]>
AuthorDate: Tue Apr 16 09:04:29 2024 -0700
HDDS-10671. Wrong size of response data in om-echo. (#6504)
---
...OzoneManagerProtocolClientSideTranslatorPB.java | 4 +--
.../apache/hadoop/ozone/util}/PayloadUtils.java | 32 ++++++++++++++--------
.../hadoop/ozone/util/TestPayloadUtils.java} | 32 ++++++++--------------
.../om/request/util/OMEchoRPCWriteRequest.java | 4 +--
.../protocolPB/OzoneManagerRequestHandler.java | 4 +--
.../hadoop/ozone/freon/OmRPCLoadGenerator.java | 20 ++++----------
6 files changed, 44 insertions(+), 52 deletions(-)
diff --git
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
index 08fa029833..3c6b6647a6 100644
---
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
+++
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
@@ -2491,12 +2491,12 @@ public final class
OzoneManagerProtocolClientSideTranslatorPB
}
@Override
- public EchoRPCResponse echoRPCReq(byte[] payloadReq, int payloadSizeResp,
+ public EchoRPCResponse echoRPCReq(byte[] payloadReq, int
payloadSizeRespBytes,
boolean writeToRatis) throws IOException {
EchoRPCRequest echoRPCRequest =
EchoRPCRequest.newBuilder()
.setPayloadReq(ByteString.copyFrom(payloadReq))
- .setPayloadSizeResp(payloadSizeResp)
+ .setPayloadSizeResp(payloadSizeRespBytes)
.setReadOnly(!writeToRatis)
.build();
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/common/PayloadUtils.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/util/PayloadUtils.java
similarity index 55%
copy from
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/common/PayloadUtils.java
copy to
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/util/PayloadUtils.java
index d198bfcbfc..6ca827832d 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/common/PayloadUtils.java
+++
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/util/PayloadUtils.java
@@ -15,30 +15,40 @@
* the License.
*/
-package org.apache.hadoop.ozone.common;
+package org.apache.hadoop.ozone.util;
-import org.apache.commons.lang3.RandomUtils;
+import org.apache.ratis.util.Preconditions;
+
+import java.util.Random;
/**
* Utility class for payload operations.
*/
public final class PayloadUtils {
+ private static final int MAX_SIZE = 2097151 * 1024;
+ private static final byte[] SEED = new byte[1024];
- private static final int RPC_PAYLOAD_MULTIPLICATION_FACTOR = 1024;
- private static final int MAX_SIZE_KB = 2097151;
+ static {
+ new Random().nextBytes(SEED);
+ }
private PayloadUtils() {
}
- public static byte[] generatePayloadBytes(int payloadSize) {
+ public static byte[] generatePayload(int payloadSizeBytes) {
+ byte[] result = new byte[Math.min(payloadSizeBytes, MAX_SIZE)];
- byte[] payloadBytes = new byte[0];
- int payloadRespSize =
- Math.min(payloadSize * RPC_PAYLOAD_MULTIPLICATION_FACTOR, MAX_SIZE_KB);
- if (payloadRespSize > 0) {
- payloadBytes = RandomUtils.nextBytes(payloadRespSize);
+ // duplicate SEED to create the required payload.
+ int curIdx = 0;
+ while (curIdx < result.length) {
+ int remaining = result.length - curIdx;
+ int copySize = Math.min(SEED.length, remaining);
+ System.arraycopy(SEED, 0, result, curIdx, copySize);
+ curIdx += copySize;
}
- return payloadBytes;
+ Preconditions.assertTrue(curIdx == result.length);
+
+ return result;
}
}
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/common/PayloadUtils.java
b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/util/TestPayloadUtils.java
similarity index 56%
rename from
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/common/PayloadUtils.java
rename to
hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/util/TestPayloadUtils.java
index d198bfcbfc..221a0ce6a9 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/common/PayloadUtils.java
+++
b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/util/TestPayloadUtils.java
@@ -14,31 +14,21 @@
* License for the specific language governing permissions and limitations
under
* the License.
*/
+package org.apache.hadoop.ozone.util;
-package org.apache.hadoop.ozone.common;
-
-import org.apache.commons.lang3.RandomUtils;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
/**
- * Utility class for payload operations.
+ * Tests {@link PayloadUtils}.
*/
-public final class PayloadUtils {
-
- private static final int RPC_PAYLOAD_MULTIPLICATION_FACTOR = 1024;
- private static final int MAX_SIZE_KB = 2097151;
-
- private PayloadUtils() {
- }
-
- public static byte[] generatePayloadBytes(int payloadSize) {
-
- byte[] payloadBytes = new byte[0];
- int payloadRespSize =
- Math.min(payloadSize * RPC_PAYLOAD_MULTIPLICATION_FACTOR, MAX_SIZE_KB);
- if (payloadRespSize > 0) {
- payloadBytes = RandomUtils.nextBytes(payloadRespSize);
- }
+public class TestPayloadUtils {
- return payloadBytes;
+ @ParameterizedTest
+ @ValueSource(ints = {0, 1, 1023, 1024, 1025, 2048})
+ public void testGeneratePayload(int payload) {
+ byte[] generated = PayloadUtils.generatePayload(payload);
+ Assertions.assertEquals(payload, generated.length);
}
}
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/util/OMEchoRPCWriteRequest.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/util/OMEchoRPCWriteRequest.java
index aba8f6f6fc..62de1601be 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/util/OMEchoRPCWriteRequest.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/util/OMEchoRPCWriteRequest.java
@@ -20,7 +20,7 @@ package org.apache.hadoop.ozone.om.request.util;
import com.google.protobuf.ByteString;
import org.apache.ratis.server.protocol.TermIndex;
-import org.apache.hadoop.ozone.common.PayloadUtils;
+import org.apache.hadoop.ozone.util.PayloadUtils;
import org.apache.hadoop.ozone.om.OzoneManager;
import org.apache.hadoop.ozone.om.request.OMClientRequest;
import org.apache.hadoop.ozone.om.response.OMClientResponse;
@@ -45,7 +45,7 @@ public class OMEchoRPCWriteRequest extends OMClientRequest {
EchoRPCRequest echoRPCRequest = getOmRequest().getEchoRPCRequest();
byte[] payloadBytes =
- PayloadUtils.generatePayloadBytes(echoRPCRequest.getPayloadSizeResp());
+ PayloadUtils.generatePayload(echoRPCRequest.getPayloadSizeResp());
EchoRPCResponse echoRPCResponse = EchoRPCResponse.newBuilder()
.setPayload(ByteString.copyFrom(payloadBytes))
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java
index 5acb9f3651..29b30fd4ea 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java
@@ -41,7 +41,7 @@ import org.apache.hadoop.hdds.client.ReplicationConfig;
import org.apache.hadoop.hdds.scm.protocolPB.OzonePBHelper;
import org.apache.hadoop.hdds.utils.FaultInjector;
import org.apache.hadoop.ozone.OzoneAcl;
-import org.apache.hadoop.ozone.common.PayloadUtils;
+import org.apache.hadoop.ozone.util.PayloadUtils;
import org.apache.hadoop.ozone.om.OzoneManager;
import org.apache.hadoop.ozone.om.OzoneManagerPrepareState;
import org.apache.hadoop.ozone.om.exceptions.OMException;
@@ -1431,7 +1431,7 @@ public class OzoneManagerRequestHandler implements
RequestHandler {
private EchoRPCResponse echoRPC(EchoRPCRequest req) {
EchoRPCResponse.Builder builder = EchoRPCResponse.newBuilder();
byte[] payloadBytes =
- PayloadUtils.generatePayloadBytes(req.getPayloadSizeResp());
+ PayloadUtils.generatePayload(req.getPayloadSizeResp());
builder.setPayload(ByteString.copyFrom(payloadBytes));
return builder.build();
}
diff --git
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OmRPCLoadGenerator.java
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OmRPCLoadGenerator.java
index 958df4c11a..1912185297 100644
---
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OmRPCLoadGenerator.java
+++
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OmRPCLoadGenerator.java
@@ -19,11 +19,12 @@ package org.apache.hadoop.ozone.freon;
import com.codahale.metrics.Timer;
import com.google.common.base.Preconditions;
-import org.apache.commons.lang3.RandomUtils;
import org.apache.hadoop.hdds.cli.HddsVersionProvider;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import
org.apache.hadoop.ozone.om.protocolPB.OzoneManagerProtocolClientSideTranslatorPB;
import java.util.concurrent.Callable;
+
+import org.apache.hadoop.ozone.util.PayloadUtils;
import picocli.CommandLine.Command;
import picocli.CommandLine.Option;
@@ -42,8 +43,6 @@ import picocli.CommandLine.Option;
public class OmRPCLoadGenerator extends BaseFreonGenerator
implements Callable<Void> {
- private static final int RPC_PAYLOAD_MULTIPLICATION_FACTOR = 1024;
- private static final int MAX_SIZE_KB = 2097151;
private Timer timer;
private OzoneConfiguration configuration;
private OzoneManagerProtocolClientSideTranslatorPB[] clients;
@@ -88,9 +87,8 @@ public class OmRPCLoadGenerator extends BaseFreonGenerator
}
init();
- payloadReqBytes = RandomUtils.nextBytes(
- calculateMaxPayloadSize(payloadReqSizeKB));
- payloadRespSize = calculateMaxPayloadSize(payloadRespSizeKB);
+ payloadReqBytes =
PayloadUtils.generatePayload(payloadSizeInBytes(payloadReqSizeKB));
+ payloadRespSize = payloadSizeInBytes(payloadRespSizeKB);
timer = getMetrics().timer("rpc-payload");
try {
runTests(this::sendRPCReq);
@@ -104,14 +102,8 @@ public class OmRPCLoadGenerator extends BaseFreonGenerator
return null;
}
- private int calculateMaxPayloadSize(int payloadSizeKB) {
- if (payloadSizeKB > 0) {
- return Math.min(
- Math.toIntExact((long)payloadSizeKB *
- RPC_PAYLOAD_MULTIPLICATION_FACTOR),
- MAX_SIZE_KB);
- }
- return 0;
+ private int payloadSizeInBytes(int payloadSizeKB) {
+ return payloadSizeKB > 0 ? payloadSizeKB * 1024 : 0;
}
private void sendRPCReq(long l) throws Exception {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
