This is an automated email from the ASF dual-hosted git repository.
ritesh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 06a5e4d55b HDDS-9128. Ozone freon tool extension for benchmarking
listKeys operation (#5156)
06a5e4d55b is described below
commit 06a5e4d55bf7970d1ace284171476a8eaf211dec
Author: tanvipenumudy <[email protected]>
AuthorDate: Fri Aug 18 11:00:35 2023 +0530
HDDS-9128. Ozone freon tool extension for benchmarking listKeys operation
(#5156)
---
.../src/main/smoketest/freon/read-write-key.robot | 13 +++--
.../java/org/apache/hadoop/ozone/freon/Freon.java | 2 +-
...ps.java => OzoneClientKeyReadWriteListOps.java} | 62 +++++++++++++++-------
3 files changed, 53 insertions(+), 24 deletions(-)
diff --git a/hadoop-ozone/dist/src/main/smoketest/freon/read-write-key.robot
b/hadoop-ozone/dist/src/main/smoketest/freon/read-write-key.robot
index 0995f757f3..b3487ddb70 100644
--- a/hadoop-ozone/dist/src/main/smoketest/freon/read-write-key.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/freon/read-write-key.robot
@@ -28,18 +28,18 @@ Pre-generate 100 keys of size 1 byte each to Ozone
Read 10 keys from pre-generated keys
${keysCount} = BuiltIn.Set Variable 10
- ${result} = Execute ozone freon ockrw -n ${keysCount} -t
10 -r 100 -v voltest -b buckettest -p performanceTest
+ ${result} = Execute ozone freon ockrw -n ${keysCount} -t
10 -r 100 -v voltest -b buckettest -p performanceTest --percentage-read 100
--percentage-list 0
Should contain ${result} Successful executions:
${keysCount}
Read 10 keys' metadata from pre-generated keys
${keysCount} = BuiltIn.Set Variable 10
- ${result} = Execute ozone freon ockrw -n ${keysCount} -t
10 -m -r 100 -v voltest -b buckettest -p performanceTest
+ ${result} = Execute ozone freon ockrw -n ${keysCount} -t
10 -m -r 100 -v voltest -b buckettest -p performanceTest --percentage-read 100
--percentage-list 0
Should contain ${result} Successful executions:
${keysCount}
Write 10 keys of size 1 byte each from key index 0 to 99
${keysCount} = BuiltIn.Set Variable 10
${size} = BuiltIn.Set Variable 1
- ${result} = Execute ozone freon ockrw -n ${keysCount} -t
10 --percentage-read 0 --size ${size} -r 100 -v voltest -b buckettest -p
performanceTest
+ ${result} = Execute ozone freon ockrw -n ${keysCount} -t
10 --percentage-read 0 --percentage-list 0 --size ${size} -r 100 -v voltest -b
buckettest -p performanceTest
Should contain ${result} Successful executions:
${keysCount}
${keyName} = Execute echo -n '1' | md5sum | head -c 7
${result} = Execute ozone sh key info
/voltest/buckettest/performanceTest/${keyName}
@@ -48,6 +48,11 @@ Write 10 keys of size 1 byte each from key index 0 to 99
Run 90 % of read-key tasks and 10 % of write-key tasks for 10 keys from
pre-generated keys
${keysCount} = BuiltIn.Set Variable 10
- ${result} = Execute ozone freon ockrw -n ${keysCount} -t
10 --percentage-read 90 -r 100 -v voltest -b buckettest -p performanceTest
+ ${result} = Execute ozone freon ockrw -n ${keysCount} -t
10 --percentage-read 90 --percentage-list 0 -r 100 -v voltest -b buckettest -p
performanceTest
+ Should contain ${result} Successful executions:
${keysCount}
+
+Run 50 % of read-key tasks, 40 % list-key tasks and 10 % of write-key tasks
for 10 keys from pre-generated keys
+ ${keysCount} = BuiltIn.Set Variable 10
+ ${result} = Execute ozone freon ockrw -n ${keysCount} -t
10 --percentage-read 50 --percentage-list 40 -r 100 -v voltest -b buckettest -p
performanceTest
Should contain ${result} Successful executions:
${keysCount}
diff --git
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/Freon.java
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/Freon.java
index d346866352..82b667a40b 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/Freon.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/Freon.java
@@ -68,7 +68,7 @@ import picocli.CommandLine.Option;
OmBucketReadWriteFileOps.class,
OmBucketReadWriteKeyOps.class,
OmRPCLoadGenerator.class,
- OzoneClientKeyReadWriteOps.class,
+ OzoneClientKeyReadWriteListOps.class,
RangeKeysGenerator.class,
DatanodeSimulator.class,
OmMetadataGenerator.class
diff --git
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OzoneClientKeyReadWriteOps.java
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OzoneClientKeyReadWriteListOps.java
similarity index 80%
rename from
hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OzoneClientKeyReadWriteOps.java
rename to
hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OzoneClientKeyReadWriteListOps.java
index 88aec0763c..aca91e12a3 100644
---
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OzoneClientKeyReadWriteOps.java
+++
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OzoneClientKeyReadWriteListOps.java
@@ -42,13 +42,13 @@ import static
org.apache.hadoop.ozone.freon.KeyGeneratorUtil.FILE_DIR_SEPARATOR;
*/
@CommandLine.Command(name = "ockrw",
- aliases = "ozone-client-key-read-write-ops",
+ aliases = "ozone-client-key-read-write-list-ops",
description = "Generate keys with a fixed name and ranges that can"
- + " be written and read as sub-ranges from multiple clients.",
+ + " be written, read and listed as sub-ranges from multiple clients.",
versionProvider = HddsVersionProvider.class,
mixinStandardHelpOptions = true,
showDefaultValues = true)
-public class OzoneClientKeyReadWriteOps extends BaseFreonGenerator
+public class OzoneClientKeyReadWriteListOps extends BaseFreonGenerator
implements Callable<Void> {
@CommandLine.Option(names = {"-v", "--volume"},
@@ -98,11 +98,24 @@ public class OzoneClientKeyReadWriteOps extends
BaseFreonGenerator
@CommandLine.Option(names = {"--percentage-read"},
description = "Percentage of read tasks in mix workload."
- + " The remainder of the percentage will writes to keys."
- + " Example --percentage-read 90 will result in 10% writes.",
- defaultValue = "100")
+ + " The remainder of the percentage will be divided between write"
+ + " and list tasks.",
+ required = true)
private int percentageRead;
+ @CommandLine.Option(names = {"--percentage-list"},
+ description = "Percentage of list tasks in mix workload."
+ + " The remainder of the percentage will be divided between write"
+ + " and read tasks.",
+ required = true)
+ private int percentageList;
+
+ @CommandLine.Option(names = {"--max-list-result"},
+ description = "Maximum number of keys to be fetched during the list
task."
+ + " It ensures the size of the result will not exceed this limit.",
+ defaultValue = "1000")
+ private int maxListResult;
+
@CommandLine.Option(
names = "--om-service-id",
description = "OM Service ID"
@@ -118,14 +131,15 @@ public class OzoneClientKeyReadWriteOps extends
BaseFreonGenerator
private byte[] keyContent;
private static final Logger LOG =
- LoggerFactory.getLogger(OzoneClientKeyReadWriteOps.class);
+ LoggerFactory.getLogger(OzoneClientKeyReadWriteListOps.class);
/**
* Task type of read task, or write task.
*/
public enum TaskType {
READ_TASK,
- WRITE_TASK
+ WRITE_TASK,
+ LIST_TASK
}
private KeyGeneratorUtil kg;
@@ -142,14 +156,14 @@ public class OzoneClientKeyReadWriteOps extends
BaseFreonGenerator
ensureVolumeAndBucketExist(ozoneClients[0], volumeName, bucketName);
- timer = getMetrics().timer("key-read-write");
+ timer = getMetrics().timer("key-read-write-list");
if (objectSizeInBytes >= 0) {
keyContent = RandomUtils.nextBytes(objectSizeInBytes);
}
if (kg == null) {
kg = new KeyGeneratorUtil();
}
- runTests(this::readWriteKeys);
+ runTests(this::readWriteListKeys);
for (int i = 0; i < clientCount; i++) {
if (ozoneClients[i] != null) {
@@ -159,9 +173,10 @@ public class OzoneClientKeyReadWriteOps extends
BaseFreonGenerator
return null;
}
- public void readWriteKeys(long counter) throws RuntimeException, IOException
{
+ public void readWriteListKeys(long counter) throws RuntimeException,
+ IOException {
int clientIndex = (int)((counter) % clientCount);
- TaskType taskType = decideReadOrWriteTask();
+ TaskType taskType = decideReadWriteOrListTask();
String keyName = getKeyName();
timer.time(() -> {
@@ -173,6 +188,9 @@ public class OzoneClientKeyReadWriteOps extends
BaseFreonGenerator
case WRITE_TASK:
processWriteTasks(keyName, ozoneClients[clientIndex]);
break;
+ case LIST_TASK:
+ processListTasks(ozoneClients[clientIndex]);
+ break;
default:
break;
}
@@ -212,16 +230,22 @@ public class OzoneClientKeyReadWriteOps extends
BaseFreonGenerator
}
}
- public TaskType decideReadOrWriteTask() {
- if (percentageRead == 100) {
- return TaskType.READ_TASK;
- } else if (percentageRead == 0) {
- return TaskType.WRITE_TASK;
+ public void processListTasks(OzoneClient ozoneClient)
+ throws RuntimeException, IOException {
+ try {
+ ozoneClient.getProxy()
+ .listKeys(volumeName, bucketName, getPrefix(), null, maxListResult);
+ } catch (Exception ex) {
+ throw ex;
}
- //mix workload
+ }
+
+ public TaskType decideReadWriteOrListTask() {
int tmp = ThreadLocalRandom.current().nextInt(1, 101);
- if (tmp <= percentageRead) {
+ if (tmp <= percentageRead) {
return TaskType.READ_TASK;
+ } else if (tmp > percentageRead && tmp <= percentageRead + percentageList)
{
+ return TaskType.LIST_TASK;
} else {
return TaskType.WRITE_TASK;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
