This is an automated email from the ASF dual-hosted git repository.
ritesh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 40bba0126d HDDS-8693. Generate linear keys for freon ockrw (#4772)
40bba0126d is described below
commit 40bba0126d9e01b2b10e229ea426be6a78a5b8ba
Author: Ritesh H Shukla <[email protected]>
AuthorDate: Mon Sep 18 14:53:42 2023 -0700
HDDS-8693. Generate linear keys for freon ockrw (#4772)
---
.../src/main/smoketest/freon/read-write-key.robot | 10 +++++++--
.../hadoop/ozone/freon/KeyGeneratorUtil.java | 2 +-
.../freon/OzoneClientKeyReadWriteListOps.java | 25 ++++++++++++++++------
3 files changed, 27 insertions(+), 10 deletions(-)
diff --git a/hadoop-ozone/dist/src/main/smoketest/freon/read-write-key.robot
b/hadoop-ozone/dist/src/main/smoketest/freon/read-write-key.robot
index b3487ddb70..f98fdc1950 100644
--- a/hadoop-ozone/dist/src/main/smoketest/freon/read-write-key.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/freon/read-write-key.robot
@@ -31,15 +31,21 @@ Read 10 keys from pre-generated keys
${result} = Execute ozone freon ockrw -n ${keysCount} -t
10 -r 100 -v voltest -b buckettest -p performanceTest --percentage-read 100
--percentage-list 0
Should contain ${result} Successful executions:
${keysCount}
-Read 10 keys' metadata from pre-generated keys
+Read 10 keys metadata from pre-generated keys
${keysCount} = BuiltIn.Set Variable 10
${result} = Execute ozone freon ockrw -n ${keysCount} -t
10 -m -r 100 -v voltest -b buckettest -p performanceTest --percentage-read 100
--percentage-list 0
Should contain ${result} Successful executions:
${keysCount}
+Read 10 keys when generate in linear manner
+ ${keysCount} = BuiltIn.Set Variable 10
+ ${result} = Execute ozone freon ockrw -n ${keysCount} -t 1
-r 100 --size 0 -v voltest -b buckettest -p performanceTest --linear
--percentage-read=0 --percentage-list=0
+ ${result} = Execute ozone freon ockrw -n ${keysCount} -t 1
-m -r 100 -v voltest -b buckettest -p performanceTest --linear
--percentage-read=100 --percentage-list=0
+ Should contain ${result} Successful executions:
${keysCount}
+
Write 10 keys of size 1 byte each from key index 0 to 99
${keysCount} = BuiltIn.Set Variable 10
${size} = BuiltIn.Set Variable 1
- ${result} = Execute ozone freon ockrw -n ${keysCount} -t
10 --percentage-read 0 --percentage-list 0 --size ${size} -r 100 -v voltest -b
buckettest -p performanceTest
+ ${result} = Execute ozone freon ockrw -n ${keysCount} -t
10 --percentage-read 0 --percentage-list 0 --size ${size} -r 100 -v voltest -b
buckettest -p performanceTest --linear
Should contain ${result} Successful executions:
${keysCount}
${keyName} = Execute echo -n '1' | md5sum | head -c 7
${result} = Execute ozone sh key info
/voltest/buckettest/performanceTest/${keyName}
diff --git
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/KeyGeneratorUtil.java
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/KeyGeneratorUtil.java
index 9773caa440..07ac346457 100644
---
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/KeyGeneratorUtil.java
+++
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/KeyGeneratorUtil.java
@@ -38,7 +38,7 @@ public class KeyGeneratorUtil {
return number -> String.valueOf(number);
}
- public String generateMd5KeyName(int number) {
+ public String generateMd5KeyName(long number) {
String encodedStr = DigestUtils.md5Hex(String.valueOf(number));
return encodedStr.substring(0, 7);
}
diff --git
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OzoneClientKeyReadWriteListOps.java
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OzoneClientKeyReadWriteListOps.java
index aca91e12a3..d8540a0540 100644
---
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OzoneClientKeyReadWriteListOps.java
+++
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OzoneClientKeyReadWriteListOps.java
@@ -34,6 +34,7 @@ import java.io.IOException;
import java.util.concurrent.Callable;
import java.util.concurrent.ThreadLocalRandom;
import java.util.HashMap;
+import java.util.concurrent.atomic.AtomicLong;
import static
org.apache.hadoop.ozone.freon.KeyGeneratorUtil.FILE_DIR_SEPARATOR;
@@ -73,14 +74,14 @@ public class OzoneClientKeyReadWriteListOps extends
BaseFreonGenerator
+ " clients. Example: Write keys 0-1000000 followed by "
+ "keys 1000001-2000000.",
defaultValue = "0")
- private int startIndex;
+ private long startIndex;
@CommandLine.Option(names = {"-r", "--range"},
description = "Range of read/write operations. This in co-ordination"
+ " with --start-index can specify the range to read. "
+ "Example: Read from --start-index 1000 and read --range 1000
keys.",
defaultValue = "1")
- private int range;
+ private long range;
@CommandLine.Option(names = {"--size"},
description = "Object size (in bytes) " +
@@ -96,6 +97,11 @@ public class OzoneClientKeyReadWriteListOps extends
BaseFreonGenerator
defaultValue = "false")
private boolean keySorted;
+ @CommandLine.Option(names = {"--linear"},
+ description = "Generate key names in a linear manner, there is no random"
+ + "selection of keys", defaultValue = "false")
+ private boolean linear;
+
@CommandLine.Option(names = {"--percentage-read"},
description = "Percentage of read tasks in mix workload."
+ " The remainder of the percentage will be divided between write"
@@ -133,6 +139,7 @@ public class OzoneClientKeyReadWriteListOps extends
BaseFreonGenerator
private static final Logger LOG =
LoggerFactory.getLogger(OzoneClientKeyReadWriteListOps.class);
+ private static AtomicLong nextNumber = new AtomicLong();
/**
* Task type of read task, or write task.
*/
@@ -253,17 +260,21 @@ public class OzoneClientKeyReadWriteListOps extends
BaseFreonGenerator
public String getKeyName() {
StringBuilder keyNameSb = new StringBuilder();
- int randomIdxWithinRange = ThreadLocalRandom.current().
- nextInt(startIndex, startIndex + range);
+ long next;
+ if (linear) {
+ next = startIndex + nextNumber.getAndUpdate(x -> (x + 1) % range);
+ } else {
+ next = ThreadLocalRandom.current().
+ nextLong(startIndex, startIndex + range);
+ }
if (keySorted) {
keyNameSb.append(getPrefix()).append(FILE_DIR_SEPARATOR).
- append(randomIdxWithinRange);
+ append(next);
} else {
keyNameSb.append(getPrefix()).append(FILE_DIR_SEPARATOR).
- append(kg.generateMd5KeyName(randomIdxWithinRange));
+ append(kg.generateMd5KeyName(next));
}
return keyNameSb.toString();
}
-
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
