This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch branch-3
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-3 by this push:
new 52e155e0bb2 HBASE-29013 Make PerformanceEvaluation support larger data
sets (#6509)
52e155e0bb2 is described below
commit 52e155e0bb246df3121261d49eff77a82ed3630b
Author: Junegunn Choi <[email protected]>
AuthorDate: Mon Dec 23 22:53:02 2024 +0900
HBASE-29013 Make PerformanceEvaluation support larger data sets (#6509)
Use 8-byte long integers in the code to prevent integer overflows.
Use ThreadLocalRandom instead of Random.
Signed-off-by: Duo Zhang <[email protected]>
Reviewed-by: Peng Lu <[email protected]>
---
.../apache/hadoop/hbase/PerformanceEvaluation.java | 212 ++++++++++-----------
.../hadoop/hbase/TestPerformanceEvaluation.java | 13 +-
2 files changed, 107 insertions(+), 118 deletions(-)
diff --git
a/hbase-diagnostics/src/main/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
b/hbase-diagnostics/src/main/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
index 1b397546aca..9698ae9376e 100644
---
a/hbase-diagnostics/src/main/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
+++
b/hbase-diagnostics/src/main/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
@@ -466,9 +466,9 @@ public class PerformanceEvaluation extends Configured
implements Tool {
int numSplitPoints = opts.presplitRegions - 1;
byte[][] splits = new byte[numSplitPoints][];
- int jump = opts.totalRows / opts.presplitRegions;
+ long jump = opts.totalRows / opts.presplitRegions;
for (int i = 0; i < numSplitPoints; i++) {
- int rowkey = jump * (1 + i);
+ long rowkey = jump * (1 + i);
splits[i] = format(rowkey);
}
return splits;
@@ -645,7 +645,7 @@ public class PerformanceEvaluation extends Configured
implements Tool {
// Make input random.
Map<Integer, String> m = new TreeMap<>();
Hash h = MurmurHash.getInstance();
- int perClientRows = (opts.totalRows / opts.numClientThreads);
+ long perClientRows = (opts.totalRows / opts.numClientThreads);
try {
for (int j = 0; j < opts.numClientThreads; j++) {
TestOptions next = new TestOptions(opts);
@@ -704,11 +704,11 @@ public class PerformanceEvaluation extends Configured
implements Tool {
String cmdName = null;
boolean nomapred = false;
boolean filterAll = false;
- int startRow = 0;
+ long startRow = 0;
float size = 1.0f;
- int perClientRunRows = DEFAULT_ROWS_PER_GB;
+ long perClientRunRows = DEFAULT_ROWS_PER_GB;
int numClientThreads = 1;
- int totalRows = DEFAULT_ROWS_PER_GB;
+ long totalRows = DEFAULT_ROWS_PER_GB;
int measureAfter = 0;
float sampleRate = 1.0f;
/**
@@ -740,7 +740,7 @@ public class PerformanceEvaluation extends Configured
implements Tool {
boolean valueRandom = false;
boolean valueZipf = false;
int valueSize = DEFAULT_VALUE_LENGTH;
- int period = (this.perClientRunRows / 10) == 0 ? perClientRunRows :
perClientRunRows / 10;
+ long period = (this.perClientRunRows / 10) == 0 ? perClientRunRows :
perClientRunRows / 10;
int cycles = 1;
int columns = 1;
int families = 1;
@@ -897,7 +897,7 @@ public class PerformanceEvaluation extends Configured
implements Tool {
this.filterAll = filterAll;
}
- public void setStartRow(int startRow) {
+ public void setStartRow(long startRow) {
this.startRow = startRow;
}
@@ -913,7 +913,7 @@ public class PerformanceEvaluation extends Configured
implements Tool {
this.numClientThreads = numClientThreads;
}
- public void setTotalRows(int totalRows) {
+ public void setTotalRows(long totalRows) {
this.totalRows = totalRows;
}
@@ -1025,7 +1025,7 @@ public class PerformanceEvaluation extends Configured
implements Tool {
return filterAll;
}
- public int getStartRow() {
+ public long getStartRow() {
return startRow;
}
@@ -1033,7 +1033,7 @@ public class PerformanceEvaluation extends Configured
implements Tool {
return size;
}
- public int getPerClientRunRows() {
+ public long getPerClientRunRows() {
return perClientRunRows;
}
@@ -1041,7 +1041,7 @@ public class PerformanceEvaluation extends Configured
implements Tool {
return numClientThreads;
}
- public int getTotalRows() {
+ public long getTotalRows() {
return totalRows;
}
@@ -1117,7 +1117,7 @@ public class PerformanceEvaluation extends Configured
implements Tool {
return valueSize;
}
- public int getPeriod() {
+ public long getPeriod() {
return period;
}
@@ -1166,17 +1166,8 @@ public class PerformanceEvaluation extends Configured
implements Tool {
* A test. Subclass to particularize what happens per row.
*/
static abstract class TestBase {
- // Below is make it so when Tests are all running in the one
- // jvm, that they each have a differently seeded Random.
- private static final Random randomSeed = new
Random(EnvironmentEdgeManager.currentTime());
+ private final long everyN;
- private static long nextRandomSeed() {
- return randomSeed.nextLong();
- }
-
- private final int everyN;
-
- protected final Random rand = new Random(nextRandomSeed());
protected final Configuration conf;
protected final TestOptions opts;
@@ -1205,16 +1196,17 @@ public class PerformanceEvaluation extends Configured
implements Tool {
this.opts = options;
this.status = status;
this.testName = this.getClass().getSimpleName();
- everyN = (int) (opts.totalRows / (opts.totalRows * opts.sampleRate));
+ everyN = (long) (opts.totalRows / (opts.totalRows * opts.sampleRate));
if (options.isValueZipf()) {
- this.zipf = new RandomDistribution.Zipf(this.rand, 1,
options.getValueSize(), 1.2);
+ this.zipf =
+ new RandomDistribution.Zipf(ThreadLocalRandom.current(), 1,
options.getValueSize(), 1.2);
}
LOG.info("Sampling 1 every " + everyN + " out of " +
opts.perClientRunRows + " total rows.");
}
- int getValueLength(final Random r) {
+ int getValueLength() {
if (this.opts.isValueRandom()) {
- return r.nextInt(opts.valueSize);
+ return ThreadLocalRandom.current().nextInt(opts.valueSize);
} else if (this.opts.isValueZipf()) {
return Math.abs(this.zipf.nextInt());
} else {
@@ -1286,7 +1278,7 @@ public class PerformanceEvaluation extends Configured
implements Tool {
}
}
- String generateStatus(final int sr, final int i, final int lr) {
+ String generateStatus(final long sr, final long i, final long lr) {
return "row [start=" + sr + ", current=" + i + ", last=" + lr + "],
latency ["
+ getShortLatencyReport() + "]"
+ (!isRandomValueSize() ? "" : ", value size [" +
getShortValueSizeReport() + "]");
@@ -1296,7 +1288,7 @@ public class PerformanceEvaluation extends Configured
implements Tool {
return opts.valueRandom;
}
- protected int getReportingPeriod() {
+ protected long getReportingPeriod() {
return opts.period;
}
@@ -1399,11 +1391,11 @@ public class PerformanceEvaluation extends Configured
implements Tool {
return (System.nanoTime() - startTime) / 1000000;
}
- int getStartRow() {
+ long getStartRow() {
return opts.startRow;
}
- int getLastRow() {
+ long getLastRow() {
return getStartRow() + opts.perClientRunRows;
}
@@ -1411,12 +1403,12 @@ public class PerformanceEvaluation extends Configured
implements Tool {
* Provides an extension point for tests that don't want a per row
invocation.
*/
void testTimed() throws IOException, InterruptedException {
- int startRow = getStartRow();
- int lastRow = getLastRow();
+ long startRow = getStartRow();
+ long lastRow = getLastRow();
// Report on completion of 1/10th of total.
for (int ii = 0; ii < opts.cycles; ii++) {
if (opts.cycles > 1) LOG.info("Cycle=" + ii + " of " + opts.cycles);
- for (int i = startRow; i < lastRow; i++) {
+ for (long i = startRow; i < lastRow; i++) {
if (i % everyN != 0) continue;
long startTime = System.nanoTime();
boolean requestSent = false;
@@ -1462,7 +1454,7 @@ public class PerformanceEvaluation extends Configured
implements Tool {
* @return true if the row was sent to server and need to record metrics.
False if not, multiGet
* and multiPut e.g., the rows are sent to server only if enough
gets/puts are gathered.
*/
- abstract boolean testRow(final int i, final long startTime)
+ abstract boolean testRow(final long i, final long startTime)
throws IOException, InterruptedException;
}
@@ -1510,7 +1502,7 @@ public class PerformanceEvaluation extends Configured
implements Tool {
MetaTest(Connection con, TestOptions options, Status status) {
super(con, options, status);
- keyLength = Integer.toString(opts.perClientRunRows).length();
+ keyLength = Long.toString(opts.perClientRunRows).length();
}
@Override
@@ -1521,7 +1513,7 @@ public class PerformanceEvaluation extends Configured
implements Tool {
/*
* Generates Lexicographically ascending strings
*/
- protected byte[] getSplitKey(final int i) {
+ protected byte[] getSplitKey(final long i) {
return Bytes.toBytes(String.format("%0" + keyLength + "d", i));
}
@@ -1558,11 +1550,11 @@ public class PerformanceEvaluation extends Configured
implements Tool {
}
@Override
- boolean testRow(final int i, final long startTime) throws IOException,
InterruptedException {
+ boolean testRow(final long i, final long startTime) throws IOException,
InterruptedException {
if (opts.randomSleep > 0) {
Thread.sleep(ThreadLocalRandom.current().nextInt(opts.randomSleep));
}
- Get get = new Get(getRandomRow(this.rand, opts.totalRows));
+ Get get = new Get(getRandomRow(opts.totalRows));
for (int family = 0; family < opts.families; family++) {
byte[] familyName = Bytes.toBytes(FAMILY_NAME_BASE + family);
if (opts.addColumns) {
@@ -1615,8 +1607,8 @@ public class PerformanceEvaluation extends Configured
implements Tool {
}
@Override
- protected int getReportingPeriod() {
- int period = opts.perClientRunRows / 10;
+ protected long getReportingPeriod() {
+ long period = opts.perClientRunRows / 10;
return period == 0 ? opts.perClientRunRows : period;
}
@@ -1637,8 +1629,8 @@ public class PerformanceEvaluation extends Configured
implements Tool {
}
@Override
- protected byte[] generateRow(final int i) {
- return getRandomRow(this.rand, opts.totalRows);
+ protected byte[] generateRow(final long i) {
+ return getRandomRow(opts.totalRows);
}
}
@@ -1666,7 +1658,7 @@ public class PerformanceEvaluation extends Configured
implements Tool {
}
@Override
- boolean testRow(final int i, final long startTime) throws IOException {
+ boolean testRow(final long i, final long startTime) throws IOException {
if (this.testScanner == null) {
Scan scan = new
Scan().withStartRow(format(opts.startRow)).setCaching(opts.caching)
.setCacheBlocks(opts.cacheBlocks).setAsyncPrefetch(opts.asyncPrefetch)
@@ -1699,7 +1691,7 @@ public class PerformanceEvaluation extends Configured
implements Tool {
}
@Override
- boolean testRow(final int i, final long startTime) throws IOException,
InterruptedException {
+ boolean testRow(final long i, final long startTime) throws IOException,
InterruptedException {
Get get = new Get(format(i));
for (int family = 0; family < opts.families; family++) {
byte[] familyName = Bytes.toBytes(FAMILY_NAME_BASE + family);
@@ -1735,22 +1727,22 @@ public class PerformanceEvaluation extends Configured
implements Tool {
}
}
- protected byte[] generateRow(final int i) {
+ protected byte[] generateRow(final long i) {
return format(i);
}
@Override
@SuppressWarnings("ReturnValueIgnored")
- boolean testRow(final int i, final long startTime) throws IOException,
InterruptedException {
+ boolean testRow(final long i, final long startTime) throws IOException,
InterruptedException {
byte[] row = generateRow(i);
Put put = new Put(row);
for (int family = 0; family < opts.families; family++) {
byte[] familyName = Bytes.toBytes(FAMILY_NAME_BASE + family);
for (int column = 0; column < opts.columns; column++) {
byte[] qualifier = column == 0 ? COLUMN_ZERO : Bytes.toBytes("" +
column);
- byte[] value = generateData(this.rand, getValueLength(this.rand));
+ byte[] value = generateData(getValueLength());
if (opts.useTags) {
- byte[] tag = generateData(this.rand, TAG_LENGTH);
+ byte[] tag = generateData(TAG_LENGTH);
Tag[] tags = new Tag[opts.noOfTags];
for (int n = 0; n < opts.noOfTags; n++) {
Tag t = new ArrayBackedTag((byte) n, tag);
@@ -1816,11 +1808,10 @@ public class PerformanceEvaluation extends Configured
implements Tool {
}
@Override
- boolean testRow(final int i, final long startTime) throws IOException {
- Scan scan =
- new Scan().withStartRow(getRandomRow(this.rand,
opts.totalRows)).setCaching(opts.caching)
-
.setCacheBlocks(opts.cacheBlocks).setAsyncPrefetch(opts.asyncPrefetch)
- .setReadType(opts.scanReadType).setScanMetricsEnabled(true);
+ boolean testRow(final long i, final long startTime) throws IOException {
+ Scan scan = new
Scan().withStartRow(getRandomRow(opts.totalRows)).setCaching(opts.caching)
+ .setCacheBlocks(opts.cacheBlocks).setAsyncPrefetch(opts.asyncPrefetch)
+ .setReadType(opts.scanReadType).setScanMetricsEnabled(true);
FilterList list = new FilterList();
for (int family = 0; family < opts.families; family++) {
byte[] familyName = Bytes.toBytes(FAMILY_NAME_BASE + family);
@@ -1851,8 +1842,8 @@ public class PerformanceEvaluation extends Configured
implements Tool {
}
@Override
- protected int getReportingPeriod() {
- int period = opts.perClientRunRows / 100;
+ protected long getReportingPeriod() {
+ long period = opts.perClientRunRows / 100;
return period == 0 ? opts.perClientRunRows : period;
}
@@ -1864,7 +1855,7 @@ public class PerformanceEvaluation extends Configured
implements Tool {
}
@Override
- boolean testRow(final int i, final long startTime) throws IOException {
+ boolean testRow(final long i, final long startTime) throws IOException {
Pair<byte[], byte[]> startAndStopRow = getStartAndStopRow();
Scan scan = new Scan().withStartRow(startAndStopRow.getFirst())
.withStopRow(startAndStopRow.getSecond()).setCaching(opts.caching)
@@ -1906,15 +1897,15 @@ public class PerformanceEvaluation extends Configured
implements Tool {
protected abstract Pair<byte[], byte[]> getStartAndStopRow();
- protected Pair<byte[], byte[]> generateStartAndStopRows(int maxRange) {
- int start = this.rand.nextInt(Integer.MAX_VALUE) % opts.totalRows;
- int stop = start + maxRange;
+ protected Pair<byte[], byte[]> generateStartAndStopRows(long maxRange) {
+ long start = ThreadLocalRandom.current().nextLong(Long.MAX_VALUE) %
opts.totalRows;
+ long stop = start + maxRange;
return new Pair<>(format(start), format(stop));
}
@Override
- protected int getReportingPeriod() {
- int period = opts.perClientRunRows / 100;
+ protected long getReportingPeriod() {
+ long period = opts.perClientRunRows / 100;
return period == 0 ? opts.perClientRunRows : period;
}
}
@@ -1977,11 +1968,11 @@ public class PerformanceEvaluation extends Configured
implements Tool {
}
@Override
- boolean testRow(final int i, final long startTime) throws IOException,
InterruptedException {
+ boolean testRow(final long i, final long startTime) throws IOException,
InterruptedException {
if (opts.randomSleep > 0) {
Thread.sleep(ThreadLocalRandom.current().nextInt(opts.randomSleep));
}
- Get get = new Get(getRandomRow(this.rand, opts.totalRows));
+ Get get = new Get(getRandomRow(opts.totalRows));
for (int family = 0; family < opts.families; family++) {
byte[] familyName = Bytes.toBytes(FAMILY_NAME_BASE + family);
if (opts.addColumns) {
@@ -2025,8 +2016,8 @@ public class PerformanceEvaluation extends Configured
implements Tool {
}
@Override
- protected int getReportingPeriod() {
- int period = opts.perClientRunRows / 10;
+ protected long getReportingPeriod() {
+ long period = opts.perClientRunRows / 10;
return period == 0 ? opts.perClientRunRows : period;
}
@@ -2058,19 +2049,19 @@ public class PerformanceEvaluation extends Configured
implements Tool {
}
@Override
- boolean testRow(final int i, final long startTime) throws IOException,
InterruptedException {
+ boolean testRow(final long i, final long startTime) throws IOException,
InterruptedException {
if (opts.randomSleep > 0) {
- Thread.sleep(rand.nextInt(opts.randomSleep));
+ Thread.sleep(ThreadLocalRandom.current().nextInt(opts.randomSleep));
}
- HRegionLocation hRegionLocation =
-
regionLocator.getRegionLocation(getSplitKey(rand.nextInt(opts.perClientRunRows)),
true);
+ HRegionLocation hRegionLocation = regionLocator.getRegionLocation(
+
getSplitKey(ThreadLocalRandom.current().nextLong(opts.perClientRunRows)), true);
LOG.debug("get location for region: " + hRegionLocation);
return true;
}
@Override
- protected int getReportingPeriod() {
- int period = opts.perClientRunRows / 10;
+ protected long getReportingPeriod() {
+ long period = opts.perClientRunRows / 10;
return period == 0 ? opts.perClientRunRows : period;
}
@@ -2086,8 +2077,8 @@ public class PerformanceEvaluation extends Configured
implements Tool {
}
@Override
- protected byte[] generateRow(final int i) {
- return getRandomRow(this.rand, opts.totalRows);
+ protected byte[] generateRow(final long i) {
+ return getRandomRow(opts.totalRows);
}
}
@@ -2098,8 +2089,8 @@ public class PerformanceEvaluation extends Configured
implements Tool {
}
@Override
- protected byte[] generateRow(final int i) {
- return getRandomRow(this.rand, opts.totalRows);
+ protected byte[] generateRow(final long i) {
+ return getRandomRow(opts.totalRows);
}
}
@@ -2120,7 +2111,7 @@ public class PerformanceEvaluation extends Configured
implements Tool {
}
@Override
- boolean testRow(final int i, final long startTime) throws IOException {
+ boolean testRow(final long i, final long startTime) throws IOException {
if (this.testScanner == null) {
Scan scan = new
Scan().withStartRow(format(opts.startRow)).setCaching(opts.caching)
.setCacheBlocks(opts.cacheBlocks).setAsyncPrefetch(opts.asyncPrefetch)
@@ -2163,7 +2154,7 @@ public class PerformanceEvaluation extends Configured
implements Tool {
}
@Override
- boolean testRow(final int i, final long startTime) throws IOException {
+ boolean testRow(final long i, final long startTime) throws IOException {
if (this.testScanner == null) {
Scan scan = new
Scan().setCaching(opts.caching).setCacheBlocks(opts.cacheBlocks)
.setAsyncPrefetch(opts.asyncPrefetch).setReadType(opts.scanReadType)
@@ -2211,12 +2202,12 @@ public class PerformanceEvaluation extends Configured
implements Tool {
}
@Override
- int getStartRow() {
+ long getStartRow() {
return 0;
}
@Override
- int getLastRow() {
+ long getLastRow() {
return opts.perClientRunRows;
}
}
@@ -2227,7 +2218,7 @@ public class PerformanceEvaluation extends Configured
implements Tool {
}
@Override
- boolean testRow(final int i, final long startTime) throws IOException {
+ boolean testRow(final long i, final long startTime) throws IOException {
Increment increment = new Increment(format(i));
// unlike checkAndXXX tests, which make most sense to do on a single
value,
// if multiple families are specified for an increment test we assume it
is
@@ -2247,7 +2238,7 @@ public class PerformanceEvaluation extends Configured
implements Tool {
}
@Override
- boolean testRow(final int i, final long startTime) throws IOException {
+ boolean testRow(final long i, final long startTime) throws IOException {
byte[] bytes = format(i);
Append append = new Append(bytes);
// unlike checkAndXXX tests, which make most sense to do on a single
value,
@@ -2268,7 +2259,7 @@ public class PerformanceEvaluation extends Configured
implements Tool {
}
@Override
- boolean testRow(final int i, final long startTime) throws IOException {
+ boolean testRow(final long i, final long startTime) throws IOException {
final byte[] bytes = format(i);
// checkAndXXX tests operate on only a single value
// Put a known value so when we go to check it, it is there.
@@ -2289,7 +2280,7 @@ public class PerformanceEvaluation extends Configured
implements Tool {
}
@Override
- boolean testRow(final int i, final long startTime) throws IOException {
+ boolean testRow(final long i, final long startTime) throws IOException {
final byte[] bytes = format(i);
// checkAndXXX tests operate on only a single value
// Put a known value so when we go to check it, it is there.
@@ -2308,7 +2299,7 @@ public class PerformanceEvaluation extends Configured
implements Tool {
}
@Override
- boolean testRow(final int i, final long startTime) throws IOException {
+ boolean testRow(final long i, final long startTime) throws IOException {
final byte[] bytes = format(i);
// checkAndXXX tests operate on only a single value
// Put a known value so when we go to check it, it is there.
@@ -2332,7 +2323,7 @@ public class PerformanceEvaluation extends Configured
implements Tool {
}
@Override
- boolean testRow(final int i, final long startTime) throws IOException {
+ boolean testRow(final long i, final long startTime) throws IOException {
try {
RegionInfo regionInfo = connection.getRegionLocator(table.getName())
.getRegionLocation(getSplitKey(i), false).getRegion();
@@ -2357,7 +2348,7 @@ public class PerformanceEvaluation extends Configured
implements Tool {
}
@Override
- boolean testRow(final int i, final long startTime) throws IOException {
+ boolean testRow(final long i, final long startTime) throws IOException {
Get get = new Get(format(i));
for (int family = 0; family < opts.families; family++) {
byte[] familyName = Bytes.toBytes(FAMILY_NAME_BASE + family);
@@ -2389,21 +2380,21 @@ public class PerformanceEvaluation extends Configured
implements Tool {
}
}
- protected byte[] generateRow(final int i) {
+ protected byte[] generateRow(final long i) {
return format(i);
}
@Override
- boolean testRow(final int i, final long startTime) throws IOException {
+ boolean testRow(final long i, final long startTime) throws IOException {
byte[] row = generateRow(i);
Put put = new Put(row);
for (int family = 0; family < opts.families; family++) {
byte familyName[] = Bytes.toBytes(FAMILY_NAME_BASE + family);
for (int column = 0; column < opts.columns; column++) {
byte[] qualifier = column == 0 ? COLUMN_ZERO : Bytes.toBytes("" +
column);
- byte[] value = generateData(this.rand, getValueLength(this.rand));
+ byte[] value = generateData(getValueLength());
if (opts.useTags) {
- byte[] tag = generateData(this.rand, TAG_LENGTH);
+ byte[] tag = generateData(TAG_LENGTH);
Tag[] tags = new Tag[opts.noOfTags];
for (int n = 0; n < opts.noOfTags; n++) {
Tag t = new ArrayBackedTag((byte) n, tag);
@@ -2445,12 +2436,12 @@ public class PerformanceEvaluation extends Configured
implements Tool {
super(con, options, status);
}
- protected byte[] generateRow(final int i) {
+ protected byte[] generateRow(final long i) {
return format(i);
}
@Override
- boolean testRow(final int i, final long startTime) throws IOException {
+ boolean testRow(final long i, final long startTime) throws IOException {
byte[] row = generateRow(i);
Delete delete = new Delete(row);
for (int family = 0; family < opts.families; family++) {
@@ -2477,7 +2468,7 @@ public class PerformanceEvaluation extends Configured
implements Tool {
}
@Override
- boolean testRow(final int i, final long startTime) throws IOException {
+ boolean testRow(final long i, final long startTime) throws IOException {
List<RegionInfo> regionInfos = new ArrayList<RegionInfo>();
RegionInfo regionInfo =
(RegionInfoBuilder.newBuilder(TableName.valueOf(TABLE_NAME))
.setStartKey(getSplitKey(i)).setEndKey(getSplitKey(i + 1)).build());
@@ -2486,7 +2477,7 @@ public class PerformanceEvaluation extends Configured
implements Tool {
// write the serverName columns
MetaTableAccessor.updateRegionLocation(connection, regionInfo,
- ServerName.valueOf("localhost", 60010, rand.nextLong()), i,
+ ServerName.valueOf("localhost", 60010,
ThreadLocalRandom.current().nextLong()), i,
EnvironmentEdgeManager.currentTime());
return true;
}
@@ -2504,8 +2495,8 @@ public class PerformanceEvaluation extends Configured
implements Tool {
}
@Override
- boolean testRow(int i, final long startTime) throws IOException {
- byte[] value = generateData(this.rand, getValueLength(this.rand));
+ boolean testRow(long i, final long startTime) throws IOException {
+ byte[] value = generateData(getValueLength());
Scan scan = constructScan(value);
ResultScanner scanner = null;
try {
@@ -2552,7 +2543,7 @@ public class PerformanceEvaluation extends Configured
implements Tool {
* @param timeMs Time taken in milliseconds.
* @return String value with label, ie '123.76 MB/s'
*/
- private static String calculateMbps(int rows, long timeMs, final int
valueSize, int families,
+ private static String calculateMbps(long rows, long timeMs, final int
valueSize, int families,
int columns) {
BigDecimal rowSize = BigDecimal.valueOf(ROW_LENGTH
+ ((valueSize + (FAMILY_NAME_BASE.length() + 1) + COLUMN_ZERO.length) *
columns) * families);
@@ -2566,9 +2557,9 @@ public class PerformanceEvaluation extends Configured
implements Tool {
* @return Returns zero-prefixed ROW_LENGTH-byte wide decimal version of
passed number (Does
* absolute in case number is negative).
*/
- public static byte[] format(final int number) {
+ public static byte[] format(final long number) {
byte[] b = new byte[ROW_LENGTH];
- int d = Math.abs(number);
+ long d = Math.abs(number);
for (int i = b.length - 1; i >= 0; i--) {
b[i] = (byte) ((d % 10) + '0');
d /= 10;
@@ -2581,10 +2572,11 @@ public class PerformanceEvaluation extends Configured
implements Tool {
* test, generation of the key and value consumes about 30% of CPU time.
* @return Generated random value to insert into a table cell.
*/
- public static byte[] generateData(final Random r, int length) {
+ public static byte[] generateData(int length) {
byte[] b = new byte[length];
int i;
+ Random r = ThreadLocalRandom.current();
for (i = 0; i < (length - 8); i += 8) {
b[i] = (byte) (65 + r.nextInt(26));
b[i + 1] = b[i];
@@ -2603,12 +2595,12 @@ public class PerformanceEvaluation extends Configured
implements Tool {
return b;
}
- static byte[] getRandomRow(final Random random, final int totalRows) {
- return format(generateRandomRow(random, totalRows));
+ static byte[] getRandomRow(final long totalRows) {
+ return format(generateRandomRow(totalRows));
}
- static int generateRandomRow(final Random random, final int totalRows) {
- return random.nextInt(Integer.MAX_VALUE) % totalRows;
+ static long generateRandomRow(final long totalRows) {
+ return ThreadLocalRandom.current().nextLong(Long.MAX_VALUE) % totalRows;
}
static RunResult runOneClient(final Class<? extends TestBase> cmd,
Configuration conf,
@@ -2643,7 +2635,7 @@ public class PerformanceEvaluation extends Configured
implements Tool {
status.setStatus("Finished " + cmd + " in " + totalElapsedTime + "ms at
offset " + opts.startRow
+ " for " + opts.perClientRunRows + " rows" + " ("
- + calculateMbps((int) (opts.perClientRunRows * opts.sampleRate),
totalElapsedTime,
+ + calculateMbps((long) (opts.perClientRunRows * opts.sampleRate),
totalElapsedTime,
getAverageValueLength(opts), opts.families, opts.columns)
+ ")");
@@ -2838,7 +2830,7 @@ public class PerformanceEvaluation extends Configured
implements Tool {
final String rows = "--rows=";
if (cmd.startsWith(rows)) {
- opts.perClientRunRows = Integer.parseInt(cmd.substring(rows.length()));
+ opts.perClientRunRows = Long.parseLong(cmd.substring(rows.length()));
continue;
}
@@ -2862,7 +2854,7 @@ public class PerformanceEvaluation extends Configured
implements Tool {
final String startRow = "--startRow=";
if (cmd.startsWith(startRow)) {
- opts.startRow = Integer.parseInt(cmd.substring(startRow.length()));
+ opts.startRow = Long.parseLong(cmd.substring(startRow.length()));
continue;
}
@@ -3160,10 +3152,10 @@ public class PerformanceEvaluation extends Configured
implements Tool {
&& (opts.getCmdName().equals(RANDOM_READ) ||
opts.getCmdName().equals(RANDOM_SEEK_SCAN)))
&& opts.size != DEFAULT_OPTS.size && opts.perClientRunRows !=
DEFAULT_OPTS.perClientRunRows
) {
- opts.totalRows = (int) (opts.size * rowsPerGB);
+ opts.totalRows = (long) (opts.size * rowsPerGB);
} else if (opts.size != DEFAULT_OPTS.size) {
// total size in GB specified
- opts.totalRows = (int) (opts.size * rowsPerGB);
+ opts.totalRows = (long) (opts.size * rowsPerGB);
opts.perClientRunRows = opts.totalRows / opts.numClientThreads;
} else {
opts.totalRows = opts.perClientRunRows * opts.numClientThreads;
diff --git
a/hbase-diagnostics/src/test/java/org/apache/hadoop/hbase/TestPerformanceEvaluation.java
b/hbase-diagnostics/src/test/java/org/apache/hadoop/hbase/TestPerformanceEvaluation.java
index cf11510a897..91722be6164 100644
---
a/hbase-diagnostics/src/test/java/org/apache/hadoop/hbase/TestPerformanceEvaluation.java
+++
b/hbase-diagnostics/src/test/java/org/apache/hadoop/hbase/TestPerformanceEvaluation.java
@@ -39,8 +39,6 @@ import java.util.LinkedList;
import java.util.NoSuchElementException;
import java.util.Properties;
import java.util.Queue;
-import java.util.Random;
-import java.util.concurrent.ThreadLocalRandom;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -124,7 +122,7 @@ public class TestPerformanceEvaluation {
public void testSizeCalculation() {
TestOptions opts = new PerformanceEvaluation.TestOptions();
opts = PerformanceEvaluation.calculateRowsAndSize(opts);
- int rows = opts.getPerClientRunRows();
+ long rows = opts.getPerClientRunRows();
// Default row count
final int defaultPerClientRunRows = 1024 * 1024;
assertEquals(defaultPerClientRunRows, rows);
@@ -146,7 +144,7 @@ public class TestPerformanceEvaluation {
public void testRandomReadCalculation() {
TestOptions opts = new PerformanceEvaluation.TestOptions();
opts = PerformanceEvaluation.calculateRowsAndSize(opts);
- int rows = opts.getPerClientRunRows();
+ long rows = opts.getPerClientRunRows();
// Default row count
final int defaultPerClientRunRows = 1024 * 1024;
assertEquals(defaultPerClientRunRows, rows);
@@ -162,9 +160,8 @@ public class TestPerformanceEvaluation {
assertEquals(1000, opts.getPerClientRunRows());
// assuming we will get one before this loop expires
boolean foundValue = false;
- Random rand = ThreadLocalRandom.current();
for (int i = 0; i < 10000000; i++) {
- int randomRow = PerformanceEvaluation.generateRandomRow(rand,
opts.totalRows);
+ long randomRow = PerformanceEvaluation.generateRandomRow(opts.totalRows);
if (randomRow > 1000) {
foundValue = true;
break;
@@ -186,7 +183,7 @@ public class TestPerformanceEvaluation {
ctor.setAccessible(true);
Histogram histogram = (Histogram) ctor.newInstance(new
UniformReservoir(1024 * 500));
for (int i = 0; i < 100; i++) {
- histogram.update(rrt.getValueLength(null));
+ histogram.update(rrt.getValueLength());
}
Snapshot snapshot = histogram.getSnapshot();
double stddev = snapshot.getStdDev();
@@ -402,7 +399,7 @@ public class TestPerformanceEvaluation {
}
@Override
- boolean testRow(int i, long startTime) throws IOException,
InterruptedException {
+ boolean testRow(long i, long startTime) throws IOException,
InterruptedException {
return false;
}
}
