This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch branch-2.5
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2.5 by this push:
new e0dfb9c26a4 HBASE-29013 Make PerformanceEvaluation support larger data
sets (branch-2) (#6559)
e0dfb9c26a4 is described below
commit e0dfb9c26a46806aea8d87448983cd65c082a7ea
Author: Junegunn Choi <[email protected]>
AuthorDate: Sun Dec 29 18:57:39 2024 +0900
HBASE-29013 Make PerformanceEvaluation support larger data sets (branch-2)
(#6559)
Signed-off-by: Duo Zhang <[email protected]>
(cherry picked from commit 5638a38f701515c5f8c7f655a43ccf67a28a4673)
---
.../apache/hadoop/hbase/PerformanceEvaluation.java | 208 ++++++++++-----------
.../hadoop/hbase/TestPerformanceEvaluation.java | 13 +-
.../mapreduce/TestCellBasedHFileOutputFormat2.java | 8 +-
.../hbase/mapreduce/TestHFileOutputFormat2.java | 10 +-
4 files changed, 109 insertions(+), 130 deletions(-)
diff --git
a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
index 87b6532737e..1486c0c9fb0 100644
---
a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
+++
b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
@@ -439,9 +439,9 @@ public class PerformanceEvaluation extends Configured
implements Tool {
int numSplitPoints = opts.presplitRegions - 1;
byte[][] splits = new byte[numSplitPoints][];
- int jump = opts.totalRows / opts.presplitRegions;
+ long jump = opts.totalRows / opts.presplitRegions;
for (int i = 0; i < numSplitPoints; i++) {
- int rowkey = jump * (1 + i);
+ long rowkey = jump * (1 + i);
splits[i] = format(rowkey);
}
return splits;
@@ -618,7 +618,7 @@ public class PerformanceEvaluation extends Configured
implements Tool {
// Make input random.
Map<Integer, String> m = new TreeMap<>();
Hash h = MurmurHash.getInstance();
- int perClientRows = (opts.totalRows / opts.numClientThreads);
+ long perClientRows = (opts.totalRows / opts.numClientThreads);
try {
for (int j = 0; j < opts.numClientThreads; j++) {
TestOptions next = new TestOptions(opts);
@@ -677,11 +677,11 @@ public class PerformanceEvaluation extends Configured
implements Tool {
String cmdName = null;
boolean nomapred = false;
boolean filterAll = false;
- int startRow = 0;
+ long startRow = 0;
float size = 1.0f;
- int perClientRunRows = DEFAULT_ROWS_PER_GB;
+ long perClientRunRows = DEFAULT_ROWS_PER_GB;
int numClientThreads = 1;
- int totalRows = DEFAULT_ROWS_PER_GB;
+ long totalRows = DEFAULT_ROWS_PER_GB;
int measureAfter = 0;
float sampleRate = 1.0f;
/**
@@ -713,7 +713,7 @@ public class PerformanceEvaluation extends Configured
implements Tool {
boolean valueRandom = false;
boolean valueZipf = false;
int valueSize = DEFAULT_VALUE_LENGTH;
- int period = (this.perClientRunRows / 10) == 0 ? perClientRunRows :
perClientRunRows / 10;
+ long period = (this.perClientRunRows / 10) == 0 ? perClientRunRows :
perClientRunRows / 10;
int cycles = 1;
int columns = 1;
int families = 1;
@@ -870,7 +870,7 @@ public class PerformanceEvaluation extends Configured
implements Tool {
this.filterAll = filterAll;
}
- public void setStartRow(int startRow) {
+ public void setStartRow(long startRow) {
this.startRow = startRow;
}
@@ -886,7 +886,7 @@ public class PerformanceEvaluation extends Configured
implements Tool {
this.numClientThreads = numClientThreads;
}
- public void setTotalRows(int totalRows) {
+ public void setTotalRows(long totalRows) {
this.totalRows = totalRows;
}
@@ -998,7 +998,7 @@ public class PerformanceEvaluation extends Configured
implements Tool {
return filterAll;
}
- public int getStartRow() {
+ public long getStartRow() {
return startRow;
}
@@ -1006,7 +1006,7 @@ public class PerformanceEvaluation extends Configured
implements Tool {
return size;
}
- public int getPerClientRunRows() {
+ public long getPerClientRunRows() {
return perClientRunRows;
}
@@ -1014,7 +1014,7 @@ public class PerformanceEvaluation extends Configured
implements Tool {
return numClientThreads;
}
- public int getTotalRows() {
+ public long getTotalRows() {
return totalRows;
}
@@ -1090,7 +1090,7 @@ public class PerformanceEvaluation extends Configured
implements Tool {
return valueSize;
}
- public int getPeriod() {
+ public long getPeriod() {
return period;
}
@@ -1139,17 +1139,8 @@ public class PerformanceEvaluation extends Configured
implements Tool {
* A test. Subclass to particularize what happens per row.
*/
static abstract class TestBase {
- // Below is make it so when Tests are all running in the one
- // jvm, that they each have a differently seeded Random.
- private static final Random randomSeed = new
Random(EnvironmentEdgeManager.currentTime());
+ private final long everyN;
- private static long nextRandomSeed() {
- return randomSeed.nextLong();
- }
-
- private final int everyN;
-
- protected final Random rand = new Random(nextRandomSeed());
protected final Configuration conf;
protected final TestOptions opts;
@@ -1178,16 +1169,17 @@ public class PerformanceEvaluation extends Configured
implements Tool {
this.opts = options;
this.status = status;
this.testName = this.getClass().getSimpleName();
- everyN = (int) (opts.totalRows / (opts.totalRows * opts.sampleRate));
+ everyN = (long) (opts.totalRows / (opts.totalRows * opts.sampleRate));
if (options.isValueZipf()) {
- this.zipf = new RandomDistribution.Zipf(this.rand, 1,
options.getValueSize(), 1.2);
+ this.zipf =
+ new RandomDistribution.Zipf(ThreadLocalRandom.current(), 1,
options.getValueSize(), 1.2);
}
LOG.info("Sampling 1 every " + everyN + " out of " +
opts.perClientRunRows + " total rows.");
}
- int getValueLength(final Random r) {
+ int getValueLength() {
if (this.opts.isValueRandom()) {
- return r.nextInt(opts.valueSize);
+ return ThreadLocalRandom.current().nextInt(opts.valueSize);
} else if (this.opts.isValueZipf()) {
return Math.abs(this.zipf.nextInt());
} else {
@@ -1259,7 +1251,7 @@ public class PerformanceEvaluation extends Configured
implements Tool {
}
}
- String generateStatus(final int sr, final int i, final int lr) {
+ String generateStatus(final long sr, final long i, final long lr) {
return "row [start=" + sr + ", current=" + i + ", last=" + lr + "],
latency ["
+ getShortLatencyReport() + "]"
+ (!isRandomValueSize() ? "" : ", value size [" +
getShortValueSizeReport() + "]");
@@ -1269,7 +1261,7 @@ public class PerformanceEvaluation extends Configured
implements Tool {
return opts.valueRandom;
}
- protected int getReportingPeriod() {
+ protected long getReportingPeriod() {
return opts.period;
}
@@ -1372,11 +1364,11 @@ public class PerformanceEvaluation extends Configured
implements Tool {
return (System.nanoTime() - startTime) / 1000000;
}
- int getStartRow() {
+ long getStartRow() {
return opts.startRow;
}
- int getLastRow() {
+ long getLastRow() {
return getStartRow() + opts.perClientRunRows;
}
@@ -1384,12 +1376,12 @@ public class PerformanceEvaluation extends Configured
implements Tool {
* Provides an extension point for tests that don't want a per row
invocation.
*/
void testTimed() throws IOException, InterruptedException {
- int startRow = getStartRow();
- int lastRow = getLastRow();
+ long startRow = getStartRow();
+ long lastRow = getLastRow();
// Report on completion of 1/10th of total.
for (int ii = 0; ii < opts.cycles; ii++) {
if (opts.cycles > 1) LOG.info("Cycle=" + ii + " of " + opts.cycles);
- for (int i = startRow; i < lastRow; i++) {
+ for (long i = startRow; i < lastRow; i++) {
if (i % everyN != 0) continue;
long startTime = System.nanoTime();
boolean requestSent = false;
@@ -1435,7 +1427,7 @@ public class PerformanceEvaluation extends Configured
implements Tool {
* @return true if the row was sent to server and need to record metrics.
False if not, multiGet
* and multiPut e.g., the rows are sent to server only if enough
gets/puts are gathered.
*/
- abstract boolean testRow(final int i, final long startTime)
+ abstract boolean testRow(final long i, final long startTime)
throws IOException, InterruptedException;
}
@@ -1483,7 +1475,7 @@ public class PerformanceEvaluation extends Configured
implements Tool {
MetaTest(Connection con, TestOptions options, Status status) {
super(con, options, status);
- keyLength = Integer.toString(opts.perClientRunRows).length();
+ keyLength = Long.toString(opts.perClientRunRows).length();
}
@Override
@@ -1494,7 +1486,7 @@ public class PerformanceEvaluation extends Configured
implements Tool {
/*
* Generates Lexicographically ascending strings
*/
- protected byte[] getSplitKey(final int i) {
+ protected byte[] getSplitKey(final long i) {
return Bytes.toBytes(String.format("%0" + keyLength + "d", i));
}
@@ -1531,11 +1523,11 @@ public class PerformanceEvaluation extends Configured
implements Tool {
}
@Override
- boolean testRow(final int i, final long startTime) throws IOException,
InterruptedException {
+ boolean testRow(final long i, final long startTime) throws IOException,
InterruptedException {
if (opts.randomSleep > 0) {
Thread.sleep(ThreadLocalRandom.current().nextInt(opts.randomSleep));
}
- Get get = new Get(getRandomRow(this.rand, opts.totalRows));
+ Get get = new Get(getRandomRow(opts.totalRows));
for (int family = 0; family < opts.families; family++) {
byte[] familyName = Bytes.toBytes(FAMILY_NAME_BASE + family);
if (opts.addColumns) {
@@ -1588,8 +1580,8 @@ public class PerformanceEvaluation extends Configured
implements Tool {
}
@Override
- protected int getReportingPeriod() {
- int period = opts.perClientRunRows / 10;
+ protected long getReportingPeriod() {
+ long period = opts.perClientRunRows / 10;
return period == 0 ? opts.perClientRunRows : period;
}
@@ -1610,8 +1602,8 @@ public class PerformanceEvaluation extends Configured
implements Tool {
}
@Override
- protected byte[] generateRow(final int i) {
- return getRandomRow(this.rand, opts.totalRows);
+ protected byte[] generateRow(final long i) {
+ return getRandomRow(opts.totalRows);
}
}
@@ -1639,7 +1631,7 @@ public class PerformanceEvaluation extends Configured
implements Tool {
}
@Override
- boolean testRow(final int i, final long startTime) throws IOException {
+ boolean testRow(final long i, final long startTime) throws IOException {
if (this.testScanner == null) {
Scan scan = new
Scan().withStartRow(format(opts.startRow)).setCaching(opts.caching)
.setCacheBlocks(opts.cacheBlocks).setAsyncPrefetch(opts.asyncPrefetch)
@@ -1672,7 +1664,7 @@ public class PerformanceEvaluation extends Configured
implements Tool {
}
@Override
- boolean testRow(final int i, final long startTime) throws IOException,
InterruptedException {
+ boolean testRow(final long i, final long startTime) throws IOException,
InterruptedException {
Get get = new Get(format(i));
for (int family = 0; family < opts.families; family++) {
byte[] familyName = Bytes.toBytes(FAMILY_NAME_BASE + family);
@@ -1708,22 +1700,22 @@ public class PerformanceEvaluation extends Configured
implements Tool {
}
}
- protected byte[] generateRow(final int i) {
+ protected byte[] generateRow(final long i) {
return format(i);
}
@Override
@SuppressWarnings("ReturnValueIgnored")
- boolean testRow(final int i, final long startTime) throws IOException,
InterruptedException {
+ boolean testRow(final long i, final long startTime) throws IOException,
InterruptedException {
byte[] row = generateRow(i);
Put put = new Put(row);
for (int family = 0; family < opts.families; family++) {
byte[] familyName = Bytes.toBytes(FAMILY_NAME_BASE + family);
for (int column = 0; column < opts.columns; column++) {
byte[] qualifier = column == 0 ? COLUMN_ZERO : Bytes.toBytes("" +
column);
- byte[] value = generateData(this.rand, getValueLength(this.rand));
+ byte[] value = generateData(getValueLength());
if (opts.useTags) {
- byte[] tag = generateData(this.rand, TAG_LENGTH);
+ byte[] tag = generateData(TAG_LENGTH);
Tag[] tags = new Tag[opts.noOfTags];
for (int n = 0; n < opts.noOfTags; n++) {
Tag t = new ArrayBackedTag((byte) n, tag);
@@ -1789,11 +1781,10 @@ public class PerformanceEvaluation extends Configured
implements Tool {
}
@Override
- boolean testRow(final int i, final long startTime) throws IOException {
- Scan scan =
- new Scan().withStartRow(getRandomRow(this.rand,
opts.totalRows)).setCaching(opts.caching)
-
.setCacheBlocks(opts.cacheBlocks).setAsyncPrefetch(opts.asyncPrefetch)
- .setReadType(opts.scanReadType).setScanMetricsEnabled(true);
+ boolean testRow(final long i, final long startTime) throws IOException {
+ Scan scan = new
Scan().withStartRow(getRandomRow(opts.totalRows)).setCaching(opts.caching)
+ .setCacheBlocks(opts.cacheBlocks).setAsyncPrefetch(opts.asyncPrefetch)
+ .setReadType(opts.scanReadType).setScanMetricsEnabled(true);
FilterList list = new FilterList();
for (int family = 0; family < opts.families; family++) {
byte[] familyName = Bytes.toBytes(FAMILY_NAME_BASE + family);
@@ -1824,8 +1815,8 @@ public class PerformanceEvaluation extends Configured
implements Tool {
}
@Override
- protected int getReportingPeriod() {
- int period = opts.perClientRunRows / 100;
+ protected long getReportingPeriod() {
+ long period = opts.perClientRunRows / 100;
return period == 0 ? opts.perClientRunRows : period;
}
@@ -1837,7 +1828,7 @@ public class PerformanceEvaluation extends Configured
implements Tool {
}
@Override
- boolean testRow(final int i, final long startTime) throws IOException {
+ boolean testRow(final long i, final long startTime) throws IOException {
Pair<byte[], byte[]> startAndStopRow = getStartAndStopRow();
Scan scan = new Scan().withStartRow(startAndStopRow.getFirst())
.withStopRow(startAndStopRow.getSecond()).setCaching(opts.caching)
@@ -1879,15 +1870,15 @@ public class PerformanceEvaluation extends Configured
implements Tool {
protected abstract Pair<byte[], byte[]> getStartAndStopRow();
- protected Pair<byte[], byte[]> generateStartAndStopRows(int maxRange) {
- int start = this.rand.nextInt(Integer.MAX_VALUE) % opts.totalRows;
- int stop = start + maxRange;
+ protected Pair<byte[], byte[]> generateStartAndStopRows(long maxRange) {
+ long start = ThreadLocalRandom.current().nextLong(Long.MAX_VALUE) %
opts.totalRows;
+ long stop = start + maxRange;
return new Pair<>(format(start), format(stop));
}
@Override
- protected int getReportingPeriod() {
- int period = opts.perClientRunRows / 100;
+ protected long getReportingPeriod() {
+ long period = opts.perClientRunRows / 100;
return period == 0 ? opts.perClientRunRows : period;
}
}
@@ -1950,11 +1941,11 @@ public class PerformanceEvaluation extends Configured
implements Tool {
}
@Override
- boolean testRow(final int i, final long startTime) throws IOException,
InterruptedException {
+ boolean testRow(final long i, final long startTime) throws IOException,
InterruptedException {
if (opts.randomSleep > 0) {
Thread.sleep(ThreadLocalRandom.current().nextInt(opts.randomSleep));
}
- Get get = new Get(getRandomRow(this.rand, opts.totalRows));
+ Get get = new Get(getRandomRow(opts.totalRows));
for (int family = 0; family < opts.families; family++) {
byte[] familyName = Bytes.toBytes(FAMILY_NAME_BASE + family);
if (opts.addColumns) {
@@ -1998,8 +1989,8 @@ public class PerformanceEvaluation extends Configured
implements Tool {
}
@Override
- protected int getReportingPeriod() {
- int period = opts.perClientRunRows / 10;
+ protected long getReportingPeriod() {
+ long period = opts.perClientRunRows / 10;
return period == 0 ? opts.perClientRunRows : period;
}
@@ -2032,19 +2023,19 @@ public class PerformanceEvaluation extends Configured
implements Tool {
}
@Override
- boolean testRow(final int i, final long startTime) throws IOException,
InterruptedException {
+ boolean testRow(final long i, final long startTime) throws IOException,
InterruptedException {
if (opts.randomSleep > 0) {
Thread.sleep(rd.nextInt(opts.randomSleep));
}
- HRegionLocation hRegionLocation =
-
regionLocator.getRegionLocation(getSplitKey(rd.nextInt(opts.perClientRunRows)),
true);
+ HRegionLocation hRegionLocation = regionLocator.getRegionLocation(
+
getSplitKey(ThreadLocalRandom.current().nextLong(opts.perClientRunRows)), true);
LOG.debug("get location for region: " + hRegionLocation);
return true;
}
@Override
- protected int getReportingPeriod() {
- int period = opts.perClientRunRows / 10;
+ protected long getReportingPeriod() {
+ long period = opts.perClientRunRows / 10;
return period == 0 ? opts.perClientRunRows : period;
}
@@ -2060,8 +2051,8 @@ public class PerformanceEvaluation extends Configured
implements Tool {
}
@Override
- protected byte[] generateRow(final int i) {
- return getRandomRow(this.rand, opts.totalRows);
+ protected byte[] generateRow(final long i) {
+ return getRandomRow(opts.totalRows);
}
}
@@ -2072,8 +2063,8 @@ public class PerformanceEvaluation extends Configured
implements Tool {
}
@Override
- protected byte[] generateRow(final int i) {
- return getRandomRow(this.rand, opts.totalRows);
+ protected byte[] generateRow(final long i) {
+ return getRandomRow(opts.totalRows);
}
}
@@ -2094,7 +2085,7 @@ public class PerformanceEvaluation extends Configured
implements Tool {
}
@Override
- boolean testRow(final int i, final long startTime) throws IOException {
+ boolean testRow(final long i, final long startTime) throws IOException {
if (this.testScanner == null) {
Scan scan = new
Scan().withStartRow(format(opts.startRow)).setCaching(opts.caching)
.setCacheBlocks(opts.cacheBlocks).setAsyncPrefetch(opts.asyncPrefetch)
@@ -2142,12 +2133,12 @@ public class PerformanceEvaluation extends Configured
implements Tool {
}
@Override
- int getStartRow() {
+ long getStartRow() {
return 0;
}
@Override
- int getLastRow() {
+ long getLastRow() {
return opts.perClientRunRows;
}
}
@@ -2158,7 +2149,7 @@ public class PerformanceEvaluation extends Configured
implements Tool {
}
@Override
- boolean testRow(final int i, final long startTime) throws IOException {
+ boolean testRow(final long i, final long startTime) throws IOException {
Increment increment = new Increment(format(i));
// unlike checkAndXXX tests, which make most sense to do on a single
value,
// if multiple families are specified for an increment test we assume it
is
@@ -2178,7 +2169,7 @@ public class PerformanceEvaluation extends Configured
implements Tool {
}
@Override
- boolean testRow(final int i, final long startTime) throws IOException {
+ boolean testRow(final long i, final long startTime) throws IOException {
byte[] bytes = format(i);
Append append = new Append(bytes);
// unlike checkAndXXX tests, which make most sense to do on a single
value,
@@ -2199,7 +2190,7 @@ public class PerformanceEvaluation extends Configured
implements Tool {
}
@Override
- boolean testRow(final int i, final long startTime) throws IOException {
+ boolean testRow(final long i, final long startTime) throws IOException {
final byte[] bytes = format(i);
// checkAndXXX tests operate on only a single value
// Put a known value so when we go to check it, it is there.
@@ -2220,7 +2211,7 @@ public class PerformanceEvaluation extends Configured
implements Tool {
}
@Override
- boolean testRow(final int i, final long startTime) throws IOException {
+ boolean testRow(final long i, final long startTime) throws IOException {
final byte[] bytes = format(i);
// checkAndXXX tests operate on only a single value
// Put a known value so when we go to check it, it is there.
@@ -2239,7 +2230,7 @@ public class PerformanceEvaluation extends Configured
implements Tool {
}
@Override
- boolean testRow(final int i, final long startTime) throws IOException {
+ boolean testRow(final long i, final long startTime) throws IOException {
final byte[] bytes = format(i);
// checkAndXXX tests operate on only a single value
// Put a known value so when we go to check it, it is there.
@@ -2263,7 +2254,7 @@ public class PerformanceEvaluation extends Configured
implements Tool {
}
@Override
- boolean testRow(final int i, final long startTime) throws IOException {
+ boolean testRow(final long i, final long startTime) throws IOException {
try {
RegionInfo regionInfo = connection.getRegionLocator(table.getName())
.getRegionLocation(getSplitKey(i), false).getRegion();
@@ -2288,7 +2279,7 @@ public class PerformanceEvaluation extends Configured
implements Tool {
}
@Override
- boolean testRow(final int i, final long startTime) throws IOException {
+ boolean testRow(final long i, final long startTime) throws IOException {
Get get = new Get(format(i));
for (int family = 0; family < opts.families; family++) {
byte[] familyName = Bytes.toBytes(FAMILY_NAME_BASE + family);
@@ -2320,21 +2311,21 @@ public class PerformanceEvaluation extends Configured
implements Tool {
}
}
- protected byte[] generateRow(final int i) {
+ protected byte[] generateRow(final long i) {
return format(i);
}
@Override
- boolean testRow(final int i, final long startTime) throws IOException {
+ boolean testRow(final long i, final long startTime) throws IOException {
byte[] row = generateRow(i);
Put put = new Put(row);
for (int family = 0; family < opts.families; family++) {
byte familyName[] = Bytes.toBytes(FAMILY_NAME_BASE + family);
for (int column = 0; column < opts.columns; column++) {
byte[] qualifier = column == 0 ? COLUMN_ZERO : Bytes.toBytes("" +
column);
- byte[] value = generateData(this.rand, getValueLength(this.rand));
+ byte[] value = generateData(getValueLength());
if (opts.useTags) {
- byte[] tag = generateData(this.rand, TAG_LENGTH);
+ byte[] tag = generateData(TAG_LENGTH);
Tag[] tags = new Tag[opts.noOfTags];
for (int n = 0; n < opts.noOfTags; n++) {
Tag t = new ArrayBackedTag((byte) n, tag);
@@ -2376,12 +2367,12 @@ public class PerformanceEvaluation extends Configured
implements Tool {
super(con, options, status);
}
- protected byte[] generateRow(final int i) {
+ protected byte[] generateRow(final long i) {
return format(i);
}
@Override
- boolean testRow(final int i, final long startTime) throws IOException {
+ boolean testRow(final long i, final long startTime) throws IOException {
byte[] row = generateRow(i);
Delete delete = new Delete(row);
for (int family = 0; family < opts.families; family++) {
@@ -2408,7 +2399,7 @@ public class PerformanceEvaluation extends Configured
implements Tool {
}
@Override
- boolean testRow(final int i, final long startTime) throws IOException {
+ boolean testRow(final long i, final long startTime) throws IOException {
List<RegionInfo> regionInfos = new ArrayList<RegionInfo>();
RegionInfo regionInfo =
(RegionInfoBuilder.newBuilder(TableName.valueOf(TABLE_NAME))
.setStartKey(getSplitKey(i)).setEndKey(getSplitKey(i + 1)).build());
@@ -2417,7 +2408,7 @@ public class PerformanceEvaluation extends Configured
implements Tool {
// write the serverName columns
MetaTableAccessor.updateRegionLocation(connection, regionInfo,
- ServerName.valueOf("localhost", 60010, rand.nextLong()), i,
+ ServerName.valueOf("localhost", 60010,
ThreadLocalRandom.current().nextLong()), i,
EnvironmentEdgeManager.currentTime());
return true;
}
@@ -2435,8 +2426,8 @@ public class PerformanceEvaluation extends Configured
implements Tool {
}
@Override
- boolean testRow(int i, final long startTime) throws IOException {
- byte[] value = generateData(this.rand, getValueLength(this.rand));
+ boolean testRow(long i, final long startTime) throws IOException {
+ byte[] value = generateData(getValueLength());
Scan scan = constructScan(value);
ResultScanner scanner = null;
try {
@@ -2483,7 +2474,7 @@ public class PerformanceEvaluation extends Configured
implements Tool {
* @param timeMs Time taken in milliseconds.
* @return String value with label, ie '123.76 MB/s'
*/
- private static String calculateMbps(int rows, long timeMs, final int
valueSize, int families,
+ private static String calculateMbps(long rows, long timeMs, final int
valueSize, int families,
int columns) {
BigDecimal rowSize = BigDecimal.valueOf(ROW_LENGTH
+ ((valueSize + (FAMILY_NAME_BASE.length() + 1) + COLUMN_ZERO.length) *
columns) * families);
@@ -2497,9 +2488,9 @@ public class PerformanceEvaluation extends Configured
implements Tool {
* @return Returns zero-prefixed ROW_LENGTH-byte wide decimal version of
passed number (Does
* absolute in case number is negative).
*/
- public static byte[] format(final int number) {
+ public static byte[] format(final long number) {
byte[] b = new byte[ROW_LENGTH];
- int d = Math.abs(number);
+ long d = Math.abs(number);
for (int i = b.length - 1; i >= 0; i--) {
b[i] = (byte) ((d % 10) + '0');
d /= 10;
@@ -2512,10 +2503,11 @@ public class PerformanceEvaluation extends Configured
implements Tool {
* test, generation of the key and value consumes about 30% of CPU time.
* @return Generated random value to insert into a table cell.
*/
- public static byte[] generateData(final Random r, int length) {
+ public static byte[] generateData(int length) {
byte[] b = new byte[length];
int i;
+ Random r = ThreadLocalRandom.current();
for (i = 0; i < (length - 8); i += 8) {
b[i] = (byte) (65 + r.nextInt(26));
b[i + 1] = b[i];
@@ -2534,12 +2526,12 @@ public class PerformanceEvaluation extends Configured
implements Tool {
return b;
}
- static byte[] getRandomRow(final Random random, final int totalRows) {
- return format(generateRandomRow(random, totalRows));
+ static byte[] getRandomRow(final long totalRows) {
+ return format(generateRandomRow(totalRows));
}
- static int generateRandomRow(final Random random, final int totalRows) {
- return random.nextInt(Integer.MAX_VALUE) % totalRows;
+ static long generateRandomRow(final long totalRows) {
+ return ThreadLocalRandom.current().nextLong(Long.MAX_VALUE) % totalRows;
}
static RunResult runOneClient(final Class<? extends TestBase> cmd,
Configuration conf,
@@ -2574,7 +2566,7 @@ public class PerformanceEvaluation extends Configured
implements Tool {
status.setStatus("Finished " + cmd + " in " + totalElapsedTime + "ms at
offset " + opts.startRow
+ " for " + opts.perClientRunRows + " rows" + " ("
- + calculateMbps((int) (opts.perClientRunRows * opts.sampleRate),
totalElapsedTime,
+ + calculateMbps((long) (opts.perClientRunRows * opts.sampleRate),
totalElapsedTime,
getAverageValueLength(opts), opts.families, opts.columns)
+ ")");
@@ -2769,7 +2761,7 @@ public class PerformanceEvaluation extends Configured
implements Tool {
final String rows = "--rows=";
if (cmd.startsWith(rows)) {
- opts.perClientRunRows = Integer.parseInt(cmd.substring(rows.length()));
+ opts.perClientRunRows = Long.parseLong(cmd.substring(rows.length()));
continue;
}
@@ -2793,7 +2785,7 @@ public class PerformanceEvaluation extends Configured
implements Tool {
final String startRow = "--startRow=";
if (cmd.startsWith(startRow)) {
- opts.startRow = Integer.parseInt(cmd.substring(startRow.length()));
+ opts.startRow = Long.parseLong(cmd.substring(startRow.length()));
continue;
}
@@ -3091,10 +3083,10 @@ public class PerformanceEvaluation extends Configured
implements Tool {
&& (opts.getCmdName().equals(RANDOM_READ) ||
opts.getCmdName().equals(RANDOM_SEEK_SCAN)))
&& opts.size != DEFAULT_OPTS.size && opts.perClientRunRows !=
DEFAULT_OPTS.perClientRunRows
) {
- opts.totalRows = (int) opts.size * rowsPerGB;
+ opts.totalRows = (long) (opts.size * rowsPerGB);
} else if (opts.size != DEFAULT_OPTS.size) {
// total size in GB specified
- opts.totalRows = (int) opts.size * rowsPerGB;
+ opts.totalRows = (long) (opts.size * rowsPerGB);
opts.perClientRunRows = opts.totalRows / opts.numClientThreads;
} else {
opts.totalRows = opts.perClientRunRows * opts.numClientThreads;
diff --git
a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/TestPerformanceEvaluation.java
b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/TestPerformanceEvaluation.java
index 7e4babe58b9..1e4493a4700 100644
---
a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/TestPerformanceEvaluation.java
+++
b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/TestPerformanceEvaluation.java
@@ -39,8 +39,6 @@ import java.util.LinkedList;
import java.util.NoSuchElementException;
import java.util.Properties;
import java.util.Queue;
-import java.util.Random;
-import java.util.concurrent.ThreadLocalRandom;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -122,7 +120,7 @@ public class TestPerformanceEvaluation {
public void testSizeCalculation() {
TestOptions opts = new PerformanceEvaluation.TestOptions();
opts = PerformanceEvaluation.calculateRowsAndSize(opts);
- int rows = opts.getPerClientRunRows();
+ long rows = opts.getPerClientRunRows();
// Default row count
final int defaultPerClientRunRows = 1024 * 1024;
assertEquals(defaultPerClientRunRows, rows);
@@ -144,7 +142,7 @@ public class TestPerformanceEvaluation {
public void testRandomReadCalculation() {
TestOptions opts = new PerformanceEvaluation.TestOptions();
opts = PerformanceEvaluation.calculateRowsAndSize(opts);
- int rows = opts.getPerClientRunRows();
+ long rows = opts.getPerClientRunRows();
// Default row count
final int defaultPerClientRunRows = 1024 * 1024;
assertEquals(defaultPerClientRunRows, rows);
@@ -160,9 +158,8 @@ public class TestPerformanceEvaluation {
assertEquals(1000, opts.getPerClientRunRows());
// assuming we will get one before this loop expires
boolean foundValue = false;
- Random rand = ThreadLocalRandom.current();
for (int i = 0; i < 10000000; i++) {
- int randomRow = PerformanceEvaluation.generateRandomRow(rand,
opts.totalRows);
+ long randomRow = PerformanceEvaluation.generateRandomRow(opts.totalRows);
if (randomRow > 1000) {
foundValue = true;
break;
@@ -184,7 +181,7 @@ public class TestPerformanceEvaluation {
ctor.setAccessible(true);
Histogram histogram = (Histogram) ctor.newInstance(new
UniformReservoir(1024 * 500));
for (int i = 0; i < 100; i++) {
- histogram.update(rrt.getValueLength(null));
+ histogram.update(rrt.getValueLength());
}
Snapshot snapshot = histogram.getSnapshot();
double stddev = snapshot.getStdDev();
@@ -400,7 +397,7 @@ public class TestPerformanceEvaluation {
}
@Override
- boolean testRow(int i, long startTime) throws IOException,
InterruptedException {
+ boolean testRow(long i, long startTime) throws IOException,
InterruptedException {
return false;
}
}
diff --git
a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestCellBasedHFileOutputFormat2.java
b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestCellBasedHFileOutputFormat2.java
index 9b207158817..58f600a8a85 100644
---
a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestCellBasedHFileOutputFormat2.java
+++
b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestCellBasedHFileOutputFormat2.java
@@ -506,23 +506,19 @@ public class TestCellBasedHFileOutputFormat2 {
}
private byte[][] generateRandomStartKeys(int numKeys) {
- Random random = new Random();
byte[][] ret = new byte[numKeys][];
// first region start key is always empty
ret[0] = HConstants.EMPTY_BYTE_ARRAY;
for (int i = 1; i < numKeys; i++) {
- ret[i] =
- PerformanceEvaluation.generateData(random,
PerformanceEvaluation.DEFAULT_VALUE_LENGTH);
+ ret[i] =
PerformanceEvaluation.generateData(PerformanceEvaluation.DEFAULT_VALUE_LENGTH);
}
return ret;
}
private byte[][] generateRandomSplitKeys(int numKeys) {
- Random random = new Random();
byte[][] ret = new byte[numKeys][];
for (int i = 0; i < numKeys; i++) {
- ret[i] =
- PerformanceEvaluation.generateData(random,
PerformanceEvaluation.DEFAULT_VALUE_LENGTH);
+ ret[i] =
PerformanceEvaluation.generateData(PerformanceEvaluation.DEFAULT_VALUE_LENGTH);
}
return ret;
}
diff --git
a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2.java
b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2.java
index 8156302fc48..40c03492f6d 100644
---
a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2.java
+++
b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2.java
@@ -34,14 +34,12 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
-import java.util.Random;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.Callable;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.ExecutorService;
-import java.util.concurrent.ThreadLocalRandom;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.apache.hadoop.conf.Configuration;
@@ -542,23 +540,19 @@ public class TestHFileOutputFormat2 {
}
private byte[][] generateRandomStartKeys(int numKeys) {
- Random random = ThreadLocalRandom.current();
byte[][] ret = new byte[numKeys][];
// first region start key is always empty
ret[0] = HConstants.EMPTY_BYTE_ARRAY;
for (int i = 1; i < numKeys; i++) {
- ret[i] =
- PerformanceEvaluation.generateData(random,
PerformanceEvaluation.DEFAULT_VALUE_LENGTH);
+ ret[i] =
PerformanceEvaluation.generateData(PerformanceEvaluation.DEFAULT_VALUE_LENGTH);
}
return ret;
}
private byte[][] generateRandomSplitKeys(int numKeys) {
- Random random = ThreadLocalRandom.current();
byte[][] ret = new byte[numKeys][];
for (int i = 0; i < numKeys; i++) {
- ret[i] =
- PerformanceEvaluation.generateData(random,
PerformanceEvaluation.DEFAULT_VALUE_LENGTH);
+ ret[i] =
PerformanceEvaluation.generateData(PerformanceEvaluation.DEFAULT_VALUE_LENGTH);
}
return ret;
}
