This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch branch-2.6
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2.6 by this push:
new 1102bdc7c1b HBASE-29013 Make PerformanceEvaluation support larger data
sets (branch-2) (#6559)
1102bdc7c1b is described below
commit 1102bdc7c1b2168e3ee72cf8b50cc8558a345220
Author: Junegunn Choi <[email protected]>
AuthorDate: Sun Dec 29 18:57:39 2024 +0900
HBASE-29013 Make PerformanceEvaluation support larger data sets (branch-2)
(#6559)
Signed-off-by: Duo Zhang <[email protected]>
(cherry picked from commit 5638a38f701515c5f8c7f655a43ccf67a28a4673)
---
.../apache/hadoop/hbase/PerformanceEvaluation.java | 210 ++++++++++-----------
.../hadoop/hbase/TestPerformanceEvaluation.java | 13 +-
.../mapreduce/TestCellBasedHFileOutputFormat2.java | 8 +-
.../hbase/mapreduce/TestHFileOutputFormat2.java | 10 +-
4 files changed, 110 insertions(+), 131 deletions(-)
diff --git
a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
index aa7b1302231..83d339fefc7 100644
---
a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
+++
b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
@@ -441,9 +441,9 @@ public class PerformanceEvaluation extends Configured
implements Tool {
int numSplitPoints = opts.presplitRegions - 1;
byte[][] splits = new byte[numSplitPoints][];
- int jump = opts.totalRows / opts.presplitRegions;
+ long jump = opts.totalRows / opts.presplitRegions;
for (int i = 0; i < numSplitPoints; i++) {
- int rowkey = jump * (1 + i);
+ long rowkey = jump * (1 + i);
splits[i] = format(rowkey);
}
return splits;
@@ -620,7 +620,7 @@ public class PerformanceEvaluation extends Configured
implements Tool {
// Make input random.
Map<Integer, String> m = new TreeMap<>();
Hash h = MurmurHash.getInstance();
- int perClientRows = (opts.totalRows / opts.numClientThreads);
+ long perClientRows = (opts.totalRows / opts.numClientThreads);
try {
for (int j = 0; j < opts.numClientThreads; j++) {
TestOptions next = new TestOptions(opts);
@@ -679,11 +679,11 @@ public class PerformanceEvaluation extends Configured
implements Tool {
String cmdName = null;
boolean nomapred = false;
boolean filterAll = false;
- int startRow = 0;
+ long startRow = 0;
float size = 1.0f;
- int perClientRunRows = DEFAULT_ROWS_PER_GB;
+ long perClientRunRows = DEFAULT_ROWS_PER_GB;
int numClientThreads = 1;
- int totalRows = DEFAULT_ROWS_PER_GB;
+ long totalRows = DEFAULT_ROWS_PER_GB;
int measureAfter = 0;
float sampleRate = 1.0f;
/**
@@ -715,7 +715,7 @@ public class PerformanceEvaluation extends Configured
implements Tool {
boolean valueRandom = false;
boolean valueZipf = false;
int valueSize = DEFAULT_VALUE_LENGTH;
- int period = (this.perClientRunRows / 10) == 0 ? perClientRunRows :
perClientRunRows / 10;
+ long period = (this.perClientRunRows / 10) == 0 ? perClientRunRows :
perClientRunRows / 10;
int cycles = 1;
int columns = 1;
int families = 1;
@@ -872,7 +872,7 @@ public class PerformanceEvaluation extends Configured
implements Tool {
this.filterAll = filterAll;
}
- public void setStartRow(int startRow) {
+ public void setStartRow(long startRow) {
this.startRow = startRow;
}
@@ -888,7 +888,7 @@ public class PerformanceEvaluation extends Configured
implements Tool {
this.numClientThreads = numClientThreads;
}
- public void setTotalRows(int totalRows) {
+ public void setTotalRows(long totalRows) {
this.totalRows = totalRows;
}
@@ -1000,7 +1000,7 @@ public class PerformanceEvaluation extends Configured
implements Tool {
return filterAll;
}
- public int getStartRow() {
+ public long getStartRow() {
return startRow;
}
@@ -1008,7 +1008,7 @@ public class PerformanceEvaluation extends Configured
implements Tool {
return size;
}
- public int getPerClientRunRows() {
+ public long getPerClientRunRows() {
return perClientRunRows;
}
@@ -1016,7 +1016,7 @@ public class PerformanceEvaluation extends Configured
implements Tool {
return numClientThreads;
}
- public int getTotalRows() {
+ public long getTotalRows() {
return totalRows;
}
@@ -1092,7 +1092,7 @@ public class PerformanceEvaluation extends Configured
implements Tool {
return valueSize;
}
- public int getPeriod() {
+ public long getPeriod() {
return period;
}
@@ -1141,17 +1141,8 @@ public class PerformanceEvaluation extends Configured
implements Tool {
* A test. Subclass to particularize what happens per row.
*/
static abstract class TestBase {
- // Below is make it so when Tests are all running in the one
- // jvm, that they each have a differently seeded Random.
- private static final Random randomSeed = new
Random(EnvironmentEdgeManager.currentTime());
+ private final long everyN;
- private static long nextRandomSeed() {
- return randomSeed.nextLong();
- }
-
- private final int everyN;
-
- protected final Random rand = new Random(nextRandomSeed());
protected final Configuration conf;
protected final TestOptions opts;
@@ -1180,16 +1171,17 @@ public class PerformanceEvaluation extends Configured
implements Tool {
this.opts = options;
this.status = status;
this.testName = this.getClass().getSimpleName();
- everyN = (int) (opts.totalRows / (opts.totalRows * opts.sampleRate));
+ everyN = (long) (opts.totalRows / (opts.totalRows * opts.sampleRate));
if (options.isValueZipf()) {
- this.zipf = new RandomDistribution.Zipf(this.rand, 1,
options.getValueSize(), 1.2);
+ this.zipf =
+ new RandomDistribution.Zipf(ThreadLocalRandom.current(), 1,
options.getValueSize(), 1.2);
}
LOG.info("Sampling 1 every " + everyN + " out of " +
opts.perClientRunRows + " total rows.");
}
- int getValueLength(final Random r) {
+ int getValueLength() {
if (this.opts.isValueRandom()) {
- return r.nextInt(opts.valueSize);
+ return ThreadLocalRandom.current().nextInt(opts.valueSize);
} else if (this.opts.isValueZipf()) {
return Math.abs(this.zipf.nextInt());
} else {
@@ -1261,7 +1253,7 @@ public class PerformanceEvaluation extends Configured
implements Tool {
}
}
- String generateStatus(final int sr, final int i, final int lr) {
+ String generateStatus(final long sr, final long i, final long lr) {
return "row [start=" + sr + ", current=" + i + ", last=" + lr + "],
latency ["
+ getShortLatencyReport() + "]"
+ (!isRandomValueSize() ? "" : ", value size [" +
getShortValueSizeReport() + "]");
@@ -1271,7 +1263,7 @@ public class PerformanceEvaluation extends Configured
implements Tool {
return opts.valueRandom;
}
- protected int getReportingPeriod() {
+ protected long getReportingPeriod() {
return opts.period;
}
@@ -1374,11 +1366,11 @@ public class PerformanceEvaluation extends Configured
implements Tool {
return (System.nanoTime() - startTime) / 1000000;
}
- int getStartRow() {
+ long getStartRow() {
return opts.startRow;
}
- int getLastRow() {
+ long getLastRow() {
return getStartRow() + opts.perClientRunRows;
}
@@ -1386,12 +1378,12 @@ public class PerformanceEvaluation extends Configured
implements Tool {
* Provides an extension point for tests that don't want a per row
invocation.
*/
void testTimed() throws IOException, InterruptedException {
- int startRow = getStartRow();
- int lastRow = getLastRow();
+ long startRow = getStartRow();
+ long lastRow = getLastRow();
// Report on completion of 1/10th of total.
for (int ii = 0; ii < opts.cycles; ii++) {
if (opts.cycles > 1) LOG.info("Cycle=" + ii + " of " + opts.cycles);
- for (int i = startRow; i < lastRow; i++) {
+ for (long i = startRow; i < lastRow; i++) {
if (i % everyN != 0) continue;
long startTime = System.nanoTime();
boolean requestSent = false;
@@ -1437,7 +1429,7 @@ public class PerformanceEvaluation extends Configured
implements Tool {
* @return true if the row was sent to server and need to record metrics.
False if not, multiGet
* and multiPut e.g., the rows are sent to server only if enough
gets/puts are gathered.
*/
- abstract boolean testRow(final int i, final long startTime)
+ abstract boolean testRow(final long i, final long startTime)
throws IOException, InterruptedException;
}
@@ -1485,7 +1477,7 @@ public class PerformanceEvaluation extends Configured
implements Tool {
MetaTest(Connection con, TestOptions options, Status status) {
super(con, options, status);
- keyLength = Integer.toString(opts.perClientRunRows).length();
+ keyLength = Long.toString(opts.perClientRunRows).length();
}
@Override
@@ -1496,7 +1488,7 @@ public class PerformanceEvaluation extends Configured
implements Tool {
/*
* Generates Lexicographically ascending strings
*/
- protected byte[] getSplitKey(final int i) {
+ protected byte[] getSplitKey(final long i) {
return Bytes.toBytes(String.format("%0" + keyLength + "d", i));
}
@@ -1533,11 +1525,11 @@ public class PerformanceEvaluation extends Configured
implements Tool {
}
@Override
- boolean testRow(final int i, final long startTime) throws IOException,
InterruptedException {
+ boolean testRow(final long i, final long startTime) throws IOException,
InterruptedException {
if (opts.randomSleep > 0) {
Thread.sleep(ThreadLocalRandom.current().nextInt(opts.randomSleep));
}
- Get get = new Get(getRandomRow(this.rand, opts.totalRows));
+ Get get = new Get(getRandomRow(opts.totalRows));
for (int family = 0; family < opts.families; family++) {
byte[] familyName = Bytes.toBytes(FAMILY_NAME_BASE + family);
if (opts.addColumns) {
@@ -1590,8 +1582,8 @@ public class PerformanceEvaluation extends Configured
implements Tool {
}
@Override
- protected int getReportingPeriod() {
- int period = opts.perClientRunRows / 10;
+ protected long getReportingPeriod() {
+ long period = opts.perClientRunRows / 10;
return period == 0 ? opts.perClientRunRows : period;
}
@@ -1612,8 +1604,8 @@ public class PerformanceEvaluation extends Configured
implements Tool {
}
@Override
- protected byte[] generateRow(final int i) {
- return getRandomRow(this.rand, opts.totalRows);
+ protected byte[] generateRow(final long i) {
+ return getRandomRow(opts.totalRows);
}
}
@@ -1641,7 +1633,7 @@ public class PerformanceEvaluation extends Configured
implements Tool {
}
@Override
- boolean testRow(final int i, final long startTime) throws IOException {
+ boolean testRow(final long i, final long startTime) throws IOException {
if (this.testScanner == null) {
Scan scan = new
Scan().withStartRow(format(opts.startRow)).setCaching(opts.caching)
.setCacheBlocks(opts.cacheBlocks).setAsyncPrefetch(opts.asyncPrefetch)
@@ -1674,7 +1666,7 @@ public class PerformanceEvaluation extends Configured
implements Tool {
}
@Override
- boolean testRow(final int i, final long startTime) throws IOException,
InterruptedException {
+ boolean testRow(final long i, final long startTime) throws IOException,
InterruptedException {
Get get = new Get(format(i));
for (int family = 0; family < opts.families; family++) {
byte[] familyName = Bytes.toBytes(FAMILY_NAME_BASE + family);
@@ -1710,22 +1702,22 @@ public class PerformanceEvaluation extends Configured
implements Tool {
}
}
- protected byte[] generateRow(final int i) {
+ protected byte[] generateRow(final long i) {
return format(i);
}
@Override
@SuppressWarnings("ReturnValueIgnored")
- boolean testRow(final int i, final long startTime) throws IOException,
InterruptedException {
+ boolean testRow(final long i, final long startTime) throws IOException,
InterruptedException {
byte[] row = generateRow(i);
Put put = new Put(row);
for (int family = 0; family < opts.families; family++) {
byte[] familyName = Bytes.toBytes(FAMILY_NAME_BASE + family);
for (int column = 0; column < opts.columns; column++) {
byte[] qualifier = column == 0 ? COLUMN_ZERO : Bytes.toBytes("" +
column);
- byte[] value = generateData(this.rand, getValueLength(this.rand));
+ byte[] value = generateData(getValueLength());
if (opts.useTags) {
- byte[] tag = generateData(this.rand, TAG_LENGTH);
+ byte[] tag = generateData(TAG_LENGTH);
Tag[] tags = new Tag[opts.noOfTags];
for (int n = 0; n < opts.noOfTags; n++) {
Tag t = new ArrayBackedTag((byte) n, tag);
@@ -1791,11 +1783,10 @@ public class PerformanceEvaluation extends Configured
implements Tool {
}
@Override
- boolean testRow(final int i, final long startTime) throws IOException {
- Scan scan =
- new Scan().withStartRow(getRandomRow(this.rand,
opts.totalRows)).setCaching(opts.caching)
-
.setCacheBlocks(opts.cacheBlocks).setAsyncPrefetch(opts.asyncPrefetch)
- .setReadType(opts.scanReadType).setScanMetricsEnabled(true);
+ boolean testRow(final long i, final long startTime) throws IOException {
+ Scan scan = new
Scan().withStartRow(getRandomRow(opts.totalRows)).setCaching(opts.caching)
+ .setCacheBlocks(opts.cacheBlocks).setAsyncPrefetch(opts.asyncPrefetch)
+ .setReadType(opts.scanReadType).setScanMetricsEnabled(true);
FilterList list = new FilterList();
for (int family = 0; family < opts.families; family++) {
byte[] familyName = Bytes.toBytes(FAMILY_NAME_BASE + family);
@@ -1826,8 +1817,8 @@ public class PerformanceEvaluation extends Configured
implements Tool {
}
@Override
- protected int getReportingPeriod() {
- int period = opts.perClientRunRows / 100;
+ protected long getReportingPeriod() {
+ long period = opts.perClientRunRows / 100;
return period == 0 ? opts.perClientRunRows : period;
}
@@ -1839,7 +1830,7 @@ public class PerformanceEvaluation extends Configured
implements Tool {
}
@Override
- boolean testRow(final int i, final long startTime) throws IOException {
+ boolean testRow(final long i, final long startTime) throws IOException {
Pair<byte[], byte[]> startAndStopRow = getStartAndStopRow();
Scan scan = new Scan().withStartRow(startAndStopRow.getFirst())
.withStopRow(startAndStopRow.getSecond()).setCaching(opts.caching)
@@ -1881,15 +1872,15 @@ public class PerformanceEvaluation extends Configured
implements Tool {
protected abstract Pair<byte[], byte[]> getStartAndStopRow();
- protected Pair<byte[], byte[]> generateStartAndStopRows(int maxRange) {
- int start = this.rand.nextInt(Integer.MAX_VALUE) % opts.totalRows;
- int stop = start + maxRange;
+ protected Pair<byte[], byte[]> generateStartAndStopRows(long maxRange) {
+ long start = ThreadLocalRandom.current().nextLong(Long.MAX_VALUE) %
opts.totalRows;
+ long stop = start + maxRange;
return new Pair<>(format(start), format(stop));
}
@Override
- protected int getReportingPeriod() {
- int period = opts.perClientRunRows / 100;
+ protected long getReportingPeriod() {
+ long period = opts.perClientRunRows / 100;
return period == 0 ? opts.perClientRunRows : period;
}
}
@@ -1952,11 +1943,11 @@ public class PerformanceEvaluation extends Configured
implements Tool {
}
@Override
- boolean testRow(final int i, final long startTime) throws IOException,
InterruptedException {
+ boolean testRow(final long i, final long startTime) throws IOException,
InterruptedException {
if (opts.randomSleep > 0) {
Thread.sleep(ThreadLocalRandom.current().nextInt(opts.randomSleep));
}
- Get get = new Get(getRandomRow(this.rand, opts.totalRows));
+ Get get = new Get(getRandomRow(opts.totalRows));
for (int family = 0; family < opts.families; family++) {
byte[] familyName = Bytes.toBytes(FAMILY_NAME_BASE + family);
if (opts.addColumns) {
@@ -2000,8 +1991,8 @@ public class PerformanceEvaluation extends Configured
implements Tool {
}
@Override
- protected int getReportingPeriod() {
- int period = opts.perClientRunRows / 10;
+ protected long getReportingPeriod() {
+ long period = opts.perClientRunRows / 10;
return period == 0 ? opts.perClientRunRows : period;
}
@@ -2034,19 +2025,19 @@ public class PerformanceEvaluation extends Configured
implements Tool {
}
@Override
- boolean testRow(final int i, final long startTime) throws IOException,
InterruptedException {
+ boolean testRow(final long i, final long startTime) throws IOException,
InterruptedException {
if (opts.randomSleep > 0) {
Thread.sleep(rd.nextInt(opts.randomSleep));
}
- HRegionLocation hRegionLocation =
-
regionLocator.getRegionLocation(getSplitKey(rd.nextInt(opts.perClientRunRows)),
true);
+ HRegionLocation hRegionLocation = regionLocator.getRegionLocation(
+
getSplitKey(ThreadLocalRandom.current().nextLong(opts.perClientRunRows)), true);
LOG.debug("get location for region: " + hRegionLocation);
return true;
}
@Override
- protected int getReportingPeriod() {
- int period = opts.perClientRunRows / 10;
+ protected long getReportingPeriod() {
+ long period = opts.perClientRunRows / 10;
return period == 0 ? opts.perClientRunRows : period;
}
@@ -2062,8 +2053,8 @@ public class PerformanceEvaluation extends Configured
implements Tool {
}
@Override
- protected byte[] generateRow(final int i) {
- return getRandomRow(this.rand, opts.totalRows);
+ protected byte[] generateRow(final long i) {
+ return getRandomRow(opts.totalRows);
}
}
@@ -2074,8 +2065,8 @@ public class PerformanceEvaluation extends Configured
implements Tool {
}
@Override
- protected byte[] generateRow(final int i) {
- return getRandomRow(this.rand, opts.totalRows);
+ protected byte[] generateRow(final long i) {
+ return getRandomRow(opts.totalRows);
}
}
@@ -2096,7 +2087,7 @@ public class PerformanceEvaluation extends Configured
implements Tool {
}
@Override
- boolean testRow(final int i, final long startTime) throws IOException {
+ boolean testRow(final long i, final long startTime) throws IOException {
if (this.testScanner == null) {
Scan scan = new
Scan().withStartRow(format(opts.startRow)).setCaching(opts.caching)
.setCacheBlocks(opts.cacheBlocks).setAsyncPrefetch(opts.asyncPrefetch)
@@ -2139,7 +2130,7 @@ public class PerformanceEvaluation extends Configured
implements Tool {
}
@Override
- boolean testRow(final int i, final long startTime) throws IOException {
+ boolean testRow(final long i, final long startTime) throws IOException {
if (this.testScanner == null) {
Scan scan = new
Scan().setCaching(opts.caching).setCacheBlocks(opts.cacheBlocks)
.setAsyncPrefetch(opts.asyncPrefetch).setReadType(opts.scanReadType)
@@ -2187,12 +2178,12 @@ public class PerformanceEvaluation extends Configured
implements Tool {
}
@Override
- int getStartRow() {
+ long getStartRow() {
return 0;
}
@Override
- int getLastRow() {
+ long getLastRow() {
return opts.perClientRunRows;
}
}
@@ -2203,7 +2194,7 @@ public class PerformanceEvaluation extends Configured
implements Tool {
}
@Override
- boolean testRow(final int i, final long startTime) throws IOException {
+ boolean testRow(final long i, final long startTime) throws IOException {
Increment increment = new Increment(format(i));
// unlike checkAndXXX tests, which make most sense to do on a single
value,
// if multiple families are specified for an increment test we assume it
is
@@ -2223,7 +2214,7 @@ public class PerformanceEvaluation extends Configured
implements Tool {
}
@Override
- boolean testRow(final int i, final long startTime) throws IOException {
+ boolean testRow(final long i, final long startTime) throws IOException {
byte[] bytes = format(i);
Append append = new Append(bytes);
// unlike checkAndXXX tests, which make most sense to do on a single
value,
@@ -2244,7 +2235,7 @@ public class PerformanceEvaluation extends Configured
implements Tool {
}
@Override
- boolean testRow(final int i, final long startTime) throws IOException {
+ boolean testRow(final long i, final long startTime) throws IOException {
final byte[] bytes = format(i);
// checkAndXXX tests operate on only a single value
// Put a known value so when we go to check it, it is there.
@@ -2265,7 +2256,7 @@ public class PerformanceEvaluation extends Configured
implements Tool {
}
@Override
- boolean testRow(final int i, final long startTime) throws IOException {
+ boolean testRow(final long i, final long startTime) throws IOException {
final byte[] bytes = format(i);
// checkAndXXX tests operate on only a single value
// Put a known value so when we go to check it, it is there.
@@ -2284,7 +2275,7 @@ public class PerformanceEvaluation extends Configured
implements Tool {
}
@Override
- boolean testRow(final int i, final long startTime) throws IOException {
+ boolean testRow(final long i, final long startTime) throws IOException {
final byte[] bytes = format(i);
// checkAndXXX tests operate on only a single value
// Put a known value so when we go to check it, it is there.
@@ -2308,7 +2299,7 @@ public class PerformanceEvaluation extends Configured
implements Tool {
}
@Override
- boolean testRow(final int i, final long startTime) throws IOException {
+ boolean testRow(final long i, final long startTime) throws IOException {
try {
RegionInfo regionInfo = connection.getRegionLocator(table.getName())
.getRegionLocation(getSplitKey(i), false).getRegion();
@@ -2333,7 +2324,7 @@ public class PerformanceEvaluation extends Configured
implements Tool {
}
@Override
- boolean testRow(final int i, final long startTime) throws IOException {
+ boolean testRow(final long i, final long startTime) throws IOException {
Get get = new Get(format(i));
for (int family = 0; family < opts.families; family++) {
byte[] familyName = Bytes.toBytes(FAMILY_NAME_BASE + family);
@@ -2365,21 +2356,21 @@ public class PerformanceEvaluation extends Configured
implements Tool {
}
}
- protected byte[] generateRow(final int i) {
+ protected byte[] generateRow(final long i) {
return format(i);
}
@Override
- boolean testRow(final int i, final long startTime) throws IOException {
+ boolean testRow(final long i, final long startTime) throws IOException {
byte[] row = generateRow(i);
Put put = new Put(row);
for (int family = 0; family < opts.families; family++) {
byte familyName[] = Bytes.toBytes(FAMILY_NAME_BASE + family);
for (int column = 0; column < opts.columns; column++) {
byte[] qualifier = column == 0 ? COLUMN_ZERO : Bytes.toBytes("" +
column);
- byte[] value = generateData(this.rand, getValueLength(this.rand));
+ byte[] value = generateData(getValueLength());
if (opts.useTags) {
- byte[] tag = generateData(this.rand, TAG_LENGTH);
+ byte[] tag = generateData(TAG_LENGTH);
Tag[] tags = new Tag[opts.noOfTags];
for (int n = 0; n < opts.noOfTags; n++) {
Tag t = new ArrayBackedTag((byte) n, tag);
@@ -2421,12 +2412,12 @@ public class PerformanceEvaluation extends Configured
implements Tool {
super(con, options, status);
}
- protected byte[] generateRow(final int i) {
+ protected byte[] generateRow(final long i) {
return format(i);
}
@Override
- boolean testRow(final int i, final long startTime) throws IOException {
+ boolean testRow(final long i, final long startTime) throws IOException {
byte[] row = generateRow(i);
Delete delete = new Delete(row);
for (int family = 0; family < opts.families; family++) {
@@ -2453,7 +2444,7 @@ public class PerformanceEvaluation extends Configured
implements Tool {
}
@Override
- boolean testRow(final int i, final long startTime) throws IOException {
+ boolean testRow(final long i, final long startTime) throws IOException {
List<RegionInfo> regionInfos = new ArrayList<RegionInfo>();
RegionInfo regionInfo =
(RegionInfoBuilder.newBuilder(TableName.valueOf(TABLE_NAME))
.setStartKey(getSplitKey(i)).setEndKey(getSplitKey(i + 1)).build());
@@ -2462,7 +2453,7 @@ public class PerformanceEvaluation extends Configured
implements Tool {
// write the serverName columns
MetaTableAccessor.updateRegionLocation(connection, regionInfo,
- ServerName.valueOf("localhost", 60010, rand.nextLong()), i,
+ ServerName.valueOf("localhost", 60010,
ThreadLocalRandom.current().nextLong()), i,
EnvironmentEdgeManager.currentTime());
return true;
}
@@ -2480,8 +2471,8 @@ public class PerformanceEvaluation extends Configured
implements Tool {
}
@Override
- boolean testRow(int i, final long startTime) throws IOException {
- byte[] value = generateData(this.rand, getValueLength(this.rand));
+ boolean testRow(long i, final long startTime) throws IOException {
+ byte[] value = generateData(getValueLength());
Scan scan = constructScan(value);
ResultScanner scanner = null;
try {
@@ -2528,7 +2519,7 @@ public class PerformanceEvaluation extends Configured
implements Tool {
* @param timeMs Time taken in milliseconds.
* @return String value with label, ie '123.76 MB/s'
*/
- private static String calculateMbps(int rows, long timeMs, final int
valueSize, int families,
+ private static String calculateMbps(long rows, long timeMs, final int
valueSize, int families,
int columns) {
BigDecimal rowSize = BigDecimal.valueOf(ROW_LENGTH
+ ((valueSize + (FAMILY_NAME_BASE.length() + 1) + COLUMN_ZERO.length) *
columns) * families);
@@ -2542,9 +2533,9 @@ public class PerformanceEvaluation extends Configured
implements Tool {
* @return Returns zero-prefixed ROW_LENGTH-byte wide decimal version of
passed number (Does
* absolute in case number is negative).
*/
- public static byte[] format(final int number) {
+ public static byte[] format(final long number) {
byte[] b = new byte[ROW_LENGTH];
- int d = Math.abs(number);
+ long d = Math.abs(number);
for (int i = b.length - 1; i >= 0; i--) {
b[i] = (byte) ((d % 10) + '0');
d /= 10;
@@ -2557,10 +2548,11 @@ public class PerformanceEvaluation extends Configured
implements Tool {
* test, generation of the key and value consumes about 30% of CPU time.
* @return Generated random value to insert into a table cell.
*/
- public static byte[] generateData(final Random r, int length) {
+ public static byte[] generateData(int length) {
byte[] b = new byte[length];
int i;
+ Random r = ThreadLocalRandom.current();
for (i = 0; i < (length - 8); i += 8) {
b[i] = (byte) (65 + r.nextInt(26));
b[i + 1] = b[i];
@@ -2579,12 +2571,12 @@ public class PerformanceEvaluation extends Configured
implements Tool {
return b;
}
- static byte[] getRandomRow(final Random random, final int totalRows) {
- return format(generateRandomRow(random, totalRows));
+ static byte[] getRandomRow(final long totalRows) {
+ return format(generateRandomRow(totalRows));
}
- static int generateRandomRow(final Random random, final int totalRows) {
- return random.nextInt(Integer.MAX_VALUE) % totalRows;
+ static long generateRandomRow(final long totalRows) {
+ return ThreadLocalRandom.current().nextLong(Long.MAX_VALUE) % totalRows;
}
static RunResult runOneClient(final Class<? extends TestBase> cmd,
Configuration conf,
@@ -2619,7 +2611,7 @@ public class PerformanceEvaluation extends Configured
implements Tool {
status.setStatus("Finished " + cmd + " in " + totalElapsedTime + "ms at
offset " + opts.startRow
+ " for " + opts.perClientRunRows + " rows" + " ("
- + calculateMbps((int) (opts.perClientRunRows * opts.sampleRate),
totalElapsedTime,
+ + calculateMbps((long) (opts.perClientRunRows * opts.sampleRate),
totalElapsedTime,
getAverageValueLength(opts), opts.families, opts.columns)
+ ")");
@@ -2814,7 +2806,7 @@ public class PerformanceEvaluation extends Configured
implements Tool {
final String rows = "--rows=";
if (cmd.startsWith(rows)) {
- opts.perClientRunRows = Integer.parseInt(cmd.substring(rows.length()));
+ opts.perClientRunRows = Long.parseLong(cmd.substring(rows.length()));
continue;
}
@@ -2838,7 +2830,7 @@ public class PerformanceEvaluation extends Configured
implements Tool {
final String startRow = "--startRow=";
if (cmd.startsWith(startRow)) {
- opts.startRow = Integer.parseInt(cmd.substring(startRow.length()));
+ opts.startRow = Long.parseLong(cmd.substring(startRow.length()));
continue;
}
@@ -3136,10 +3128,10 @@ public class PerformanceEvaluation extends Configured
implements Tool {
&& (opts.getCmdName().equals(RANDOM_READ) ||
opts.getCmdName().equals(RANDOM_SEEK_SCAN)))
&& opts.size != DEFAULT_OPTS.size && opts.perClientRunRows !=
DEFAULT_OPTS.perClientRunRows
) {
- opts.totalRows = (int) opts.size * rowsPerGB;
+ opts.totalRows = (long) (opts.size * rowsPerGB);
} else if (opts.size != DEFAULT_OPTS.size) {
// total size in GB specified
- opts.totalRows = (int) opts.size * rowsPerGB;
+ opts.totalRows = (long) (opts.size * rowsPerGB);
opts.perClientRunRows = opts.totalRows / opts.numClientThreads;
} else {
opts.totalRows = opts.perClientRunRows * opts.numClientThreads;
diff --git
a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/TestPerformanceEvaluation.java
b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/TestPerformanceEvaluation.java
index 7e4babe58b9..1e4493a4700 100644
---
a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/TestPerformanceEvaluation.java
+++
b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/TestPerformanceEvaluation.java
@@ -39,8 +39,6 @@ import java.util.LinkedList;
import java.util.NoSuchElementException;
import java.util.Properties;
import java.util.Queue;
-import java.util.Random;
-import java.util.concurrent.ThreadLocalRandom;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -122,7 +120,7 @@ public class TestPerformanceEvaluation {
public void testSizeCalculation() {
TestOptions opts = new PerformanceEvaluation.TestOptions();
opts = PerformanceEvaluation.calculateRowsAndSize(opts);
- int rows = opts.getPerClientRunRows();
+ long rows = opts.getPerClientRunRows();
// Default row count
final int defaultPerClientRunRows = 1024 * 1024;
assertEquals(defaultPerClientRunRows, rows);
@@ -144,7 +142,7 @@ public class TestPerformanceEvaluation {
public void testRandomReadCalculation() {
TestOptions opts = new PerformanceEvaluation.TestOptions();
opts = PerformanceEvaluation.calculateRowsAndSize(opts);
- int rows = opts.getPerClientRunRows();
+ long rows = opts.getPerClientRunRows();
// Default row count
final int defaultPerClientRunRows = 1024 * 1024;
assertEquals(defaultPerClientRunRows, rows);
@@ -160,9 +158,8 @@ public class TestPerformanceEvaluation {
assertEquals(1000, opts.getPerClientRunRows());
// assuming we will get one before this loop expires
boolean foundValue = false;
- Random rand = ThreadLocalRandom.current();
for (int i = 0; i < 10000000; i++) {
- int randomRow = PerformanceEvaluation.generateRandomRow(rand,
opts.totalRows);
+ long randomRow = PerformanceEvaluation.generateRandomRow(opts.totalRows);
if (randomRow > 1000) {
foundValue = true;
break;
@@ -184,7 +181,7 @@ public class TestPerformanceEvaluation {
ctor.setAccessible(true);
Histogram histogram = (Histogram) ctor.newInstance(new
UniformReservoir(1024 * 500));
for (int i = 0; i < 100; i++) {
- histogram.update(rrt.getValueLength(null));
+ histogram.update(rrt.getValueLength());
}
Snapshot snapshot = histogram.getSnapshot();
double stddev = snapshot.getStdDev();
@@ -400,7 +397,7 @@ public class TestPerformanceEvaluation {
}
@Override
- boolean testRow(int i, long startTime) throws IOException,
InterruptedException {
+ boolean testRow(long i, long startTime) throws IOException,
InterruptedException {
return false;
}
}
diff --git
a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestCellBasedHFileOutputFormat2.java
b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestCellBasedHFileOutputFormat2.java
index f535ccf7e3e..8c55f938147 100644
---
a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestCellBasedHFileOutputFormat2.java
+++
b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestCellBasedHFileOutputFormat2.java
@@ -506,23 +506,19 @@ public class TestCellBasedHFileOutputFormat2 {
}
private byte[][] generateRandomStartKeys(int numKeys) {
- Random random = new Random();
byte[][] ret = new byte[numKeys][];
// first region start key is always empty
ret[0] = HConstants.EMPTY_BYTE_ARRAY;
for (int i = 1; i < numKeys; i++) {
- ret[i] =
- PerformanceEvaluation.generateData(random,
PerformanceEvaluation.DEFAULT_VALUE_LENGTH);
+ ret[i] =
PerformanceEvaluation.generateData(PerformanceEvaluation.DEFAULT_VALUE_LENGTH);
}
return ret;
}
private byte[][] generateRandomSplitKeys(int numKeys) {
- Random random = new Random();
byte[][] ret = new byte[numKeys][];
for (int i = 0; i < numKeys; i++) {
- ret[i] =
- PerformanceEvaluation.generateData(random,
PerformanceEvaluation.DEFAULT_VALUE_LENGTH);
+ ret[i] =
PerformanceEvaluation.generateData(PerformanceEvaluation.DEFAULT_VALUE_LENGTH);
}
return ret;
}
diff --git
a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2.java
b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2.java
index 67bb489ce29..7d4e0361533 100644
---
a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2.java
+++
b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2.java
@@ -34,14 +34,12 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
-import java.util.Random;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.Callable;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.ExecutorService;
-import java.util.concurrent.ThreadLocalRandom;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.apache.hadoop.conf.Configuration;
@@ -542,23 +540,19 @@ public class TestHFileOutputFormat2 {
}
private byte[][] generateRandomStartKeys(int numKeys) {
- Random random = ThreadLocalRandom.current();
byte[][] ret = new byte[numKeys][];
// first region start key is always empty
ret[0] = HConstants.EMPTY_BYTE_ARRAY;
for (int i = 1; i < numKeys; i++) {
- ret[i] =
- PerformanceEvaluation.generateData(random,
PerformanceEvaluation.DEFAULT_VALUE_LENGTH);
+ ret[i] =
PerformanceEvaluation.generateData(PerformanceEvaluation.DEFAULT_VALUE_LENGTH);
}
return ret;
}
private byte[][] generateRandomSplitKeys(int numKeys) {
- Random random = ThreadLocalRandom.current();
byte[][] ret = new byte[numKeys][];
for (int i = 0; i < numKeys; i++) {
- ret[i] =
- PerformanceEvaluation.generateData(random,
PerformanceEvaluation.DEFAULT_VALUE_LENGTH);
+ ret[i] =
PerformanceEvaluation.generateData(PerformanceEvaluation.DEFAULT_VALUE_LENGTH);
}
return ret;
}
