HBASE-5401 PerformanceEvaluation generates 10x the number of expected mappers (Yi Liang)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/d787155f Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/d787155f Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/d787155f Branch: refs/heads/hbase-12439 Commit: d787155fd24c576b66663220372dbb7286d5e291 Parents: a1d2ff4 Author: Michael Stack <[email protected]> Authored: Wed Dec 21 11:55:48 2016 -0800 Committer: Michael Stack <[email protected]> Committed: Wed Dec 21 11:55:48 2016 -0800 ---------------------------------------------------------------------- .../hadoop/hbase/PerformanceEvaluation.java | 24 ++++++++------------ .../hadoop/hbase/TestPerformanceEvaluation.java | 4 ++-- 2 files changed, 12 insertions(+), 16 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/d787155f/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java index 5439bae..7d94a02 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java @@ -509,10 +509,8 @@ public class PerformanceEvaluation extends Configured implements Tool { } /** - * Per client, how many tasks will we run? We divide number of rows by this number and have the - * client do the resulting count in a map task. + * Each client has one mapper to do the work, and client do the resulting count in a map task. */ - static int TASKS_PER_CLIENT = 10; static String JOB_INPUT_FILENAME = "input.txt"; @@ -542,17 +540,15 @@ public class PerformanceEvaluation extends Configured implements Tool { Hash h = MurmurHash.getInstance(); int perClientRows = (opts.totalRows / opts.numClientThreads); try { - for (int i = 0; i < TASKS_PER_CLIENT; i++) { - for (int j = 0; j < opts.numClientThreads; j++) { - TestOptions next = new TestOptions(opts); - next.startRow = (j * perClientRows) + (i * (perClientRows/10)); - next.perClientRunRows = perClientRows / 10; - String s = MAPPER.writeValueAsString(next); - LOG.info("Client=" + j + ", maptask=" + i + ", input=" + s); - byte[] b = Bytes.toBytes(s); - int hash = h.hash(new ByteArrayHashKey(b, 0, b.length), -1); - m.put(hash, s); - } + for (int j = 0; j < opts.numClientThreads; j++) { + TestOptions next = new TestOptions(opts); + next.startRow = j * perClientRows; + next.perClientRunRows = perClientRows; + String s = MAPPER.writeValueAsString(next); + LOG.info("Client=" + j + ", input=" + s); + byte[] b = Bytes.toBytes(s); + int hash = h.hash(new ByteArrayHashKey(b, 0, b.length), -1); + m.put(hash, s); } for (Map.Entry<Integer, String> e: m.entrySet()) { out.println(e.getValue()); http://git-wip-us.apache.org/repos/asf/hbase/blob/d787155f/hbase-server/src/test/java/org/apache/hadoop/hbase/TestPerformanceEvaluation.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestPerformanceEvaluation.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestPerformanceEvaluation.java index f774742..cb7fdad 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestPerformanceEvaluation.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestPerformanceEvaluation.java @@ -90,7 +90,7 @@ public class TestPerformanceEvaluation { while (br.readLine() != null) { count++; } - assertEquals(clients * PerformanceEvaluation.TASKS_PER_CLIENT, count); + assertEquals(clients, count); } finally { dis.close(); } @@ -183,4 +183,4 @@ public class TestPerformanceEvaluation { assertTrue(e.getCause() instanceof NoSuchElementException); } } -} \ No newline at end of file +}
