HADOOP-15098. TestClusterTopology#testChooseRandom fails intermittently. Contributed by Zsolt Venczel.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/acb92904 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/acb92904 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/acb92904 Branch: refs/heads/YARN-6592 Commit: acb92904d04053551d3010937e133f74450043df Parents: 67662e2 Author: Sean Mackrory <[email protected]> Authored: Thu Dec 7 10:52:02 2017 -0700 Committer: Sean Mackrory <[email protected]> Committed: Thu Dec 7 12:30:58 2017 -0700 ---------------------------------------------------------------------- .../apache/hadoop/net/TestClusterTopology.java | 58 ++++++++++++-------- 1 file changed, 35 insertions(+), 23 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/acb92904/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestClusterTopology.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestClusterTopology.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestClusterTopology.java index 09f0914..6aad6c5 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestClusterTopology.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestClusterTopology.java @@ -139,38 +139,50 @@ public class TestClusterTopology extends Assert { NodeElement node4 = getNewNode("node4", "/d1/r3"); cluster.add(node4); + // Number of test runs + int numTestRuns = 3; + int chiSquareTestRejectedCounter = 0; + // Number of iterations to do the test int numIterations = 100; - // Pick random nodes - HashMap<String,Integer> histogram = new HashMap<String,Integer>(); - for (int i=0; i<numIterations; i++) { - String randomNode = cluster.chooseRandom(NodeBase.ROOT).getName(); - if (!histogram.containsKey(randomNode)) { - histogram.put(randomNode, 0); + for (int testRun = 0; testRun < numTestRuns; ++testRun) { + + // Pick random nodes + HashMap<String, Integer> histogram = new HashMap<String, Integer>(); + for (int i = 0; i < numIterations; i++) { + String randomNode = cluster.chooseRandom(NodeBase.ROOT).getName(); + if (!histogram.containsKey(randomNode)) { + histogram.put(randomNode, 0); + } + histogram.put(randomNode, histogram.get(randomNode) + 1); + } + assertEquals("Random is not selecting all nodes", 4, histogram.size()); + + // Check with 99% confidence alpha=0.01 as confidence = 100 * (1 - alpha) + ChiSquareTest chiSquareTest = new ChiSquareTest(); + double[] expected = new double[histogram.size()]; + long[] observed = new long[histogram.size()]; + int j = 0; + for (Integer occurrence : histogram.values()) { + expected[j] = 1.0 * numIterations / histogram.size(); + observed[j] = occurrence; + j++; + } + boolean chiSquareTestRejected = + chiSquareTest.chiSquareTest(expected, observed, 0.01); + + if (chiSquareTestRejected) { + ++chiSquareTestRejectedCounter; } - histogram.put(randomNode, histogram.get(randomNode) + 1); - } - assertEquals("Random is not selecting all nodes", 4, histogram.size()); - - // Check with 99% confidence (alpha=0.01 as confidence = (100 * (1 - alpha) - ChiSquareTest chiSquareTest = new ChiSquareTest(); - double[] expected = new double[histogram.size()]; - long[] observed = new long[histogram.size()]; - int j=0; - for (Integer occurrence : histogram.values()) { - expected[j] = 1.0 * numIterations / histogram.size(); - observed[j] = occurrence; - j++; } - boolean chiSquareTestRejected = - chiSquareTest.chiSquareTest(expected, observed, 0.01); // Check that they have the proper distribution - assertFalse("Not choosing nodes randomly", chiSquareTestRejected); + assertFalse("Random not choosing nodes with proper distribution", + chiSquareTestRejectedCounter==3); // Pick random nodes excluding the 2 nodes in /d1/r3 - histogram = new HashMap<String,Integer>(); + HashMap<String, Integer> histogram = new HashMap<String, Integer>(); for (int i=0; i<numIterations; i++) { String randomNode = cluster.chooseRandom("~/d1/r3").getName(); if (!histogram.containsKey(randomNode)) { --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
