Repository: hadoop Updated Branches: refs/heads/trunk 30acb7372 -> 61dc2ea3f
HDFS-8063: Fix intermittent test failures in TestTracing (Masatake Iwasaki via Colin P. McCabe) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/61dc2ea3 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/61dc2ea3 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/61dc2ea3 Branch: refs/heads/trunk Commit: 61dc2ea3fee4085b19cd2d01de9eacdc4c42e21f Parents: 30acb73 Author: Colin Patrick Mccabe <cmcc...@cloudera.com> Authored: Thu Apr 9 11:28:02 2015 -0700 Committer: Colin Patrick Mccabe <cmcc...@cloudera.com> Committed: Thu Apr 9 11:28:02 2015 -0700 ---------------------------------------------------------------------- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 + .../org/apache/hadoop/tracing/TestTracing.java | 125 +++++++------------ 2 files changed, 45 insertions(+), 83 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/61dc2ea3/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 59cab03..4b22fa4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -1427,6 +1427,9 @@ Release 2.7.0 - UNRELEASED HDFS-8072. Reserved RBW space is not released if client terminates while writing block. (Arpit Agarwal) + HDFS-8063: Fix intermittent test failures in TestTracing (Masatake Iwasaki + via Colin P. McCabe) + BREAKDOWN OF HDFS-7584 SUBTASKS AND RELATED JIRAS HDFS-7720. Quota by Storage Type API, tools and ClientNameNode http://git-wip-us.apache.org/repos/asf/hadoop/blob/61dc2ea3/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tracing/TestTracing.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tracing/TestTracing.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tracing/TestTracing.java index 01361b5..f6fef5a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tracing/TestTracing.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tracing/TestTracing.java @@ -31,7 +31,7 @@ import org.apache.htrace.Span; import org.apache.htrace.SpanReceiver; import org.apache.htrace.Trace; import org.apache.htrace.TraceScope; -import org.junit.AfterClass; +import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.BeforeClass; @@ -56,27 +56,26 @@ public class TestTracing { private static SpanReceiverHost spanReceiverHost; @Test - public void testGetSpanReceiverHost() throws Exception { - Configuration c = new Configuration(); + public void testTracing() throws Exception { // getting instance already loaded. - c.set(SpanReceiverHost.SPAN_RECEIVERS_CONF_KEY, ""); - SpanReceiverHost s = SpanReceiverHost.getInstance(c); - Assert.assertEquals(spanReceiverHost, s); + Assert.assertEquals(spanReceiverHost, + SpanReceiverHost.getInstance(new Configuration())); + + // write and read without tracing started + String fileName = "testTracingDisabled.dat"; + writeTestFile(fileName); + Assert.assertTrue(SetSpanReceiver.SetHolder.size() == 0); + readTestFile(fileName); + Assert.assertTrue(SetSpanReceiver.SetHolder.size() == 0); + + writeWithTracing(); + readWithTracing(); } - @Test - public void testWriteTraceHooks() throws Exception { + public void writeWithTracing() throws Exception { long startTime = System.currentTimeMillis(); TraceScope ts = Trace.startSpan("testWriteTraceHooks", Sampler.ALWAYS); - Path file = new Path("traceWriteTest.dat"); - FSDataOutputStream stream = dfs.create(file); - - for (int i = 0; i < 10; i++) { - byte[] data = RandomStringUtils.randomAlphabetic(102400).getBytes(); - stream.write(data); - } - stream.hflush(); - stream.close(); + writeTestFile("testWriteTraceHooks.dat"); long endTime = System.currentTimeMillis(); ts.close(); @@ -125,55 +124,17 @@ public class TestTracing { Assert.assertEquals(ts.getSpan().getTraceId(), span.getTraceId()); } } + SetSpanReceiver.SetHolder.spans.clear(); } - @Test - public void testWriteWithoutTraceHooks() throws Exception { - Path file = new Path("withoutTraceWriteTest.dat"); - FSDataOutputStream stream = dfs.create(file); - for (int i = 0; i < 10; i++) { - byte[] data = RandomStringUtils.randomAlphabetic(102400).getBytes(); - stream.write(data); - } - stream.hflush(); - stream.close(); - Assert.assertTrue(SetSpanReceiver.SetHolder.size() == 0); - } - - @Test - public void testReadTraceHooks() throws Exception { - String fileName = "traceReadTest.dat"; - Path filePath = new Path(fileName); - - // Create the file. - FSDataOutputStream ostream = dfs.create(filePath); - for (int i = 0; i < 50; i++) { - byte[] data = RandomStringUtils.randomAlphabetic(10240).getBytes(); - ostream.write(data); - } - ostream.close(); - - + public void readWithTracing() throws Exception { + String fileName = "testReadTraceHooks.dat"; + writeTestFile(fileName); long startTime = System.currentTimeMillis(); TraceScope ts = Trace.startSpan("testReadTraceHooks", Sampler.ALWAYS); - FSDataInputStream istream = dfs.open(filePath, 10240); - ByteBuffer buf = ByteBuffer.allocate(10240); - - int count = 0; - try { - while (istream.read(buf) > 0) { - count += 1; - buf.clear(); - istream.seek(istream.getPos() + 5); - } - } catch (IOException ioe) { - // Ignore this it's probably a seek after eof. - } finally { - istream.close(); - } - ts.getSpan().addTimelineAnnotation("count: " + count); - long endTime = System.currentTimeMillis(); + readTestFile(fileName); ts.close(); + long endTime = System.currentTimeMillis(); String[] expectedSpanNames = { "testReadTraceHooks", @@ -198,21 +159,22 @@ public class TestTracing { for (Span span : SetSpanReceiver.SetHolder.spans.values()) { Assert.assertEquals(ts.getSpan().getTraceId(), span.getTraceId()); } + SetSpanReceiver.SetHolder.spans.clear(); } - @Test - public void testReadWithoutTraceHooks() throws Exception { - String fileName = "withoutTraceReadTest.dat"; - Path filePath = new Path(fileName); - - // Create the file. - FSDataOutputStream ostream = dfs.create(filePath); - for (int i = 0; i < 50; i++) { - byte[] data = RandomStringUtils.randomAlphabetic(10240).getBytes(); - ostream.write(data); + private void writeTestFile(String testFileName) throws Exception { + Path filePath = new Path(testFileName); + FSDataOutputStream stream = dfs.create(filePath); + for (int i = 0; i < 10; i++) { + byte[] data = RandomStringUtils.randomAlphabetic(102400).getBytes(); + stream.write(data); } - ostream.close(); + stream.hsync(); + stream.close(); + } + private void readTestFile(String testFileName) throws Exception { + Path filePath = new Path(testFileName); FSDataInputStream istream = dfs.open(filePath, 10240); ByteBuffer buf = ByteBuffer.allocate(10240); @@ -228,32 +190,29 @@ public class TestTracing { } finally { istream.close(); } - Assert.assertTrue(SetSpanReceiver.SetHolder.size() == 0); - } - - @Before - public void cleanSet() { - SetSpanReceiver.SetHolder.spans.clear(); } @BeforeClass - public static void setupCluster() throws IOException { + public static void setup() throws IOException { conf = new Configuration(); conf.setLong("dfs.blocksize", 100 * 1024); conf.set(SpanReceiverHost.SPAN_RECEIVERS_CONF_KEY, SetSpanReceiver.class.getName()); + spanReceiverHost = SpanReceiverHost.getInstance(conf); + } + @Before + public void startCluster() throws IOException { cluster = new MiniDFSCluster.Builder(conf) .numDataNodes(3) .build(); cluster.waitActive(); - dfs = cluster.getFileSystem(); - spanReceiverHost = SpanReceiverHost.getInstance(conf); + SetSpanReceiver.SetHolder.spans.clear(); } - @AfterClass - public static void shutDown() throws IOException { + @After + public void shutDown() throws IOException { cluster.shutdown(); }
