Added: hadoop/common/trunk/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/metrics/TestAzureFileSystemInstrumentation.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/metrics/TestAzureFileSystemInstrumentation.java?rev=1605187&view=auto ============================================================================== --- hadoop/common/trunk/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/metrics/TestAzureFileSystemInstrumentation.java (added) +++ hadoop/common/trunk/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/metrics/TestAzureFileSystemInstrumentation.java Tue Jun 24 20:52:44 2014 @@ -0,0 +1,546 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.azure.metrics; + +import static org.apache.hadoop.fs.azure.metrics.AzureFileSystemInstrumentation.WASB_CLIENT_ERRORS; +import static org.apache.hadoop.fs.azure.metrics.AzureFileSystemInstrumentation.WASB_DIRECTORIES_CREATED; +import static org.apache.hadoop.fs.azure.metrics.AzureFileSystemInstrumentation.WASB_DOWNLOAD_LATENCY; +import static org.apache.hadoop.fs.azure.metrics.AzureFileSystemInstrumentation.WASB_DOWNLOAD_RATE; +import static org.apache.hadoop.fs.azure.metrics.AzureFileSystemInstrumentation.WASB_FILES_CREATED; +import static org.apache.hadoop.fs.azure.metrics.AzureFileSystemInstrumentation.WASB_FILES_DELETED; +import static org.apache.hadoop.fs.azure.metrics.AzureFileSystemInstrumentation.WASB_SERVER_ERRORS; +import static org.apache.hadoop.fs.azure.metrics.AzureFileSystemInstrumentation.WASB_UPLOAD_LATENCY; +import static org.apache.hadoop.fs.azure.metrics.AzureFileSystemInstrumentation.WASB_UPLOAD_RATE; +import static org.apache.hadoop.fs.azure.metrics.AzureFileSystemInstrumentation.WASB_WEB_RESPONSES; +import static org.apache.hadoop.test.MetricsAsserts.assertCounter; +import static org.apache.hadoop.test.MetricsAsserts.getMetrics; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assume.assumeNotNull; +import static org.mockito.Matchers.argThat; +import static org.mockito.Mockito.verify; + +import java.io.InputStream; +import java.io.OutputStream; +import java.util.Arrays; +import java.util.Date; + +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.azure.AzureBlobStorageTestAccount; +import org.apache.hadoop.fs.azure.AzureException; +import org.apache.hadoop.fs.azure.AzureNativeFileSystemStore; +import org.apache.hadoop.fs.azure.NativeAzureFileSystem; +import org.apache.hadoop.metrics2.MetricsRecordBuilder; +import org.apache.hadoop.metrics2.MetricsTag; +import org.hamcrest.BaseMatcher; +import org.hamcrest.Description; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +public class TestAzureFileSystemInstrumentation { + private FileSystem fs; + private AzureBlobStorageTestAccount testAccount; + + @Before + public void setUp() throws Exception { + testAccount = AzureBlobStorageTestAccount.create(); + if (testAccount != null) { + fs = testAccount.getFileSystem(); + } + assumeNotNull(testAccount); + } + + @After + public void tearDown() throws Exception { + if (testAccount != null) { + testAccount.cleanup(); + testAccount = null; + fs = null; + } + } + + @Test + public void testMetricTags() throws Exception { + String accountName = + testAccount.getRealAccount().getBlobEndpoint() + .getAuthority(); + String containerName = + testAccount.getRealContainer().getName(); + MetricsRecordBuilder myMetrics = getMyMetrics(); + verify(myMetrics).add(argThat( + new TagMatcher("accountName", accountName) + )); + verify(myMetrics).add(argThat( + new TagMatcher("containerName", containerName) + )); + verify(myMetrics).add(argThat( + new TagMatcher("Context", "azureFileSystem") + )); + verify(myMetrics).add(argThat( + new TagExistsMatcher("wasbFileSystemId") + )); + } + + + @Test + public void testMetricsOnMkdirList() throws Exception { + long base = getBaseWebResponses(); + + // Create a directory + assertTrue(fs.mkdirs(new Path("a"))); + // At the time of writing, it takes 1 request to create the actual directory, + // plus 2 requests per level to check that there's no blob with that name and + // 1 request per level above to create it if it doesn't exist. + // So for the path above (/user/<name>/a), it takes 2 requests each to check + // there's no blob called /user, no blob called /user/<name> and no blob + // called /user/<name>/a, and then 3 request for the creation of the three + // levels, and then 2 requests for checking/stamping the version of AS, + // totaling 11. + // Also, there's the initial 1 request for container check so total is 12. + base = assertWebResponsesInRange(base, 1, 12); + assertEquals(1, + AzureMetricsTestUtil.getLongCounterValue(getInstrumentation(), WASB_DIRECTORIES_CREATED)); + + // List the root contents + assertEquals(1, fs.listStatus(new Path("/")).length); + base = assertWebResponsesEquals(base, 1); + + assertNoErrors(); + } + + private BandwidthGaugeUpdater getBandwidthGaugeUpdater() { + NativeAzureFileSystem azureFs = (NativeAzureFileSystem)fs; + AzureNativeFileSystemStore azureStore = azureFs.getStore(); + return azureStore.getBandwidthGaugeUpdater(); + } + + private static byte[] nonZeroByteArray(int size) { + byte[] data = new byte[size]; + Arrays.fill(data, (byte)5); + return data; + } + + @Test + public void testMetricsOnFileCreateRead() throws Exception { + long base = getBaseWebResponses(); + + assertEquals(0, AzureMetricsTestUtil.getCurrentBytesWritten(getInstrumentation())); + + Path filePath = new Path("/metricsTest_webResponses"); + final int FILE_SIZE = 1000; + + // Suppress auto-update of bandwidth metrics so we get + // to update them exactly when we want to. + getBandwidthGaugeUpdater().suppressAutoUpdate(); + + // Create a file + Date start = new Date(); + OutputStream outputStream = fs.create(filePath); + outputStream.write(nonZeroByteArray(FILE_SIZE)); + outputStream.close(); + long uploadDurationMs = new Date().getTime() - start.getTime(); + + // The exact number of requests/responses that happen to create a file + // can vary - at the time of writing this code it takes 10 + // requests/responses for the 1000 byte file (33 for 100 MB), + // plus the initial container-check request but that + // can very easily change in the future. Just assert that we do roughly + // more than 2 but less than 15. + logOpResponseCount("Creating a 1K file", base); + base = assertWebResponsesInRange(base, 2, 15); + getBandwidthGaugeUpdater().triggerUpdate(true); + long bytesWritten = AzureMetricsTestUtil.getCurrentBytesWritten(getInstrumentation()); + assertTrue("The bytes written in the last second " + bytesWritten + + " is pretty far from the expected range of around " + FILE_SIZE + + " bytes plus a little overhead.", + bytesWritten > (FILE_SIZE / 2) && bytesWritten < (FILE_SIZE * 2)); + long totalBytesWritten = AzureMetricsTestUtil.getCurrentTotalBytesWritten(getInstrumentation()); + assertTrue("The total bytes written " + totalBytesWritten + + " is pretty far from the expected range of around " + FILE_SIZE + + " bytes plus a little overhead.", + totalBytesWritten >= FILE_SIZE && totalBytesWritten < (FILE_SIZE * 2)); + long uploadRate = AzureMetricsTestUtil.getLongGaugeValue(getInstrumentation(), WASB_UPLOAD_RATE); + System.out.println("Upload rate: " + uploadRate + " bytes/second."); + long expectedRate = (FILE_SIZE * 1000L) / uploadDurationMs; + assertTrue("The upload rate " + uploadRate + + " is below the expected range of around " + expectedRate + + " bytes/second that the unit test observed. This should never be" + + " the case since the test underestimates the rate by looking at " + + " end-to-end time instead of just block upload time.", + uploadRate >= expectedRate); + long uploadLatency = AzureMetricsTestUtil.getLongGaugeValue(getInstrumentation(), + WASB_UPLOAD_LATENCY); + System.out.println("Upload latency: " + uploadLatency); + long expectedLatency = uploadDurationMs; // We're uploading less than a block. + assertTrue("The upload latency " + uploadLatency + + " should be greater than zero now that I've just uploaded a file.", + uploadLatency > 0); + assertTrue("The upload latency " + uploadLatency + + " is more than the expected range of around " + expectedLatency + + " milliseconds that the unit test observed. This should never be" + + " the case since the test overestimates the latency by looking at " + + " end-to-end time instead of just block upload time.", + uploadLatency <= expectedLatency); + + // Read the file + start = new Date(); + InputStream inputStream = fs.open(filePath); + int count = 0; + while (inputStream.read() >= 0) { + count++; + } + inputStream.close(); + long downloadDurationMs = new Date().getTime() - start.getTime(); + assertEquals(FILE_SIZE, count); + + // Again, exact number varies. At the time of writing this code + // it takes 4 request/responses, so just assert a rough range between + // 1 and 10. + logOpResponseCount("Reading a 1K file", base); + base = assertWebResponsesInRange(base, 1, 10); + getBandwidthGaugeUpdater().triggerUpdate(false); + long totalBytesRead = AzureMetricsTestUtil.getCurrentTotalBytesRead(getInstrumentation()); + assertEquals(FILE_SIZE, totalBytesRead); + long bytesRead = AzureMetricsTestUtil.getCurrentBytesRead(getInstrumentation()); + assertTrue("The bytes read in the last second " + bytesRead + + " is pretty far from the expected range of around " + FILE_SIZE + + " bytes plus a little overhead.", + bytesRead > (FILE_SIZE / 2) && bytesRead < (FILE_SIZE * 2)); + long downloadRate = AzureMetricsTestUtil.getLongGaugeValue(getInstrumentation(), WASB_DOWNLOAD_RATE); + System.out.println("Download rate: " + downloadRate + " bytes/second."); + expectedRate = (FILE_SIZE * 1000L) / downloadDurationMs; + assertTrue("The download rate " + downloadRate + + " is below the expected range of around " + expectedRate + + " bytes/second that the unit test observed. This should never be" + + " the case since the test underestimates the rate by looking at " + + " end-to-end time instead of just block download time.", + downloadRate >= expectedRate); + long downloadLatency = AzureMetricsTestUtil.getLongGaugeValue(getInstrumentation(), + WASB_DOWNLOAD_LATENCY); + System.out.println("Download latency: " + downloadLatency); + expectedLatency = downloadDurationMs; // We're downloading less than a block. + assertTrue("The download latency " + downloadLatency + + " should be greater than zero now that I've just downloaded a file.", + downloadLatency > 0); + assertTrue("The download latency " + downloadLatency + + " is more than the expected range of around " + expectedLatency + + " milliseconds that the unit test observed. This should never be" + + " the case since the test overestimates the latency by looking at " + + " end-to-end time instead of just block download time.", + downloadLatency <= expectedLatency); + + assertNoErrors(); + } + + @Test + public void testMetricsOnBigFileCreateRead() throws Exception { + long base = getBaseWebResponses(); + + assertEquals(0, AzureMetricsTestUtil.getCurrentBytesWritten(getInstrumentation())); + + Path filePath = new Path("/metricsTest_webResponses"); + final int FILE_SIZE = 100 * 1024 * 1024; + + // Suppress auto-update of bandwidth metrics so we get + // to update them exactly when we want to. + getBandwidthGaugeUpdater().suppressAutoUpdate(); + + // Create a file + OutputStream outputStream = fs.create(filePath); + outputStream.write(new byte[FILE_SIZE]); + outputStream.close(); + + // The exact number of requests/responses that happen to create a file + // can vary - at the time of writing this code it takes 34 + // requests/responses for the 100 MB file, + // plus the initial container check request, but that + // can very easily change in the future. Just assert that we do roughly + // more than 20 but less than 50. + logOpResponseCount("Creating a 100 MB file", base); + base = assertWebResponsesInRange(base, 20, 50); + getBandwidthGaugeUpdater().triggerUpdate(true); + long totalBytesWritten = AzureMetricsTestUtil.getCurrentTotalBytesWritten(getInstrumentation()); + assertTrue("The total bytes written " + totalBytesWritten + + " is pretty far from the expected range of around " + FILE_SIZE + + " bytes plus a little overhead.", + totalBytesWritten >= FILE_SIZE && totalBytesWritten < (FILE_SIZE * 2)); + long uploadRate = AzureMetricsTestUtil.getLongGaugeValue(getInstrumentation(), WASB_UPLOAD_RATE); + System.out.println("Upload rate: " + uploadRate + " bytes/second."); + long uploadLatency = AzureMetricsTestUtil.getLongGaugeValue(getInstrumentation(), + WASB_UPLOAD_LATENCY); + System.out.println("Upload latency: " + uploadLatency); + assertTrue("The upload latency " + uploadLatency + + " should be greater than zero now that I've just uploaded a file.", + uploadLatency > 0); + + // Read the file + InputStream inputStream = fs.open(filePath); + int count = 0; + while (inputStream.read() >= 0) { + count++; + } + inputStream.close(); + assertEquals(FILE_SIZE, count); + + // Again, exact number varies. At the time of writing this code + // it takes 27 request/responses, so just assert a rough range between + // 20 and 40. + logOpResponseCount("Reading a 100 MB file", base); + base = assertWebResponsesInRange(base, 20, 40); + getBandwidthGaugeUpdater().triggerUpdate(false); + long totalBytesRead = AzureMetricsTestUtil.getCurrentTotalBytesRead(getInstrumentation()); + assertEquals(FILE_SIZE, totalBytesRead); + long downloadRate = AzureMetricsTestUtil.getLongGaugeValue(getInstrumentation(), WASB_DOWNLOAD_RATE); + System.out.println("Download rate: " + downloadRate + " bytes/second."); + long downloadLatency = AzureMetricsTestUtil.getLongGaugeValue(getInstrumentation(), + WASB_DOWNLOAD_LATENCY); + System.out.println("Download latency: " + downloadLatency); + assertTrue("The download latency " + downloadLatency + + " should be greater than zero now that I've just downloaded a file.", + downloadLatency > 0); + } + + @Test + public void testMetricsOnFileRename() throws Exception { + long base = getBaseWebResponses(); + + Path originalPath = new Path("/metricsTest_RenameStart"); + Path destinationPath = new Path("/metricsTest_RenameFinal"); + + // Create an empty file + assertEquals(0, AzureMetricsTestUtil.getLongCounterValue(getInstrumentation(), WASB_FILES_CREATED)); + assertTrue(fs.createNewFile(originalPath)); + logOpResponseCount("Creating an empty file", base); + base = assertWebResponsesInRange(base, 2, 20); + assertEquals(1, AzureMetricsTestUtil.getLongCounterValue(getInstrumentation(), WASB_FILES_CREATED)); + + // Rename the file + assertTrue(fs.rename(originalPath, destinationPath)); + // Varies: at the time of writing this code it takes 7 requests/responses. + logOpResponseCount("Renaming a file", base); + base = assertWebResponsesInRange(base, 2, 15); + + assertNoErrors(); + } + + @Test + public void testMetricsOnFileExistsDelete() throws Exception { + long base = getBaseWebResponses(); + + Path filePath = new Path("/metricsTest_delete"); + + // Check existence + assertFalse(fs.exists(filePath)); + // At the time of writing this code it takes 2 requests/responses to + // check existence, which seems excessive, plus initial request for + // container check. + logOpResponseCount("Checking file existence for non-existent file", base); + base = assertWebResponsesInRange(base, 1, 3); + + // Create an empty file + assertTrue(fs.createNewFile(filePath)); + base = getCurrentWebResponses(); + + // Check existence again + assertTrue(fs.exists(filePath)); + logOpResponseCount("Checking file existence for existent file", base); + base = assertWebResponsesInRange(base, 1, 2); + + // Delete the file + assertEquals(0, AzureMetricsTestUtil.getLongCounterValue(getInstrumentation(), WASB_FILES_DELETED)); + assertTrue(fs.delete(filePath, false)); + // At the time of writing this code it takes 4 requests/responses to + // delete, which seems excessive. Check for range 1-4 for now. + logOpResponseCount("Deleting a file", base); + base = assertWebResponsesInRange(base, 1, 4); + assertEquals(1, AzureMetricsTestUtil.getLongCounterValue(getInstrumentation(), WASB_FILES_DELETED)); + + assertNoErrors(); + } + + @Test + public void testMetricsOnDirRename() throws Exception { + long base = getBaseWebResponses(); + + Path originalDirName = new Path("/metricsTestDirectory_RenameStart"); + Path innerFileName = new Path(originalDirName, "innerFile"); + Path destDirName = new Path("/metricsTestDirectory_RenameFinal"); + + // Create an empty directory + assertTrue(fs.mkdirs(originalDirName)); + base = getCurrentWebResponses(); + + // Create an inner file + assertTrue(fs.createNewFile(innerFileName)); + base = getCurrentWebResponses(); + + // Rename the directory + assertTrue(fs.rename(originalDirName, destDirName)); + // At the time of writing this code it takes 11 requests/responses + // to rename the directory with one file. Check for range 1-20 for now. + logOpResponseCount("Renaming a directory", base); + base = assertWebResponsesInRange(base, 1, 20); + + assertNoErrors(); + } + + @Test + public void testClientErrorMetrics() throws Exception { + String directoryName = "metricsTestDirectory_ClientError"; + Path directoryPath = new Path("/" + directoryName); + assertTrue(fs.mkdirs(directoryPath)); + String leaseID = testAccount.acquireShortLease(directoryName); + try { + try { + fs.delete(directoryPath, true); + assertTrue("Should've thrown.", false); + } catch (AzureException ex) { + assertTrue("Unexpected exception: " + ex, + ex.getMessage().contains("lease")); + } + assertEquals(1, AzureMetricsTestUtil.getLongCounterValue(getInstrumentation(), WASB_CLIENT_ERRORS)); + assertEquals(0, AzureMetricsTestUtil.getLongCounterValue(getInstrumentation(), WASB_SERVER_ERRORS)); + } finally { + testAccount.releaseLease(leaseID, directoryName); + } + } + + private void logOpResponseCount(String opName, long base) { + System.out.println(opName + " took " + (getCurrentWebResponses() - base) + + " web responses to complete."); + } + + /** + * Gets (and asserts) the value of the wasb_web_responses counter just + * after the creation of the file system object. + */ + private long getBaseWebResponses() { + // The number of requests should start at 0 + return assertWebResponsesEquals(0, 0); + } + + /** + * Gets the current value of the wasb_web_responses counter. + */ + private long getCurrentWebResponses() { + return AzureMetricsTestUtil.getCurrentWebResponses(getInstrumentation()); + } + + /** + * Checks that the wasb_web_responses counter is at the given value. + * @param base The base value (before the operation of interest). + * @param expected The expected value for the operation of interest. + * @return The new base value now. + */ + private long assertWebResponsesEquals(long base, long expected) { + assertCounter(WASB_WEB_RESPONSES, base + expected, getMyMetrics()); + return base + expected; + } + + private void assertNoErrors() { + assertEquals(0, AzureMetricsTestUtil.getLongCounterValue(getInstrumentation(), WASB_CLIENT_ERRORS)); + assertEquals(0, AzureMetricsTestUtil.getLongCounterValue(getInstrumentation(), WASB_SERVER_ERRORS)); + } + + /** + * Checks that the wasb_web_responses counter is in the given range. + * @param base The base value (before the operation of interest). + * @param inclusiveLowerLimit The lower limit for what it should increase by. + * @param inclusiveUpperLimit The upper limit for what it should increase by. + * @return The new base value now. + */ + private long assertWebResponsesInRange(long base, + long inclusiveLowerLimit, + long inclusiveUpperLimit) { + long currentResponses = getCurrentWebResponses(); + long justOperation = currentResponses - base; + assertTrue(String.format( + "Web responses expected in range [%d, %d], but was %d.", + inclusiveLowerLimit, inclusiveUpperLimit, justOperation), + justOperation >= inclusiveLowerLimit && + justOperation <= inclusiveUpperLimit); + return currentResponses; + } + + /** + * Gets the metrics for the file system object. + * @return The metrics record. + */ + private MetricsRecordBuilder getMyMetrics() { + return getMetrics(getInstrumentation()); + } + + private AzureFileSystemInstrumentation getInstrumentation() { + return ((NativeAzureFileSystem)fs).getInstrumentation(); + } + + /** + * A matcher class for asserting that we got a tag with a given + * value. + */ + private static class TagMatcher extends TagExistsMatcher { + private final String tagValue; + + public TagMatcher(String tagName, String tagValue) { + super(tagName); + this.tagValue = tagValue; + } + + @Override + public boolean matches(MetricsTag toMatch) { + return toMatch.value().equals(tagValue); + } + + @Override + public void describeTo(Description desc) { + super.describeTo(desc); + desc.appendText(" with value " + tagValue); + } + } + + /** + * A matcher class for asserting that we got a tag with any value. + */ + private static class TagExistsMatcher extends BaseMatcher<MetricsTag> { + private final String tagName; + + public TagExistsMatcher(String tagName) { + this.tagName = tagName; + } + + @Override + public boolean matches(Object toMatch) { + MetricsTag asTag = (MetricsTag)toMatch; + return asTag.name().equals(tagName) && matches(asTag); + } + + protected boolean matches(MetricsTag toMatch) { + return true; + } + + @Override + public void describeTo(Description desc) { + desc.appendText("Has tag " + tagName); + } + } + +}
Added: hadoop/common/trunk/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/metrics/TestBandwidthGaugeUpdater.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/metrics/TestBandwidthGaugeUpdater.java?rev=1605187&view=auto ============================================================================== --- hadoop/common/trunk/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/metrics/TestBandwidthGaugeUpdater.java (added) +++ hadoop/common/trunk/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/metrics/TestBandwidthGaugeUpdater.java Tue Jun 24 20:52:44 2014 @@ -0,0 +1,125 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.azure.metrics; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assume.assumeNotNull; + +import java.util.Date; +import java.util.Map; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.azure.AzureBlobStorageTestAccount; +import org.junit.Assume; +import org.junit.Test; + +public class TestBandwidthGaugeUpdater { + @Test + public void testSingleThreaded() throws Exception { + AzureFileSystemInstrumentation instrumentation = + new AzureFileSystemInstrumentation(new Configuration()); + BandwidthGaugeUpdater updater = + new BandwidthGaugeUpdater(instrumentation, 1000, true); + updater.triggerUpdate(true); + assertEquals(0, AzureMetricsTestUtil.getCurrentBytesWritten(instrumentation)); + updater.blockUploaded(new Date(), new Date(), 150); + updater.triggerUpdate(true); + assertEquals(150, AzureMetricsTestUtil.getCurrentBytesWritten(instrumentation)); + updater.blockUploaded(new Date(new Date().getTime() - 10000), + new Date(), 200); + updater.triggerUpdate(true); + long currentBytes = AzureMetricsTestUtil.getCurrentBytesWritten(instrumentation); + assertTrue( + "We expect around (200/10 = 20) bytes written as the gauge value." + + "Got " + currentBytes, + currentBytes > 18 && currentBytes < 22); + updater.close(); + } + + @Test + public void testMultiThreaded() throws Exception { + final AzureFileSystemInstrumentation instrumentation = + new AzureFileSystemInstrumentation(new Configuration()); + final BandwidthGaugeUpdater updater = + new BandwidthGaugeUpdater(instrumentation, 1000, true); + Thread[] threads = new Thread[10]; + for (int i = 0; i < threads.length; i++) { + threads[i] = new Thread(new Runnable() { + @Override + public void run() { + updater.blockDownloaded(new Date(), new Date(), 10); + updater.blockDownloaded(new Date(0), new Date(0), 10); + } + }); + } + for (Thread t : threads) { + t.start(); + } + for (Thread t : threads) { + t.join(); + } + updater.triggerUpdate(false); + assertEquals(10 * threads.length, AzureMetricsTestUtil.getCurrentBytesRead(instrumentation)); + updater.close(); + } + + @Test + public void testFinalizerThreadShutdown() throws Exception { + + // force cleanup of any existing wasb filesystems + System.gc(); + System.runFinalization(); + + int nUpdaterThreadsStart = getWasbThreadCount(); + assertTrue("Existing WASB threads have not been cleared", nUpdaterThreadsStart == 0); + + final int nFilesystemsToSpawn = 10; + AzureBlobStorageTestAccount testAccount = null; + + for(int i = 0; i < nFilesystemsToSpawn; i++){ + testAccount = AzureBlobStorageTestAccount.createMock(); + testAccount.getFileSystem(); + } + + int nUpdaterThreadsAfterSpawn = getWasbThreadCount(); + Assume.assumeTrue("Background threads should have spawned.", nUpdaterThreadsAfterSpawn == 10); + + testAccount = null; //clear the last reachable reference + + // force cleanup + System.gc(); + System.runFinalization(); + + int nUpdaterThreadsAfterCleanup = getWasbThreadCount(); + assertTrue("Finalizers should have reduced the thread count. ", nUpdaterThreadsAfterCleanup == 0 ); + } + + private int getWasbThreadCount() { + int c = 0; + Map<Thread, StackTraceElement[]> stacksStart = Thread.getAllStackTraces(); + for (Thread t : stacksStart.keySet()){ + if(t.getName().equals(BandwidthGaugeUpdater.THREAD_NAME)) + { + c++; + } + } + return c; + } +} Added: hadoop/common/trunk/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/metrics/TestNativeAzureFileSystemMetricsSystem.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/metrics/TestNativeAzureFileSystemMetricsSystem.java?rev=1605187&view=auto ============================================================================== --- hadoop/common/trunk/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/metrics/TestNativeAzureFileSystemMetricsSystem.java (added) +++ hadoop/common/trunk/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/metrics/TestNativeAzureFileSystemMetricsSystem.java Tue Jun 24 20:52:44 2014 @@ -0,0 +1,77 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.azure.metrics; + +import static org.junit.Assert.*; + +import org.apache.hadoop.fs.*; +import org.apache.hadoop.fs.azure.AzureBlobStorageTestAccount; +import org.apache.hadoop.fs.azure.NativeAzureFileSystem; +import org.junit.*; + +/** + * Tests that the WASB-specific metrics system is working correctly. + */ +public class TestNativeAzureFileSystemMetricsSystem { + private static final String WASB_FILES_CREATED = "wasb_files_created"; + + private static int getFilesCreated(AzureBlobStorageTestAccount testAccount) { + return testAccount.getLatestMetricValue(WASB_FILES_CREATED, 0).intValue(); + } + + /** + * Tests that when we have multiple file systems created/destroyed + * metrics from each are published correctly. + * @throws Exception + */ + @Test + public void testMetricsAcrossFileSystems() + throws Exception { + AzureBlobStorageTestAccount a1, a2, a3; + + a1 = AzureBlobStorageTestAccount.createMock(); + assertEquals(0, getFilesCreated(a1)); + a2 = AzureBlobStorageTestAccount.createMock(); + assertEquals(0, getFilesCreated(a2)); + a1.getFileSystem().create(new Path("/foo")).close(); + a1.getFileSystem().create(new Path("/bar")).close(); + a2.getFileSystem().create(new Path("/baz")).close(); + assertEquals(0, getFilesCreated(a1)); + assertEquals(0, getFilesCreated(a2)); + a1.closeFileSystem(); // Causes the file system to close, which publishes metrics + a2.closeFileSystem(); + + assertEquals(2, getFilesCreated(a1)); + assertEquals(1, getFilesCreated(a2)); + a3 = AzureBlobStorageTestAccount.createMock(); + assertEquals(0, getFilesCreated(a3)); + a3.closeFileSystem(); + assertEquals(0, getFilesCreated(a3)); + } + + + @Test + public void testMetricsSourceNames() { + String name1 = NativeAzureFileSystem.newMetricsSourceName(); + String name2 = NativeAzureFileSystem.newMetricsSourceName(); + assertTrue(name1.startsWith("AzureFileSystemMetrics")); + assertTrue(name2.startsWith("AzureFileSystemMetrics")); + assertTrue(!name1.equals(name2)); + } +} Added: hadoop/common/trunk/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/metrics/TestRollingWindowAverage.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/metrics/TestRollingWindowAverage.java?rev=1605187&view=auto ============================================================================== --- hadoop/common/trunk/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/metrics/TestRollingWindowAverage.java (added) +++ hadoop/common/trunk/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/metrics/TestRollingWindowAverage.java Tue Jun 24 20:52:44 2014 @@ -0,0 +1,42 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.azure.metrics; + +import static org.junit.Assert.*; +import org.junit.*; + +public class TestRollingWindowAverage { + /** + * Tests the basic functionality of the class. + */ + @Test + public void testBasicFunctionality() throws Exception { + RollingWindowAverage average = new RollingWindowAverage(100); + assertEquals(0, average.getCurrentAverage()); // Nothing there yet. + average.addPoint(5); + assertEquals(5, average.getCurrentAverage()); // One point in there. + Thread.sleep(50); + average.addPoint(15); + assertEquals(10, average.getCurrentAverage()); // Two points in there. + Thread.sleep(60); + assertEquals(15, average.getCurrentAverage()); // One point retired. + Thread.sleep(50); + assertEquals(0, average.getCurrentAverage()); // Both points retired. + } +} Modified: hadoop/common/trunk/hadoop-tools/hadoop-azure/src/test/resources/azure-test.xml URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-tools/hadoop-azure/src/test/resources/azure-test.xml?rev=1605187&r1=1605186&r2=1605187&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-tools/hadoop-azure/src/test/resources/azure-test.xml (original) +++ hadoop/common/trunk/hadoop-tools/hadoop-azure/src/test/resources/azure-test.xml Tue Jun 24 20:52:44 2014 @@ -26,6 +26,7 @@ <value>org.apache.hadoop.fs.azure.NativeAzureFileSystem</value> </property> + <!-- For tests against live azure, provide the following account information --> <!-- Added: hadoop/common/trunk/hadoop-tools/hadoop-azure/src/test/resources/hadoop-metrics2-azure-file-system.properties URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-tools/hadoop-azure/src/test/resources/hadoop-metrics2-azure-file-system.properties?rev=1605187&view=auto ============================================================================== --- hadoop/common/trunk/hadoop-tools/hadoop-azure/src/test/resources/hadoop-metrics2-azure-file-system.properties (added) +++ hadoop/common/trunk/hadoop-tools/hadoop-azure/src/test/resources/hadoop-metrics2-azure-file-system.properties Tue Jun 24 20:52:44 2014 @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +azure-file-system.sink.azuretestcollector.class = org.apache.hadoop.fs.azure.AzureBlobStorageTestAccount$StandardCollector
