http://git-wip-us.apache.org/repos/asf/hadoop/blob/3fbcf763/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestFileSystemOperationsExceptionHandlingMultiThreaded.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestFileSystemOperationsExceptionHandlingMultiThreaded.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestFileSystemOperationsExceptionHandlingMultiThreaded.java deleted file mode 100644 index 1cd18ee..0000000 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestFileSystemOperationsExceptionHandlingMultiThreaded.java +++ /dev/null @@ -1,330 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.fs.azure; - -import java.io.FileNotFoundException; - -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.FSDataInputStream; -import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.permission.FsAction; -import org.apache.hadoop.fs.permission.FsPermission; -import org.junit.After; -import org.junit.Test; - -public class TestFileSystemOperationsExceptionHandlingMultiThreaded - extends AbstractWasbTestBase { - - FSDataInputStream inputStream = null; - - private static Path testPath = new Path("testfile.dat"); - private static Path testFolderPath = new Path("testfolder"); - - - /* - * Helper method to creates an input stream to test various scenarios. - */ - private void getInputStreamToTest(FileSystem fs, Path testPath) throws Throwable { - - FSDataOutputStream outputStream = fs.create(testPath); - String testString = "This is a test string"; - outputStream.write(testString.getBytes()); - outputStream.close(); - - inputStream = fs.open(testPath); - } - - /* - * Test to validate correct exception is thrown for Multithreaded read - * scenario for block blobs - */ - @Test(expected=FileNotFoundException.class) - public void testMultiThreadedBlockBlobReadScenario() throws Throwable { - - AzureBlobStorageTestAccount testAccount = createTestAccount(); - fs = testAccount.getFileSystem(); - Path testFilePath1 = new Path("test1.dat"); - - getInputStreamToTest(fs, testFilePath1); - Thread renameThread = new Thread(new RenameThread(fs, testFilePath1)); - renameThread.start(); - - renameThread.join(); - - byte[] readBuffer = new byte[512]; - inputStream.read(readBuffer); - } - - /* - * Test to validate correct exception is thrown for Multithreaded seek - * scenario for block blobs - */ - - @Test(expected=FileNotFoundException.class) - public void testMultiThreadBlockBlobSeekScenario() throws Throwable { - - AzureBlobStorageTestAccount testAccount = createTestAccount(); - fs = testAccount.getFileSystem(); - Path testFilePath1 = new Path("test1.dat"); - - getInputStreamToTest(fs, testFilePath1); - Thread renameThread = new Thread(new RenameThread(fs, testFilePath1)); - renameThread.start(); - - renameThread.join(); - - inputStream.seek(5); - inputStream.read(); - } - - @Test(expected=FileNotFoundException.class) - /* - * Tests basic multi threaded setPermission scenario - */ - public void testMultiThreadedPageBlobSetPermissionScenario() throws Throwable { - ExceptionHandlingTestHelper.createEmptyFile(ExceptionHandlingTestHelper.getPageBlobTestStorageAccount(), - testPath); - Thread t = new Thread(new DeleteThread(fs, testPath)); - t.start(); - while (t.isAlive()) { - fs.setPermission(testPath, new FsPermission(FsAction.EXECUTE, FsAction.READ, FsAction.READ)); - } - fs.setPermission(testPath, new FsPermission(FsAction.EXECUTE, FsAction.READ, FsAction.READ)); - } - - @Test(expected=FileNotFoundException.class) - /* - * Tests basic multi threaded setPermission scenario - */ - public void testMultiThreadedBlockBlobSetPermissionScenario() throws Throwable { - ExceptionHandlingTestHelper.createEmptyFile(createTestAccount(), - testPath); - Thread t = new Thread(new DeleteThread(fs, testPath)); - t.start(); - while (t.isAlive()) { - fs.setPermission(testPath, new FsPermission(FsAction.EXECUTE, FsAction.READ, FsAction.READ)); - } - fs.setPermission(testPath, new FsPermission(FsAction.EXECUTE, FsAction.READ, FsAction.READ)); - } - - @Test(expected=FileNotFoundException.class) - /* - * Tests basic multi threaded setPermission scenario - */ - public void testMultiThreadedPageBlobOpenScenario() throws Throwable { - - ExceptionHandlingTestHelper.createEmptyFile(createTestAccount(), - testPath); - Thread t = new Thread(new DeleteThread(fs, testPath)); - t.start(); - while (t.isAlive()) { - inputStream = fs.open(testPath); - inputStream.close(); - } - - inputStream = fs.open(testPath); - inputStream.close(); - } - - @Test(expected=FileNotFoundException.class) - /* - * Tests basic multi threaded setPermission scenario - */ - public void testMultiThreadedBlockBlobOpenScenario() throws Throwable { - - ExceptionHandlingTestHelper.createEmptyFile(ExceptionHandlingTestHelper.getPageBlobTestStorageAccount(), - testPath); - Thread t = new Thread(new DeleteThread(fs, testPath)); - t.start(); - - while (t.isAlive()) { - inputStream = fs.open(testPath); - inputStream.close(); - } - inputStream = fs.open(testPath); - inputStream.close(); - } - - @Test(expected=FileNotFoundException.class) - /* - * Tests basic multi threaded setOwner scenario - */ - public void testMultiThreadedBlockBlobSetOwnerScenario() throws Throwable { - - ExceptionHandlingTestHelper.createEmptyFile(createTestAccount(), testPath); - Thread t = new Thread(new DeleteThread(fs, testPath)); - t.start(); - while (t.isAlive()) { - fs.setOwner(testPath, "testowner", "testgroup"); - } - fs.setOwner(testPath, "testowner", "testgroup"); - } - - @Test(expected=FileNotFoundException.class) - /* - * Tests basic multi threaded setOwner scenario - */ - public void testMultiThreadedPageBlobSetOwnerScenario() throws Throwable { - ExceptionHandlingTestHelper.createEmptyFile(ExceptionHandlingTestHelper.getPageBlobTestStorageAccount(), - testPath); - Thread t = new Thread(new DeleteThread(fs, testPath)); - t.start(); - while (t.isAlive()) { - fs.setOwner(testPath, "testowner", "testgroup"); - } - fs.setOwner(testPath, "testowner", "testgroup"); - } - - @Test(expected=FileNotFoundException.class) - /* - * Tests basic multi threaded listStatus scenario - */ - public void testMultiThreadedBlockBlobListStatusScenario() throws Throwable { - - ExceptionHandlingTestHelper.createTestFolder(createTestAccount(), testFolderPath); - Thread t = new Thread(new DeleteThread(fs, testFolderPath)); - t.start(); - while (t.isAlive()) { - fs.listStatus(testFolderPath); - } - fs.listStatus(testFolderPath); - } - - @Test(expected=FileNotFoundException.class) - /* - * Tests basic multi threaded listStatus scenario - */ - public void testMultiThreadedPageBlobListStatusScenario() throws Throwable { - - ExceptionHandlingTestHelper.createTestFolder(ExceptionHandlingTestHelper.getPageBlobTestStorageAccount(), - testFolderPath); - Thread t = new Thread(new DeleteThread(fs, testFolderPath)); - t.start(); - while (t.isAlive()) { - fs.listStatus(testFolderPath); - } - fs.listStatus(testFolderPath); - } - - /* - * Test to validate correct exception is thrown for Multithreaded read - * scenario for page blobs - */ - - @Test(expected=FileNotFoundException.class) - public void testMultiThreadedPageBlobReadScenario() throws Throwable { - - AzureBlobStorageTestAccount testAccount = ExceptionHandlingTestHelper.getPageBlobTestStorageAccount(); - fs = testAccount.getFileSystem(); - Path testFilePath1 = new Path("test1.dat"); - - getInputStreamToTest(fs, testFilePath1); - Thread renameThread = new Thread(new RenameThread(fs, testFilePath1)); - renameThread.start(); - - renameThread.join(); - byte[] readBuffer = new byte[512]; - inputStream.read(readBuffer); - } - - /* - * Test to validate correct exception is thrown for Multithreaded seek - * scenario for page blobs - */ - - @Test(expected=FileNotFoundException.class) - public void testMultiThreadedPageBlobSeekScenario() throws Throwable { - - AzureBlobStorageTestAccount testAccount = ExceptionHandlingTestHelper.getPageBlobTestStorageAccount(); - fs = testAccount.getFileSystem(); - Path testFilePath1 = new Path("test1.dat"); - - getInputStreamToTest(fs, testFilePath1); - Thread renameThread = new Thread(new RenameThread(fs, testFilePath1)); - renameThread.start(); - - renameThread.join(); - inputStream.seek(5); - } - - @Override - protected AzureBlobStorageTestAccount createTestAccount() throws Exception { - return AzureBlobStorageTestAccount.create(); - } - - @After - public void tearDown() throws Exception { - - if (inputStream != null) { - inputStream.close(); - } - - if (fs != null && fs.exists(testPath)) { - fs.delete(testPath, true); - } - } -} - -/* - * Helper thread that just renames the test file. - */ -class RenameThread implements Runnable { - - private FileSystem fs; - private Path testPath; - private Path renamePath = new Path("test2.dat"); - - public RenameThread(FileSystem fs, Path testPath) { - this.fs = fs; - this.testPath = testPath; - } - - @Override - public void run(){ - try { - fs.rename(testPath, renamePath); - }catch (Exception e) { - // Swallowing the exception as the - // correctness of the test is controlled - // by the other thread - } - } -} - -class DeleteThread implements Runnable { - private FileSystem fs; - private Path testPath; - - public DeleteThread(FileSystem fs, Path testPath) { - this.fs = fs; - this.testPath = testPath; - } - - @Override - public void run() { - try { - fs.delete(testPath, true); - } catch (Exception e) { - // Swallowing the exception as the - // correctness of the test is controlled - // by the other thread - } - } -}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3fbcf763/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestFileSystemOperationsWithThreads.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestFileSystemOperationsWithThreads.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestFileSystemOperationsWithThreads.java deleted file mode 100644 index fd3690c..0000000 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestFileSystemOperationsWithThreads.java +++ /dev/null @@ -1,821 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.fs.azure; - -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -import java.io.IOException; -import java.net.URI; -import java.util.concurrent.RejectedExecutionException; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; - -import org.apache.commons.logging.impl.Log4JLogger; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.azure.NativeAzureFileSystem.FolderRenamePending; -import org.apache.hadoop.test.GenericTestUtils.LogCapturer; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; -import org.mockito.Mockito; -import org.mockito.invocation.InvocationOnMock; -import org.mockito.stubbing.Answer; - -/** - * Tests the Native Azure file system (WASB) using parallel threads for rename and delete operations. - */ -public class TestFileSystemOperationsWithThreads extends AbstractWasbTestBase { - - private final int renameThreads = 10; - private final int deleteThreads = 20; - private int iterations = 1; - private LogCapturer logs = null; - - @Rule - public ExpectedException exception = ExpectedException.none(); - - @Before - public void setUp() throws Exception { - super.setUp(); - Configuration conf = fs.getConf(); - - // By default enable parallel threads for rename and delete operations. - // Also enable flat listing of blobs for these operations. - conf.setInt(NativeAzureFileSystem.AZURE_RENAME_THREADS, renameThreads); - conf.setInt(NativeAzureFileSystem.AZURE_DELETE_THREADS, deleteThreads); - conf.setBoolean(AzureNativeFileSystemStore.KEY_ENABLE_FLAT_LISTING, true); - - URI uri = fs.getUri(); - fs.initialize(uri, conf); - - // Capture logs - logs = LogCapturer.captureLogs(new Log4JLogger(org.apache.log4j.Logger - .getRootLogger())); - } - - /* - * Helper method to create sub directory and different types of files - * for multiple iterations. - */ - private void createFolder(FileSystem fs, String root) throws Exception { - fs.mkdirs(new Path(root)); - for (int i = 0; i < this.iterations; i++) { - fs.mkdirs(new Path(root + "/" + i)); - fs.createNewFile(new Path(root + "/" + i + "/fileToRename")); - fs.createNewFile(new Path(root + "/" + i + "/file/to/rename")); - fs.createNewFile(new Path(root + "/" + i + "/file+to%rename")); - fs.createNewFile(new Path(root + "/fileToRename" + i)); - } - } - - /* - * Helper method to do rename operation and validate all files in source folder - * doesn't exists and similar files exists in new folder. - */ - private void validateRenameFolder(FileSystem fs, String source, String dest) throws Exception { - // Create source folder with files. - createFolder(fs, source); - Path sourceFolder = new Path(source); - Path destFolder = new Path(dest); - - // rename operation - assertTrue(fs.rename(sourceFolder, destFolder)); - assertTrue(fs.exists(destFolder)); - - for (int i = 0; i < this.iterations; i++) { - // Check destination folder and files exists. - assertTrue(fs.exists(new Path(dest + "/" + i))); - assertTrue(fs.exists(new Path(dest + "/" + i + "/fileToRename"))); - assertTrue(fs.exists(new Path(dest + "/" + i + "/file/to/rename"))); - assertTrue(fs.exists(new Path(dest + "/" + i + "/file+to%rename"))); - assertTrue(fs.exists(new Path(dest + "/fileToRename" + i))); - - // Check source folder and files doesn't exists. - assertFalse(fs.exists(new Path(source + "/" + i))); - assertFalse(fs.exists(new Path(source + "/" + i + "/fileToRename"))); - assertFalse(fs.exists(new Path(source + "/" + i + "/file/to/rename"))); - assertFalse(fs.exists(new Path(source + "/" + i + "/file+to%rename"))); - assertFalse(fs.exists(new Path(source + "/fileToRename" + i))); - } - } - - /* - * Test case for rename operation with multiple threads and flat listing enabled. - */ - @Test - public void testRenameSmallFolderWithThreads() throws Exception { - - validateRenameFolder(fs, "root", "rootnew"); - - // With single iteration, we would have created 7 blobs. - int expectedThreadsCreated = Math.min(7, renameThreads); - - // Validate from logs that threads are created. - String content = logs.getOutput(); - assertInLog(content, "ms with threads: " + expectedThreadsCreated); - - // Validate thread executions - for (int i = 0; i < expectedThreadsCreated; i++) { - assertInLog(content, - "AzureBlobRenameThread-" + Thread.currentThread().getName() + "-" + i); - } - - // Also ensure that we haven't spawned extra threads. - if (expectedThreadsCreated < renameThreads) { - for (int i = expectedThreadsCreated; i < renameThreads; i++) { - assertNotInLog(content, - "AzureBlobRenameThread-" + Thread.currentThread().getName() + "-" + i); - } - } - } - - /* - * Test case for rename operation with multiple threads and flat listing enabled. - */ - @Test - public void testRenameLargeFolderWithThreads() throws Exception { - - // Populate source folder with large number of files and directories. - this.iterations = 10; - validateRenameFolder(fs, "root", "rootnew"); - - // Validate from logs that threads are created. - String content = logs.getOutput(); - assertInLog(content, "ms with threads: " + renameThreads); - - // Validate thread executions - for (int i = 0; i < renameThreads; i++) { - assertInLog(content, - "AzureBlobRenameThread-" + Thread.currentThread().getName() + "-" + i); - } - } - - /* - * Test case for rename operation with threads disabled and flat listing enabled. - */ - @Test - public void testRenameLargeFolderDisableThreads() throws Exception { - Configuration conf = fs.getConf(); - - // Number of threads set to 0 or 1 disables threads. - conf.setInt(NativeAzureFileSystem.AZURE_RENAME_THREADS, 0); - URI uri = fs.getUri(); - fs.initialize(uri, conf); - - // Populate source folder with large number of files and directories. - this.iterations = 10; - validateRenameFolder(fs, "root", "rootnew"); - - // Validate from logs that threads are disabled. - String content = logs.getOutput(); - assertInLog(content, - "Disabling threads for Rename operation as thread count 0"); - - // Validate no thread executions - for (int i = 0; i < renameThreads; i++) { - String term = "AzureBlobRenameThread-" - + Thread.currentThread().getName() - + "-" + i; - assertNotInLog(content, term); - } - } - - /** - * Assert that a log contains the given term. - * @param content log output - * @param term search term - */ - protected void assertInLog(String content, String term) { - assertTrue("Empty log", !content.isEmpty()); - if (!content.contains(term)) { - String message = "No " + term + " found in logs"; - LOG.error(message); - System.err.println(content); - fail(message); - } - } - - /** - * Assert that a log does not contain the given term. - * @param content log output - * @param term search term - */ - protected void assertNotInLog(String content, String term) { - assertTrue("Empty log", !content.isEmpty()); - if (content.contains(term)) { - String message = term + " found in logs"; - LOG.error(message); - System.err.println(content); - fail(message); - } - } - - /* - * Test case for rename operation with threads and flat listing disabled. - */ - @Test - public void testRenameSmallFolderDisableThreadsDisableFlatListing() throws Exception { - Configuration conf = fs.getConf(); - conf = fs.getConf(); - - // Number of threads set to 0 or 1 disables threads. - conf.setInt(NativeAzureFileSystem.AZURE_RENAME_THREADS, 1); - conf.setBoolean(AzureNativeFileSystemStore.KEY_ENABLE_FLAT_LISTING, false); - URI uri = fs.getUri(); - fs.initialize(uri, conf); - - validateRenameFolder(fs, "root", "rootnew"); - - // Validate from logs that threads are disabled. - String content = logs.getOutput(); - assertInLog(content, - "Disabling threads for Rename operation as thread count 1"); - - // Validate no thread executions - for (int i = 0; i < renameThreads; i++) { - assertNotInLog(content, - "AzureBlobRenameThread-" + Thread.currentThread().getName() + "-" + i); - } - } - - /* - * Helper method to do delete operation and validate all files in source folder - * doesn't exists after delete operation. - */ - private void validateDeleteFolder(FileSystem fs, String source) throws Exception { - // Create folder with files. - createFolder(fs, "root"); - Path sourceFolder = new Path(source); - - // Delete operation - assertTrue(fs.delete(sourceFolder, true)); - assertFalse(fs.exists(sourceFolder)); - - for (int i = 0; i < this.iterations; i++) { - // check that source folder and files doesn't exists - assertFalse(fs.exists(new Path(source + "/" + i))); - assertFalse(fs.exists(new Path(source + "/" + i + "/fileToRename"))); - assertFalse(fs.exists(new Path(source + "/" + i + "/file/to/rename"))); - assertFalse(fs.exists(new Path(source + "/" + i + "/file+to%rename"))); - assertFalse(fs.exists(new Path(source + "/fileToRename" + i))); - } - } - - /* - * Test case for delete operation with multiple threads and flat listing enabled. - */ - @Test - public void testDeleteSmallFolderWithThreads() throws Exception { - - validateDeleteFolder(fs, "root"); - - // With single iteration, we would have created 7 blobs. - int expectedThreadsCreated = Math.min(7, deleteThreads); - - // Validate from logs that threads are enabled. - String content = logs.getOutput(); - assertInLog(content, "ms with threads: " + expectedThreadsCreated); - - // Validate thread executions - for (int i = 0; i < expectedThreadsCreated; i++) { - assertInLog(content, - "AzureBlobDeleteThread-" + Thread.currentThread().getName() + "-" + i); - } - - // Also ensure that we haven't spawned extra threads. - if (expectedThreadsCreated < deleteThreads) { - for (int i = expectedThreadsCreated; i < deleteThreads; i++) { - assertNotInLog(content, - "AzureBlobDeleteThread-" + Thread.currentThread().getName() + "-" + i); - } - } - } - - /* - * Test case for delete operation with multiple threads and flat listing enabled. - */ - @Test - public void testDeleteLargeFolderWithThreads() throws Exception { - // Populate source folder with large number of files and directories. - this.iterations = 10; - validateDeleteFolder(fs, "root"); - - // Validate from logs that threads are enabled. - String content = logs.getOutput(); - assertInLog(content, "ms with threads: " + deleteThreads); - - // Validate thread executions - for (int i = 0; i < deleteThreads; i++) { - assertInLog(content, - "AzureBlobDeleteThread-" + Thread.currentThread().getName() + "-" + i); - } - } - - /* - * Test case for delete operation with threads disabled and flat listing enabled. - */ - @Test - public void testDeleteLargeFolderDisableThreads() throws Exception { - Configuration conf = fs.getConf(); - conf.setInt(NativeAzureFileSystem.AZURE_DELETE_THREADS, 0); - URI uri = fs.getUri(); - fs.initialize(uri, conf); - - // Populate source folder with large number of files and directories. - this.iterations = 10; - validateDeleteFolder(fs, "root"); - - // Validate from logs that threads are disabled. - String content = logs.getOutput(); - assertInLog(content, - "Disabling threads for Delete operation as thread count 0"); - - // Validate no thread executions - for (int i = 0; i < deleteThreads; i++) { - assertNotInLog(content, - "AzureBlobDeleteThread-" + Thread.currentThread().getName() + "-" + i); - } - } - - /* - * Test case for rename operation with threads and flat listing disabled. - */ - @Test - public void testDeleteSmallFolderDisableThreadsDisableFlatListing() throws Exception { - Configuration conf = fs.getConf(); - - // Number of threads set to 0 or 1 disables threads. - conf.setInt(NativeAzureFileSystem.AZURE_DELETE_THREADS, 1); - conf.setBoolean(AzureNativeFileSystemStore.KEY_ENABLE_FLAT_LISTING, false); - URI uri = fs.getUri(); - fs.initialize(uri, conf); - - validateDeleteFolder(fs, "root"); - - // Validate from logs that threads are disabled. - String content = logs.getOutput(); - assertInLog(content, - "Disabling threads for Delete operation as thread count 1"); - - // Validate no thread executions - for (int i = 0; i < deleteThreads; i++) { - assertNotInLog(content, - "AzureBlobDeleteThread-" + Thread.currentThread().getName() + "-" + i); - } - } - - /* - * Test case for delete operation with multiple threads and flat listing enabled. - */ - @Test - public void testDeleteThreadPoolExceptionFailure() throws Exception { - - // Spy azure file system object and raise exception for new thread pool - NativeAzureFileSystem mockFs = Mockito.spy((NativeAzureFileSystem) fs); - - String path = mockFs.pathToKey(mockFs.makeAbsolute(new Path("root"))); - - AzureFileSystemThreadPoolExecutor mockThreadPoolExecutor = Mockito.spy( - mockFs.getThreadPoolExecutor(deleteThreads, "AzureBlobDeleteThread", "Delete", - path, NativeAzureFileSystem.AZURE_DELETE_THREADS)); - Mockito.when(mockThreadPoolExecutor.getThreadPool(7)).thenThrow(new Exception()); - - // With single iteration, we would have created 7 blobs resulting 7 threads. - Mockito.when(mockFs.getThreadPoolExecutor(deleteThreads, "AzureBlobDeleteThread", "Delete", - path, NativeAzureFileSystem.AZURE_DELETE_THREADS)).thenReturn(mockThreadPoolExecutor); - - validateDeleteFolder(mockFs, "root"); - - // Validate from logs that threads are disabled. - String content = logs.getOutput(); - assertInLog(content, "Failed to create thread pool with threads"); - assertInLog(content, "Serializing the Delete operation"); - } - - /* - * Test case for delete operation with multiple threads and flat listing enabled. - */ - @Test - public void testDeleteThreadPoolExecuteFailure() throws Exception { - - // Mock thread pool executor to throw exception for all requests. - ThreadPoolExecutor mockThreadExecutor = Mockito.mock(ThreadPoolExecutor.class); - Mockito.doThrow(new RejectedExecutionException()).when(mockThreadExecutor).execute(Mockito.any(Runnable.class)); - - // Spy azure file system object and return mocked thread pool - NativeAzureFileSystem mockFs = Mockito.spy((NativeAzureFileSystem) fs); - - String path = mockFs.pathToKey(mockFs.makeAbsolute(new Path("root"))); - - AzureFileSystemThreadPoolExecutor mockThreadPoolExecutor = Mockito.spy( - mockFs.getThreadPoolExecutor(deleteThreads, "AzureBlobDeleteThread", "Delete", - path, NativeAzureFileSystem.AZURE_DELETE_THREADS)); - Mockito.when(mockThreadPoolExecutor.getThreadPool(7)).thenReturn(mockThreadExecutor); - - // With single iteration, we would have created 7 blobs resulting 7 threads. - Mockito.when(mockFs.getThreadPoolExecutor(deleteThreads, "AzureBlobDeleteThread", "Delete", - path, NativeAzureFileSystem.AZURE_DELETE_THREADS)).thenReturn(mockThreadPoolExecutor); - - validateDeleteFolder(mockFs, "root"); - - // Validate from logs that threads are disabled. - String content = logs.getOutput(); - assertInLog(content, - "Rejected execution of thread for Delete operation on blob"); - assertInLog(content, "Serializing the Delete operation"); - } - - /* - * Test case for delete operation with multiple threads and flat listing enabled. - */ - @Test - public void testDeleteThreadPoolExecuteSingleThreadFailure() throws Exception { - - // Spy azure file system object and return mocked thread pool - NativeAzureFileSystem mockFs = Mockito.spy((NativeAzureFileSystem) fs); - - // Spy a thread pool executor and link it to azure file system object. - String path = mockFs.pathToKey(mockFs.makeAbsolute(new Path("root"))); - AzureFileSystemThreadPoolExecutor mockThreadPoolExecutor = Mockito.spy( - mockFs.getThreadPoolExecutor(deleteThreads, "AzureBlobDeleteThread", "Delete", - path, NativeAzureFileSystem.AZURE_DELETE_THREADS)); - - // With single iteration, we would have created 7 blobs resulting 7 threads. - Mockito.when(mockFs.getThreadPoolExecutor(deleteThreads, "AzureBlobDeleteThread", "Delete", - path, NativeAzureFileSystem.AZURE_DELETE_THREADS)).thenReturn(mockThreadPoolExecutor); - - // Create a thread executor and link it to mocked thread pool executor object. - ThreadPoolExecutor mockThreadExecutor = Mockito.spy(mockThreadPoolExecutor.getThreadPool(7)); - Mockito.when(mockThreadPoolExecutor.getThreadPool(7)).thenReturn(mockThreadExecutor); - - // Mock thread executor to throw exception for all requests. - Mockito.doCallRealMethod().doThrow(new RejectedExecutionException()).when(mockThreadExecutor).execute(Mockito.any(Runnable.class)); - - validateDeleteFolder(mockFs, "root"); - - // Validate from logs that threads are enabled and unused threads. - String content = logs.getOutput(); - assertInLog(content, - "Using thread pool for Delete operation with threads 7"); - assertInLog(content, - "6 threads not used for Delete operation on blob"); - } - - /* - * Test case for delete operation with multiple threads and flat listing enabled. - */ - @Test - public void testDeleteThreadPoolTerminationFailure() throws Exception { - - // Spy azure file system object and return mocked thread pool - NativeAzureFileSystem mockFs = Mockito.spy((NativeAzureFileSystem) fs); - - // Spy a thread pool executor and link it to azure file system object. - String path = mockFs.pathToKey(mockFs.makeAbsolute(new Path("root"))); - AzureFileSystemThreadPoolExecutor mockThreadPoolExecutor = Mockito.spy( - ((NativeAzureFileSystem) fs).getThreadPoolExecutor(deleteThreads, "AzureBlobDeleteThread", "Delete", - path, NativeAzureFileSystem.AZURE_DELETE_THREADS)); - - // Create a thread executor and link it to mocked thread pool executor object. - // Mock thread executor to throw exception for terminating threads. - ThreadPoolExecutor mockThreadExecutor = Mockito.mock(ThreadPoolExecutor.class); - Mockito.doNothing().when(mockThreadExecutor).execute(Mockito.any(Runnable.class)); - Mockito.when(mockThreadExecutor.awaitTermination(Long.MAX_VALUE, TimeUnit.DAYS)).thenThrow(new InterruptedException()); - - Mockito.when(mockThreadPoolExecutor.getThreadPool(7)).thenReturn(mockThreadExecutor); - - // With single iteration, we would have created 7 blobs resulting 7 threads. - Mockito.when(mockFs.getThreadPoolExecutor(deleteThreads, "AzureBlobDeleteThread", "Delete", - path, NativeAzureFileSystem.AZURE_DELETE_THREADS)).thenReturn(mockThreadPoolExecutor); - - createFolder(mockFs, "root"); - Path sourceFolder = new Path("root"); - boolean exception = false; - try { - mockFs.delete(sourceFolder, true); - } catch (IOException e){ - exception = true; - } - - assertTrue(exception); - assertTrue(mockFs.exists(sourceFolder)); - - // Validate from logs that threads are enabled and delete operation is failed. - String content = logs.getOutput(); - assertInLog(content, - "Using thread pool for Delete operation with threads"); - assertInLog(content, "Threads got interrupted Delete blob operation"); - assertInLog(content, - "Delete failed as operation on subfolders and files failed."); - } - - /* - * Validate that when a directory is deleted recursively, the operation succeeds - * even if a child directory delete fails because the directory does not exist. - * This can happen if a child directory is deleted by an external agent while - * the parent is in progress of being deleted recursively. - */ - @Test - public void testRecursiveDirectoryDeleteWhenChildDirectoryDeleted() - throws Exception { - testRecusiveDirectoryDelete(true); - } - - /* - * Validate that when a directory is deleted recursively, the operation succeeds - * even if a file delete fails because it does not exist. - * This can happen if a file is deleted by an external agent while - * the parent directory is in progress of being deleted. - */ - @Test - public void testRecursiveDirectoryDeleteWhenDeletingChildFileReturnsFalse() - throws Exception { - testRecusiveDirectoryDelete(false); - } - - private void testRecusiveDirectoryDelete(boolean useDir) throws Exception { - String childPathToBeDeletedByExternalAgent = (useDir) - ? "root/0" - : "root/0/fileToRename"; - // Spy azure file system object and return false for deleting one file - NativeAzureFileSystem mockFs = Mockito.spy((NativeAzureFileSystem) fs); - String path = mockFs.pathToKey(mockFs.makeAbsolute(new Path( - childPathToBeDeletedByExternalAgent))); - - Answer<Boolean> answer = new Answer<Boolean>() { - public Boolean answer(InvocationOnMock invocation) throws Throwable { - String path = (String) invocation.getArguments()[0]; - boolean isDir = (boolean) invocation.getArguments()[1]; - boolean realResult = fs.deleteFile(path, isDir); - assertTrue(realResult); - boolean fakeResult = false; - return fakeResult; - } - }; - - Mockito.when(mockFs.deleteFile(path, useDir)).thenAnswer(answer); - - createFolder(mockFs, "root"); - Path sourceFolder = new Path("root"); - - assertTrue(mockFs.delete(sourceFolder, true)); - assertFalse(mockFs.exists(sourceFolder)); - - // Validate from logs that threads are enabled, that a child directory was - // deleted by an external caller, and the parent delete operation still - // succeeds. - String content = logs.getOutput(); - assertInLog(content, - "Using thread pool for Delete operation with threads"); - assertInLog(content, String.format("Attempt to delete non-existent %s %s", - useDir ? "directory" : "file", path)); - } - - /* - * Test case for delete operation with multiple threads and flat listing enabled. - */ - @Test - public void testDeleteSingleDeleteException() throws Exception { - - // Spy azure file system object and raise exception for deleting one file - NativeAzureFileSystem mockFs = Mockito.spy((NativeAzureFileSystem) fs); - String path = mockFs.pathToKey(mockFs.makeAbsolute(new Path("root/0"))); - Mockito.doThrow(new IOException()).when(mockFs).deleteFile(path, true); - - createFolder(mockFs, "root"); - Path sourceFolder = new Path("root"); - - boolean exception = false; - try { - mockFs.delete(sourceFolder, true); - } catch (IOException e){ - exception = true; - } - - assertTrue(exception); - assertTrue(mockFs.exists(sourceFolder)); - - // Validate from logs that threads are enabled and delete operation failed. - String content = logs.getOutput(); - assertInLog(content, - "Using thread pool for Delete operation with threads"); - assertInLog(content, - "Encountered Exception for Delete operation for file " + path); - assertInLog(content, - "Terminating execution of Delete operation now as some other thread already got exception or operation failed"); - } - - /* - * Test case for rename operation with multiple threads and flat listing enabled. - */ - @Test - public void testRenameThreadPoolExceptionFailure() throws Exception { - - // Spy azure file system object and raise exception for new thread pool - NativeAzureFileSystem mockFs = Mockito.spy((NativeAzureFileSystem) fs); - - String path = mockFs.pathToKey(mockFs.makeAbsolute(new Path("root"))); - AzureFileSystemThreadPoolExecutor mockThreadPoolExecutor = Mockito.spy( - ((NativeAzureFileSystem) fs).getThreadPoolExecutor(renameThreads, "AzureBlobRenameThread", "Rename", - path, NativeAzureFileSystem.AZURE_RENAME_THREADS)); - Mockito.when(mockThreadPoolExecutor.getThreadPool(7)).thenThrow(new Exception()); - - // With single iteration, we would have created 7 blobs resulting 7 threads. - Mockito.doReturn(mockThreadPoolExecutor).when(mockFs).getThreadPoolExecutor(renameThreads, "AzureBlobRenameThread", "Rename", - path, NativeAzureFileSystem.AZURE_RENAME_THREADS); - - validateRenameFolder(mockFs, "root", "rootnew"); - - // Validate from logs that threads are disabled. - String content = logs.getOutput(); - assertInLog(content, "Failed to create thread pool with threads"); - assertInLog(content, "Serializing the Rename operation"); - } - - /* - * Test case for rename operation with multiple threads and flat listing enabled. - */ - @Test - public void testRenameThreadPoolExecuteFailure() throws Exception { - - // Mock thread pool executor to throw exception for all requests. - ThreadPoolExecutor mockThreadExecutor = Mockito.mock(ThreadPoolExecutor.class); - Mockito.doThrow(new RejectedExecutionException()).when(mockThreadExecutor).execute(Mockito.any(Runnable.class)); - - // Spy azure file system object and return mocked thread pool - NativeAzureFileSystem mockFs = Mockito.spy((NativeAzureFileSystem) fs); - - String path = mockFs.pathToKey(mockFs.makeAbsolute(new Path("root"))); - AzureFileSystemThreadPoolExecutor mockThreadPoolExecutor = Mockito.spy( - mockFs.getThreadPoolExecutor(renameThreads, "AzureBlobRenameThread", "Rename", - path, NativeAzureFileSystem.AZURE_RENAME_THREADS)); - Mockito.when(mockThreadPoolExecutor.getThreadPool(7)).thenReturn(mockThreadExecutor); - - // With single iteration, we would have created 7 blobs resulting 7 threads. - Mockito.when(mockFs.getThreadPoolExecutor(renameThreads, "AzureBlobRenameThread", "Rename", - path, NativeAzureFileSystem.AZURE_RENAME_THREADS)).thenReturn(mockThreadPoolExecutor); - - validateRenameFolder(mockFs, "root", "rootnew"); - - // Validate from logs that threads are disabled. - String content = logs.getOutput(); - assertInLog(content, - "Rejected execution of thread for Rename operation on blob"); - assertInLog(content, "Serializing the Rename operation"); - } - - /* - * Test case for rename operation with multiple threads and flat listing enabled. - */ - @Test - public void testRenameThreadPoolExecuteSingleThreadFailure() throws Exception { - - // Spy azure file system object and return mocked thread pool - NativeAzureFileSystem mockFs = Mockito.spy((NativeAzureFileSystem) fs); - - // Spy a thread pool executor and link it to azure file system object. - String path = mockFs.pathToKey(mockFs.makeAbsolute(new Path("root"))); - AzureFileSystemThreadPoolExecutor mockThreadPoolExecutor = Mockito.spy( - mockFs.getThreadPoolExecutor(renameThreads, "AzureBlobRenameThread", "Rename", - path, NativeAzureFileSystem.AZURE_RENAME_THREADS)); - - // With single iteration, we would have created 7 blobs resulting 7 threads. - Mockito.when(mockFs.getThreadPoolExecutor(renameThreads, "AzureBlobRenameThread", "Rename", - path, NativeAzureFileSystem.AZURE_RENAME_THREADS)).thenReturn(mockThreadPoolExecutor); - - // Create a thread executor and link it to mocked thread pool executor object. - ThreadPoolExecutor mockThreadExecutor = Mockito.spy(mockThreadPoolExecutor.getThreadPool(7)); - Mockito.when(mockThreadPoolExecutor.getThreadPool(7)).thenReturn(mockThreadExecutor); - - // Mock thread executor to throw exception for all requests. - Mockito.doCallRealMethod().doThrow(new RejectedExecutionException()).when(mockThreadExecutor).execute(Mockito.any(Runnable.class)); - - validateRenameFolder(mockFs, "root", "rootnew"); - - // Validate from logs that threads are enabled and unused threads exists. - String content = logs.getOutput(); - assertInLog(content, - "Using thread pool for Rename operation with threads 7"); - assertInLog(content, - "6 threads not used for Rename operation on blob"); - } - - /* - * Test case for rename operation with multiple threads and flat listing enabled. - */ - @Test - public void testRenameThreadPoolTerminationFailure() throws Exception { - - // Spy azure file system object and return mocked thread pool - NativeAzureFileSystem mockFs = Mockito.spy((NativeAzureFileSystem) fs); - - // Spy a thread pool executor and link it to azure file system object. - String path = mockFs.pathToKey(mockFs.makeAbsolute(new Path("root"))); - AzureFileSystemThreadPoolExecutor mockThreadPoolExecutor = Mockito.spy( - mockFs.getThreadPoolExecutor(renameThreads, "AzureBlobRenameThread", "Rename", - path, NativeAzureFileSystem.AZURE_RENAME_THREADS)); - - // With single iteration, we would have created 7 blobs resulting 7 threads. - Mockito.when(mockFs.getThreadPoolExecutor(renameThreads, "AzureBlobRenameThread", "Rename", - path, NativeAzureFileSystem.AZURE_RENAME_THREADS)).thenReturn(mockThreadPoolExecutor); - - // Mock thread executor to throw exception for all requests. - ThreadPoolExecutor mockThreadExecutor = Mockito.mock(ThreadPoolExecutor.class); - Mockito.doNothing().when(mockThreadExecutor).execute(Mockito.any(Runnable.class)); - Mockito.when(mockThreadExecutor.awaitTermination(Long.MAX_VALUE, TimeUnit.DAYS)).thenThrow(new InterruptedException()); - Mockito.when(mockThreadPoolExecutor.getThreadPool(7)).thenReturn(mockThreadExecutor); - - - createFolder(mockFs, "root"); - Path sourceFolder = new Path("root"); - Path destFolder = new Path("rootnew"); - boolean exception = false; - try { - mockFs.rename(sourceFolder, destFolder); - } catch (IOException e){ - exception = true; - } - - assertTrue(exception); - assertTrue(mockFs.exists(sourceFolder)); - - // Validate from logs that threads are enabled and rename operation is failed. - String content = logs.getOutput(); - assertInLog(content, - "Using thread pool for Rename operation with threads"); - assertInLog(content, "Threads got interrupted Rename blob operation"); - assertInLog(content, - "Rename failed as operation on subfolders and files failed."); - } - - /* - * Test case for rename operation with multiple threads and flat listing enabled. - */ - @Test - public void testRenameSingleRenameException() throws Exception { - - // Spy azure file system object and raise exception for deleting one file - Path sourceFolder = new Path("root"); - Path destFolder = new Path("rootnew"); - - // Spy azure file system object and populate rename pending spy object. - NativeAzureFileSystem mockFs = Mockito.spy((NativeAzureFileSystem) fs); - - // Populate data now only such that rename pending spy object would see this data. - createFolder(mockFs, "root"); - - String srcKey = mockFs.pathToKey(mockFs.makeAbsolute(sourceFolder)); - String dstKey = mockFs.pathToKey(mockFs.makeAbsolute(destFolder)); - - FolderRenamePending mockRenameFs = Mockito.spy(mockFs.prepareAtomicFolderRename(srcKey, dstKey)); - Mockito.when(mockFs.prepareAtomicFolderRename(srcKey, dstKey)).thenReturn(mockRenameFs); - String path = mockFs.pathToKey(mockFs.makeAbsolute(new Path("root/0"))); - Mockito.doThrow(new IOException()).when(mockRenameFs).renameFile(Mockito.any(FileMetadata.class)); - - boolean exception = false; - try { - mockFs.rename(sourceFolder, destFolder); - } catch (IOException e){ - exception = true; - } - - assertTrue(exception); - assertTrue(mockFs.exists(sourceFolder)); - - // Validate from logs that threads are enabled and delete operation failed. - String content = logs.getOutput(); - assertInLog(content, - "Using thread pool for Rename operation with threads"); - assertInLog(content, - "Encountered Exception for Rename operation for file " + path); - assertInLog(content, - "Terminating execution of Rename operation now as some other thread already got exception or operation failed"); - } - - @Override - protected AzureBlobStorageTestAccount createTestAccount() throws Exception { - return AzureBlobStorageTestAccount.create(); - } - -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/3fbcf763/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFSAuthWithBlobSpecificKeys.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFSAuthWithBlobSpecificKeys.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFSAuthWithBlobSpecificKeys.java deleted file mode 100644 index 013fd00..0000000 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFSAuthWithBlobSpecificKeys.java +++ /dev/null @@ -1,44 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.fs.azure; - -import org.apache.hadoop.conf.Configuration; - -import static org.apache.hadoop.fs.azure.SecureStorageInterfaceImpl.KEY_USE_CONTAINER_SASKEY_FOR_ALL_ACCESS; - -/** - * Test class to hold all WASB authorization tests that use blob-specific keys - * to access storage. - */ -public class TestNativeAzureFSAuthWithBlobSpecificKeys - extends TestNativeAzureFileSystemAuthorization { - - @Override - public Configuration getConfiguration() { - Configuration conf = super.getConfiguration(); - conf.set(KEY_USE_CONTAINER_SASKEY_FOR_ALL_ACCESS, "false"); - return conf; - } - - @Override - protected AzureBlobStorageTestAccount createTestAccount() throws Exception { - Configuration conf = getConfiguration(); - return AzureBlobStorageTestAccount.create(conf); - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/3fbcf763/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFSAuthorizationCaching.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFSAuthorizationCaching.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFSAuthorizationCaching.java deleted file mode 100644 index 51e0fd1..0000000 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFSAuthorizationCaching.java +++ /dev/null @@ -1,60 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.fs.azure; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.contract.ContractTestUtils; -import org.junit.Test; - -import static org.apache.hadoop.fs.azure.CachingAuthorizer.KEY_AUTH_SERVICE_CACHING_ENABLE; - -/** - * Test class to hold all WASB authorization caching related tests. - */ -public class TestNativeAzureFSAuthorizationCaching - extends TestNativeAzureFileSystemAuthorization { - - private static final int DUMMY_TTL_VALUE = 5000; - - @Override - public Configuration getConfiguration() { - Configuration conf = super.getConfiguration(); - conf.set(KEY_AUTH_SERVICE_CACHING_ENABLE, "true"); - return conf; - } - - @Override - protected AzureBlobStorageTestAccount createTestAccount() throws Exception { - Configuration conf = getConfiguration(); - return AzureBlobStorageTestAccount.create(conf); - } - - /** - * Test to verify cache behavior -- assert that PUT overwrites value if present - */ - @Test - public void testCachePut() throws Throwable { - CachingAuthorizer<String, Integer> cache = new CachingAuthorizer<>(DUMMY_TTL_VALUE, "TEST"); - cache.init(getConfiguration()); - cache.put("TEST", 1); - cache.put("TEST", 3); - int result = cache.get("TEST"); - ContractTestUtils.assertTrue("Cache returned unexpected result", result == 3); - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/3fbcf763/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFSPageBlobLive.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFSPageBlobLive.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFSPageBlobLive.java deleted file mode 100644 index 208cff3..0000000 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFSPageBlobLive.java +++ /dev/null @@ -1,43 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.fs.azure; - -import org.apache.hadoop.conf.Configuration; - -/** - * Run the base Azure file system tests strictly on page blobs to make sure fundamental - * operations on page blob files and folders work as expected. - * These operations include create, delete, rename, list, and so on. - */ -public class TestNativeAzureFSPageBlobLive extends - NativeAzureFileSystemBaseTest { - - @Override - protected AzureBlobStorageTestAccount createTestAccount() - throws Exception { - Configuration conf = new Configuration(); - - // Configure the page blob directories key so every file created is a page blob. - conf.set(AzureNativeFileSystemStore.KEY_PAGE_BLOB_DIRECTORIES, "/"); - - // Configure the atomic rename directories key so every folder will have - // atomic rename applied. - conf.set(AzureNativeFileSystemStore.KEY_ATOMIC_RENAME_DIRECTORIES, "/"); - return AzureBlobStorageTestAccount.create(conf); - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/3fbcf763/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemAppend.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemAppend.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemAppend.java deleted file mode 100644 index a2b35cb..0000000 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemAppend.java +++ /dev/null @@ -1,362 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.fs.azure; - -import java.io.IOException; -import java.net.URI; -import java.util.Arrays; - -import org.apache.commons.lang.RandomStringUtils; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FSDataInputStream; -import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.test.GenericTestUtils; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; - -public class TestNativeAzureFileSystemAppend extends AbstractWasbTestBase { - - private static final String TEST_FILE = "test.dat"; - private static final Path TEST_PATH = new Path(TEST_FILE); - - private AzureBlobStorageTestAccount testAccount = null; - - @Before - public void setUp() throws Exception { - super.setUp(); - testAccount = createTestAccount(); - fs = testAccount.getFileSystem(); - Configuration conf = fs.getConf(); - conf.setBoolean(NativeAzureFileSystem.APPEND_SUPPORT_ENABLE_PROPERTY_NAME, true); - URI uri = fs.getUri(); - fs.initialize(uri, conf); - } - - /* - * Helper method that creates test data of size provided by the - * "size" parameter. - */ - private static byte[] getTestData(int size) { - byte[] testData = new byte[size]; - System.arraycopy(RandomStringUtils.randomAlphabetic(size).getBytes(), 0, testData, 0, size); - return testData; - } - - // Helper method to create file and write fileSize bytes of data on it. - private byte[] createBaseFileWithData(int fileSize, Path testPath) throws Throwable { - - FSDataOutputStream createStream = null; - try { - createStream = fs.create(testPath); - byte[] fileData = null; - - if (fileSize != 0) { - fileData = getTestData(fileSize); - createStream.write(fileData); - } - return fileData; - } finally { - if (createStream != null) { - createStream.close(); - } - } - } - - /* - * Helper method to verify a file data equal to "dataLength" parameter - */ - private boolean verifyFileData(int dataLength, byte[] testData, int testDataIndex, - FSDataInputStream srcStream) { - - try { - - byte[] fileBuffer = new byte[dataLength]; - byte[] testDataBuffer = new byte[dataLength]; - - int fileBytesRead = srcStream.read(fileBuffer); - - if (fileBytesRead < dataLength) { - return false; - } - - System.arraycopy(testData, testDataIndex, testDataBuffer, 0, dataLength); - - if (!Arrays.equals(fileBuffer, testDataBuffer)) { - return false; - } - - return true; - - } catch (Exception ex) { - return false; - } - - } - - /* - * Helper method to verify Append on a testFile. - */ - private boolean verifyAppend(byte[] testData, Path testFile) { - - FSDataInputStream srcStream = null; - try { - - srcStream = fs.open(testFile); - int baseBufferSize = 2048; - int testDataSize = testData.length; - int testDataIndex = 0; - - while (testDataSize > baseBufferSize) { - - if (!verifyFileData(baseBufferSize, testData, testDataIndex, srcStream)) { - return false; - } - testDataIndex += baseBufferSize; - testDataSize -= baseBufferSize; - } - - if (!verifyFileData(testDataSize, testData, testDataIndex, srcStream)) { - return false; - } - - return true; - } catch(Exception ex) { - return false; - } finally { - if (srcStream != null) { - try { - srcStream.close(); - } catch(IOException ioe) { - // Swallowing - } - } - } - } - - /* - * Test case to verify if an append on small size data works. This tests - * append E2E - */ - @Test - public void testSingleAppend() throws Throwable{ - - FSDataOutputStream appendStream = null; - try { - int baseDataSize = 50; - byte[] baseDataBuffer = createBaseFileWithData(baseDataSize, TEST_PATH); - - int appendDataSize = 20; - byte[] appendDataBuffer = getTestData(appendDataSize); - appendStream = fs.append(TEST_PATH, 10); - appendStream.write(appendDataBuffer); - appendStream.close(); - byte[] testData = new byte[baseDataSize + appendDataSize]; - System.arraycopy(baseDataBuffer, 0, testData, 0, baseDataSize); - System.arraycopy(appendDataBuffer, 0, testData, baseDataSize, appendDataSize); - - Assert.assertTrue(verifyAppend(testData, TEST_PATH)); - } finally { - if (appendStream != null) { - appendStream.close(); - } - } - } - - /* - * Test case to verify append to an empty file. - */ - @Test - public void testSingleAppendOnEmptyFile() throws Throwable { - - FSDataOutputStream appendStream = null; - - try { - createBaseFileWithData(0, TEST_PATH); - - int appendDataSize = 20; - byte[] appendDataBuffer = getTestData(appendDataSize); - appendStream = fs.append(TEST_PATH, 10); - appendStream.write(appendDataBuffer); - appendStream.close(); - - Assert.assertTrue(verifyAppend(appendDataBuffer, TEST_PATH)); - } finally { - if (appendStream != null) { - appendStream.close(); - } - } - } - - /* - * Test to verify that we can open only one Append stream on a File. - */ - @Test - public void testSingleAppenderScenario() throws Throwable { - - FSDataOutputStream appendStream1 = null; - FSDataOutputStream appendStream2 = null; - IOException ioe = null; - try { - createBaseFileWithData(0, TEST_PATH); - appendStream1 = fs.append(TEST_PATH, 10); - boolean encounteredException = false; - try { - appendStream2 = fs.append(TEST_PATH, 10); - } catch(IOException ex) { - encounteredException = true; - ioe = ex; - } - - appendStream1.close(); - - Assert.assertTrue(encounteredException); - GenericTestUtils.assertExceptionContains("Unable to set Append lease on the Blob", ioe); - } finally { - if (appendStream1 != null) { - appendStream1.close(); - } - - if (appendStream2 != null) { - appendStream2.close(); - } - } - } - - /* - * Tests to verify multiple appends on a Blob. - */ - @Test - public void testMultipleAppends() throws Throwable { - - int baseDataSize = 50; - byte[] baseDataBuffer = createBaseFileWithData(baseDataSize, TEST_PATH); - - int appendDataSize = 100; - int targetAppendCount = 50; - byte[] testData = new byte[baseDataSize + (appendDataSize*targetAppendCount)]; - int testDataIndex = 0; - System.arraycopy(baseDataBuffer, 0, testData, testDataIndex, baseDataSize); - testDataIndex += baseDataSize; - - int appendCount = 0; - - FSDataOutputStream appendStream = null; - - try { - while (appendCount < targetAppendCount) { - - byte[] appendDataBuffer = getTestData(appendDataSize); - appendStream = fs.append(TEST_PATH, 30); - appendStream.write(appendDataBuffer); - appendStream.close(); - - System.arraycopy(appendDataBuffer, 0, testData, testDataIndex, appendDataSize); - testDataIndex += appendDataSize; - appendCount++; - } - - Assert.assertTrue(verifyAppend(testData, TEST_PATH)); - - } finally { - if (appendStream != null) { - appendStream.close(); - } - } - } - - /* - * Test to verify we multiple appends on the same stream. - */ - @Test - public void testMultipleAppendsOnSameStream() throws Throwable { - - int baseDataSize = 50; - byte[] baseDataBuffer = createBaseFileWithData(baseDataSize, TEST_PATH); - int appendDataSize = 100; - int targetAppendCount = 50; - byte[] testData = new byte[baseDataSize + (appendDataSize*targetAppendCount)]; - int testDataIndex = 0; - System.arraycopy(baseDataBuffer, 0, testData, testDataIndex, baseDataSize); - testDataIndex += baseDataSize; - int appendCount = 0; - - FSDataOutputStream appendStream = null; - - try { - - while (appendCount < targetAppendCount) { - - appendStream = fs.append(TEST_PATH, 50); - - int singleAppendChunkSize = 20; - int appendRunSize = 0; - while (appendRunSize < appendDataSize) { - - byte[] appendDataBuffer = getTestData(singleAppendChunkSize); - appendStream.write(appendDataBuffer); - System.arraycopy(appendDataBuffer, 0, testData, - testDataIndex + appendRunSize, singleAppendChunkSize); - - appendRunSize += singleAppendChunkSize; - } - - appendStream.close(); - testDataIndex += appendDataSize; - appendCount++; - } - - Assert.assertTrue(verifyAppend(testData, TEST_PATH)); - } finally { - if (appendStream != null) { - appendStream.close(); - } - } - } - - @Test(expected=UnsupportedOperationException.class) - /* - * Test to verify the behavior when Append Support configuration flag is set to false - */ - public void testFalseConfigurationFlagBehavior() throws Throwable { - - fs = testAccount.getFileSystem(); - Configuration conf = fs.getConf(); - conf.setBoolean(NativeAzureFileSystem.APPEND_SUPPORT_ENABLE_PROPERTY_NAME, false); - URI uri = fs.getUri(); - fs.initialize(uri, conf); - - FSDataOutputStream appendStream = null; - - try { - createBaseFileWithData(0, TEST_PATH); - appendStream = fs.append(TEST_PATH, 10); - } finally { - if (appendStream != null) { - appendStream.close(); - } - } - } - - @Override - protected AzureBlobStorageTestAccount createTestAccount() throws Exception { - return AzureBlobStorageTestAccount.create(); - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/3fbcf763/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemAtomicRenameDirList.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemAtomicRenameDirList.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemAtomicRenameDirList.java deleted file mode 100644 index 602c1f7..0000000 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemAtomicRenameDirList.java +++ /dev/null @@ -1,50 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.fs.azure; - -import java.io.IOException; -import java.net.URI; - -import org.apache.hadoop.conf.Configuration; -import org.junit.Test; - -public class TestNativeAzureFileSystemAtomicRenameDirList - extends AbstractWasbTestBase { - private AzureBlobStorageTestAccount testAccount; - - // HBase-site config controlling HBase root dir - private static final String HBASE_ROOT_DIR_CONF_STRING = "hbase.rootdir"; - private static final String HBASE_ROOT_DIR_VALUE_ON_DIFFERENT_FS = "wasb://somedifferentfilesystem.blob.core.windows.net/hbase"; - @Override - protected AzureBlobStorageTestAccount createTestAccount() throws Exception { - testAccount = AzureBlobStorageTestAccount.create(); - return testAccount; - } - - @Test - public void testAzureNativeStoreIsAtomicRenameKeyDoesNotThrowNPEOnInitializingWithNonDefaultURI () throws IOException { - NativeAzureFileSystem azureFs = (NativeAzureFileSystem)fs; - AzureNativeFileSystemStore azureStore = azureFs.getStore(); - Configuration conf = fs.getConf(); - conf.set(HBASE_ROOT_DIR_CONF_STRING, HBASE_ROOT_DIR_VALUE_ON_DIFFERENT_FS); - URI uri = fs.getUri(); - fs.initialize(uri, conf); - azureStore.isAtomicRenameKey("anyrandomkey"); - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/3fbcf763/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemAuthorization.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemAuthorization.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemAuthorization.java index d0e78ae..21c9ef3 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemAuthorization.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemAuthorization.java @@ -41,7 +41,6 @@ import org.apache.hadoop.test.LambdaTestUtils; import org.apache.hadoop.util.StringUtils; import org.junit.Assume; -import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -51,7 +50,7 @@ import com.google.common.annotations.VisibleForTesting; import static org.apache.hadoop.fs.azure.AzureNativeFileSystemStore.KEY_USE_SECURE_MODE; import static org.apache.hadoop.fs.azure.CachingAuthorizer.KEY_AUTH_SERVICE_CACHING_ENABLE; import static org.apache.hadoop.fs.contract.ContractTestUtils.*; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; /** * Test class to hold all WASB authorization tests. @@ -72,8 +71,8 @@ public class TestNativeAzureFileSystemAuthorization protected static final String WRITE = WasbAuthorizationOperations.WRITE.toString(); @Override - public Configuration getConfiguration() { - Configuration conf = super.getConfiguration(); + public Configuration createConfiguration() { + Configuration conf = super.createConfiguration(); conf.set(NativeAzureFileSystem.KEY_AZURE_AUTHORIZATION, "true"); conf.set(KEY_USE_SECURE_MODE, "true"); conf.set(RemoteWasbAuthorizerImpl.KEY_REMOTE_AUTH_SERVICE_URLS, "http://localhost/";); @@ -86,13 +85,12 @@ public class TestNativeAzureFileSystemAuthorization @Override protected AzureBlobStorageTestAccount createTestAccount() throws Exception { - Configuration conf = getConfiguration(); - return AzureBlobStorageTestAccount.create(conf); + return AzureBlobStorageTestAccount.create(createConfiguration()); } - - @Before - public void beforeMethod() { + @Override + public void setUp() throws Exception { + super.setUp(); boolean useSecureMode = fs.getConf().getBoolean(KEY_USE_SECURE_MODE, false); boolean useAuthorization = fs.getConf().getBoolean(NativeAzureFileSystem.KEY_AZURE_AUTHORIZATION, false); Assume.assumeTrue("Test valid when both SecureMode and Authorization are enabled .. skipping", @@ -103,7 +101,6 @@ public class TestNativeAzureFileSystemAuthorization fs.updateWasbAuthorizer(authorizer); } - @Rule public ExpectedException expectedEx = ExpectedException.none(); @@ -124,7 +121,7 @@ public class TestNativeAzureFileSystemAuthorization } /** - * Setup the expected exception class, and exception message that the test is supposed to fail with + * Setup the expected exception class, and exception message that the test is supposed to fail with. */ protected void setExpectedFailureMessage(String operation, Path path) { expectedEx.expect(WasbAuthorizationException.class); @@ -164,7 +161,7 @@ public class TestNativeAzureFileSystemAuthorization } /** - * Positive test to verify Create access check + * Positive test to verify Create access check. * The test tries to create a file whose parent is non-existent to ensure that * the intermediate folders between ancestor and direct parent are being created * when proper ranger policies are configured. @@ -191,7 +188,7 @@ public class TestNativeAzureFileSystemAuthorization /** - * Negative test to verify that create fails when trying to overwrite an existing file + * Negative test to verify that create fails when trying to overwrite an existing file. * without proper write permissions on the file being overwritten. * @throws Throwable */ @@ -217,7 +214,7 @@ public class TestNativeAzureFileSystemAuthorization } /** - * Positive test to verify that create succeeds when trying to overwrite an existing file + * Positive test to verify that create succeeds when trying to overwrite an existing file. * when proper write permissions on the file being overwritten are provided. * @throws Throwable */ @@ -268,7 +265,7 @@ public class TestNativeAzureFileSystemAuthorization } /** - * Positive test to verify listStatus access check + * Positive test to verify listStatus access check. * @throws Throwable */ @Test @@ -293,7 +290,7 @@ public class TestNativeAzureFileSystemAuthorization } /** - * Negative test to verify listStatus access check + * Negative test to verify listStatus access check. * @throws Throwable */ @@ -380,7 +377,7 @@ public class TestNativeAzureFileSystemAuthorization } /** - * Negative test to verify rename access check - the dstFolder disallows rename + * Negative test to verify rename access check - the dstFolder disallows rename. * @throws Throwable */ @Test //(expected=WasbAuthorizationException.class) @@ -411,7 +408,7 @@ public class TestNativeAzureFileSystemAuthorization } /** - * Positive test to verify rename access check - the dstFolder allows rename + * Positive test to verify rename access check - the dstFolder allows rename. * @throws Throwable */ @Test @@ -810,7 +807,7 @@ public class TestNativeAzureFileSystemAuthorization } /** - * Positive test to verify file delete access check + * Positive test to verify file delete access check. * @throws Throwable */ @Test @@ -832,7 +829,7 @@ public class TestNativeAzureFileSystemAuthorization } /** - * Negative test to verify file delete access check + * Negative test to verify file delete access check. * @throws Throwable */ @Test //(expected=WasbAuthorizationException.class) @@ -870,7 +867,7 @@ public class TestNativeAzureFileSystemAuthorization /** * Positive test to verify file delete access check, with intermediate folders - * Uses wildcard recursive permissions + * Uses wildcard recursive permissions. * @throws Throwable */ @Test @@ -1270,7 +1267,7 @@ public class TestNativeAzureFileSystemAuthorization } /** - * Positive test for mkdirs access check + * Positive test for mkdirs access check. * @throws Throwable */ @Test @@ -1357,7 +1354,7 @@ public class TestNativeAzureFileSystemAuthorization } } /** - * Negative test for mkdirs access check + * Negative test for mkdirs access check. * @throws Throwable */ @Test //(expected=WasbAuthorizationException.class) @@ -1381,7 +1378,7 @@ public class TestNativeAzureFileSystemAuthorization } /** - * Positive test triple slash format (wasb:///) access check + * Positive test triple slash format (wasb:///) access check. * @throws Throwable */ @Test @@ -1478,7 +1475,7 @@ public class TestNativeAzureFileSystemAuthorization } /** - * Negative test for setOwner when Authorization is enabled + * Negative test for setOwner when Authorization is enabled. */ @Test public void testSetOwnerThrowsForUnauthorisedUsers() throws Throwable { @@ -1515,7 +1512,7 @@ public class TestNativeAzureFileSystemAuthorization /** * Test for setOwner when Authorization is enabled and - * the user is specified in chown allowed user list + * the user is specified in chown allowed user list. * */ @Test public void testSetOwnerSucceedsForAuthorisedUsers() throws Throwable { @@ -1556,7 +1553,7 @@ public class TestNativeAzureFileSystemAuthorization /** * Test for setOwner when Authorization is enabled and - * the userlist is specified as '*' + * the userlist is specified as '*'. * */ @Test public void testSetOwnerSucceedsForAnyUserWhenWildCardIsSpecified() throws Throwable { @@ -1598,7 +1595,7 @@ public class TestNativeAzureFileSystemAuthorization } /** Test for setOwner throws for illegal setup of chown - * allowed testSetOwnerSucceedsForAuthorisedUsers + * allowed testSetOwnerSucceedsForAuthorisedUsers. */ @Test public void testSetOwnerFailsForIllegalSetup() throws Throwable { http://git-wip-us.apache.org/repos/asf/hadoop/blob/3fbcf763/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemBlockLocations.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemBlockLocations.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemBlockLocations.java index b2660bb..b280cac 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemBlockLocations.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemBlockLocations.java @@ -18,8 +18,6 @@ package org.apache.hadoop.fs.azure; -import static org.junit.Assert.assertEquals; - import java.io.OutputStream; import org.apache.hadoop.conf.Configuration; @@ -29,7 +27,11 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.junit.Test; -public class TestNativeAzureFileSystemBlockLocations { +/** + * Test block location logic. + */ +public class TestNativeAzureFileSystemBlockLocations + extends AbstractWasbTestWithTimeout { @Test public void testNumberOfBlocks() throws Exception { Configuration conf = new Configuration(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/3fbcf763/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemClientLogging.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemClientLogging.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemClientLogging.java deleted file mode 100644 index 4114e60..0000000 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemClientLogging.java +++ /dev/null @@ -1,140 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.fs.azure; - -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -import java.net.URI; -import java.util.StringTokenizer; - -import org.apache.commons.logging.impl.Log4JLogger; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.test.GenericTestUtils.LogCapturer; -import org.apache.log4j.Logger; -import org.junit.Test; - -/** - * Test to validate Azure storage client side logging. Tests works only when - * testing with Live Azure storage because Emulator does not have support for - * client-side logging. - * - */ -public class TestNativeAzureFileSystemClientLogging - extends AbstractWasbTestBase { - - private AzureBlobStorageTestAccount testAccount; - - // Core-site config controlling Azure Storage Client logging - private static final String KEY_LOGGING_CONF_STRING = "fs.azure.storage.client.logging"; - - // Temporary directory created using WASB. - private static final String TEMP_DIR = "tempDir"; - - /* - * Helper method to verify the client logging is working. This check primarily - * checks to make sure we see a line in the logs corresponding to the entity - * that is created during test run. - */ - private boolean verifyStorageClientLogs(String capturedLogs, String entity) - throws Exception { - - URI uri = testAccount.getRealAccount().getBlobEndpoint(); - String container = testAccount.getRealContainer().getName(); - String validateString = uri + Path.SEPARATOR + container + Path.SEPARATOR - + entity; - boolean entityFound = false; - - StringTokenizer tokenizer = new StringTokenizer(capturedLogs, "\n"); - - while (tokenizer.hasMoreTokens()) { - String token = tokenizer.nextToken(); - if (token.contains(validateString)) { - entityFound = true; - break; - } - } - return entityFound; - } - - /* - * Helper method that updates the core-site config to enable/disable logging. - */ - private void updateFileSystemConfiguration(Boolean loggingFlag) - throws Exception { - - Configuration conf = fs.getConf(); - conf.set(KEY_LOGGING_CONF_STRING, loggingFlag.toString()); - URI uri = fs.getUri(); - fs.initialize(uri, conf); - } - - // Using WASB code to communicate with Azure Storage. - private void performWASBOperations() throws Exception { - - Path tempDir = new Path(Path.SEPARATOR + TEMP_DIR); - fs.mkdirs(tempDir); - fs.delete(tempDir, true); - } - - @Test - public void testLoggingEnabled() throws Exception { - - LogCapturer logs = LogCapturer.captureLogs(new Log4JLogger(Logger - .getRootLogger())); - - // Update configuration based on the Test. - updateFileSystemConfiguration(true); - - performWASBOperations(); - - String output = getLogOutput(logs); - assertTrue("Log entry " + TEMP_DIR + " not found in " + output, - verifyStorageClientLogs(output, TEMP_DIR)); - } - - protected String getLogOutput(LogCapturer logs) { - String output = logs.getOutput(); - assertTrue("No log created/captured", !output.isEmpty()); - return output; - } - - @Test - public void testLoggingDisabled() throws Exception { - - LogCapturer logs = LogCapturer.captureLogs(new Log4JLogger(Logger - .getRootLogger())); - - // Update configuration based on the Test. - updateFileSystemConfiguration(false); - - performWASBOperations(); - String output = getLogOutput(logs); - - assertFalse("Log entry " + TEMP_DIR + " found in " + output, - verifyStorageClientLogs(output, TEMP_DIR)); - } - - @Override - protected AzureBlobStorageTestAccount createTestAccount() throws Exception { - testAccount = AzureBlobStorageTestAccount.create(); - return testAccount; - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/3fbcf763/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemConcurrency.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemConcurrency.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemConcurrency.java index cbfc563..655ae90 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemConcurrency.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemConcurrency.java @@ -18,11 +18,6 @@ package org.apache.hadoop.fs.azure; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; - import java.io.OutputStream; import java.io.PrintWriter; import java.io.StringWriter; @@ -33,32 +28,30 @@ import java.util.concurrent.ConcurrentLinkedQueue; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.util.StringUtils; -import org.junit.After; -import org.junit.Before; import org.junit.Test; -public class TestNativeAzureFileSystemConcurrency { - private AzureBlobStorageTestAccount testAccount; - private FileSystem fs; +public class TestNativeAzureFileSystemConcurrency extends AbstractWasbTestBase { private InMemoryBlockBlobStore backingStore; - @Before + @Override public void setUp() throws Exception { - testAccount = AzureBlobStorageTestAccount.createMock(); - fs = testAccount.getFileSystem(); - backingStore = testAccount.getMockStorage().getBackingStore(); + super.setUp(); + backingStore = getTestAccount().getMockStorage().getBackingStore(); } - @After + @Override public void tearDown() throws Exception { - testAccount.cleanup(); - fs = null; + super.tearDown(); backingStore = null; } + @Override + protected AzureBlobStorageTestAccount createTestAccount() throws Exception { + return AzureBlobStorageTestAccount.createMock(); + } + @Test public void testLinkBlobs() throws Exception { Path filePath = new Path("/inProgress"); --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
