http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d2d97fa/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemAuthorization.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemAuthorization.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemAuthorization.java index a3f2843..4bf6f04 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemAuthorization.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemAuthorization.java @@ -29,7 +29,6 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.StringUtils; import org.junit.Assume; -import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -49,8 +48,8 @@ public class TestNativeAzureFileSystemAuthorization protected MockWasbAuthorizerImpl authorizer; @Override - public Configuration getConfiguration() { - Configuration conf = super.getConfiguration(); + public Configuration createConfiguration() { + Configuration conf = super.createConfiguration(); conf.set(NativeAzureFileSystem.KEY_AZURE_AUTHORIZATION, "true"); conf.set(RemoteWasbAuthorizerImpl.KEY_REMOTE_AUTH_SERVICE_URLS, "http://localhost/";); conf.set(NativeAzureFileSystem.AZURE_CHOWN_USERLIST_PROPERTY_NAME, "user1 , user2"); @@ -59,13 +58,12 @@ public class TestNativeAzureFileSystemAuthorization @Override protected AzureBlobStorageTestAccount createTestAccount() throws Exception { - Configuration conf = getConfiguration(); - return AzureBlobStorageTestAccount.create(conf); + return AzureBlobStorageTestAccount.create(createConfiguration()); } - - @Before - public void beforeMethod() { + @Override + public void setUp() throws Exception { + super.setUp(); boolean useSecureMode = fs.getConf().getBoolean(KEY_USE_SECURE_MODE, false); boolean useAuthorization = fs.getConf().getBoolean(NativeAzureFileSystem.KEY_AZURE_AUTHORIZATION, false); Assume.assumeTrue("Test valid when both SecureMode and Authorization are enabled .. skipping", @@ -76,7 +74,6 @@ public class TestNativeAzureFileSystemAuthorization fs.updateWasbAuthorizer(authorizer); } - @Rule public ExpectedException expectedEx = ExpectedException.none(); @@ -95,7 +92,7 @@ public class TestNativeAzureFileSystemAuthorization } /** - * Setup the expected exception class, and exception message that the test is supposed to fail with + * Setup the expected exception class, and exception message that the test is supposed to fail with. */ protected void setExpectedFailureMessage(String operation, Path path) { expectedEx.expect(WasbAuthorizationException.class); @@ -104,7 +101,7 @@ public class TestNativeAzureFileSystemAuthorization } /** - * Positive test to verify Create access check + * Positive test to verify Create access check. * The file is created directly under an existing folder. * No intermediate folders need to be created. * @throws Throwable @@ -128,7 +125,7 @@ public class TestNativeAzureFileSystemAuthorization } /** - * Positive test to verify Create access check + * Positive test to verify Create access check. * The test tries to create a file whose parent is non-existent to ensure that * the intermediate folders between ancestor and direct parent are being created * when proper ranger policies are configured. @@ -155,7 +152,7 @@ public class TestNativeAzureFileSystemAuthorization /** - * Negative test to verify that create fails when trying to overwrite an existing file + * Negative test to verify that create fails when trying to overwrite an existing file. * without proper write permissions on the file being overwritten. * @throws Throwable */ @@ -181,7 +178,7 @@ public class TestNativeAzureFileSystemAuthorization } /** - * Positive test to verify that create succeeds when trying to overwrite an existing file + * Positive test to verify that create succeeds when trying to overwrite an existing file. * when proper write permissions on the file being overwritten are provided. * @throws Throwable */ @@ -232,7 +229,7 @@ public class TestNativeAzureFileSystemAuthorization } /** - * Positive test to verify listStatus access check + * Positive test to verify listStatus access check. * @throws Throwable */ @Test @@ -257,7 +254,7 @@ public class TestNativeAzureFileSystemAuthorization } /** - * Negative test to verify listStatus access check + * Negative test to verify listStatus access check. * @throws Throwable */ @@ -342,7 +339,7 @@ public class TestNativeAzureFileSystemAuthorization } /** - * Negative test to verify rename access check - the dstFolder disallows rename + * Negative test to verify rename access check - the dstFolder disallows rename. * @throws Throwable */ @Test //(expected=WasbAuthorizationException.class) @@ -373,7 +370,7 @@ public class TestNativeAzureFileSystemAuthorization } /** - * Positive test to verify rename access check - the dstFolder allows rename + * Positive test to verify rename access check - the dstFolder allows rename. * @throws Throwable */ @Test @@ -484,7 +481,7 @@ public class TestNativeAzureFileSystemAuthorization } /** - * Positive test to verify file delete access check + * Positive test to verify file delete access check. * @throws Throwable */ @Test @@ -506,7 +503,7 @@ public class TestNativeAzureFileSystemAuthorization } /** - * Negative test to verify file delete access check + * Negative test to verify file delete access check. * @throws Throwable */ @Test //(expected=WasbAuthorizationException.class) @@ -544,7 +541,7 @@ public class TestNativeAzureFileSystemAuthorization /** * Positive test to verify file delete access check, with intermediate folders - * Uses wildcard recursive permissions + * Uses wildcard recursive permissions. * @throws Throwable */ @Test @@ -582,7 +579,7 @@ public class TestNativeAzureFileSystemAuthorization } /** - * Positive test for mkdirs access check + * Positive test for mkdirs access check. * @throws Throwable */ @Test @@ -668,7 +665,7 @@ public class TestNativeAzureFileSystemAuthorization } } /** - * Negative test for mkdirs access check + * Negative test for mkdirs access check. * @throws Throwable */ @Test //(expected=WasbAuthorizationException.class) @@ -692,7 +689,7 @@ public class TestNativeAzureFileSystemAuthorization } /** - * Positive test triple slash format (wasb:///) access check + * Positive test triple slash format (wasb:///) access check. * @throws Throwable */ @Test @@ -708,7 +705,7 @@ public class TestNativeAzureFileSystemAuthorization } /** - * Negative test for setOwner when Authorization is enabled + * Negative test for setOwner when Authorization is enabled. */ @Test public void testSetOwnerThrowsForUnauthorisedUsers() throws Throwable { @@ -744,7 +741,7 @@ public class TestNativeAzureFileSystemAuthorization /** * Test for setOwner when Authorization is enabled and - * the user is specified in chown allowed user list + * the user is specified in chown allowed user list. * */ @Test public void testSetOwnerSucceedsForAuthorisedUsers() throws Throwable { @@ -785,7 +782,7 @@ public class TestNativeAzureFileSystemAuthorization /** * Test for setOwner when Authorization is enabled and - * the userlist is specified as '*' + * the userlist is specified as '*'. * */ @Test public void testSetOwnerSucceedsForAnyUserWhenWildCardIsSpecified() throws Throwable { @@ -829,7 +826,7 @@ public class TestNativeAzureFileSystemAuthorization } /** Test for setOwner throws for illegal setup of chown - * allowed testSetOwnerSucceedsForAuthorisedUsers + * allowed testSetOwnerSucceedsForAuthorisedUsers. */ @Test public void testSetOwnerFailsForIllegalSetup() throws Throwable {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d2d97fa/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemAuthorizationWithOwner.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemAuthorizationWithOwner.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemAuthorizationWithOwner.java deleted file mode 100644 index 4bd4633..0000000 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemAuthorizationWithOwner.java +++ /dev/null @@ -1,122 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.fs.azure; - -import org.apache.hadoop.fs.contract.ContractTestUtils; -import org.apache.hadoop.security.UserGroupInformation; -import java.security.PrivilegedExceptionAction; - -import org.apache.hadoop.fs.Path; -import org.junit.Test; -import org.junit.Before; -import static org.junit.Assert.assertEquals; - -/** - * Test class that runs wasb authorization tests with owner check enabled. - */ -public class TestNativeAzureFileSystemAuthorizationWithOwner - extends TestNativeAzureFileSystemAuthorization { - - @Before - public void beforeMethod() { - super.beforeMethod(); - authorizer.init(fs.getConf(), true); - } - - /** - * Test case when owner matches current user - */ - @Test - public void testOwnerPermissionPositive() throws Throwable { - - Path parentDir = new Path("/testOwnerPermissionPositive"); - Path testPath = new Path(parentDir, "test.data"); - - authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), true); - authorizer.addAuthRule(testPath.toString(), WasbAuthorizationOperations.READ.toString(), true); - authorizer.addAuthRule(parentDir.toString(), WasbAuthorizationOperations.WRITE.toString(), true); - // additional rule used for assertPathExists - authorizer.addAuthRule(parentDir.toString(), WasbAuthorizationOperations.READ.toString(), true); - fs.updateWasbAuthorizer(authorizer); - - try { - // creates parentDir with owner as current user - fs.mkdirs(parentDir); - ContractTestUtils.assertPathExists(fs, "parentDir does not exist", parentDir); - - fs.create(testPath); - fs.getFileStatus(testPath); - ContractTestUtils.assertPathExists(fs, "testPath does not exist", testPath); - - } finally { - allowRecursiveDelete(fs, parentDir.toString()); - fs.delete(parentDir, true); - } - } - - /** - * Negative test case for owner does not match current user - */ - @Test - public void testOwnerPermissionNegative() throws Throwable { - expectedEx.expect(WasbAuthorizationException.class); - - Path parentDir = new Path("/testOwnerPermissionNegative"); - Path childDir = new Path(parentDir, "childDir"); - - setExpectedFailureMessage("mkdirs", childDir); - - authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), true); - authorizer.addAuthRule(parentDir.toString(), WasbAuthorizationOperations.WRITE.toString(), true); - - fs.updateWasbAuthorizer(authorizer); - - try{ - fs.mkdirs(parentDir); - UserGroupInformation ugiSuperUser = UserGroupInformation.createUserForTesting( - "testuser", new String[] {}); - - ugiSuperUser.doAs(new PrivilegedExceptionAction<Void>() { - @Override - public Void run() throws Exception { - fs.mkdirs(childDir); - return null; - } - }); - - } finally { - allowRecursiveDelete(fs, parentDir.toString()); - fs.delete(parentDir, true); - } - } - - /** - * Test to verify that retrieving owner information does not - * throw when file/folder does not exist - */ - @Test - public void testRetrievingOwnerDoesNotFailWhenFileDoesNotExist() throws Throwable { - - Path testdirectory = new Path("/testDirectory123454565"); - - String owner = fs.getOwnerForPath(testdirectory); - assertEquals("", owner); - } -} - http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d2d97fa/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemBlockLocations.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemBlockLocations.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemBlockLocations.java index b2660bb..b280cac 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemBlockLocations.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemBlockLocations.java @@ -18,8 +18,6 @@ package org.apache.hadoop.fs.azure; -import static org.junit.Assert.assertEquals; - import java.io.OutputStream; import org.apache.hadoop.conf.Configuration; @@ -29,7 +27,11 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.junit.Test; -public class TestNativeAzureFileSystemBlockLocations { +/** + * Test block location logic. + */ +public class TestNativeAzureFileSystemBlockLocations + extends AbstractWasbTestWithTimeout { @Test public void testNumberOfBlocks() throws Exception { Configuration conf = new Configuration(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d2d97fa/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemClientLogging.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemClientLogging.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemClientLogging.java deleted file mode 100644 index 4114e60..0000000 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemClientLogging.java +++ /dev/null @@ -1,140 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.fs.azure; - -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -import java.net.URI; -import java.util.StringTokenizer; - -import org.apache.commons.logging.impl.Log4JLogger; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.test.GenericTestUtils.LogCapturer; -import org.apache.log4j.Logger; -import org.junit.Test; - -/** - * Test to validate Azure storage client side logging. Tests works only when - * testing with Live Azure storage because Emulator does not have support for - * client-side logging. - * - */ -public class TestNativeAzureFileSystemClientLogging - extends AbstractWasbTestBase { - - private AzureBlobStorageTestAccount testAccount; - - // Core-site config controlling Azure Storage Client logging - private static final String KEY_LOGGING_CONF_STRING = "fs.azure.storage.client.logging"; - - // Temporary directory created using WASB. - private static final String TEMP_DIR = "tempDir"; - - /* - * Helper method to verify the client logging is working. This check primarily - * checks to make sure we see a line in the logs corresponding to the entity - * that is created during test run. - */ - private boolean verifyStorageClientLogs(String capturedLogs, String entity) - throws Exception { - - URI uri = testAccount.getRealAccount().getBlobEndpoint(); - String container = testAccount.getRealContainer().getName(); - String validateString = uri + Path.SEPARATOR + container + Path.SEPARATOR - + entity; - boolean entityFound = false; - - StringTokenizer tokenizer = new StringTokenizer(capturedLogs, "\n"); - - while (tokenizer.hasMoreTokens()) { - String token = tokenizer.nextToken(); - if (token.contains(validateString)) { - entityFound = true; - break; - } - } - return entityFound; - } - - /* - * Helper method that updates the core-site config to enable/disable logging. - */ - private void updateFileSystemConfiguration(Boolean loggingFlag) - throws Exception { - - Configuration conf = fs.getConf(); - conf.set(KEY_LOGGING_CONF_STRING, loggingFlag.toString()); - URI uri = fs.getUri(); - fs.initialize(uri, conf); - } - - // Using WASB code to communicate with Azure Storage. - private void performWASBOperations() throws Exception { - - Path tempDir = new Path(Path.SEPARATOR + TEMP_DIR); - fs.mkdirs(tempDir); - fs.delete(tempDir, true); - } - - @Test - public void testLoggingEnabled() throws Exception { - - LogCapturer logs = LogCapturer.captureLogs(new Log4JLogger(Logger - .getRootLogger())); - - // Update configuration based on the Test. - updateFileSystemConfiguration(true); - - performWASBOperations(); - - String output = getLogOutput(logs); - assertTrue("Log entry " + TEMP_DIR + " not found in " + output, - verifyStorageClientLogs(output, TEMP_DIR)); - } - - protected String getLogOutput(LogCapturer logs) { - String output = logs.getOutput(); - assertTrue("No log created/captured", !output.isEmpty()); - return output; - } - - @Test - public void testLoggingDisabled() throws Exception { - - LogCapturer logs = LogCapturer.captureLogs(new Log4JLogger(Logger - .getRootLogger())); - - // Update configuration based on the Test. - updateFileSystemConfiguration(false); - - performWASBOperations(); - String output = getLogOutput(logs); - - assertFalse("Log entry " + TEMP_DIR + " found in " + output, - verifyStorageClientLogs(output, TEMP_DIR)); - } - - @Override - protected AzureBlobStorageTestAccount createTestAccount() throws Exception { - testAccount = AzureBlobStorageTestAccount.create(); - return testAccount; - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d2d97fa/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemConcurrency.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemConcurrency.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemConcurrency.java index cbfc563..655ae90 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemConcurrency.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemConcurrency.java @@ -18,11 +18,6 @@ package org.apache.hadoop.fs.azure; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; - import java.io.OutputStream; import java.io.PrintWriter; import java.io.StringWriter; @@ -33,32 +28,30 @@ import java.util.concurrent.ConcurrentLinkedQueue; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.util.StringUtils; -import org.junit.After; -import org.junit.Before; import org.junit.Test; -public class TestNativeAzureFileSystemConcurrency { - private AzureBlobStorageTestAccount testAccount; - private FileSystem fs; +public class TestNativeAzureFileSystemConcurrency extends AbstractWasbTestBase { private InMemoryBlockBlobStore backingStore; - @Before + @Override public void setUp() throws Exception { - testAccount = AzureBlobStorageTestAccount.createMock(); - fs = testAccount.getFileSystem(); - backingStore = testAccount.getMockStorage().getBackingStore(); + super.setUp(); + backingStore = getTestAccount().getMockStorage().getBackingStore(); } - @After + @Override public void tearDown() throws Exception { - testAccount.cleanup(); - fs = null; + super.tearDown(); backingStore = null; } + @Override + protected AzureBlobStorageTestAccount createTestAccount() throws Exception { + return AzureBlobStorageTestAccount.createMock(); + } + @Test public void testLinkBlobs() throws Exception { Path filePath = new Path("/inProgress"); http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d2d97fa/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemConcurrencyLive.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemConcurrencyLive.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemConcurrencyLive.java deleted file mode 100644 index 7c5899d..0000000 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemConcurrencyLive.java +++ /dev/null @@ -1,184 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.fs.azure; - - -import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.junit.Assert; -import org.junit.Test; - -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.Callable; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; - -/*** - * Test class to hold all Live Azure storage concurrency tests. - */ -public class TestNativeAzureFileSystemConcurrencyLive - extends AbstractWasbTestBase { - - private static final int THREAD_COUNT = 102; - private static final int TEST_EXECUTION_TIMEOUT = 5000; - @Override - protected AzureBlobStorageTestAccount createTestAccount() throws Exception { - return AzureBlobStorageTestAccount.create(); - } - - /** - * Validate contract for FileSystem.create when overwrite is true and there - * are concurrent callers of FileSystem.delete. An existing file should be - * overwritten, even if the original destination exists but is deleted by an - * external agent during the create operation. - */ - @Test(timeout = TEST_EXECUTION_TIMEOUT) - public void testConcurrentCreateDeleteFile() throws Exception { - Path testFile = new Path("test.dat"); - - List<CreateFileTask> tasks = new ArrayList<>(THREAD_COUNT); - - for (int i = 0; i < THREAD_COUNT; i++) { - tasks.add(new CreateFileTask(fs, testFile)); - } - - ExecutorService es = null; - - try { - es = Executors.newFixedThreadPool(THREAD_COUNT); - - List<Future<Void>> futures = es.invokeAll(tasks); - - for (Future<Void> future : futures) { - Assert.assertTrue(future.isDone()); - - // we are using Callable<V>, so if an exception - // occurred during the operation, it will be thrown - // when we call get - Assert.assertEquals(null, future.get()); - } - } finally { - if (es != null) { - es.shutdownNow(); - } - } - } - - /** - * Validate contract for FileSystem.delete when invoked concurrently. - * One of the threads should successfully delete the file and return true; - * all other threads should return false. - */ - @Test(timeout = TEST_EXECUTION_TIMEOUT) - public void testConcurrentDeleteFile() throws Exception { - Path testFile = new Path("test.dat"); - fs.create(testFile).close(); - - List<DeleteFileTask> tasks = new ArrayList<>(THREAD_COUNT); - - for (int i = 0; i < THREAD_COUNT; i++) { - tasks.add(new DeleteFileTask(fs, testFile)); - } - - ExecutorService es = null; - try { - es = Executors.newFixedThreadPool(THREAD_COUNT); - - List<Future<Boolean>> futures = es.invokeAll(tasks); - - int successCount = 0; - for (Future<Boolean> future : futures) { - Assert.assertTrue(future.isDone()); - - // we are using Callable<V>, so if an exception - // occurred during the operation, it will be thrown - // when we call get - Boolean success = future.get(); - if (success) { - successCount++; - } - } - - Assert.assertEquals( - "Exactly one delete operation should return true.", - 1, - successCount); - } finally { - if (es != null) { - es.shutdownNow(); - } - } - } -} - -abstract class FileSystemTask<V> implements Callable<V> { - private final FileSystem fileSystem; - private final Path path; - - protected FileSystem getFileSystem() { - return this.fileSystem; - } - - protected Path getFilePath() { - return this.path; - } - - FileSystemTask(FileSystem fs, Path p) { - this.fileSystem = fs; - this.path = p; - } - - public abstract V call() throws Exception; -} - -class DeleteFileTask extends FileSystemTask<Boolean> { - - DeleteFileTask(FileSystem fs, Path p) { - super(fs, p); - } - - @Override - public Boolean call() throws Exception { - return this.getFileSystem().delete(this.getFilePath(), false); - } -} - -class CreateFileTask extends FileSystemTask<Void> { - CreateFileTask(FileSystem fs, Path p) { - super(fs, p); - } - - public Void call() throws Exception { - FileSystem fs = getFileSystem(); - Path p = getFilePath(); - - // Create an empty file and close the stream. - FSDataOutputStream stream = fs.create(p, true); - stream.close(); - - // Delete the file. We don't care if delete returns true or false. - // We just want to ensure the file does not exist. - this.getFileSystem().delete(this.getFilePath(), false); - - return null; - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d2d97fa/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemContractEmulator.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemContractEmulator.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemContractEmulator.java deleted file mode 100644 index 217ca81..0000000 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemContractEmulator.java +++ /dev/null @@ -1,48 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.fs.azure; - -import static org.junit.Assume.assumeNotNull; - -import org.apache.hadoop.fs.FileSystemContractBaseTest; -import org.junit.After; -import org.junit.Before; - -public class TestNativeAzureFileSystemContractEmulator extends - FileSystemContractBaseTest { - private AzureBlobStorageTestAccount testAccount; - - @Before - public void setUp() throws Exception { - testAccount = AzureBlobStorageTestAccount.createForEmulator(); - if (testAccount != null) { - fs = testAccount.getFileSystem(); - } - assumeNotNull(fs); - } - - @After - public void tearDown() throws Exception { - if (testAccount != null) { - testAccount.cleanup(); - testAccount = null; - fs = null; - } - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d2d97fa/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemContractLive.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemContractLive.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemContractLive.java deleted file mode 100644 index b546009..0000000 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemContractLive.java +++ /dev/null @@ -1,80 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.fs.azure; - -import static org.junit.Assume.assumeNotNull; - -import org.apache.hadoop.fs.FileSystemContractBaseTest; -import org.junit.After; -import org.junit.Before; -import org.junit.Ignore; -import org.junit.Test; - -public class TestNativeAzureFileSystemContractLive extends - FileSystemContractBaseTest { - private AzureBlobStorageTestAccount testAccount; - - @Before - public void setUp() throws Exception { - testAccount = AzureBlobStorageTestAccount.create(); - if (testAccount != null) { - fs = testAccount.getFileSystem(); - } - assumeNotNull(fs); - } - - @After - public void tearDown() throws Exception { - if (testAccount != null) { - testAccount.cleanup(); - testAccount = null; - fs = null; - } - } - - /** - * The following tests are failing on Azure and the Azure - * file system code needs to be modified to make them pass. - * A separate work item has been opened for this. - */ - @Ignore - @Test - public void testMoveFileUnderParent() throws Throwable { - } - - @Ignore - @Test - public void testRenameFileToSelf() throws Throwable { - } - - @Ignore - @Test - public void testRenameChildDirForbidden() throws Exception { - } - - @Ignore - @Test - public void testMoveDirUnderParent() throws Throwable { - } - - @Ignore - @Test - public void testRenameDirToSelf() throws Throwable { - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d2d97fa/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemContractMocked.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemContractMocked.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemContractMocked.java index f458bb3..2809260 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemContractMocked.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemContractMocked.java @@ -23,6 +23,9 @@ import org.junit.Before; import org.junit.Ignore; import org.junit.Test; +/** + * Mocked testing of FileSystemContractBaseTest. + */ public class TestNativeAzureFileSystemContractMocked extends FileSystemContractBaseTest { http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d2d97fa/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemContractPageBlobLive.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemContractPageBlobLive.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemContractPageBlobLive.java deleted file mode 100644 index 2a88ad2..0000000 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemContractPageBlobLive.java +++ /dev/null @@ -1,93 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.fs.azure; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystemContractBaseTest; -import org.junit.After; -import static org.junit.Assume.assumeNotNull; -import org.junit.Before; -import org.junit.Ignore; -import org.junit.Test; - -public class TestNativeAzureFileSystemContractPageBlobLive extends - FileSystemContractBaseTest { - private AzureBlobStorageTestAccount testAccount; - - private AzureBlobStorageTestAccount createTestAccount() - throws Exception { - Configuration conf = new Configuration(); - - // Configure the page blob directories key so every file created is a page blob. - conf.set(AzureNativeFileSystemStore.KEY_PAGE_BLOB_DIRECTORIES, "/"); - - // Configure the atomic rename directories key so every folder will have - // atomic rename applied. - conf.set(AzureNativeFileSystemStore.KEY_ATOMIC_RENAME_DIRECTORIES, "/"); - return AzureBlobStorageTestAccount.create(conf); - } - - @Before - public void setUp() throws Exception { - testAccount = createTestAccount(); - if (testAccount != null) { - fs = testAccount.getFileSystem(); - } - assumeNotNull(fs); - } - - @After - public void tearDown() throws Exception { - if (testAccount != null) { - testAccount.cleanup(); - testAccount = null; - fs = null; - } - } - - /** - * The following tests are failing on Azure and the Azure - * file system code needs to be modified to make them pass. - * A separate work item has been opened for this. - */ - @Ignore - @Test - public void testMoveFileUnderParent() throws Throwable { - } - - @Ignore - @Test - public void testRenameFileToSelf() throws Throwable { - } - - @Ignore - @Test - public void testRenameChildDirForbidden() throws Exception { - } - - @Ignore - @Test - public void testMoveDirUnderParent() throws Throwable { - } - - @Ignore - @Test - public void testRenameDirToSelf() throws Throwable { - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d2d97fa/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemFileNameCheck.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemFileNameCheck.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemFileNameCheck.java index 82eabaa..0dfbb37 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemFileNameCheck.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemFileNameCheck.java @@ -18,17 +18,11 @@ package org.apache.hadoop.fs.azure; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - import java.io.IOException; import java.util.HashMap; -import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.junit.After; -import org.junit.Before; + import org.junit.Test; /** @@ -38,24 +32,18 @@ import org.junit.Test; * creation/rename of files/directories through WASB that have colons in the * names. */ -public class TestNativeAzureFileSystemFileNameCheck { - private FileSystem fs = null; - private AzureBlobStorageTestAccount testAccount = null; +public class TestNativeAzureFileSystemFileNameCheck extends AbstractWasbTestBase { private String root = null; - @Before + @Override public void setUp() throws Exception { - testAccount = AzureBlobStorageTestAccount.createMock(); - fs = testAccount.getFileSystem(); + super.setUp(); root = fs.getUri().toString(); } - @After - public void tearDown() throws Exception { - testAccount.cleanup(); - root = null; - fs = null; - testAccount = null; + @Override + protected AzureBlobStorageTestAccount createTestAccount() throws Exception { + return AzureBlobStorageTestAccount.createMock(); } @Test @@ -138,4 +126,4 @@ public class TestNativeAzureFileSystemFileNameCheck { fsck.run(new String[] { p.toString() }); return fsck.getPathNameWarning(); } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d2d97fa/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemLive.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemLive.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemLive.java deleted file mode 100644 index 6baba33..0000000 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemLive.java +++ /dev/null @@ -1,242 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.fs.azure; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -import java.util.concurrent.CountDownLatch; - -import org.apache.hadoop.io.IOUtils; -import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.permission.FsPermission; - -import org.junit.Test; - -import com.microsoft.azure.storage.StorageException; - -/* - * Tests the Native Azure file system (WASB) against an actual blob store if - * provided in the environment. - */ -public class TestNativeAzureFileSystemLive extends - NativeAzureFileSystemBaseTest { - - @Override - protected AzureBlobStorageTestAccount createTestAccount() throws Exception { - return AzureBlobStorageTestAccount.create(); - } - - @Test - public void testLazyRenamePendingCanOverwriteExistingFile() - throws Exception { - final String SRC_FILE_KEY = "srcFile"; - final String DST_FILE_KEY = "dstFile"; - Path srcPath = new Path(SRC_FILE_KEY); - FSDataOutputStream srcStream = fs.create(srcPath); - assertTrue(fs.exists(srcPath)); - Path dstPath = new Path(DST_FILE_KEY); - FSDataOutputStream dstStream = fs.create(dstPath); - assertTrue(fs.exists(dstPath)); - NativeAzureFileSystem nfs = (NativeAzureFileSystem)fs; - final String fullSrcKey = nfs.pathToKey(nfs.makeAbsolute(srcPath)); - final String fullDstKey = nfs.pathToKey(nfs.makeAbsolute(dstPath)); - nfs.getStoreInterface().rename(fullSrcKey, fullDstKey, true, null); - assertTrue(fs.exists(dstPath)); - assertFalse(fs.exists(srcPath)); - IOUtils.cleanup(null, srcStream); - IOUtils.cleanup(null, dstStream); - } - /** - * Tests fs.delete() function to delete a blob when another blob is holding a - * lease on it. Delete if called without a lease should fail if another process - * is holding a lease and throw appropriate exception - * This is a scenario that would happen in HMaster startup when it tries to - * clean up the temp dirs while the HMaster process which was killed earlier - * held lease on the blob when doing some DDL operation - */ - @Test - public void testDeleteThrowsExceptionWithLeaseExistsErrorMessage() - throws Exception { - LOG.info("Starting test"); - final String FILE_KEY = "fileWithLease"; - // Create the file - Path path = new Path(FILE_KEY); - fs.create(path); - assertTrue(fs.exists(path)); - NativeAzureFileSystem nfs = (NativeAzureFileSystem)fs; - final String fullKey = nfs.pathToKey(nfs.makeAbsolute(path)); - final AzureNativeFileSystemStore store = nfs.getStore(); - - // Acquire the lease on the file in a background thread - final CountDownLatch leaseAttemptComplete = new CountDownLatch(1); - final CountDownLatch beginningDeleteAttempt = new CountDownLatch(1); - Thread t = new Thread() { - @Override - public void run() { - // Acquire the lease and then signal the main test thread. - SelfRenewingLease lease = null; - try { - lease = store.acquireLease(fullKey); - LOG.info("Lease acquired: " + lease.getLeaseID()); - } catch (AzureException e) { - LOG.warn("Lease acqusition thread unable to acquire lease", e); - } finally { - leaseAttemptComplete.countDown(); - } - - // Wait for the main test thread to signal it will attempt the delete. - try { - beginningDeleteAttempt.await(); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - - // Keep holding the lease past the lease acquisition retry interval, so - // the test covers the case of delete retrying to acquire the lease. - try { - Thread.sleep(SelfRenewingLease.LEASE_ACQUIRE_RETRY_INTERVAL * 3); - } catch (InterruptedException ex) { - Thread.currentThread().interrupt(); - } - - try { - if (lease != null){ - LOG.info("Freeing lease"); - lease.free(); - } - } catch (StorageException se) { - LOG.warn("Unable to free lease.", se); - } - } - }; - - // Start the background thread and wait for it to signal the lease is held. - t.start(); - try { - leaseAttemptComplete.await(); - } catch (InterruptedException ex) { - Thread.currentThread().interrupt(); - } - - // Try to delete the same file - beginningDeleteAttempt.countDown(); - store.delete(fullKey); - - // At this point file SHOULD BE DELETED - assertFalse(fs.exists(path)); - } - - /** - * Check that isPageBlobKey works as expected. This assumes that - * in the test configuration, the list of supported page blob directories - * only includes "pageBlobs". That's why this test is made specific - * to this subclass. - */ - @Test - public void testIsPageBlobKey() { - AzureNativeFileSystemStore store = ((NativeAzureFileSystem) fs).getStore(); - - // Use literal strings so it's easier to understand the tests. - // In case the constant changes, we want to know about it so we can update this test. - assertEquals(AzureBlobStorageTestAccount.DEFAULT_PAGE_BLOB_DIRECTORY, "pageBlobs"); - - // URI prefix for test environment. - String uriPrefix = "file:///"; - - // negative tests - String[] negativeKeys = { "", "/", "bar", "bar/", "bar/pageBlobs", "bar/pageBlobs/foo", - "bar/pageBlobs/foo/", "/pageBlobs/", "/pageBlobs", "pageBlobs", "pageBlobsxyz/" }; - for (String s : negativeKeys) { - assertFalse(store.isPageBlobKey(s)); - assertFalse(store.isPageBlobKey(uriPrefix + s)); - } - - // positive tests - String[] positiveKeys = { "pageBlobs/", "pageBlobs/foo/", "pageBlobs/foo/bar/" }; - for (String s : positiveKeys) { - assertTrue(store.isPageBlobKey(s)); - assertTrue(store.isPageBlobKey(uriPrefix + s)); - } - } - - /** - * Test that isAtomicRenameKey() works as expected. - */ - @Test - public void testIsAtomicRenameKey() { - - AzureNativeFileSystemStore store = ((NativeAzureFileSystem) fs).getStore(); - - // We want to know if the default configuration changes so we can fix - // this test. - assertEquals(AzureBlobStorageTestAccount.DEFAULT_ATOMIC_RENAME_DIRECTORIES, - "/atomicRenameDir1,/atomicRenameDir2"); - - // URI prefix for test environment. - String uriPrefix = "file:///"; - - // negative tests - String[] negativeKeys = { "", "/", "bar", "bar/", "bar/hbase", - "bar/hbase/foo", "bar/hbase/foo/", "/hbase/", "/hbase", "hbase", - "hbasexyz/", "foo/atomicRenameDir1/"}; - for (String s : negativeKeys) { - assertFalse(store.isAtomicRenameKey(s)); - assertFalse(store.isAtomicRenameKey(uriPrefix + s)); - } - - // Positive tests. The directories for atomic rename are /hbase - // plus the ones in the configuration (DEFAULT_ATOMIC_RENAME_DIRECTORIES - // for this test). - String[] positiveKeys = { "hbase/", "hbase/foo/", "hbase/foo/bar/", - "atomicRenameDir1/foo/", "atomicRenameDir2/bar/"}; - for (String s : positiveKeys) { - assertTrue(store.isAtomicRenameKey(s)); - assertTrue(store.isAtomicRenameKey(uriPrefix + s)); - } - } - - /** - * Tests fs.mkdir() function to create a target blob while another thread - * is holding the lease on the blob. mkdir should not fail since the blob - * already exists. - * This is a scenario that would happen in HBase distributed log splitting. - * Multiple threads will try to create and update "recovered.edits" folder - * under the same path. - */ - @Test - public void testMkdirOnExistingFolderWithLease() throws Exception { - SelfRenewingLease lease; - final String FILE_KEY = "folderWithLease"; - // Create the folder - fs.mkdirs(new Path(FILE_KEY)); - NativeAzureFileSystem nfs = (NativeAzureFileSystem) fs; - String fullKey = nfs.pathToKey(nfs.makeAbsolute(new Path(FILE_KEY))); - AzureNativeFileSystemStore store = nfs.getStore(); - // Acquire the lease on the folder - lease = store.acquireLease(fullKey); - assertTrue(lease.getLeaseID() != null); - // Try to create the same folder - store.storeEmptyFolder(fullKey, - nfs.createPermissionStatus(FsPermission.getDirDefault())); - lease.free(); - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d2d97fa/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemMocked.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemMocked.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemMocked.java index aa1e4f7..20d45b2 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemMocked.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemMocked.java @@ -21,6 +21,10 @@ package org.apache.hadoop.fs.azure; import java.io.IOException; import org.junit.Ignore; +/** + * Run {@link NativeAzureFileSystemBaseTest} tests against a mocked store, + * skipping tests of unsupported features + */ public class TestNativeAzureFileSystemMocked extends NativeAzureFileSystemBaseTest { http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d2d97fa/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemUploadLogic.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemUploadLogic.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemUploadLogic.java index 4c2df8d..7f63295 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemUploadLogic.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemUploadLogic.java @@ -18,41 +18,27 @@ package org.apache.hadoop.fs.azure; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; - import java.io.ByteArrayInputStream; import java.io.InputStream; import java.io.OutputStream; import org.apache.hadoop.fs.Path; -import org.junit.After; -import org.junit.Before; import org.junit.Ignore; import org.junit.Test; /** * Tests for the upload, buffering and flush logic in WASB. */ -public class TestNativeAzureFileSystemUploadLogic { - private AzureBlobStorageTestAccount testAccount; +public class TestNativeAzureFileSystemUploadLogic extends AbstractWasbTestBase { // Just an arbitrary number so that the values I write have a predictable // pattern: 0, 1, 2, .. , 45, 46, 0, 1, 2, ... static final int byteValuePeriod = 47; - @Before - public void setUp() throws Exception { - testAccount = AzureBlobStorageTestAccount.createMock(); - } - - @After - public void tearDown() throws Exception { - if (testAccount != null) { - testAccount.cleanup(); - testAccount = null; - } + @Override + protected AzureBlobStorageTestAccount createTestAccount() throws Exception { + return AzureBlobStorageTestAccount.createMock(); } /** @@ -126,9 +112,9 @@ public class TestNativeAzureFileSystemUploadLogic { * @param expectedSize The expected size of the data in there. */ private void assertDataInFile(Path file, int expectedSize) throws Exception { - InputStream inStream = testAccount.getFileSystem().open(file); - assertDataInStream(inStream, expectedSize); - inStream.close(); + try(InputStream inStream = getFileSystem().open(file)) { + assertDataInStream(inStream, expectedSize); + } } /** @@ -139,7 +125,7 @@ public class TestNativeAzureFileSystemUploadLogic { private void assertDataInTempBlob(int expectedSize) throws Exception { // Look for the temporary upload blob in the backing store. InMemoryBlockBlobStore backingStore = - testAccount.getMockStorage().getBackingStore(); + getTestAccount().getMockStorage().getBackingStore(); String tempKey = null; for (String key : backingStore.getKeys()) { if (key.contains(NativeAzureFileSystem.AZURE_TEMP_FOLDER)) { @@ -149,9 +135,10 @@ public class TestNativeAzureFileSystemUploadLogic { } } assertNotNull(tempKey); - InputStream inStream = new ByteArrayInputStream(backingStore.getContent(tempKey)); - assertDataInStream(inStream, expectedSize); - inStream.close(); + try (InputStream inStream = new ByteArrayInputStream( + backingStore.getContent(tempKey))) { + assertDataInStream(inStream, expectedSize); + } } /** @@ -162,25 +149,30 @@ public class TestNativeAzureFileSystemUploadLogic { */ private void testConsistencyAfterManyFlushes(FlushFrequencyVariation variation) throws Exception { - Path uploadedFile = new Path("/uploadedFile"); - OutputStream outStream = testAccount.getFileSystem().create(uploadedFile); - final int totalSize = 9123; - int flushPeriod; - switch (variation) { - case BeforeSingleBufferFull: flushPeriod = 300; break; - case AfterSingleBufferFull: flushPeriod = 600; break; - case AfterAllRingBufferFull: flushPeriod = 1600; break; - default: - throw new IllegalArgumentException("Unknown variation: " + variation); - } - for (int i = 0; i < totalSize; i++) { - outStream.write(i % byteValuePeriod); - if ((i + 1) % flushPeriod == 0) { - outStream.flush(); - assertDataInTempBlob(i + 1); + Path uploadedFile = methodPath(); + try { + OutputStream outStream = getFileSystem().create(uploadedFile); + final int totalSize = 9123; + int flushPeriod; + switch (variation) { + case BeforeSingleBufferFull: flushPeriod = 300; break; + case AfterSingleBufferFull: flushPeriod = 600; break; + case AfterAllRingBufferFull: flushPeriod = 1600; break; + default: + throw new IllegalArgumentException("Unknown variation: " + variation); } + for (int i = 0; i < totalSize; i++) { + outStream.write(i % byteValuePeriod); + if ((i + 1) % flushPeriod == 0) { + outStream.flush(); + assertDataInTempBlob(i + 1); + } + } + outStream.close(); + assertDataInFile(uploadedFile, totalSize); + } finally { + getFileSystem().delete(uploadedFile, false); + } - outStream.close(); - assertDataInFile(uploadedFile, totalSize); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d2d97fa/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestOutOfBandAzureBlobOperations.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestOutOfBandAzureBlobOperations.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestOutOfBandAzureBlobOperations.java index 544d6ab..303a89a 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestOutOfBandAzureBlobOperations.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestOutOfBandAzureBlobOperations.java @@ -18,11 +18,6 @@ package org.apache.hadoop.fs.azure; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; - import java.util.HashMap; import org.apache.hadoop.fs.FileStatus; @@ -37,7 +32,8 @@ import org.junit.Test; * Tests that WASB handles things gracefully when users add blobs to the Azure * Storage container from outside WASB's control. */ -public class TestOutOfBandAzureBlobOperations { +public class TestOutOfBandAzureBlobOperations + extends AbstractWasbTestWithTimeout { private AzureBlobStorageTestAccount testAccount; private FileSystem fs; private InMemoryBlockBlobStore backingStore; http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d2d97fa/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestOutOfBandAzureBlobOperationsLive.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestOutOfBandAzureBlobOperationsLive.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestOutOfBandAzureBlobOperationsLive.java deleted file mode 100644 index 60b01c6..0000000 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestOutOfBandAzureBlobOperationsLive.java +++ /dev/null @@ -1,203 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.fs.azure; - -import static org.junit.Assert.assertTrue; -import static org.junit.Assume.assumeNotNull; - -import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.security.UserGroupInformation; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -import com.microsoft.azure.storage.blob.BlobOutputStream; -import com.microsoft.azure.storage.blob.CloudBlockBlob; - -public class TestOutOfBandAzureBlobOperationsLive { - private FileSystem fs; - private AzureBlobStorageTestAccount testAccount; - - @Before - public void setUp() throws Exception { - testAccount = AzureBlobStorageTestAccount.create(); - if (testAccount != null) { - fs = testAccount.getFileSystem(); - } - assumeNotNull(testAccount); - } - - @After - public void tearDown() throws Exception { - if (testAccount != null) { - testAccount.cleanup(); - testAccount = null; - fs = null; - } - } - - // scenario for this particular test described at MONARCH-HADOOP-764 - // creating a file out-of-band would confuse mkdirs("<oobfilesUncleFolder>") - // eg oob creation of "user/<name>/testFolder/a/input/file" - // Then wasb creation of "user/<name>/testFolder/a/output" fails - @Test - public void outOfBandFolder_uncleMkdirs() throws Exception { - - // NOTE: manual use of CloubBlockBlob targets working directory explicitly. - // WASB driver methods prepend working directory implicitly. - String workingDir = "user/" - + UserGroupInformation.getCurrentUser().getShortUserName() + "/"; - - CloudBlockBlob blob = testAccount.getBlobReference(workingDir - + "testFolder1/a/input/file"); - BlobOutputStream s = blob.openOutputStream(); - s.close(); - assertTrue(fs.exists(new Path("testFolder1/a/input/file"))); - - Path targetFolder = new Path("testFolder1/a/output"); - assertTrue(fs.mkdirs(targetFolder)); - } - - // scenario for this particular test described at MONARCH-HADOOP-764 - @Test - public void outOfBandFolder_parentDelete() throws Exception { - - // NOTE: manual use of CloubBlockBlob targets working directory explicitly. - // WASB driver methods prepend working directory implicitly. - String workingDir = "user/" - + UserGroupInformation.getCurrentUser().getShortUserName() + "/"; - CloudBlockBlob blob = testAccount.getBlobReference(workingDir - + "testFolder2/a/input/file"); - BlobOutputStream s = blob.openOutputStream(); - s.close(); - assertTrue(fs.exists(new Path("testFolder2/a/input/file"))); - - Path targetFolder = new Path("testFolder2/a/input"); - assertTrue(fs.delete(targetFolder, true)); - } - - @Test - public void outOfBandFolder_rootFileDelete() throws Exception { - - CloudBlockBlob blob = testAccount.getBlobReference("fileY"); - BlobOutputStream s = blob.openOutputStream(); - s.close(); - assertTrue(fs.exists(new Path("/fileY"))); - assertTrue(fs.delete(new Path("/fileY"), true)); - } - - @Test - public void outOfBandFolder_firstLevelFolderDelete() throws Exception { - - CloudBlockBlob blob = testAccount.getBlobReference("folderW/file"); - BlobOutputStream s = blob.openOutputStream(); - s.close(); - assertTrue(fs.exists(new Path("/folderW"))); - assertTrue(fs.exists(new Path("/folderW/file"))); - assertTrue(fs.delete(new Path("/folderW"), true)); - } - - // scenario for this particular test described at MONARCH-HADOOP-764 - @Test - public void outOfBandFolder_siblingCreate() throws Exception { - - // NOTE: manual use of CloubBlockBlob targets working directory explicitly. - // WASB driver methods prepend working directory implicitly. - String workingDir = "user/" - + UserGroupInformation.getCurrentUser().getShortUserName() + "/"; - CloudBlockBlob blob = testAccount.getBlobReference(workingDir - + "testFolder3/a/input/file"); - BlobOutputStream s = blob.openOutputStream(); - s.close(); - assertTrue(fs.exists(new Path("testFolder3/a/input/file"))); - - Path targetFile = new Path("testFolder3/a/input/file2"); - FSDataOutputStream s2 = fs.create(targetFile); - s2.close(); - } - - // scenario for this particular test described at MONARCH-HADOOP-764 - // creating a new file in the root folder - @Test - public void outOfBandFolder_create_rootDir() throws Exception { - Path targetFile = new Path("/newInRoot"); - FSDataOutputStream s2 = fs.create(targetFile); - s2.close(); - } - - // scenario for this particular test described at MONARCH-HADOOP-764 - @Test - public void outOfBandFolder_rename() throws Exception { - - // NOTE: manual use of CloubBlockBlob targets working directory explicitly. - // WASB driver methods prepend working directory implicitly. - String workingDir = "user/" - + UserGroupInformation.getCurrentUser().getShortUserName() + "/"; - CloudBlockBlob blob = testAccount.getBlobReference(workingDir - + "testFolder4/a/input/file"); - BlobOutputStream s = blob.openOutputStream(); - s.close(); - - Path srcFilePath = new Path("testFolder4/a/input/file"); - assertTrue(fs.exists(srcFilePath)); - - Path destFilePath = new Path("testFolder4/a/input/file2"); - fs.rename(srcFilePath, destFilePath); - } - - // Verify that you can rename a file which is the only file in an implicit folder in the - // WASB file system. - // scenario for this particular test described at MONARCH-HADOOP-892 - @Test - public void outOfBandSingleFile_rename() throws Exception { - - //NOTE: manual use of CloubBlockBlob targets working directory explicitly. - // WASB driver methods prepend working directory implicitly. - String workingDir = "user/" + UserGroupInformation.getCurrentUser().getShortUserName() + "/"; - CloudBlockBlob blob = testAccount.getBlobReference(workingDir + "testFolder5/a/input/file"); - BlobOutputStream s = blob.openOutputStream(); - s.close(); - - Path srcFilePath = new Path("testFolder5/a/input/file"); - assertTrue(fs.exists(srcFilePath)); - - Path destFilePath = new Path("testFolder5/file2"); - fs.rename(srcFilePath, destFilePath); - } - - // WASB must force explicit parent directories in create, delete, mkdirs, rename. - // scenario for this particular test described at MONARCH-HADOOP-764 - @Test - public void outOfBandFolder_rename_rootLevelFiles() throws Exception { - - // NOTE: manual use of CloubBlockBlob targets working directory explicitly. - // WASB driver methods prepend working directory implicitly. - CloudBlockBlob blob = testAccount.getBlobReference("fileX"); - BlobOutputStream s = blob.openOutputStream(); - s.close(); - - Path srcFilePath = new Path("/fileX"); - assertTrue(fs.exists(srcFilePath)); - - Path destFilePath = new Path("/fileXrename"); - fs.rename(srcFilePath, destFilePath); - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d2d97fa/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestReadAndSeekPageBlobAfterWrite.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestReadAndSeekPageBlobAfterWrite.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestReadAndSeekPageBlobAfterWrite.java deleted file mode 100644 index 41b8386..0000000 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestReadAndSeekPageBlobAfterWrite.java +++ /dev/null @@ -1,355 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.fs.azure; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import static org.junit.Assume.assumeNotNull; - -import java.io.IOException; -import java.io.OutputStream; -import java.util.Arrays; -import java.util.Random; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.fs.FSDataInputStream; -import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.azure.AzureException; -import org.apache.hadoop.util.Time; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -/** - * Write data into a page blob and verify you can read back all of it - * or just a part of it. - */ -public class TestReadAndSeekPageBlobAfterWrite { - private static final Log LOG = LogFactory.getLog(TestReadAndSeekPageBlobAfterWrite.class); - - private FileSystem fs; - private AzureBlobStorageTestAccount testAccount; - private byte[] randomData; - - // Page blob physical page size - private static final int PAGE_SIZE = PageBlobFormatHelpers.PAGE_SIZE; - - // Size of data on page (excluding header) - private static final int PAGE_DATA_SIZE = PAGE_SIZE - PageBlobFormatHelpers.PAGE_HEADER_SIZE; - private static final int MAX_BYTES = 33554432; // maximum bytes in a file that we'll test - private static final int MAX_PAGES = MAX_BYTES / PAGE_SIZE; // maximum number of pages we'll test - private Random rand = new Random(); - - // A key with a prefix under /pageBlobs, which for the test file system will - // force use of a page blob. - private static final String KEY = "/pageBlobs/file.dat"; - private static final Path PATH = new Path(KEY); // path of page blob file to read and write - - protected AzureBlobStorageTestAccount createTestAccount() throws Exception { - return AzureBlobStorageTestAccount.create(); - } - - @Before - public void setUp() throws Exception { - testAccount = createTestAccount(); - if (testAccount != null) { - fs = testAccount.getFileSystem(); - } - assumeNotNull(testAccount); - - // Make sure we are using an integral number of pages. - assertEquals(0, MAX_BYTES % PAGE_SIZE); - - // load an in-memory array of random data - randomData = new byte[PAGE_SIZE * MAX_PAGES]; - rand.nextBytes(randomData); - } - - @After - public void tearDown() throws Exception { - if (testAccount != null) { - testAccount.cleanup(); - testAccount = null; - fs = null; - } - } - - /** - * Make sure the file name (key) is a page blob file name. If anybody changes that, - * we need to come back and update this test class. - */ - @Test - public void testIsPageBlobFileName() { - AzureNativeFileSystemStore store = ((NativeAzureFileSystem) fs).getStore(); - String[] a = KEY.split("/"); - String key2 = a[1] + "/"; - assertTrue(store.isPageBlobKey(key2)); - } - - /** - * For a set of different file sizes, write some random data to a page blob, - * read it back, and compare that what was read is the same as what was written. - */ - @Test - public void testReadAfterWriteRandomData() throws IOException { - - // local shorthand - final int PDS = PAGE_DATA_SIZE; - - // Test for sizes at and near page boundaries - int[] dataSizes = { - - // on first page - 0, 1, 2, 3, - - // Near first physical page boundary (because the implementation - // stores PDS + the page header size bytes on each page). - PDS - 1, PDS, PDS + 1, PDS + 2, PDS + 3, - - // near second physical page boundary - (2 * PDS) - 1, (2 * PDS), (2 * PDS) + 1, (2 * PDS) + 2, (2 * PDS) + 3, - - // near tenth physical page boundary - (10 * PDS) - 1, (10 * PDS), (10 * PDS) + 1, (10 * PDS) + 2, (10 * PDS) + 3, - - // test one big size, >> 4MB (an internal buffer size in the code) - MAX_BYTES - }; - - for (int i : dataSizes) { - testReadAfterWriteRandomData(i); - } - } - - private void testReadAfterWriteRandomData(int size) throws IOException { - writeRandomData(size); - readRandomDataAndVerify(size); - } - - /** - * Read "size" bytes of data and verify that what was read and what was written - * are the same. - */ - private void readRandomDataAndVerify(int size) throws AzureException, IOException { - byte[] b = new byte[size]; - FSDataInputStream stream = fs.open(PATH); - int bytesRead = stream.read(b); - stream.close(); - assertEquals(bytesRead, size); - - // compare the data read to the data written - assertTrue(comparePrefix(randomData, b, size)); - } - - // return true if the beginning "size" values of the arrays are the same - private boolean comparePrefix(byte[] a, byte[] b, int size) { - if (a.length < size || b.length < size) { - return false; - } - for (int i = 0; i < size; i++) { - if (a[i] != b[i]) { - return false; - } - } - return true; - } - - // Write a specified amount of random data to the file path for this test class. - private void writeRandomData(int size) throws IOException { - OutputStream output = fs.create(PATH); - output.write(randomData, 0, size); - output.close(); - } - - /** - * Write data to a page blob, open it, seek, and then read a range of data. - * Then compare that the data read from that range is the same as the data originally written. - */ - @Test - public void testPageBlobSeekAndReadAfterWrite() throws IOException { - writeRandomData(PAGE_SIZE * MAX_PAGES); - int recordSize = 100; - byte[] b = new byte[recordSize]; - FSDataInputStream stream = fs.open(PATH); - - // Seek to a boundary around the middle of the 6th page - int seekPosition = 5 * PAGE_SIZE + 250; - stream.seek(seekPosition); - - // Read a record's worth of bytes and verify results - int bytesRead = stream.read(b); - verifyReadRandomData(b, bytesRead, seekPosition, recordSize); - - // Seek to another spot and read a record greater than a page - seekPosition = 10 * PAGE_SIZE + 250; - stream.seek(seekPosition); - recordSize = 1000; - b = new byte[recordSize]; - bytesRead = stream.read(b); - verifyReadRandomData(b, bytesRead, seekPosition, recordSize); - - // Read the last 100 bytes of the file - recordSize = 100; - seekPosition = PAGE_SIZE * MAX_PAGES - recordSize; - stream.seek(seekPosition); - b = new byte[recordSize]; - bytesRead = stream.read(b); - verifyReadRandomData(b, bytesRead, seekPosition, recordSize); - - // Read past the end of the file and we should get only partial data. - recordSize = 100; - seekPosition = PAGE_SIZE * MAX_PAGES - recordSize + 50; - stream.seek(seekPosition); - b = new byte[recordSize]; - bytesRead = stream.read(b); - assertEquals(50, bytesRead); - - // compare last 50 bytes written with those read - byte[] tail = Arrays.copyOfRange(randomData, seekPosition, randomData.length); - assertTrue(comparePrefix(tail, b, 50)); - } - - // Verify that reading a record of data after seeking gives the expected data. - private void verifyReadRandomData(byte[] b, int bytesRead, int seekPosition, int recordSize) { - byte[] originalRecordData = - Arrays.copyOfRange(randomData, seekPosition, seekPosition + recordSize + 1); - assertEquals(recordSize, bytesRead); - assertTrue(comparePrefix(originalRecordData, b, recordSize)); - } - - // Test many small flushed writes interspersed with periodic hflush calls. - // For manual testing, increase NUM_WRITES to a large number. - // The goal for a long-running manual test is to make sure that it finishes - // and the close() call does not time out. It also facilitates debugging into - // hflush/hsync. - @Test - public void testManySmallWritesWithHFlush() throws IOException { - writeAndReadOneFile(50, 100, 20); - } - - /** - * Write a total of numWrites * recordLength data to a file, read it back, - * and check to make sure what was read is the same as what was written. - * The syncInterval is the number of writes after which to call hflush to - * force the data to storage. - */ - private void writeAndReadOneFile(int numWrites, int recordLength, int syncInterval) throws IOException { - final int NUM_WRITES = numWrites; - final int RECORD_LENGTH = recordLength; - final int SYNC_INTERVAL = syncInterval; - - // A lower bound on the minimum time we think it will take to do - // a write to Azure storage. - final long MINIMUM_EXPECTED_TIME = 20; - LOG.info("Writing " + NUM_WRITES * RECORD_LENGTH + " bytes to " + PATH.getName()); - FSDataOutputStream output = fs.create(PATH); - int writesSinceHFlush = 0; - try { - - // Do a flush and hflush to exercise case for empty write queue in PageBlobOutputStream, - // to test concurrent execution gates. - output.flush(); - output.hflush(); - for (int i = 0; i < NUM_WRITES; i++) { - output.write(randomData, i * RECORD_LENGTH, RECORD_LENGTH); - writesSinceHFlush++; - output.flush(); - if ((i % SYNC_INTERVAL) == 0) { - output.hflush(); - writesSinceHFlush = 0; - } - } - } finally { - long start = Time.monotonicNow(); - output.close(); - long end = Time.monotonicNow(); - LOG.debug("close duration = " + (end - start) + " msec."); - if (writesSinceHFlush > 0) { - assertTrue(String.format( - "close duration with >= 1 pending write is %d, less than minimum expected of %d", - end - start, MINIMUM_EXPECTED_TIME), - end - start >= MINIMUM_EXPECTED_TIME); - } - } - - // Read the data back and check it. - FSDataInputStream stream = fs.open(PATH); - int SIZE = NUM_WRITES * RECORD_LENGTH; - byte[] b = new byte[SIZE]; - try { - stream.seek(0); - stream.read(b, 0, SIZE); - verifyReadRandomData(b, SIZE, 0, SIZE); - } finally { - stream.close(); - } - - // delete the file - fs.delete(PATH, false); - } - - // Test writing to a large file repeatedly as a stress test. - // Set the repetitions to a larger number for manual testing - // for a longer stress run. - @Test - public void testLargeFileStress() throws IOException { - int numWrites = 32; - int recordSize = 1024 * 1024; - int syncInterval = 10; - int repetitions = 1; - for (int i = 0; i < repetitions; i++) { - writeAndReadOneFile(numWrites, recordSize, syncInterval); - } - } - - // Write to a file repeatedly to verify that it extends. - // The page blob file should start out at 128MB and finish at 256MB. - @Test(timeout=300000) - public void testFileSizeExtension() throws IOException { - final int writeSize = 1024 * 1024; - final int numWrites = 129; - final byte dataByte = 5; - byte[] data = new byte[writeSize]; - Arrays.fill(data, dataByte); - FSDataOutputStream output = fs.create(PATH); - try { - for (int i = 0; i < numWrites; i++) { - output.write(data); - output.hflush(); - LOG.debug("total writes = " + (i + 1)); - } - } finally { - output.close(); - } - - // Show that we wrote more than the default page blob file size. - assertTrue(numWrites * writeSize > PageBlobOutputStream.PAGE_BLOB_MIN_SIZE); - - // Verify we can list the new size. That will prove we expanded the file. - FileStatus[] status = fs.listStatus(PATH); - assertTrue(status[0].getLen() == numWrites * writeSize); - LOG.debug("Total bytes written to " + PATH + " = " + status[0].getLen()); - fs.delete(PATH, false); - } - -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d2d97fa/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestShellDecryptionKeyProvider.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestShellDecryptionKeyProvider.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestShellDecryptionKeyProvider.java index 0bf33d8..0334c39 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestShellDecryptionKeyProvider.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestShellDecryptionKeyProvider.java @@ -19,20 +19,23 @@ package org.apache.hadoop.fs.azure; import static org.apache.hadoop.test.PlatformAssumptions.assumeWindows; -import static org.junit.Assert.assertEquals; import java.io.File; import org.apache.commons.io.FileUtils; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.junit.Assert; import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -public class TestShellDecryptionKeyProvider { - public static final Log LOG = LogFactory - .getLog(TestShellDecryptionKeyProvider.class); +/** + * Windows only tests of shell scripts to provide decryption keys. + */ +public class TestShellDecryptionKeyProvider + extends AbstractWasbTestWithTimeout { + public static final Logger LOG = LoggerFactory + .getLogger(TestShellDecryptionKeyProvider.class); private static File TEST_ROOT_DIR = new File(System.getProperty( "test.build.data", "/tmp"), "TestShellDecryptionKeyProvider"); http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d2d97fa/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestWasbFsck.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestWasbFsck.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestWasbFsck.java index 467424b..9d32fb2 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestWasbFsck.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestWasbFsck.java @@ -18,10 +18,6 @@ package org.apache.hadoop.fs.azure; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileStatus; @@ -32,7 +28,10 @@ import org.junit.Before; import org.junit.Ignore; import org.junit.Test; -public class TestWasbFsck { +/** + * Tests which look at fsck recovery. + */ +public class TestWasbFsck extends AbstractWasbTestWithTimeout { private AzureBlobStorageTestAccount testAccount; private FileSystem fs; private InMemoryBlockBlobStore backingStore; --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
