HADOOP-11685. StorageException complaining " no lease ID" during HBase distributed log splitting. Contributed by Duo Xu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1f7ecb0c Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1f7ecb0c Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1f7ecb0c Branch: refs/heads/HDFS-8966 Commit: 1f7ecb0c84042783f9fcf3f77d7d889dc58c9ead Parents: 73822de Author: cnauroth <[email protected]> Authored: Tue Oct 27 22:56:22 2015 -0700 Committer: cnauroth <[email protected]> Committed: Tue Oct 27 22:56:22 2015 -0700 ---------------------------------------------------------------------- hadoop-common-project/hadoop-common/CHANGES.txt | 3 ++ .../fs/azure/AzureNativeFileSystemStore.java | 15 +++++++++- .../hadoop/fs/azure/NativeAzureFileSystem.java | 5 ++-- .../fs/azure/TestNativeAzureFileSystemLive.java | 29 +++++++++++++++++++- 4 files changed, 48 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/1f7ecb0c/hadoop-common-project/hadoop-common/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 016fec8..25a3a60 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -919,6 +919,9 @@ Release 2.8.0 - UNRELEASED HADOOP-12520. Use XInclude in hadoop-azure test configuration to isolate Azure Storage account keys for service integration tests. (cnauroth) + HADOOP-11685. StorageException complaining " no lease ID" during HBase + distributed log splitting (Duo Xu via cnauroth) + OPTIMIZATIONS HADOOP-11785. Reduce the number of listStatus operation in distcp http://git-wip-us.apache.org/repos/asf/hadoop/blob/1f7ecb0c/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java index 8a33742..6412714 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java @@ -1503,10 +1503,23 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore { storePermissionStatus(blob, permissionStatus); storeFolderAttribute(blob); openOutputStream(blob).close(); - } catch (Exception e) { + } catch (StorageException e) { // Caught exception while attempting upload. Re-throw as an Azure // storage exception. throw new AzureException(e); + } catch (URISyntaxException e) { + throw new AzureException(e); + } catch (IOException e) { + Throwable t = e.getCause(); + if (t != null && t instanceof StorageException) { + StorageException se = (StorageException) t; + // If we got this exception, the blob should have already been created + if (!se.getErrorCode().equals("LeaseIdMissing")) { + throw new AzureException(e); + } + } else { + throw new AzureException(e); + } } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/1f7ecb0c/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java index 9305b24..7c5a504 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java @@ -1870,8 +1870,9 @@ public class NativeAzureFileSystem extends FileSystem { * @throws IOException * If login fails in getCurrentUser */ - private PermissionStatus createPermissionStatus(FsPermission permission) - throws IOException { + @VisibleForTesting + PermissionStatus createPermissionStatus(FsPermission permission) + throws IOException { // Create the permission status for this file based on current user return new PermissionStatus( UserGroupInformation.getCurrentUser().getShortUserName(), http://git-wip-us.apache.org/repos/asf/hadoop/blob/1f7ecb0c/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemLive.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemLive.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemLive.java index 985c0e9d..b033460 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemLive.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemLive.java @@ -21,7 +21,8 @@ package org.apache.hadoop.fs.azure; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; - +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsPermission; import org.junit.Test; /* @@ -104,4 +105,30 @@ public class TestNativeAzureFileSystemLive extends assertTrue(store.isAtomicRenameKey(uriPrefix + s)); } } + + /** + * Tests fs.mkdir() function to create a target blob while another thread + * is holding the lease on the blob. mkdir should not fail since the blob + * already exists. + * This is a scenario that would happen in HBase distributed log splitting. + * Multiple threads will try to create and update "recovered.edits" folder + * under the same path. + */ + @Test + public void testMkdirOnExistingFolderWithLease() throws Exception { + SelfRenewingLease lease; + final String FILE_KEY = "folderWithLease"; + // Create the folder + fs.mkdirs(new Path(FILE_KEY)); + NativeAzureFileSystem nfs = (NativeAzureFileSystem) fs; + String fullKey = nfs.pathToKey(nfs.makeAbsolute(new Path(FILE_KEY))); + AzureNativeFileSystemStore store = nfs.getStore(); + // Acquire the lease on the folder + lease = store.acquireLease(fullKey); + assertTrue(lease.getLeaseID() != null); + // Try to create the same folder + store.storeEmptyFolder(fullKey, + nfs.createPermissionStatus(FsPermission.getDirDefault())); + lease.free(); + } }
