HADOOP-12774. s3a should use UGI.getCurrentUser.getShortname() for username. Contributed by Steve Loughran.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3372e940 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3372e940 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3372e940 Branch: refs/heads/YARN-4752 Commit: 3372e940303149d6258e0b72c54d72f080f0daa2 Parents: d8fa1cf Author: Chris Nauroth <cnaur...@apache.org> Authored: Mon Oct 24 21:54:06 2016 -0700 Committer: Chris Nauroth <cnaur...@apache.org> Committed: Mon Oct 24 21:54:06 2016 -0700 ---------------------------------------------------------------------- .../java/org/apache/hadoop/fs/s3a/Listing.java | 5 +-- .../org/apache/hadoop/fs/s3a/S3AFileStatus.java | 33 ++++++++++++++------ .../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 29 ++++++++++++----- .../java/org/apache/hadoop/fs/s3a/S3AUtils.java | 8 +++-- .../hadoop/fs/s3a/ITestS3AConfiguration.java | 23 +++++++++++++- 5 files changed, 74 insertions(+), 24 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/3372e940/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Listing.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Listing.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Listing.java index 4120b20..30d8e6f 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Listing.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Listing.java @@ -298,7 +298,7 @@ public class Listing { // Skip over keys that are ourselves and old S3N _$folder$ files if (acceptor.accept(keyPath, summary) && filter.accept(keyPath)) { FileStatus status = createFileStatus(keyPath, summary, - owner.getDefaultBlockSize(keyPath)); + owner.getDefaultBlockSize(keyPath), owner.getUsername()); LOG.debug("Adding: {}", status); stats.add(status); added++; @@ -312,7 +312,8 @@ public class Listing { for (String prefix : objects.getCommonPrefixes()) { Path keyPath = owner.keyToQualifiedPath(prefix); if (acceptor.accept(keyPath, prefix) && filter.accept(keyPath)) { - FileStatus status = new S3AFileStatus(true, false, keyPath); + FileStatus status = new S3AFileStatus(false, keyPath, + owner.getUsername()); LOG.debug("Adding directory: {}", status); added++; stats.add(status); http://git-wip-us.apache.org/repos/asf/hadoop/blob/3372e940/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileStatus.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileStatus.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileStatus.java index 75a6500..b0f08e3 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileStatus.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileStatus.java @@ -33,28 +33,41 @@ import org.apache.hadoop.fs.Path; public class S3AFileStatus extends FileStatus { private boolean isEmptyDirectory; - // Directories - public S3AFileStatus(boolean isdir, boolean isemptydir, Path path) { - super(0, isdir, 1, 0, 0, path); + /** + * Create a directory status. + * @param isemptydir is this an empty directory? + * @param path the path + * @param owner the owner + */ + public S3AFileStatus(boolean isemptydir, + Path path, + String owner) { + super(0, true, 1, 0, 0, path); isEmptyDirectory = isemptydir; + setOwner(owner); + setGroup(owner); } - // Files + /** + * A simple file. + * @param length file length + * @param modification_time mod time + * @param path path + * @param blockSize block size + * @param owner owner + */ public S3AFileStatus(long length, long modification_time, Path path, - long blockSize) { + long blockSize, String owner) { super(length, false, 1, blockSize, modification_time, path); isEmptyDirectory = false; + setOwner(owner); + setGroup(owner); } public boolean isEmptyDirectory() { return isEmptyDirectory; } - @Override - public String getOwner() { - return System.getProperty("user.name"); - } - /** Compare if this object is equal to another object. * @param o the object to be compared. * @return true if two file status has the same path name; false if not. http://git-wip-us.apache.org/repos/asf/hadoop/blob/3372e940/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java index 9908ba7..6030fe4 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java @@ -87,6 +87,7 @@ import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.fs.StorageStatistics; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.fs.s3native.S3xLoginHelper; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.Progressable; import org.apache.hadoop.util.ReflectionUtils; @@ -120,6 +121,7 @@ public class S3AFileSystem extends FileSystem { public static final int DEFAULT_BLOCKSIZE = 32 * 1024 * 1024; private URI uri; private Path workingDir; + private String username; private AmazonS3 s3; private String bucket; private int maxKeys; @@ -160,7 +162,9 @@ public class S3AFileSystem extends FileSystem { instrumentation = new S3AInstrumentation(name); uri = S3xLoginHelper.buildFSURI(name); - workingDir = new Path("/user", System.getProperty("user.name")) + // Username is the current user at the time the FS was instantiated. + username = UserGroupInformation.getCurrentUser().getShortUserName(); + workingDir = new Path("/user", username) .makeQualified(this.uri, this.getWorkingDirectory()); bucket = name.getHost(); @@ -1389,6 +1393,14 @@ public class S3AFileSystem extends FileSystem { } /** + * Get the username of the FS. + * @return the short name of the user who instantiated the FS + */ + public String getUsername() { + return username; + } + + /** * * Make the given path and all non-existent parents into * directories. Has the semantics of Unix {@code 'mkdir -p'}. @@ -1479,14 +1491,14 @@ public class S3AFileSystem extends FileSystem { if (objectRepresentsDirectory(key, meta.getContentLength())) { LOG.debug("Found exact file: fake directory"); - return new S3AFileStatus(true, true, - path); + return new S3AFileStatus(true, path, username); } else { LOG.debug("Found exact file: normal file"); return new S3AFileStatus(meta.getContentLength(), dateToLong(meta.getLastModified()), path, - getDefaultBlockSize(path)); + getDefaultBlockSize(path), + username); } } catch (AmazonServiceException e) { if (e.getStatusCode() != 404) { @@ -1504,7 +1516,7 @@ public class S3AFileSystem extends FileSystem { if (objectRepresentsDirectory(newKey, meta.getContentLength())) { LOG.debug("Found file (with /): fake directory"); - return new S3AFileStatus(true, true, path); + return new S3AFileStatus(true, path, username); } else { LOG.warn("Found file (with /): real file? should not happen: {}", key); @@ -1512,7 +1524,8 @@ public class S3AFileSystem extends FileSystem { return new S3AFileStatus(meta.getContentLength(), dateToLong(meta.getLastModified()), path, - getDefaultBlockSize(path)); + getDefaultBlockSize(path), + username); } } catch (AmazonServiceException e) { if (e.getStatusCode() != 404) { @@ -1549,10 +1562,10 @@ public class S3AFileSystem extends FileSystem { } } - return new S3AFileStatus(true, false, path); + return new S3AFileStatus(false, path, username); } else if (key.isEmpty()) { LOG.debug("Found root directory"); - return new S3AFileStatus(true, true, path); + return new S3AFileStatus(true, path, username); } } catch (AmazonServiceException e) { if (e.getStatusCode() != 404) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/3372e940/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java index f926f34..56e0c37 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java @@ -241,17 +241,19 @@ public final class S3AUtils { * @param keyPath path to entry * @param summary summary from AWS * @param blockSize block size to declare. + * @param owner owner of the file * @return a status entry */ public static S3AFileStatus createFileStatus(Path keyPath, S3ObjectSummary summary, - long blockSize) { + long blockSize, + String owner) { if (objectRepresentsDirectory(summary.getKey(), summary.getSize())) { - return new S3AFileStatus(true, true, keyPath); + return new S3AFileStatus(true, keyPath, owner); } else { return new S3AFileStatus(summary.getSize(), dateToLong(summary.getLastModified()), keyPath, - blockSize); + blockSize, owner); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/3372e940/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java index 30d4bf6..04057a9 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java @@ -44,8 +44,10 @@ import static org.junit.Assert.fail; import java.io.File; import java.net.URI; +import java.security.PrivilegedExceptionAction; import org.apache.hadoop.security.ProviderUtils; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.alias.CredentialProvider; import org.apache.hadoop.security.alias.CredentialProviderFactory; import org.apache.hadoop.util.VersionInfo; @@ -436,7 +438,7 @@ public class ITestS3AConfiguration { dir1.mkdirs(); dir2.mkdirs(); conf = new Configuration(); - conf.set(Constants.BUFFER_DIR, dir1 +", " + dir2); + conf.set(Constants.BUFFER_DIR, dir1 + ", " + dir2); fs = S3ATestUtils.createTestFileSystem(conf); File tmp1 = fs.createTmpFileForWrite("out-", 1024, conf); tmp1.delete(); @@ -446,6 +448,25 @@ public class ITestS3AConfiguration { tmp1.getParent(), tmp2.getParent()); } + @Test + public void testUsernameFromUGI() throws Throwable { + final String alice = "alice"; + UserGroupInformation fakeUser = + UserGroupInformation.createUserForTesting(alice, + new String[]{"users", "administrators"}); + conf = new Configuration(); + fs = fakeUser.doAs(new PrivilegedExceptionAction<S3AFileSystem>() { + @Override + public S3AFileSystem run() throws Exception{ + return S3ATestUtils.createTestFileSystem(conf); + } + }); + assertEquals("username", alice, fs.getUsername()); + S3AFileStatus status = fs.getFileStatus(new Path("/")); + assertEquals("owner in " + status, alice, status.getOwner()); + assertEquals("group in " + status, alice, status.getGroup()); + } + /** * Reads and returns a field from an object using reflection. If the field * cannot be found, is null, or is not the expected type, then this method --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
