Repository: hadoop Updated Branches: refs/heads/trunk 80e59e787 -> 3ff8580f2
HADOOP-15428. s3guard bucket-info will create s3guard table if FS is set to do this automatically. (Contributed by Gabor Bota) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3ff8580f Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3ff8580f Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3ff8580f Branch: refs/heads/trunk Commit: 3ff8580f2289664a54f369b500ad77f31a07c326 Parents: 80e59e7 Author: Sean Mackrory <mackror...@apache.org> Authored: Fri Dec 7 09:56:16 2018 -0700 Committer: Sean Mackrory <mackror...@apache.org> Committed: Mon Dec 10 14:03:08 2018 -0700 ---------------------------------------------------------------------- .../apache/hadoop/fs/s3a/s3guard/S3GuardTool.java | 12 ++++++++++-- .../src/site/markdown/tools/hadoop-aws/s3guard.md | 2 +- .../s3a/s3guard/AbstractS3GuardToolTestBase.java | 18 ++++++++++++++++++ 3 files changed, 29 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/3ff8580f/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java index aea57a6..8bef357 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java @@ -1061,7 +1061,7 @@ public abstract class S3GuardTool extends Configured implements Tool { + "\t" + PURPOSE + "\n\n" + "Common options:\n" + " -" + GUARDED_FLAG + " - Require S3Guard\n" - + " -" + UNGUARDED_FLAG + " - Require S3Guard to be disabled\n" + + " -" + UNGUARDED_FLAG + " - Force S3Guard to be disabled\n" + " -" + AUTH_FLAG + " - Require the S3Guard mode to be \"authoritative\"\n" + " -" + NONAUTH_FLAG + " - Require the S3Guard mode to be \"non-authoritative\"\n" + " -" + MAGIC_FLAG + " - Require the S3 filesystem to be support the \"magic\" committer\n" @@ -1092,6 +1092,15 @@ public abstract class S3GuardTool extends Configured implements Tool { throw invalidArgs("No bucket specified"); } String s3Path = paths.get(0); + CommandFormat commands = getCommandFormat(); + + // check if UNGUARDED_FLAG is passed and use NullMetadataStore in + // config to avoid side effects like creating the table if not exists + if (commands.getOpt(UNGUARDED_FLAG)) { + LOG.debug("Unguarded flag is passed to command :" + this.getName()); + getConf().set(S3_METADATA_STORE_IMPL, S3GUARD_METASTORE_NULL); + } + S3AFileSystem fs = (S3AFileSystem) FileSystem.newInstance( toUri(s3Path), getConf()); setFilesystem(fs); @@ -1128,7 +1137,6 @@ public abstract class S3GuardTool extends Configured implements Tool { "none"); printOption(out, "\tInput seek policy", INPUT_FADVISE, INPUT_FADV_NORMAL); - CommandFormat commands = getCommandFormat(); if (usingS3Guard) { if (commands.getOpt(UNGUARDED_FLAG)) { throw badState("S3Guard is enabled for %s", fsUri); http://git-wip-us.apache.org/repos/asf/hadoop/blob/3ff8580f/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/s3guard.md ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/s3guard.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/s3guard.md index b4cbd29..c098eea 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/s3guard.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/s3guard.md @@ -505,7 +505,7 @@ Options | argument | meaning | |-----------|-------------| | `-guarded` | Require S3Guard to be enabled | -| `-unguarded` | Require S3Guard to be disabled | +| `-unguarded` | Force S3Guard to be disabled | | `-auth` | Require the S3Guard mode to be "authoritative" | | `-nonauth` | Require the S3Guard mode to be "non-authoritative" | | `-magic` | Require the S3 filesystem to be support the "magic" committer | http://git-wip-us.apache.org/repos/asf/hadoop/blob/3ff8580f/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java index 632676f..9378358 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java @@ -317,6 +317,24 @@ public abstract class AbstractS3GuardToolTestBase extends AbstractS3ATestBase { } @Test + public void testBucketInfoUnguarded() throws Exception { + final Configuration conf = getConfiguration(); + conf.set(S3GUARD_DDB_TABLE_CREATE_KEY, Boolean.FALSE.toString()); + conf.set(S3GUARD_DDB_TABLE_NAME_KEY, + "testBucketInfoUnguarded-" + UUID.randomUUID()); + + // run a bucket info command and look for + // confirmation that it got the output from DDB diags + S3GuardTool.BucketInfo infocmd = new S3GuardTool.BucketInfo(conf); + String info = exec(infocmd, S3GuardTool.BucketInfo.NAME, + "-" + S3GuardTool.BucketInfo.UNGUARDED_FLAG, + getFileSystem().getUri().toString()); + + assertTrue("Output should contain information about S3A client " + info, + info.contains("S3A Client")); + } + + @Test public void testSetCapacityFailFastIfNotGuarded() throws Exception{ Configuration conf = getConfiguration(); conf.set(S3GUARD_DDB_TABLE_NAME_KEY, UUID.randomUUID().toString()); --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
