HADOOP-15845. Require explicit URI on CLI for s3guard init and destroy. Contributed by Gabor Bota.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1a25bbe9 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1a25bbe9 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1a25bbe9 Branch: refs/heads/HDFS-13891 Commit: 1a25bbe9ec7155712a82b157a8379826f8c79d4b Parents: 21c9080 Author: Sean Mackrory <[email protected]> Authored: Mon Dec 10 17:10:03 2018 -0700 Committer: Sean Mackrory <[email protected]> Committed: Tue Dec 11 08:33:13 2018 -0700 ---------------------------------------------------------------------- .../hadoop/fs/s3a/s3guard/S3GuardTool.java | 32 +++++++++++++++++--- .../s3guard/AbstractS3GuardToolTestBase.java | 13 ++++++++ .../s3a/s3guard/ITestS3GuardToolDynamoDB.java | 3 +- 3 files changed, 43 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/1a25bbe9/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java index 8bef357..70ea1f4 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java @@ -218,7 +218,8 @@ public abstract class S3GuardTool extends Configured implements Tool { format.addOptionWithValue(SECONDS_FLAG); } - protected void checkMetadataStoreUri(List<String> paths) throws IOException { + protected void checkIfS3BucketIsGuarded(List<String> paths) + throws IOException { // be sure that path is provided in params, so there's no IOoBE String s3Path = ""; if(!paths.isEmpty()) { @@ -240,6 +241,23 @@ public abstract class S3GuardTool extends Configured implements Tool { } /** + * Check if bucket or DDB table name is set. + */ + protected void checkBucketNameOrDDBTableNameProvided(List<String> paths) { + String s3Path = null; + if(!paths.isEmpty()) { + s3Path = paths.get(0); + } + + String metadataStoreUri = getCommandFormat().getOptValue(META_FLAG); + + if(metadataStoreUri == null && s3Path == null) { + throw invalidArgs("S3 bucket url or DDB table name have to be provided " + + "explicitly to use " + getName() + " command."); + } + } + + /** * Parse metadata store from command line option or HDFS configuration. * * @param forceCreate override the auto-creation setting to true. @@ -433,6 +451,12 @@ public abstract class S3GuardTool extends Configured implements Tool { @Override public int run(String[] args, PrintStream out) throws Exception { List<String> paths = parseArgs(args); + try { + checkBucketNameOrDDBTableNameProvided(paths); + } catch (ExitUtil.ExitException e) { + errorln(USAGE); + throw e; + } String readCap = getCommandFormat().getOptValue(READ_FLAG); if (readCap != null && !readCap.isEmpty()) { @@ -521,7 +545,7 @@ public abstract class S3GuardTool extends Configured implements Tool { public int run(String[] args, PrintStream out) throws Exception { List<String> paths = parseArgs(args); Map<String, String> options = new HashMap<>(); - checkMetadataStoreUri(paths); + checkIfS3BucketIsGuarded(paths); String readCap = getCommandFormat().getOptValue(READ_FLAG); if (StringUtils.isNotEmpty(readCap)) { @@ -592,14 +616,14 @@ public abstract class S3GuardTool extends Configured implements Tool { public int run(String[] args, PrintStream out) throws Exception { List<String> paths = parseArgs(args); try { + checkBucketNameOrDDBTableNameProvided(paths); + checkIfS3BucketIsGuarded(paths); parseDynamoDBRegion(paths); } catch (ExitUtil.ExitException e) { errorln(USAGE); throw e; } - checkMetadataStoreUri(paths); - try { initMetadataStore(false); } catch (FileNotFoundException e) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/1a25bbe9/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java index 9378358..48c60ce 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java @@ -381,6 +381,19 @@ public abstract class AbstractS3GuardToolTestBase extends AbstractS3ATestBase { } } + @Test + public void testDestroyFailsIfNoBucketNameOrDDBTableSet() + throws Exception { + intercept(ExitUtil.ExitException.class, + () -> run(S3GuardTool.Destroy.NAME)); + } + + @Test + public void testInitFailsIfNoBucketNameOrDDBTableSet() throws Exception { + intercept(ExitUtil.ExitException.class, + () -> run(S3GuardTool.Init.NAME)); + } + /** * Get the test CSV file; assume() that it is not modified (i.e. we haven't * switched to a new storage infrastructure where the bucket is no longer http://git-wip-us.apache.org/repos/asf/hadoop/blob/1a25bbe9/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolDynamoDB.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolDynamoDB.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolDynamoDB.java index 65e2619..97173fe 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolDynamoDB.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolDynamoDB.java @@ -126,7 +126,8 @@ public class ITestS3GuardToolDynamoDB extends AbstractS3GuardToolTestBase { String[] argsR = new String[]{ cmdR.getName(), - "-tag", tagMapToStringParams(tagMap) + "-tag", tagMapToStringParams(tagMap), + getFileSystem().getBucket() }; // run --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
