Repository: hadoop Updated Branches: refs/heads/HADOOP-13345 566c8b74f -> 322a2bc74
HADOOP-14181. S3Guard: Add validation of DynamoDB region Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/322a2bc7 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/322a2bc7 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/322a2bc7 Branch: refs/heads/HADOOP-13345 Commit: 322a2bc74433e5dbd395ab77691966c9ca48cedd Parents: 566c8b7 Author: Sean Mackrory <[email protected]> Authored: Tue Mar 14 10:00:48 2017 -0600 Committer: Sean Mackrory <[email protected]> Committed: Wed Mar 15 08:55:49 2017 -0600 ---------------------------------------------------------------------- .../fs/s3a/s3guard/DynamoDBClientFactory.java | 25 ++++++++++++++++++-- .../hadoop/fs/s3a/s3guard/S3GuardTool.java | 4 ++-- .../s3a/s3guard/ITestS3GuardToolDynamoDB.java | 21 ++++++++++++++++ 3 files changed, 46 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/322a2bc7/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBClientFactory.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBClientFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBClientFactory.java index 8f1b9e3..c2fe6a0 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBClientFactory.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBClientFactory.java @@ -22,6 +22,7 @@ import java.io.IOException; import com.amazonaws.ClientConfiguration; import com.amazonaws.auth.AWSCredentialsProvider; +import com.amazonaws.regions.Regions; import com.amazonaws.services.dynamodbv2.AmazonDynamoDB; import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClientBuilder; import com.google.common.base.Preconditions; @@ -80,8 +81,14 @@ interface DynamoDBClientFactory extends Configurable { if (StringUtils.isEmpty(region)) { region = defaultRegion; } - Preconditions.checkState(StringUtils.isNotEmpty(region), - "No DynamoDB region is provided!"); + try { + Regions.fromName(region); + } catch (IllegalArgumentException | NullPointerException e) { + throw new IOException("Invalid region specified: " + region + "; " + + "Region can be configured with " + S3GUARD_DDB_REGION_KEY +": " + + validRegionsString()); + } + LOG.debug("Creating DynamoDB client in region {}", region); return AmazonDynamoDBClientBuilder.standard() @@ -90,6 +97,20 @@ interface DynamoDBClientFactory extends Configurable { .withRegion(region) .build(); } + + private static String validRegionsString() { + final String DELIMITER = ", "; + Regions[] regions = Regions.values(); + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < regions.length; i++) { + if (i > 0) { + sb.append(DELIMITER); + } + sb.append(regions[i].getName()); + } + return sb.toString(); + + } } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/322a2bc7/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java index 862f871..9579839 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java @@ -79,8 +79,8 @@ public abstract class S3GuardTool extends Configured implements Tool { // Exit codes static final int SUCCESS = 0; - static final int INVALID_ARGUMENT = -1; - static final int ERROR = -99; + static final int INVALID_ARGUMENT = 1; + static final int ERROR = 99; protected S3AFileSystem s3a; protected MetadataStore ms; http://git-wip-us.apache.org/repos/asf/hadoop/blob/322a2bc7/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolDynamoDB.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolDynamoDB.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolDynamoDB.java index 33cb001..c2e4f5c 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolDynamoDB.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolDynamoDB.java @@ -29,6 +29,7 @@ import org.junit.Test; import java.io.IOException; import java.util.Random; +import static org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.INVALID_ARGUMENT; import static org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.SUCCESS; /** @@ -56,6 +57,26 @@ public class ITestS3GuardToolDynamoDB extends S3GuardToolTestBase { } @Test + public void testInvalidRegion() throws Exception { + String testTableName = "testInvalidRegion" + new Random().nextInt(); + String testRegion = "invalidRegion"; + // Initialize MetadataStore + Init initCmd = new Init(getFs().getConf()); + try { + initCmd.run(new String[]{ + "init", + "-region", testRegion, + "-meta", "dynamodb://" + testTableName + }); + } catch (IOException e) { + // Expected + return; + } + fail("Use of invalid region did not fail - table may have been " + + "created and not cleaned up: " + testTableName); + } + + @Test public void testDynamoDBInitDestroyCycle() throws IOException, InterruptedException { String testTableName = "testDynamoDBInitDestroy" + new Random().nextInt(); --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
