Repository: hadoop Updated Branches: refs/heads/HADOOP-13345 220414fcf -> 3a8f307a8
HADOOP-14027. S3Guard: Implicitly creating DynamoDB table ignores endpoint config. Contributed by Sean Mackrory Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3a8f307a Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3a8f307a Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3a8f307a Branch: refs/heads/HADOOP-13345 Commit: 3a8f307a845442e68e0049c7cd88a51a712d56f0 Parents: 220414f Author: Mingliang Liu <[email protected]> Authored: Wed Mar 8 11:46:27 2017 -0800 Committer: Mingliang Liu <[email protected]> Committed: Wed Mar 8 11:46:50 2017 -0800 ---------------------------------------------------------------------- .../fs/s3a/s3guard/DynamoDBMetadataStore.java | 10 +++++++++- .../fs/s3a/ITestS3AAWSCredentialsProvider.java | 5 +++++ .../hadoop/fs/s3a/ITestS3ACredentialsInURL.java | 2 +- .../s3a/s3guard/TestDynamoDBMetadataStore.java | 20 ++++++++++++++------ 4 files changed, 29 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a8f307a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java index bc54edf..13f8ee0 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java @@ -216,7 +216,15 @@ public class DynamoDBMetadataStore implements MetadataStore { "DynamoDBMetadataStore only supports S3A filesystem."); final S3AFileSystem s3afs = (S3AFileSystem) fs; final String bucket = s3afs.getBucket(); - region = s3afs.getBucketLocation(); + String confRegion = s3afs.getConf().getTrimmed(S3GUARD_DDB_REGION_KEY); + if (!StringUtils.isEmpty(confRegion)) { + region = confRegion; + LOG.debug("Overriding S3 region with configured DynamoDB region: {}", + region); + } else { + region = s3afs.getBucketLocation(); + LOG.debug("Inferring DynamoDB region from S3 bucket: {}", region); + } username = s3afs.getUsername(); conf = s3afs.getConf(); dynamoDB = createDynamoDB(conf, region); http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a8f307a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AAWSCredentialsProvider.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AAWSCredentialsProvider.java index 99454f4..ead770a 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AAWSCredentialsProvider.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AAWSCredentialsProvider.java @@ -26,6 +26,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.test.GenericTestUtils; import org.junit.Rule; import org.junit.Test; import org.junit.rules.Timeout; @@ -102,6 +103,10 @@ public class ITestS3AAWSCredentialsProvider { createFailingFS(conf); } catch (AccessDeniedException e) { // expected + } catch (AWSServiceIOException e) { + GenericTestUtils.assertExceptionContains( + "UnrecognizedClientException", e); + // expected } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a8f307a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ACredentialsInURL.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ACredentialsInURL.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ACredentialsInURL.java index be21994..894bda9 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ACredentialsInURL.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ACredentialsInURL.java @@ -126,7 +126,7 @@ public class ITestS3ACredentialsInURL extends Assert { Configuration conf = new Configuration(); String fsname = conf.getTrimmed(TEST_FS_S3A_NAME, ""); Assume.assumeNotNull(fsname); - Assume.assumeFalse(S3ATestUtils.isMetadataStoreAuthoritative(conf)); + assumeS3GuardNotEnabled(conf); URI original = new URI(fsname); URI testURI = createUriWithEmbeddedSecrets(original, "user", "//"); http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a8f307a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDynamoDBMetadataStore.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDynamoDBMetadataStore.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDynamoDBMetadataStore.java index b5568eb..8200c02 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDynamoDBMetadataStore.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDynamoDBMetadataStore.java @@ -244,8 +244,11 @@ public class TestDynamoDBMetadataStore extends MetadataStoreTestBase { verifyTableInitialized(tableName); assertNotNull(ddbms.getTable()); assertEquals(tableName, ddbms.getTable().getTableName()); - assertEquals("DynamoDB table should be in the same region as S3 bucket", - s3afs.getBucketLocation(tableName), + String expectedRegion = conf.get(Constants.S3GUARD_DDB_REGION_KEY, + s3afs.getBucketLocation(tableName)); + assertEquals("DynamoDB table should be in configured region or the same" + + " region as S3 bucket", + expectedRegion, ddbms.getRegion()); } } @@ -259,6 +262,8 @@ public class TestDynamoDBMetadataStore extends MetadataStoreTestBase { final String tableName = "testInitializeWithConfiguration"; final Configuration conf = getFileSystem().getConf(); conf.unset(Constants.S3GUARD_DDB_TABLE_NAME_KEY); + String savedRegion = conf.get(Constants.S3GUARD_DDB_REGION_KEY, + getFileSystem().getBucketLocation()); conf.unset(Constants.S3GUARD_DDB_REGION_KEY); try { DynamoDBMetadataStore ddbms = new DynamoDBMetadataStore(); @@ -275,8 +280,7 @@ public class TestDynamoDBMetadataStore extends MetadataStoreTestBase { } catch (IllegalArgumentException ignored) { } // config region - conf.set(Constants.S3GUARD_DDB_REGION_KEY, - getFileSystem().getBucketLocation()); + conf.set(Constants.S3GUARD_DDB_REGION_KEY, savedRegion); try (DynamoDBMetadataStore ddbms = new DynamoDBMetadataStore()) { ddbms.initialize(conf); verifyTableInitialized(tableName); @@ -390,7 +394,9 @@ public class TestDynamoDBMetadataStore extends MetadataStoreTestBase { @Test public void testTableVersionRequired() throws Exception { final DynamoDBMetadataStore ddbms = createContract().getMetadataStore(); - Table table = verifyTableInitialized(BUCKET); + String tableName = getFileSystem().getConf().get(Constants + .S3GUARD_DDB_TABLE_NAME_KEY, BUCKET); + Table table = verifyTableInitialized(tableName); table.deleteItem(VERSION_MARKER_PRIMARY_KEY); // create existing table @@ -405,7 +411,9 @@ public class TestDynamoDBMetadataStore extends MetadataStoreTestBase { @Test public void testTableVersionMismatch() throws Exception { final DynamoDBMetadataStore ddbms = createContract().getMetadataStore(); - Table table = verifyTableInitialized(BUCKET); + String tableName = getFileSystem().getConf().get(Constants + .S3GUARD_DDB_TABLE_NAME_KEY, BUCKET); + Table table = verifyTableInitialized(tableName); table.deleteItem(VERSION_MARKER_PRIMARY_KEY); Item v200 = createVersionMarker(VERSION_MARKER, 200, 0); table.putItem(v200); --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
