Repository: hadoop Updated Branches: refs/heads/HADOOP-13345 e3f20027f -> a5cc315db
HADOOP-13908. S3Guard: Existing tables may not be initialized correctly in DynamoDBMetadataStore. Contributed by Mingliang Liu. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a5cc315d Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a5cc315d Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a5cc315d Branch: refs/heads/HADOOP-13345 Commit: a5cc315dbef15e8f708663d45800fdc957797cf2 Parents: e3f2002 Author: Chris Nauroth <[email protected]> Authored: Mon Jan 9 15:48:30 2017 -0800 Committer: Chris Nauroth <[email protected]> Committed: Mon Jan 9 15:48:30 2017 -0800 ---------------------------------------------------------------------- .../fs/s3a/s3guard/DynamoDBMetadataStore.java | 83 ++++++++++++-------- .../s3a/s3guard/TestDynamoDBMetadataStore.java | 27 ++++++- 2 files changed, 73 insertions(+), 37 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/a5cc315d/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java index 45ecaff..6ff0ee1 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java @@ -190,10 +190,7 @@ public class DynamoDBMetadataStore implements MetadataStore { // use the bucket as the DynamoDB table name if not specified in config tableName = conf.getTrimmed(S3GUARD_DDB_TABLE_NAME_KEY, bucket); - // create the table unless it's explicitly told not to do so - if (conf.getBoolean(S3GUARD_DDB_TABLE_CREATE_KEY, false)) { - createTable(); - } + initTable(); } /** @@ -230,7 +227,7 @@ public class DynamoDBMetadataStore implements MetadataStore { dynamoDB = new DynamoDB(dynamoDBClient); region = dynamoDBClient.getEndpointPrefix(); - createTable(); + initTable(); } @Override @@ -510,46 +507,64 @@ public class DynamoDBMetadataStore implements MetadataStore { /** * Create a table if it does not exist and wait for it to become active. * - * If a table with the intended name already exists, then it logs the - * {@link ResourceInUseException} and uses that table. The DynamoDB table - * creation API is asynchronous. This method wait for the table to become - * active after sending the creation request, so overall, this method is - * synchronous, and the table is guaranteed to exist after this method - * returns successfully. + * If a table with the intended name already exists, then it uses that table. + * Otherwise, it will automatically create the table if the config + * {@link org.apache.hadoop.fs.s3a.Constants#S3GUARD_DDB_TABLE_CREATE_KEY} is + * enabled. The DynamoDB table creation API is asynchronous. This method wait + * for the table to become active after sending the creation request, so + * overall, this method is synchronous, and the table is guaranteed to exist + * after this method returns successfully. + * + * @throws IOException if table does not exist and auto-creation is disabled; + * or any other I/O exception occurred. */ @VisibleForTesting - void createTable() throws IOException { + void initTable() throws IOException { final ProvisionedThroughput capacity = new ProvisionedThroughput( conf.getLong(S3GUARD_DDB_TABLE_CAPACITY_READ_KEY, S3GUARD_DDB_TABLE_CAPACITY_READ_DEFAULT), conf.getLong(S3GUARD_DDB_TABLE_CAPACITY_WRITE_KEY, S3GUARD_DDB_TABLE_CAPACITY_WRITE_DEFAULT)); + table = dynamoDB.getTable(tableName); try { - LOG.info("Creating DynamoDB table {} in region {}", tableName, region); - table = dynamoDB.createTable(new CreateTableRequest() - .withTableName(tableName) - .withKeySchema(keySchema()) - .withAttributeDefinitions(attributeDefinitions()) - .withProvisionedThroughput(capacity)); - } catch (ResourceInUseException e) { - LOG.info("ResourceInUseException while creating DynamoDB table {} in " - + "region {}. This may indicate that the table was created by " - + "another concurrent thread or process.", - tableName, region); - table = dynamoDB.getTable(tableName); - } + try { + table.describe(); + LOG.debug("Using existing DynamoDB table {} in region {}", + tableName, region); + } catch (ResourceNotFoundException rnfe) { + if (conf.getBoolean(S3GUARD_DDB_TABLE_CREATE_KEY, false)) { + try { + LOG.info("Creating non-existent DynamoDB table {} in region {}", + tableName, region); + dynamoDB.createTable(new CreateTableRequest() + .withTableName(tableName) + .withKeySchema(keySchema()) + .withAttributeDefinitions(attributeDefinitions()) + .withProvisionedThroughput(capacity)); + } catch (ResourceInUseException e) { + LOG.debug("ResourceInUseException while creating DynamoDB table {} " + + "in region {}. This may indicate that the table was " + + "created by another concurrent thread or process.", + tableName, region); + } + } else { + throw new IOException("DynamoDB table '" + tableName + "' does not " + + "exist in region " + region + "; auto-creation is turned off"); + } + } - try { - table.waitForActive(); - } catch (InterruptedException e) { - LOG.warn("Interrupted while waiting for DynamoDB table {} active", - tableName, e); - Thread.currentThread().interrupt(); - throw new InterruptedIOException("DynamoDB table '" + tableName + "'" + - " is not active yet in region " + region); + try { + table.waitForActive(); + } catch (InterruptedException e) { + LOG.warn("Interrupted while waiting for DynamoDB table {} active", + tableName, e); + Thread.currentThread().interrupt(); + throw new InterruptedIOException("DynamoDB table '" + tableName + "'" + + " is not active yet in region " + region); + } } catch (AmazonClientException e) { - throw translateException("createTable", (String) null, e); + throw translateException("initTable", (String) null, e); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/a5cc315d/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDynamoDBMetadataStore.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDynamoDBMetadataStore.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDynamoDBMetadataStore.java index f88137b..fe38c12 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDynamoDBMetadataStore.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDynamoDBMetadataStore.java @@ -214,7 +214,6 @@ public class TestDynamoDBMetadataStore extends MetadataStoreTestBase { public void testInitializeWithConfiguration() throws IOException { final String tableName = "testInitializeWithConfiguration"; final Configuration conf = new Configuration(); - String a = conf.get(Constants.S3GUARD_DDB_ENDPOINT_KEY); try { DynamoDBMetadataStore ddbms = new DynamoDBMetadataStore(); ddbms.initialize(conf); @@ -234,6 +233,7 @@ public class TestDynamoDBMetadataStore extends MetadataStoreTestBase { // config credentials conf.set(Constants.ACCESS_KEY, "dummy-access-key"); conf.set(Constants.SECRET_KEY, "dummy-secret-key"); + conf.setBoolean(Constants.S3GUARD_DDB_TABLE_CREATE_KEY, true); try (DynamoDBMetadataStore ddbms = new DynamoDBMetadataStore()) { ddbms.initialize(conf); verifyTableInitialized(tableName); @@ -312,15 +312,36 @@ public class TestDynamoDBMetadataStore extends MetadataStoreTestBase { } @Test - public void testCreateExistingTable() throws IOException { + public void testInitExistingTable() throws IOException { final DynamoDBMetadataStore ddbms = createContract().getMetadataStore(); verifyTableInitialized(BUCKET); // create existing table - ddbms.createTable(); + ddbms.initTable(); verifyTableInitialized(BUCKET); } /** + * Test that initTable fails with IOException when table does not exist and + * table auto-creation is disabled. + */ + @Test + public void testFailNonexistentTable() throws IOException { + final String tableName = "testFailNonexistentTable"; + final DynamoDBMSContract contract = createContract(); + final S3AFileSystem s3afs = contract.getFileSystem(); + final Configuration conf = s3afs.getConf(); + conf.set(Constants.S3GUARD_DDB_TABLE_NAME_KEY, tableName); + conf.unset(Constants.S3GUARD_DDB_TABLE_CREATE_KEY); + try { + final DynamoDBMetadataStore ddbms = new DynamoDBMetadataStore(); + ddbms.initialize(s3afs); + fail("Should have failed as table does not exist and table auto-creation " + + "is disabled"); + } catch (IOException ignored) { + } + } + + /** * Test cases about root directory as it is not in the DynamoDB table. */ @Test --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
