This is an automated email from the ASF dual-hosted git repository.
gabota pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new d5e9971 HADOOP-16653. S3Guard DDB overreacts to no tag access
(#1660). Contributed by Gabor Bota.
d5e9971 is described below
commit d5e9971e6d98b50de64acbf46154f82208919930
Author: Gabor Bota <gabor.b...@cloudera.com>
AuthorDate: Mon Oct 28 11:22:41 2019 +0100
HADOOP-16653. S3Guard DDB overreacts to no tag access (#1660). Contributed
by Gabor Bota.
---
.../s3guard/DynamoDBMetadataStoreTableManager.java | 45 +++++++++++++++-------
.../src/site/markdown/tools/hadoop-aws/s3guard.md | 5 +--
.../apache/hadoop/fs/s3a/auth/ITestAssumeRole.java | 24 ++++++++++++
3 files changed, 57 insertions(+), 17 deletions(-)
diff --git
a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStoreTableManager.java
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStoreTableManager.java
index d9f297c..fe08ecd 100644
---
a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStoreTableManager.java
+++
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStoreTableManager.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.fs.s3a.s3guard;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InterruptedIOException;
+import java.nio.file.AccessDeniedException;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
@@ -68,6 +69,7 @@ import static
org.apache.hadoop.fs.s3a.Constants.S3GUARD_DDB_TABLE_CAPACITY_WRIT
import static
org.apache.hadoop.fs.s3a.Constants.S3GUARD_DDB_TABLE_CAPACITY_WRITE_KEY;
import static org.apache.hadoop.fs.s3a.Constants.S3GUARD_DDB_TABLE_CREATE_KEY;
import static org.apache.hadoop.fs.s3a.Constants.S3GUARD_DDB_TABLE_TAG;
+import static org.apache.hadoop.fs.s3a.S3AUtils.translateDynamoDBException;
import static org.apache.hadoop.fs.s3a.S3AUtils.translateException;
import static
org.apache.hadoop.fs.s3a.s3guard.DynamoDBMetadataStore.E_ON_DEMAND_NO_SET_CAPACITY;
import static org.apache.hadoop.fs.s3a.s3guard.DynamoDBMetadataStore.VERSION;
@@ -228,19 +230,19 @@ public class DynamoDBMetadataStoreTableManager {
return table;
}
- protected void tagTableWithVersionMarker() {
+ protected void tagTableWithVersionMarker() throws AmazonDynamoDBException {
try {
TagResourceRequest tagResourceRequest = new TagResourceRequest()
.withResourceArn(table.getDescription().getTableArn())
.withTags(newVersionMarkerTag());
amazonDynamoDB.tagResource(tagResourceRequest);
} catch (AmazonDynamoDBException e) {
- LOG.warn("Exception during tagging table: {}", e.getMessage());
+ LOG.debug("Exception during tagging table: {}", e.getMessage(), e);
}
}
protected static Item getVersionMarkerFromTags(Table table,
- AmazonDynamoDB addb) {
+ AmazonDynamoDB addb) throws IOException {
List<Tag> tags = null;
try {
final TableDescription description = table.describe();
@@ -252,8 +254,10 @@ public class DynamoDBMetadataStoreTableManager {
LOG.error("Table: {} not found.", table.getTableName());
throw e;
} catch (AmazonDynamoDBException e) {
- LOG.warn("Exception while getting tags from the dynamo table: {}",
- e.getMessage());
+ LOG.debug("Exception while getting tags from the dynamo table: {}",
+ e.getMessage(), e);
+ throw translateDynamoDBException(table.getTableName(),
+ "Retrieving tags.", e);
}
if (tags == null) {
@@ -374,8 +378,15 @@ public class DynamoDBMetadataStoreTableManager {
@VisibleForTesting
protected void verifyVersionCompatibility() throws IOException {
final Item versionMarkerItem = getVersionMarkerItem();
- final Item versionMarkerFromTag =
- getVersionMarkerFromTags(table, amazonDynamoDB);
+ Item versionMarkerFromTag = null;
+ boolean canReadDdbTags = true;
+
+ try {
+ versionMarkerFromTag = getVersionMarkerFromTags(table, amazonDynamoDB);
+ } catch (AccessDeniedException e) {
+ LOG.debug("Can not read tags of table.");
+ canReadDdbTags = false;
+ }
LOG.debug("versionMarkerItem: {}; versionMarkerFromTag: {}",
versionMarkerItem, versionMarkerFromTag);
@@ -387,12 +398,19 @@ public class DynamoDBMetadataStoreTableManager {
+ " Table: " + tableName);
}
- LOG.info("Table {} contains no version marker item or tag. " +
- "The table is empty, so the version marker will be added " +
- "as TAG and ITEM.", tableName);
+ if (canReadDdbTags) {
+ LOG.info("Table {} contains no version marker item and tag. " +
+ "The table is empty, so the version marker will be added " +
+ "as TAG and ITEM.", tableName);
+ putVersionMarkerItemToTable();
+ tagTableWithVersionMarker();
+ }
- tagTableWithVersionMarker();
- putVersionMarkerItemToTable();
+ if (!canReadDdbTags) {
+ LOG.info("Table {} contains no version marker item and the tags are
not readable. " +
+ "The table is empty, so the ITEM version marker will be added .",
tableName);
+ putVersionMarkerItemToTable();
+ }
}
if (versionMarkerItem == null && versionMarkerFromTag != null) {
@@ -408,7 +426,8 @@ public class DynamoDBMetadataStoreTableManager {
putVersionMarkerItemToTable();
}
- if (versionMarkerItem != null && versionMarkerFromTag == null) {
+ if (versionMarkerItem != null && versionMarkerFromTag == null
+ && canReadDdbTags) {
final int itemVersionMarker =
extractVersionFromMarker(versionMarkerItem);
throwExceptionOnVersionMismatch(itemVersionMarker, tableName,
diff --git
a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/s3guard.md
b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/s3guard.md
index 571f223..0a05790 100644
--- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/s3guard.md
+++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/s3guard.md
@@ -977,10 +977,7 @@ the check throws IOException
*Note*: If the user does not have sufficient rights to tag the table the
initialization of S3Guard will not fail, but there will be no version marker
tag
-on the dynamo table and the following message will be logged on WARN level:
-```
-Exception during tagging table: {AmazonDynamoDBException exception message}
-```
+on the dynamo table.
*Versioning policy*
diff --git
a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestAssumeRole.java
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestAssumeRole.java
index a4c6a6e..f155aa0 100644
---
a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestAssumeRole.java
+++
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestAssumeRole.java
@@ -756,4 +756,28 @@ public class ITestAssumeRole extends AbstractS3ATestBase {
Assertions.assertThat(info)
.contains(S3GuardTool.BucketInfo.LOCATION_UNKNOWN);
}
+ /**
+ * Turn off access to dynamo DB Tags and see how DDB table init copes.
+ * There's no testing of the codepath other than checking the logs
+ * - this test does make sure that no regression stops the tag permission
+ * failures from halting the client
+ */
+ @Test
+ public void testRestrictDDBTagAccess() throws Throwable {
+
+ describe("extra policies in assumed roles need;"
+ + " all required policies stated");
+ Configuration conf = createAssumedRoleConfig();
+
+ bindRolePolicyStatements(conf,
+ STATEMENT_S3GUARD_CLIENT,
+ STATEMENT_ALLOW_SSE_KMS_RW,
+ STATEMENT_ALL_S3,
+ new Statement(Effects.Deny)
+ .addActions(S3_PATH_RW_OPERATIONS)
+ .addResources(ALL_DDB_TABLES));
+ Path path = path("testRestrictDDBTagAccess");
+
+ roleFS = (S3AFileSystem) path.getFileSystem(conf);
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
