Repository: hadoop Updated Branches: refs/heads/HADOOP-13345 587e4c728 -> 26e1eaa60
Revert "HADOOP-14215. S3Guard: DynamoDB client should waitForActive on existing" Previous commit message is inaccurate and should not have been pushed. This reverts commit 587e4c7285bca7dd4629240f1b908b9c15edd935. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/26e1eaa6 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/26e1eaa6 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/26e1eaa6 Branch: refs/heads/HADOOP-13345 Commit: 26e1eaa606db7a83e68d23c36dfa07dd3957ce7f Parents: 587e4c7 Author: Sean Mackrory <[email protected]> Authored: Fri Apr 7 17:38:26 2017 -0600 Committer: Sean Mackrory <[email protected]> Committed: Fri Apr 7 17:41:06 2017 -0600 ---------------------------------------------------------------------- .../fs/s3a/s3guard/DynamoDBMetadataStore.java | 9 ++++---- .../fs/s3a/s3guard/MetadataStoreTestBase.java | 11 +--------- .../s3a/s3guard/TestDynamoDBMetadataStore.java | 23 ++++---------------- .../fs/s3a/s3guard/TestLocalMetadataStore.java | 11 +--------- .../fs/s3a/s3guard/TestNullMetadataStore.java | 5 ----- 5 files changed, 10 insertions(+), 49 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/26e1eaa6/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java index 2b28c58..e0a171d 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java @@ -191,7 +191,7 @@ public class DynamoDBMetadataStore implements MetadataStore { private Configuration conf; private String username; - private RetryPolicy dataAccessRetryPolicy; + private RetryPolicy batchRetryPolicy; /** * A utility function to create DynamoDB instance. @@ -274,7 +274,7 @@ public class DynamoDBMetadataStore implements MetadataStore { private void setMaxRetries(Configuration config) { int maxRetries = config.getInt(S3GUARD_DDB_MAX_RETRIES, S3GUARD_DDB_MAX_RETRIES_DEFAULT); - dataAccessRetryPolicy = RetryPolicies + batchRetryPolicy = RetryPolicies .exponentialBackoffRetry(maxRetries, MIN_RETRY_SLEEP_MSEC, TimeUnit.MILLISECONDS); } @@ -517,7 +517,7 @@ public class DynamoDBMetadataStore implements MetadataStore { private void retryBackoff(int retryCount) throws IOException { try { // Our RetryPolicy ignores everything but retryCount here. - RetryPolicy.RetryAction action = dataAccessRetryPolicy.shouldRetry(null, + RetryPolicy.RetryAction action = batchRetryPolicy.shouldRetry(null, retryCount, 0, true); if (action.action == RetryPolicy.RetryAction.RetryDecision.FAIL) { throw new IOException( @@ -764,7 +764,7 @@ public class DynamoDBMetadataStore implements MetadataStore { Item versionMarker = table.getItem(versionMarkerKey); while (versionMarker == null) { try { - RetryPolicy.RetryAction action = dataAccessRetryPolicy.shouldRetry(null, + RetryPolicy.RetryAction action = batchRetryPolicy.shouldRetry(null, retryCount, 0, true); if (action.action == RetryPolicy.RetryAction.RetryDecision.FAIL) { break; @@ -775,7 +775,6 @@ public class DynamoDBMetadataStore implements MetadataStore { } catch (Exception e) { throw new IOException("initTable: Unexpected exception", e); } - retryCount++; versionMarker = table.getItem(versionMarkerKey); } return versionMarker; http://git-wip-us.apache.org/repos/asf/hadoop/blob/26e1eaa6/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStoreTestBase.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStoreTestBase.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStoreTestBase.java index 88df45e..36a5cc5 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStoreTestBase.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStoreTestBase.java @@ -18,7 +18,6 @@ package org.apache.hadoop.fs.s3a.s3guard; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; @@ -64,18 +63,10 @@ public abstract class MetadataStoreTestBase extends Assert { private final long modTime = accessTime - 5000; /** - * Each test should override this. Will use a new Configuration instance. - * @return Contract which specifies the MetadataStore under test plus config. - */ - public abstract AbstractMSContract createContract() throws IOException; - - /** * Each test should override this. - * @param conf Base configuration instance to use. * @return Contract which specifies the MetadataStore under test plus config. */ - public abstract AbstractMSContract createContract(Configuration conf) - throws IOException; + public abstract AbstractMSContract createContract() throws IOException; /** * Tests assume that implementations will return recently set results. If http://git-wip-us.apache.org/repos/asf/hadoop/blob/26e1eaa6/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDynamoDBMetadataStore.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDynamoDBMetadataStore.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDynamoDBMetadataStore.java index 3584b54..a841e21 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDynamoDBMetadataStore.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDynamoDBMetadataStore.java @@ -126,10 +126,7 @@ public class TestDynamoDBMetadataStore extends MetadataStoreTestBase { private final DynamoDBMetadataStore ms = new DynamoDBMetadataStore(); DynamoDBMSContract() throws IOException { - this(new Configuration()); - } - - DynamoDBMSContract(Configuration conf) throws IOException { + final Configuration conf = new Configuration(); // using mocked S3 clients conf.setClass(S3_CLIENT_FACTORY_IMPL, MockS3ClientFactory.class, S3ClientFactory.class); @@ -163,12 +160,6 @@ public class TestDynamoDBMetadataStore extends MetadataStoreTestBase { } @Override - public DynamoDBMSContract createContract(Configuration conf) throws - IOException { - return new DynamoDBMSContract(conf); - } - - @Override FileStatus basicFileStatus(Path path, int size, boolean isDir) throws IOException { String owner = UserGroupInformation.getCurrentUser().getShortUserName(); @@ -349,21 +340,15 @@ public class TestDynamoDBMetadataStore extends MetadataStoreTestBase { */ @Test public void testTableVersionRequired() throws Exception { - Configuration conf = getFileSystem().getConf(); - int maxRetries = conf.getInt(Constants.S3GUARD_DDB_MAX_RETRIES, Constants - .S3GUARD_DDB_MAX_RETRIES_DEFAULT); - conf.setInt(Constants.S3GUARD_DDB_MAX_RETRIES, 3); - - final DynamoDBMetadataStore ddbms = createContract(conf).getMetadataStore(); - String tableName = conf.get(Constants.S3GUARD_DDB_TABLE_NAME_KEY, BUCKET); + final DynamoDBMetadataStore ddbms = createContract().getMetadataStore(); + String tableName = getFileSystem().getConf().get(Constants + .S3GUARD_DDB_TABLE_NAME_KEY, BUCKET); Table table = verifyTableInitialized(tableName); table.deleteItem(VERSION_MARKER_PRIMARY_KEY); // create existing table intercept(IOException.class, E_NO_VERSION_MARKER, () -> ddbms.initTable()); - - conf.setInt(Constants.S3GUARD_DDB_MAX_RETRIES, maxRetries); } /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/26e1eaa6/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestLocalMetadataStore.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestLocalMetadataStore.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestLocalMetadataStore.java index 4cffc6f..c3eb0d3 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestLocalMetadataStore.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestLocalMetadataStore.java @@ -42,10 +42,7 @@ public class TestLocalMetadataStore extends MetadataStoreTestBase { private FileSystem fs; private LocalMSContract() throws IOException { - this(new Configuration()); - } - - private LocalMSContract(Configuration config) throws IOException { + Configuration config = new Configuration(); config.set(LocalMetadataStore.CONF_MAX_RECORDS, MAX_ENTRIES_STR); fs = FileSystem.getLocal(config); } @@ -67,12 +64,6 @@ public class TestLocalMetadataStore extends MetadataStoreTestBase { return new LocalMSContract(); } - @Override - public AbstractMSContract createContract(Configuration conf) throws - IOException { - return new LocalMSContract(conf); - } - @Test public void testClearByAncestor() { Map<Path, String> map = new HashMap<>(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/26e1eaa6/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestNullMetadataStore.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestNullMetadataStore.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestNullMetadataStore.java index 5b19efa..5b18686 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestNullMetadataStore.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestNullMetadataStore.java @@ -55,9 +55,4 @@ public class TestNullMetadataStore extends MetadataStoreTestBase { public AbstractMSContract createContract() { return new NullMSContract(); } - - @Override - public AbstractMSContract createContract(Configuration conf) { - return createContract(); - } } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
