Repository: hadoop Updated Branches: refs/heads/trunk 475bff6e8 -> a13929ddc
HADOOP-15645. ITestS3GuardToolLocal.testDiffCommand fails if bucket has per-bucket binding to DDB. Contributed by Steve Loughran. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a13929dd Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a13929dd Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a13929dd Branch: refs/heads/trunk Commit: a13929ddcb3b90044350ae1c23a1150e8b4b975b Parents: 475bff6 Author: Ewan Higgs <[email protected]> Authored: Mon Aug 13 12:57:45 2018 +0200 Committer: Ewan Higgs <[email protected]> Committed: Mon Aug 13 12:57:45 2018 +0200 ---------------------------------------------------------------------- .../fs/s3a/s3guard/AbstractS3GuardToolTestBase.java | 13 ++++++++++--- .../fs/s3a/s3guard/ITestS3GuardToolDynamoDB.java | 7 ++++--- .../hadoop/fs/s3a/s3guard/ITestS3GuardToolLocal.java | 11 +++++++++++ 3 files changed, 25 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/a13929dd/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java index 9185fc5..96aac15 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java @@ -56,6 +56,7 @@ import static org.apache.hadoop.fs.s3a.Constants.METADATASTORE_AUTHORITATIVE; import static org.apache.hadoop.fs.s3a.Constants.S3GUARD_DDB_TABLE_NAME_KEY; import static org.apache.hadoop.fs.s3a.Constants.S3GUARD_METASTORE_NULL; import static org.apache.hadoop.fs.s3a.Constants.S3_METADATA_STORE_IMPL; +import static org.apache.hadoop.fs.s3a.S3AUtils.clearBucketOption; import static org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.E_BAD_STATE; import static org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.SUCCESS; import static org.apache.hadoop.test.LambdaTestUtils.intercept; @@ -142,12 +143,14 @@ public abstract class AbstractS3GuardToolTestBase extends AbstractS3ATestBase { public void setup() throws Exception { super.setup(); S3ATestUtils.assumeS3GuardState(true, getConfiguration()); - ms = getFileSystem().getMetadataStore(); + S3AFileSystem fs = getFileSystem(); + ms = fs.getMetadataStore(); // Also create a "raw" fs without any MetadataStore configured Configuration conf = new Configuration(getConfiguration()); - URI fsUri = getFileSystem().getUri(); + clearBucketOption(conf, fs.getBucket(), S3_METADATA_STORE_IMPL); conf.set(S3_METADATA_STORE_IMPL, S3GUARD_METASTORE_NULL); + URI fsUri = fs.getUri(); S3AUtils.setBucketOption(conf,fsUri.getHost(), METADATASTORE_AUTHORITATIVE, S3GUARD_METASTORE_NULL); @@ -394,13 +397,17 @@ public abstract class AbstractS3GuardToolTestBase extends AbstractS3ATestBase { } @Test - public void testDiffCommand() throws Exception { + public void + testDiffCommand() throws Exception { S3AFileSystem fs = getFileSystem(); ms = getMetadataStore(); Set<Path> filesOnS3 = new HashSet<>(); // files on S3. Set<Path> filesOnMS = new HashSet<>(); // files on metadata store. Path testPath = path("test-diff"); + // clean up through the store and behind it. + fs.delete(testPath, true); + rawFs.delete(testPath, true); mkdirs(testPath, true, true); Path msOnlyPath = new Path(testPath, "ms_only"); http://git-wip-us.apache.org/repos/asf/hadoop/blob/a13929dd/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolDynamoDB.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolDynamoDB.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolDynamoDB.java index 1a59bf1..c96cbd0 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolDynamoDB.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolDynamoDB.java @@ -50,9 +50,10 @@ public class ITestS3GuardToolDynamoDB extends AbstractS3GuardToolTestBase { @Override public void setup() throws Exception { super.setup(); - Assume.assumeTrue("Test only applies when DynamoDB is used for S3Guard", - getConfiguration().get(Constants.S3_METADATA_STORE_IMPL).equals( - Constants.S3GUARD_METASTORE_DYNAMO)); + MetadataStore ms = getMetadataStore(); + Assume.assumeTrue("Test only applies when DynamoDB is used for S3Guard;" + + "Store is " + (ms == null ? "none" : ms.toString()), + ms instanceof DynamoDBMetadataStore); } // Check the existence of a given DynamoDB table. http://git-wip-us.apache.org/repos/asf/hadoop/blob/a13929dd/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolLocal.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolLocal.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolLocal.java index f5c4b03..1ee3cde 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolLocal.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolLocal.java @@ -31,6 +31,8 @@ import java.util.concurrent.Callable; import org.apache.hadoop.test.LambdaTestUtils; import org.apache.hadoop.util.StringUtils; + +import org.junit.Assume; import org.junit.Test; import org.apache.hadoop.fs.FSDataOutputStream; @@ -52,6 +54,15 @@ public class ITestS3GuardToolLocal extends AbstractS3GuardToolTestBase { private static final String[] ABORT_FORCE_OPTIONS = new String[] {"-abort", "-force", "-verbose"}; + @Override + public void setup() throws Exception { + super.setup(); + MetadataStore ms = getMetadataStore(); + Assume.assumeTrue("Test only applies when a local store is used for S3Guard;" + + "Store is " + (ms == null ? "none" : ms.toString()), + ms instanceof LocalMetadataStore); + } + @Test public void testImportCommand() throws Exception { S3AFileSystem fs = getFileSystem(); --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
