HADOOP-15480 AbstractS3GuardToolTestBase.testDiffCommand fails when using dynamo (Gabor Bota)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5f6769f7 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5f6769f7 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5f6769f7 Branch: refs/heads/HDDS-48 Commit: 5f6769f7964ff002b6c04a95893b5baeb424b6db Parents: 135941e Author: Aaron Fabbri <fab...@apache.org> Authored: Tue May 29 19:20:22 2018 -0700 Committer: Aaron Fabbri <fab...@apache.org> Committed: Tue May 29 19:20:22 2018 -0700 ---------------------------------------------------------------------- .../s3guard/AbstractS3GuardToolTestBase.java | 37 +++++++++++++------- .../s3a/s3guard/ITestS3GuardToolDynamoDB.java | 5 --- .../fs/s3a/s3guard/ITestS3GuardToolLocal.java | 5 --- 3 files changed, 25 insertions(+), 22 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f6769f7/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java index 2b43810..7d75f52 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java @@ -25,6 +25,7 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStreamReader; import java.io.PrintStream; +import java.net.URI; import java.util.Collection; import java.util.HashSet; import java.util.Set; @@ -32,6 +33,8 @@ import java.util.concurrent.Callable; import java.util.concurrent.TimeUnit; import org.apache.hadoop.util.StopWatch; +import com.google.common.base.Preconditions; +import org.apache.hadoop.fs.FileSystem; import org.junit.Assume; import org.junit.Test; @@ -48,6 +51,8 @@ import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.util.ExitUtil; import org.apache.hadoop.util.StringUtils; +import static org.apache.hadoop.fs.s3a.Constants.S3GUARD_METASTORE_NULL; +import static org.apache.hadoop.fs.s3a.Constants.S3_METADATA_STORE_IMPL; import static org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.E_BAD_STATE; import static org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.SUCCESS; import static org.apache.hadoop.test.LambdaTestUtils.intercept; @@ -65,6 +70,7 @@ public abstract class AbstractS3GuardToolTestBase extends AbstractS3ATestBase { private static final int PRUNE_MAX_AGE_SECS = 2; private MetadataStore ms; + private S3AFileSystem rawFs; protected static void expectResult(int expected, String message, @@ -129,28 +135,34 @@ public abstract class AbstractS3GuardToolTestBase extends AbstractS3ATestBase { return ms; } - protected abstract MetadataStore newMetadataStore(); - @Override public void setup() throws Exception { super.setup(); S3ATestUtils.assumeS3GuardState(true, getConfiguration()); - ms = newMetadataStore(); - ms.initialize(getFileSystem()); + ms = getFileSystem().getMetadataStore(); + + // Also create a "raw" fs without any MetadataStore configured + Configuration conf = new Configuration(getConfiguration()); + conf.set(S3_METADATA_STORE_IMPL, S3GUARD_METASTORE_NULL); + URI fsUri = getFileSystem().getUri(); + rawFs = (S3AFileSystem) FileSystem.newInstance(fsUri, conf); } @Override public void teardown() throws Exception { super.teardown(); IOUtils.cleanupWithLogger(LOG, ms); + IOUtils.closeStream(rawFs); } protected void mkdirs(Path path, boolean onS3, boolean onMetadataStore) throws IOException { + Preconditions.checkArgument(onS3 || onMetadataStore); + // getFileSystem() returns an fs with MetadataStore configured + S3AFileSystem fs = onMetadataStore ? getFileSystem() : rawFs; if (onS3) { - getFileSystem().mkdirs(path); - } - if (onMetadataStore) { + fs.mkdirs(path); + } else if (onMetadataStore) { S3AFileStatus status = new S3AFileStatus(true, path, OWNER); ms.put(new PathMetadata(status)); } @@ -178,13 +190,14 @@ public abstract class AbstractS3GuardToolTestBase extends AbstractS3ATestBase { */ protected void createFile(Path path, boolean onS3, boolean onMetadataStore) throws IOException { + Preconditions.checkArgument(onS3 || onMetadataStore); + // getFileSystem() returns an fs with MetadataStore configured + S3AFileSystem fs = onMetadataStore ? getFileSystem() : rawFs; if (onS3) { - ContractTestUtils.touch(getFileSystem(), path); - } - - if (onMetadataStore) { + ContractTestUtils.touch(fs, path); + } else if (onMetadataStore) { S3AFileStatus status = new S3AFileStatus(100L, System.currentTimeMillis(), - getFileSystem().qualify(path), 512L, "hdfs"); + fs.qualify(path), 512L, "hdfs"); putFile(ms, status); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f6769f7/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolDynamoDB.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolDynamoDB.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolDynamoDB.java index 821bba5..1a59bf1 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolDynamoDB.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolDynamoDB.java @@ -48,11 +48,6 @@ import static org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.*; public class ITestS3GuardToolDynamoDB extends AbstractS3GuardToolTestBase { @Override - protected MetadataStore newMetadataStore() { - return new DynamoDBMetadataStore(); - } - - @Override public void setup() throws Exception { super.setup(); Assume.assumeTrue("Test only applies when DynamoDB is used for S3Guard", http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f6769f7/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolLocal.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolLocal.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolLocal.java index a40c7a5..f5c4b03 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolLocal.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolLocal.java @@ -52,11 +52,6 @@ public class ITestS3GuardToolLocal extends AbstractS3GuardToolTestBase { private static final String[] ABORT_FORCE_OPTIONS = new String[] {"-abort", "-force", "-verbose"}; - @Override - protected MetadataStore newMetadataStore() { - return new LocalMetadataStore(); - } - @Test public void testImportCommand() throws Exception { S3AFileSystem fs = getFileSystem(); --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org