Repository: hadoop Updated Branches: refs/heads/HADOOP-13345 35e4bcd43 -> b4c2ab278
HADOOP-14735. ITestS3AEncryptionSSEC failing in parallel s3guard runs. Contributed by Steve Loughran. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b4c2ab27 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b4c2ab27 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b4c2ab27 Branch: refs/heads/HADOOP-13345 Commit: b4c2ab278a4280192643469f900fa689b7b53d84 Parents: 35e4bcd Author: Steve Loughran <ste...@apache.org> Authored: Wed Aug 9 17:53:42 2017 +0100 Committer: Steve Loughran <ste...@apache.org> Committed: Wed Aug 9 17:53:42 2017 +0100 ---------------------------------------------------------------------- .../hadoop/fs/s3a/AbstractS3ATestBase.java | 13 +- .../hadoop/fs/s3a/ITestS3AEncryptionSSEC.java | 313 +++++++++---------- 2 files changed, 156 insertions(+), 170 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/b4c2ab27/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3ATestBase.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3ATestBase.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3ATestBase.java index ca928f6..f0c389d 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3ATestBase.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3ATestBase.java @@ -112,10 +112,21 @@ public abstract class AbstractS3ATestBase extends AbstractFSContractTestBase */ protected Path writeThenReadFile(String name, int len) throws IOException { Path path = path(name); + writeThenReadFile(path, len); + return path; + } + + /** + * Write a file, read it back, validate the dataset. Overwrites the file + * if it is present + * @param path path to file + * @param len length of file + * @throws IOException any IO problem + */ + protected void writeThenReadFile(Path path, int len) throws IOException { byte[] data = dataset(len, 'a', 'z'); writeDataset(getFileSystem(), path, data, data.length, 1024 * 1024, true); ContractTestUtils.verifyFileContents(getFileSystem(), path, data); - return path; } /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/b4c2ab27/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSEC.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSEC.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSEC.java index 51c8c7b..8b7e031 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSEC.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSEC.java @@ -18,20 +18,21 @@ package org.apache.hadoop.fs.s3a; -import static org.apache.hadoop.fs.contract.ContractTestUtils.dataset; -import static org.apache.hadoop.fs.contract.ContractTestUtils.rm; -import static org.apache.hadoop.fs.s3a.S3ATestUtils.assumeS3GuardState; -import static org.apache.hadoop.fs.s3a.S3ATestUtils.skipIfEncryptionTestsDisabled; -import static org.apache.hadoop.test.LambdaTestUtils.intercept; - import java.io.IOException; +import java.nio.file.AccessDeniedException; + +import org.junit.Test; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.contract.ContractTestUtils; import org.apache.hadoop.fs.contract.s3a.S3AContract; -import org.junit.Test; +import org.apache.hadoop.io.IOUtils; + +import static org.apache.hadoop.fs.contract.ContractTestUtils.dataset; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.*; +import static org.apache.hadoop.test.LambdaTestUtils.intercept; /** * Concrete class that extends {@link AbstractTestS3AEncryption} @@ -39,17 +40,39 @@ import org.junit.Test; */ public class ITestS3AEncryptionSSEC extends AbstractTestS3AEncryption { + private static final String SERVICE_AMAZON_S3_STATUS_CODE_403 + = "Service: Amazon S3; Status Code: 403;"; + private static final String KEY_1 + = "4niV/jPK5VFRHY+KNb6wtqYd4xXyMgdJ9XQJpcQUVbs="; + private static final String KEY_2 + = "G61nz31Q7+zpjJWbakxfTOZW4VS0UmQWAq2YXhcTXoo="; + private static final String KEY_3 + = "NTx0dUPrxoo9+LbNiT/gqf3z9jILqL6ilismFmJO50U="; + private static final String KEY_4 + = "msdo3VvvZznp66Gth58a91Hxe/UpExMkwU9BHkIjfW8="; + private static final int TEST_FILE_LEN = 2048; + + /** + * Filesystem created with a different key. + */ + private FileSystem fsKeyB; + @Override protected Configuration createConfiguration() { Configuration conf = super.createConfiguration(); - S3ATestUtils.disableFilesystemCaching(conf); + disableFilesystemCaching(conf); conf.set(Constants.SERVER_SIDE_ENCRYPTION_ALGORITHM, getSSEAlgorithm().getMethod()); - conf.set(Constants.SERVER_SIDE_ENCRYPTION_KEY, - "4niV/jPK5VFRHY+KNb6wtqYd4xXyMgdJ9XQJpcQUVbs="); + conf.set(Constants.SERVER_SIDE_ENCRYPTION_KEY, KEY_1); return conf; } + @Override + public void teardown() throws Exception { + super.teardown(); + IOUtils.closeStream(fsKeyB); + } + /** * This will create and write to a file using encryption key A, then attempt * to read from it again with encryption key B. This will not work as it @@ -65,26 +88,25 @@ public class ITestS3AEncryptionSSEC extends AbstractTestS3AEncryption { assumeEnabled(); skipIfEncryptionTestsDisabled(getConfiguration()); - final Path[] path = new Path[1]; - intercept(java.nio.file.AccessDeniedException.class, - "Service: Amazon S3; Status Code: 403;", () -> { - - int len = 2048; - describe("Create an encrypted file of size " + len); - String src = createFilename(len); - path[0] = writeThenReadFile(src, len); - - //extract the test FS - FileSystem fileSystem = createNewFileSystemWithSSECKey( - "kX7SdwVc/1VXJr76kfKnkQ3ONYhxianyL2+C3rPVT9s="); - byte[] data = dataset(len, 'a', 'z'); - ContractTestUtils.verifyFileContents(fileSystem, path[0], data); - throw new Exception("Fail"); - }); + intercept(AccessDeniedException.class, + SERVICE_AMAZON_S3_STATUS_CODE_403, + () -> { + int len = TEST_FILE_LEN; + describe("Create an encrypted file of size " + len); + Path src = path("testCreateFileAndReadWithDifferentEncryptionKey"); + writeThenReadFile(src, len); + + //extract the test FS + fsKeyB = createNewFileSystemWithSSECKey( + "kX7SdwVc/1VXJr76kfKnkQ3ONYhxianyL2+C3rPVT9s="); + byte[] data = dataset(len, 'a', 'z'); + ContractTestUtils.verifyFileContents(fsKeyB, src, data); + return fsKeyB.getFileStatus(src); + }); } /** - * While each object has it's own key and should be distinct, this verifies + * While each object has its own key and should be distinct, this verifies * that hadoop treats object keys as a filesystem path. So if a top level * dir is encrypted with keyA, a sublevel dir cannot be accessed with a * different keyB. @@ -99,24 +121,18 @@ public class ITestS3AEncryptionSSEC extends AbstractTestS3AEncryption { skipIfEncryptionTestsDisabled(getConfiguration()); assumeS3GuardState(false, getConfiguration()); - final Path[] path = new Path[1]; - intercept(java.nio.file.AccessDeniedException.class, - "Service: Amazon S3; Status Code: 403;", () -> { - - path[0] = S3ATestUtils.createTestPath( - new Path(createFilename("dir/")) - ); - Path nestedDirectory = S3ATestUtils.createTestPath( - new Path(createFilename("dir/nestedDir/")) - ); - FileSystem fsKeyB = createNewFileSystemWithSSECKey( - "G61nz31Q7+zpjJWbakxfTOZW4VS0UmQWAq2YXhcTXoo="); - getFileSystem().mkdirs(path[0]); - fsKeyB.mkdirs(nestedDirectory); - - throw new Exception("Exception should be thrown."); - }); - rm(getFileSystem(), path[0], true, false); + intercept(AccessDeniedException.class, + SERVICE_AMAZON_S3_STATUS_CODE_403, + () -> { + Path base = path("testCreateSubdirWithDifferentKey"); + Path nestedDirectory = new Path(base, "nestedDir"); + fsKeyB = createNewFileSystemWithSSECKey( + KEY_2); + getFileSystem().mkdirs(base); + fsKeyB.mkdirs(nestedDirectory); + // expected to fail + return fsKeyB.getFileStatus(nestedDirectory); + }); } /** @@ -132,20 +148,17 @@ public class ITestS3AEncryptionSSEC extends AbstractTestS3AEncryption { assumeEnabled(); skipIfEncryptionTestsDisabled(getConfiguration()); - final Path[] path = new Path[1]; - intercept(java.nio.file.AccessDeniedException.class, - "Service: Amazon S3; Status Code: 403;", () -> { - - int len = 2048; - String src = createFilename(len); - path[0] = writeThenReadFile(src, len); - - FileSystem fsKeyB = createNewFileSystemWithSSECKey( - "NTx0dUPrxoo9+LbNiT/gqf3z9jILqL6ilismFmJO50U="); - fsKeyB.rename(path[0], new Path(createFilename("different-path.txt"))); - - throw new Exception("Exception should be thrown."); - }); + intercept(AccessDeniedException.class, + SERVICE_AMAZON_S3_STATUS_CODE_403, + () -> { + int len = TEST_FILE_LEN; + Path src = path(createFilename(len)); + writeThenReadFile(src, len); + fsKeyB = createNewFileSystemWithSSECKey(KEY_3); + Path dest = path(createFilename("different-path.txt")); + getFileSystem().mkdirs(dest.getParent()); + return fsKeyB.rename(src, dest); + }); } /** @@ -159,11 +172,11 @@ public class ITestS3AEncryptionSSEC extends AbstractTestS3AEncryption { assumeEnabled(); skipIfEncryptionTestsDisabled(getConfiguration()); - String src = createFilename("original-path.txt"); - Path path = writeThenReadFile(src, 2048); - Path newPath = path(createFilename("different-path.txt")); - getFileSystem().rename(path, newPath); - byte[] data = dataset(2048, 'a', 'z'); + Path src = path("original-path.txt"); + writeThenReadFile(src, TEST_FILE_LEN); + Path newPath = path("different-path.txt"); + getFileSystem().rename(src, newPath); + byte[] data = dataset(TEST_FILE_LEN, 'a', 'z'); ContractTestUtils.verifyFileContents(getFileSystem(), newPath, data); } @@ -179,29 +192,24 @@ public class ITestS3AEncryptionSSEC extends AbstractTestS3AEncryption { skipIfEncryptionTestsDisabled(getConfiguration()); assumeS3GuardState(false, getConfiguration()); - Path nestedDirectory = S3ATestUtils.createTestPath( - path(createFilename("/a/b/c/")) - ); + Path pathABC = path("testListEncryptedDir/a/b/c/"); + Path pathAB = pathABC.getParent(); + Path pathA = pathAB.getParent(); + + Path nestedDirectory = createTestPath(pathABC); assertTrue(getFileSystem().mkdirs(nestedDirectory)); - FileSystem fsKeyB = createNewFileSystemWithSSECKey( - "msdo3VvvZznp66Gth58a91Hxe/UpExMkwU9BHkIjfW8="); + fsKeyB = createNewFileSystemWithSSECKey(KEY_4); - fsKeyB.listFiles(S3ATestUtils.createTestPath( - path(createFilename("/a/")) - ), true); - fsKeyB.listFiles(S3ATestUtils.createTestPath( - path(createFilename("/a/b/")) - ), true); + fsKeyB.listFiles(pathA, true); + fsKeyB.listFiles(pathAB, true); //Until this point, no exception is thrown about access - intercept(java.nio.file.AccessDeniedException.class, - "Service: Amazon S3; Status Code: 403;", () -> { - fsKeyB.listFiles(S3ATestUtils.createTestPath( - path(createFilename("/a/b/c/")) - ), false); - throw new Exception("Exception should be thrown."); - }); + intercept(AccessDeniedException.class, + SERVICE_AMAZON_S3_STATUS_CODE_403, + () -> { + fsKeyB.listFiles(pathABC, false); + }); Configuration conf = this.createConfiguration(); conf.unset(Constants.SERVER_SIDE_ENCRYPTION_ALGORITHM); @@ -212,22 +220,13 @@ public class ITestS3AEncryptionSSEC extends AbstractTestS3AEncryption { FileSystem unencryptedFileSystem = contract.getTestFileSystem(); //unencrypted can access until the final directory - unencryptedFileSystem.listFiles(S3ATestUtils.createTestPath( - path(createFilename("/a/")) - ), true); - unencryptedFileSystem.listFiles(S3ATestUtils.createTestPath( - path(createFilename("/a/b/")) - ), true); - intercept(org.apache.hadoop.fs.s3a.AWSS3IOException.class, - "Bad Request (Service: Amazon S3; Status Code: 400; Error" + - " Code: 400 Bad Request;", () -> { - - unencryptedFileSystem.listFiles(S3ATestUtils.createTestPath( - path(createFilename("/a/b/c/")) - ), false); - throw new Exception("Exception should be thrown."); - }); - rm(getFileSystem(), path(createFilename("/")), true, false); + unencryptedFileSystem.listFiles(pathA, true); + unencryptedFileSystem.listFiles(pathAB, true); + AWSS3IOException ex = intercept(AWSS3IOException.class, + () -> { + unencryptedFileSystem.listFiles(pathABC, false); + }); + assertStatusCode(ex, 400); } /** @@ -241,30 +240,25 @@ public class ITestS3AEncryptionSSEC extends AbstractTestS3AEncryption { skipIfEncryptionTestsDisabled(getConfiguration()); assumeS3GuardState(false, getConfiguration()); - Path nestedDirectory = S3ATestUtils.createTestPath( - path(createFilename("/a/b/c/")) - ); - assertTrue(getFileSystem().mkdirs(nestedDirectory)); + Path pathABC = path("testListStatusEncryptedDir/a/b/c/"); + Path pathAB = pathABC.getParent(); + Path pathA = pathAB.getParent(); + assertTrue(getFileSystem().mkdirs(pathABC)); - FileSystem fsKeyB = createNewFileSystemWithSSECKey( - "msdo3VvvZznp66Gth58a91Hxe/UpExMkwU9BHkIjfW8="); + fsKeyB = createNewFileSystemWithSSECKey(KEY_4); - fsKeyB.listStatus(S3ATestUtils.createTestPath( - path(createFilename("/a/")))); - fsKeyB.listStatus(S3ATestUtils.createTestPath( - path(createFilename("/a/b/")))); + fsKeyB.listStatus(pathA); + fsKeyB.listStatus(pathAB); //Until this point, no exception is thrown about access - intercept(java.nio.file.AccessDeniedException.class, - "Service: Amazon S3; Status Code: 403;", () -> { - fsKeyB.listStatus(S3ATestUtils.createTestPath( - path(createFilename("/a/b/c/")))); - - throw new Exception("Exception should be thrown."); + intercept(AccessDeniedException.class, + SERVICE_AMAZON_S3_STATUS_CODE_403, + () -> { + fsKeyB.listStatus(pathABC); }); //Now try it with an unencrypted filesystem. - Configuration conf = this.createConfiguration(); + Configuration conf = createConfiguration(); conf.unset(Constants.SERVER_SIDE_ENCRYPTION_ALGORITHM); conf.unset(Constants.SERVER_SIDE_ENCRYPTION_KEY); @@ -273,20 +267,14 @@ public class ITestS3AEncryptionSSEC extends AbstractTestS3AEncryption { FileSystem unencryptedFileSystem = contract.getTestFileSystem(); //unencrypted can access until the final directory - unencryptedFileSystem.listStatus(S3ATestUtils.createTestPath( - path(createFilename("/a/")))); - unencryptedFileSystem.listStatus(S3ATestUtils.createTestPath( - path(createFilename("/a/b/")))); - - intercept(org.apache.hadoop.fs.s3a.AWSS3IOException.class, - "Bad Request (Service: Amazon S3; Status Code: 400; Error Code: 400" + - " Bad Request;", () -> { - - unencryptedFileSystem.listStatus(S3ATestUtils.createTestPath( - path(createFilename("/a/b/c/")))); - throw new Exception("Exception should be thrown."); + unencryptedFileSystem.listStatus(pathA); + unencryptedFileSystem.listStatus(pathAB); + + AWSS3IOException ex = intercept(AWSS3IOException.class, + () -> { + unencryptedFileSystem.listStatus(pathABC); }); - rm(getFileSystem(), path(createFilename("/")), true, false); + assertStatusCode(ex, 400); } /** @@ -299,31 +287,23 @@ public class ITestS3AEncryptionSSEC extends AbstractTestS3AEncryption { assumeEnabled(); skipIfEncryptionTestsDisabled(getConfiguration()); assumeS3GuardState(false, getConfiguration()); + Path pathABC = path("testListStatusEncryptedFile/a/b/c/"); + assertTrue(getFileSystem().mkdirs(pathABC)); - Path nestedDirectory = S3ATestUtils.createTestPath( - path(createFilename("/a/b/c/")) - ); - assertTrue(getFileSystem().mkdirs(nestedDirectory)); - - String src = createFilename("/a/b/c/fileToStat.txt"); - Path fileToStat = writeThenReadFile(src, 2048); + Path fileToStat = new Path(pathABC, "fileToStat.txt"); + writeThenReadFile(fileToStat, TEST_FILE_LEN); - FileSystem fsKeyB = createNewFileSystemWithSSECKey( - "msdo3VvvZznp66Gth58a91Hxe/UpExMkwU9BHkIjfW8="); + fsKeyB = createNewFileSystemWithSSECKey(KEY_4); //Until this point, no exception is thrown about access - intercept(java.nio.file.AccessDeniedException.class, - "Service: Amazon S3; Status Code: 403;", () -> { - fsKeyB.listStatus(S3ATestUtils.createTestPath(fileToStat)); - - throw new Exception("Exception should be thrown."); - }); - rm(getFileSystem(), path(createFilename("/")), true, false); + intercept(AccessDeniedException.class, + SERVICE_AMAZON_S3_STATUS_CODE_403, + () -> { + fsKeyB.listStatus(fileToStat); + }); } - - /** * It is possible to delete directories without the proper encryption key and * the hierarchy above it. @@ -335,31 +315,25 @@ public class ITestS3AEncryptionSSEC extends AbstractTestS3AEncryption { assumeEnabled(); skipIfEncryptionTestsDisabled(getConfiguration()); assumeS3GuardState(false, getConfiguration()); - - Path nestedDirectory = S3ATestUtils.createTestPath( - path(createFilename("/a/b/c/")) - ); - assertTrue(getFileSystem().mkdirs(nestedDirectory)); - String src = createFilename("/a/b/c/filetobedeleted.txt"); - Path fileToDelete = writeThenReadFile(src, 2048); - - FileSystem fsKeyB = createNewFileSystemWithSSECKey( - "msdo3VvvZznp66Gth58a91Hxe/UpExMkwU9BHkIjfW8="); - intercept(java.nio.file.AccessDeniedException.class, - "Forbidden (Service: Amazon S3; Status Code: 403; Error Code: " + - "403 Forbidden", () -> { - - fsKeyB.delete(fileToDelete, false); - throw new Exception("Exception should be thrown."); - }); + Path pathABC = path("testDeleteEncryptedObjectWithDifferentKey/a/b/c/"); + + Path pathAB = pathABC.getParent(); + Path pathA = pathAB.getParent(); + assertTrue(getFileSystem().mkdirs(pathABC)); + Path fileToDelete = new Path(pathABC, "filetobedeleted.txt"); + writeThenReadFile(fileToDelete, TEST_FILE_LEN); + fsKeyB = createNewFileSystemWithSSECKey(KEY_4); + intercept(AccessDeniedException.class, + SERVICE_AMAZON_S3_STATUS_CODE_403, + () -> { + fsKeyB.delete(fileToDelete, false); + }); //This is possible - fsKeyB.delete(S3ATestUtils.createTestPath( - path(createFilename("/a/b/c/"))), true); - fsKeyB.delete(S3ATestUtils.createTestPath( - path(createFilename("/a/b/"))), true); - fsKeyB.delete(S3ATestUtils.createTestPath( - path(createFilename("/a/"))), true); + fsKeyB.delete(pathABC, true); + fsKeyB.delete(pathAB, true); + fsKeyB.delete(pathA, true); + assertPathDoesNotExist("expected recursive delete", fileToDelete); } private FileSystem createNewFileSystemWithSSECKey(String sseCKey) throws @@ -377,4 +351,5 @@ public class ITestS3AEncryptionSSEC extends AbstractTestS3AEncryption { protected S3AEncryptionMethods getSSEAlgorithm() { return S3AEncryptionMethods.SSE_C; } + } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
