Repository: hadoop Updated Branches: refs/heads/HADOOP-13345 0db7176ba -> be922ba6a
HADOOP-14527. S3Guard: ITestS3GuardListConsistency is too slow. Contributed by Aaron Fabbri. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/be922ba6 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/be922ba6 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/be922ba6 Branch: refs/heads/HADOOP-13345 Commit: be922ba6ad4d4eb68ae1df437e578e361bc3ee37 Parents: 0db7176 Author: Sean Mackrory <[email protected]> Authored: Fri Jun 23 14:40:45 2017 -0600 Committer: Sean Mackrory <[email protected]> Committed: Fri Jun 23 14:41:18 2017 -0600 ---------------------------------------------------------------------- .../fs/s3a/InconsistentAmazonS3Client.java | 29 ++++++++++++++++++++ .../fs/s3a/ITestS3GuardListConsistency.java | 27 ++++++++++-------- 2 files changed, 45 insertions(+), 11 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/be922ba6/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/InconsistentAmazonS3Client.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/InconsistentAmazonS3Client.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/InconsistentAmazonS3Client.java index 85f4a2f..ba40d34 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/InconsistentAmazonS3Client.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/InconsistentAmazonS3Client.java @@ -22,6 +22,7 @@ import com.amazonaws.AmazonClientException; import com.amazonaws.AmazonServiceException; import com.amazonaws.ClientConfiguration; import com.amazonaws.auth.AWSCredentialsProvider; +import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3Client; import com.amazonaws.services.s3.model.DeleteObjectRequest; import com.amazonaws.services.s3.model.DeleteObjectsRequest; @@ -137,6 +138,34 @@ public class InconsistentAmazonS3Client extends AmazonS3Client { delayKeyMsec, delayKeySubstring, delayKeyProbability); } + /** + * Clear all oustanding inconsistent keys. After calling this function, + * listings should behave normally (no failure injection), until additional + * keys are matched for delay, e.g. via putObject(), deleteObject(). + */ + public void clearInconsistency() { + LOG.info("clearing all delayed puts / deletes"); + delayedDeletes.clear(); + delayedPutKeys.clear(); + } + + /** + * Convenience function for test code to cast from supertype. + * @param c supertype to cast from + * @return subtype, not null + * @throws Exception on error + */ + public static InconsistentAmazonS3Client castFrom(AmazonS3 c) throws + Exception { + InconsistentAmazonS3Client ic = null; + if (c instanceof InconsistentAmazonS3Client) { + ic = (InconsistentAmazonS3Client) c; + } + Preconditions.checkNotNull(ic, "Not an instance of " + + "InconsistentAmazonS3Client"); + return ic; + } + @Override public DeleteObjectsResult deleteObjects(DeleteObjectsRequest deleteObjectsRequest) throws AmazonClientException, http://git-wip-us.apache.org/repos/asf/hadoop/blob/be922ba6/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3GuardListConsistency.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3GuardListConsistency.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3GuardListConsistency.java index b0da172..6cff533 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3GuardListConsistency.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3GuardListConsistency.java @@ -20,7 +20,6 @@ package org.apache.hadoop.fs.s3a; import com.amazonaws.services.s3.model.ObjectListing; import com.amazonaws.services.s3.AmazonS3; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileStatus; @@ -83,7 +82,7 @@ public class ITestS3GuardListConsistency extends AbstractS3ATestBase { for (Path mkdir : mkdirs) { assertTrue(fs.mkdirs(mkdir)); } - Thread.sleep(DEFAULT_DELAY_KEY_MSEC); + clearInconsistency(fs); } assertTrue("srcdirs and dstdirs must have equal length", @@ -176,7 +175,7 @@ public class ITestS3GuardListConsistency extends AbstractS3ATestBase { for (Path path : testDirs) { assertTrue(fs.mkdirs(path)); } - Thread.sleep(2 * DEFAULT_DELAY_KEY_MSEC); + clearInconsistency(fs); for (Path path : testDirs) { assertTrue(fs.delete(path, false)); } @@ -216,7 +215,7 @@ public class ITestS3GuardListConsistency extends AbstractS3ATestBase { for (Path path : testDirs) { assertTrue(fs.mkdirs(path)); } - Thread.sleep(2 * DEFAULT_DELAY_KEY_MSEC); + clearInconsistency(fs); assertTrue(fs.delete(testDirs[1], false)); assertTrue(fs.delete(testDirs[2], false)); @@ -229,8 +228,8 @@ public class ITestS3GuardListConsistency extends AbstractS3ATestBase { assertTrue(list.contains(path("a3/b/dir1"))); assertFalse(list.contains(path("a3/b/dir2"))); // This should fail without S3Guard, and succeed with it. - assertFalse(list.contains( - path("a3/b/dir3-" + DEFAULT_DELAY_KEY_SUBSTRING))); + assertFalse(list.contains(path("a3/b/dir3-" + + DEFAULT_DELAY_KEY_SUBSTRING))); try { RemoteIterator<LocatedFileStatus> old = fs.listFilesAndEmptyDirectories( @@ -287,9 +286,9 @@ public class ITestS3GuardListConsistency extends AbstractS3ATestBase { String rootDir = "doTestConsistentListLocatedStatusAfterPut"; fs.mkdirs(path(rootDir)); - final int[] numOfPaths = {0, 1, 10}; + final int[] numOfPaths = {0, 1, 5}; for (int normalPathNum : numOfPaths) { - for (int delayedPathNum : numOfPaths) { + for (int delayedPathNum : new int[] {0, 2}) { LOG.info("Testing with normalPathNum={}, delayedPathNum={}", normalPathNum, delayedPathNum); doTestConsistentListLocatedStatusAfterPut(fs, rootDir, normalPathNum, @@ -352,10 +351,10 @@ public class ITestS3GuardListConsistency extends AbstractS3ATestBase { // skip it. Assume.assumeTrue(fs.hasMetadataStore()); - final int[] numOfPaths = {0, 1, 2}; + final int[] numOfPaths = {0, 2}; for (int dirNum : numOfPaths) { for (int normalFile : numOfPaths) { - for (int delayedFile : numOfPaths) { + for (int delayedFile : new int[] {0, 1}) { for (boolean recursive : new boolean[] {true, false}) { doTestListFiles(fs, dirNum, normalFile, delayedFile, recursive); } @@ -498,7 +497,7 @@ public class ITestS3GuardListConsistency extends AbstractS3ATestBase { touch(fs, new Path(new Path(root, "dir" + i), "file" + i + "-" + j)); } } - Thread.sleep(2 * DEFAULT_DELAY_KEY_MSEC); + clearInconsistency(fs); AmazonS3 client = fs.getAmazonS3Client(); String key = fs.pathToKey(root) + "/"; @@ -536,4 +535,10 @@ public class ITestS3GuardListConsistency extends AbstractS3ATestBase { postDeleteUndelimited.getCommonPrefixes().size() ); } + + private static void clearInconsistency(S3AFileSystem fs) throws Exception { + AmazonS3 s3 = fs.getAmazonS3Client(); + InconsistentAmazonS3Client ic = InconsistentAmazonS3Client.castFrom(s3); + ic.clearInconsistency(); + } } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
