This is an automated email from the ASF dual-hosted git repository.
ahmar pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new c1414d7cdad HADOOP-19527. S3A: Skip AAL tests if encryption is set.
(#7592)
c1414d7cdad is described below
commit c1414d7cdadc09781370bba4cb3d0db6d26097dd
Author: ahmarsuhail <ahma...@amazon.co.uk>
AuthorDate: Fri Apr 11 13:54:38 2025 +0100
HADOOP-19527. S3A: Skip AAL tests if encryption is set. (#7592)
Contributed by Ahmar Suhail.
---
...TestS3AContractAnalyticsStreamVectoredRead.java | 21 ++++++++++++++----
.../ITestS3AAnalyticsAcceleratorStreamReading.java | 1 +
.../org/apache/hadoop/fs/s3a/S3ATestUtils.java | 25 ++++++++++++++++++++++
3 files changed, 43 insertions(+), 4 deletions(-)
diff --git
a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractAnalyticsStreamVectoredRead.java
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractAnalyticsStreamVectoredRead.java
index a2b053783bc..8cf182680c3 100644
---
a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractAnalyticsStreamVectoredRead.java
+++
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractAnalyticsStreamVectoredRead.java
@@ -23,12 +23,15 @@
import org.apache.hadoop.fs.contract.AbstractFSContract;
import static org.apache.hadoop.fs.s3a.S3ATestUtils.enableAnalyticsAccelerator;
+import static
org.apache.hadoop.fs.s3a.S3ATestUtils.skipForAnyEncryptionExceptSSES3;
/**
- * S3A contract tests for vectored reads with the Analytics stream. The
analytics stream does
- * not explicitly implement the vectoredRead() method, or currently do and
vectored-read specific
- * optimisations (such as range coalescing). However, this test ensures that
the base implementation
- * of readVectored {@link org.apache.hadoop.fs.PositionedReadable} still works.
+ * S3A contract tests for vectored reads with the Analytics stream.
+ * The analytics stream does not explicitly implement the vectoredRead()
method,
+ * or currently do and vectored-read specific optimisations
+ * (such as range coalescing). However, this test ensures that the base
+ * implementation of readVectored {@link
org.apache.hadoop.fs.PositionedReadable}
+ * still works.
*/
public class ITestS3AContractAnalyticsStreamVectoredRead extends
AbstractContractVectoredReadTest {
@@ -44,6 +47,16 @@ public ITestS3AContractAnalyticsStreamVectoredRead(String
bufferType) {
protected Configuration createConfiguration() {
Configuration conf = super.createConfiguration();
enableAnalyticsAccelerator(conf);
+ // If encryption is set, some AAL tests will fail.
+ // This is because AAL caches the head request response, and uses
+ // the eTag when making a GET request. When using encryption, the eTag is
+ // no longer a hash of the object content, and is not always the same when
+ // the same object is created multiple times. This test creates the file
+ // vectored_file.txt before running each test, which will have a
+ // different eTag when using encryption, leading to preconditioned
failures.
+ // This issue is tracked in:
+ // https://github.com/awslabs/analytics-accelerator-s3/issues/218
+ skipForAnyEncryptionExceptSSES3(conf);
conf.set("fs.contract.vector-io-early-eof-check", "false");
return conf;
}
diff --git
a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AAnalyticsAcceleratorStreamReading.java
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AAnalyticsAcceleratorStreamReading.java
index 9a0afdf6513..816ec90646f 100644
---
a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AAnalyticsAcceleratorStreamReading.java
+++
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AAnalyticsAcceleratorStreamReading.java
@@ -69,6 +69,7 @@ public class ITestS3AAnalyticsAcceleratorStreamReading
extends AbstractS3ATestBa
@Before
public void setUp() throws Exception {
super.setup();
+ skipIfClientSideEncryption();
externalTestFile = getExternalData(getConfiguration());
}
diff --git
a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java
index 4d97ab2179b..aa296d8bf7b 100644
---
a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java
+++
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java
@@ -106,6 +106,7 @@
import static org.apache.hadoop.fs.contract.ContractTestUtils.createFile;
import static org.apache.hadoop.fs.impl.FlagSet.createFlagSet;
+import static org.apache.hadoop.fs.s3a.S3AEncryptionMethods.SSE_S3;
import static org.apache.hadoop.fs.s3a.impl.streams.InputStreamType.Analytics;
import static org.apache.hadoop.fs.s3a.impl.streams.InputStreamType.Prefetch;
import static org.apache.hadoop.fs.s3a.impl.CallableSupplier.submit;
@@ -1720,6 +1721,30 @@ public static void skipIfEncryptionNotSet(Configuration
configuration,
}
}
+ /**
+ * Skip a test if encryption algorithm is not empty, or if it is set to
+ * anything other than AES256.
+ *
+ * @param configuration configuration
+ */
+ public static void skipForAnyEncryptionExceptSSES3(Configuration
configuration) {
+ String bucket = getTestBucketName(configuration);
+ try {
+ final EncryptionSecrets secrets = buildEncryptionSecrets(bucket,
configuration);
+ S3AEncryptionMethods s3AEncryptionMethods =
secrets.getEncryptionMethod();
+
+ if (s3AEncryptionMethods.getMethod().equals(SSE_S3.getMethod())
+ || s3AEncryptionMethods.getMethod().isEmpty()) {
+ return;
+ }
+
+ skip("Encryption method is set to " + s3AEncryptionMethods.getMethod());
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+
+ }
+
/**
* Get the input stream statistics of an input stream.
* Raises an exception if the inner stream is not an S3A input stream
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
