This is an automated email from the ASF dual-hosted git repository.
stevel pushed a commit to branch branch-3.4
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.4 by this push:
new 90051c0e450 HADOOP-19805. S3A: When creating a signer, if it is
Configurable, set its config (#8232) (#8231)
90051c0e450 is described below
commit 90051c0e450eeba208fa8612917359b5b76fa94b
Author: Steve Loughran <[email protected]>
AuthorDate: Fri Feb 6 17:41:52 2026 +0000
HADOOP-19805. S3A: When creating a signer, if it is Configurable, set its
config (#8232) (#8231)
If the signer for a filesystem implements Configurable, then after
instantiating it, call setConf() on it,
passing in the shared filesystem instance configuration.
This makes it a lot easier to configure a signer, rather than the full
AwsSignerInitializer interface, which is only applied when you have a chain
of custom signers.
Contributed by Steve Loughran.
---
.../apache/hadoop/fs/s3a/auth/SignerFactory.java | 13 +++++++++---
.../apache/hadoop/fs/s3a/impl/AWSClientConfig.java | 4 ++--
.../hadoop/fs/s3a/auth/ITestCustomSigner.java | 24 ++++++++++++++++++++--
.../hadoop/fs/s3a/auth/TestSignerManager.java | 20 +++++++++++-------
4 files changed, 47 insertions(+), 14 deletions(-)
diff --git
a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/SignerFactory.java
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/SignerFactory.java
index e46fd88e85f..e4e5d9015a0 100644
---
a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/SignerFactory.java
+++
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/SignerFactory.java
@@ -35,10 +35,12 @@
import software.amazon.awssdk.identity.spi.IdentityProvider;
import software.amazon.awssdk.identity.spi.IdentityProviders;
+import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.s3a.S3AUtils;
import org.apache.hadoop.fs.s3a.impl.InstantiationIOException;
+import static java.util.Objects.requireNonNull;
import static org.apache.hadoop.fs.s3a.Constants.HTTP_SIGNER_CLASS_NAME;
import static
org.apache.hadoop.fs.s3a.impl.InstantiationIOException.unavailable;
import static org.apache.hadoop.util.Preconditions.checkArgument;
@@ -100,15 +102,17 @@ public static boolean isSignerRegistered(String
signerType) {
/**
* Create an instance of the given signer.
- *
+ * If a signer is Configurable, call setConf on it with the passed in config.
* @param signerType The signer type.
+ * @param conf configuration to init with
* @param configKey Config key used to configure the signer.
* @return The new signer instance.
* @throws InstantiationIOException instantiation problems.
* @throws IOException on any other problem.
*
*/
- public static Signer createSigner(String signerType, String configKey)
throws IOException {
+ public static Signer createSigner(String signerType, final Configuration
conf, String configKey)
+ throws IOException {
if (S3_V2_SIGNER.equals(signerType)) {
throw unavailable(null, null, configKey, S3_V2_SIGNER + " is no longer
supported");
}
@@ -124,7 +128,10 @@ public static Signer createSigner(String signerType,
String configKey) throws IO
Signer signer =
S3AUtils.getInstanceFromReflection(className, null, null,
Signer.class, "create",
configKey);
-
+ requireNonNull(conf);
+ if (signer instanceof Configurable) {
+ ((Configurable) signer).setConf(conf);
+ }
return signer;
}
diff --git
a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/AWSClientConfig.java
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/AWSClientConfig.java
index 4ea43e7a66e..a7c6859fa23 100644
---
a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/AWSClientConfig.java
+++
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/AWSClientConfig.java
@@ -124,7 +124,7 @@ public static ClientOverrideConfiguration.Builder
createClientConfigBuilder(Conf
if (!signer.isEmpty()) {
LOG.debug("Signer override = {}", signer);
overrideConfigBuilder.putAdvancedOption(SdkAdvancedClientOption.SIGNER,
- SignerFactory.createSigner(signer, SIGNING_ALGORITHM));
+ SignerFactory.createSigner(signer, conf, SIGNING_ALGORITHM));
}
initSigner(conf, overrideConfigBuilder, awsServiceIdentifier);
@@ -407,7 +407,7 @@ private static void initSigner(Configuration conf,
if (!signerOverride.isEmpty()) {
LOG.debug("Signer override for {} = {}", awsServiceIdentifier,
signerOverride);
clientConfig.putAdvancedOption(SdkAdvancedClientOption.SIGNER,
- SignerFactory.createSigner(signerOverride, configKey));
+ SignerFactory.createSigner(signerOverride, conf, configKey));
}
}
}
diff --git
a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestCustomSigner.java
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestCustomSigner.java
index c357c8aac0f..4f8902a236e 100644
---
a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestCustomSigner.java
+++
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestCustomSigner.java
@@ -42,6 +42,7 @@
import org.slf4j.LoggerFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.ContentSummary;
import org.apache.hadoop.fs.FileSystem;
@@ -193,6 +194,9 @@ private S3AFileSystem
runStoreOperationsAndVerify(UserGroupInformation ugi,
.as("Configuration TEST_KEY mismatch in %s",
CustomSigner.description())
.isEqualTo(identifier);
+ Assertions.assertThat(CustomSigner.getLastConfiguration())
+ .isSameAs(fs.getConf());
+
// now do some more operations to make sure all is good.
final Path subdir = new Path(finalPath, "year=1970/month=1/day=1");
fs.mkdirs(subdir);
@@ -257,8 +261,8 @@ private String determineRegion(String bucketName) throws
IOException {
}
@Private
- public static final class CustomSigner extends AbstractAwsS3V4Signer
implements Signer {
-
+ public static final class CustomSigner extends AbstractAwsS3V4Signer
implements Signer,
+ Configurable {
private static final AtomicInteger INSTANTIATION_COUNT =
new AtomicInteger(0);
@@ -267,11 +271,27 @@ public static final class CustomSigner extends
AbstractAwsS3V4Signer implements
private static StoreValue lastStoreValue;
+ private static Configuration lastConfiguration;
+
public CustomSigner() {
int c = INSTANTIATION_COUNT.incrementAndGet();
LOG.info("Creating Signer #{}", c);
}
+ @Override
+ public void setConf(final Configuration conf) {
+ lastConfiguration = conf;
+ }
+
+ @Override
+ public Configuration getConf() {
+ return lastConfiguration;
+ }
+
+ public static Configuration getLastConfiguration() {
+ return lastConfiguration;
+ }
+
/**
* Method to sign the incoming request with credentials.
*
diff --git
a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/TestSignerManager.java
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/TestSignerManager.java
index b56b8c20bfe..5cfa65c30db 100644
---
a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/TestSignerManager.java
+++
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/TestSignerManager.java
@@ -34,7 +34,9 @@
import org.junit.Test;
import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
import
org.apache.hadoop.fs.s3a.auth.TestSignerManager.SignerInitializerForTest.StoreValue;
import org.apache.hadoop.fs.s3a.auth.delegation.DelegationTokenProvider;
import org.apache.hadoop.fs.s3a.impl.InstantiationIOException;
@@ -93,7 +95,7 @@ public void testCustomSignerFailureIfNotRegistered() throws
Exception {
signerManager.initCustomSigners();
// Simulate a call from the AWS SDK to create the signer.
intercept(InstantiationIOException.class,
- () -> SignerFactory.createSigner("testsignerUnregistered", null));
+ () -> SignerFactory.createSigner("testsignerUnregistered", config,
null));
}
@Test
@@ -103,7 +105,11 @@ public void testCustomSignerInitialization() throws
IOException {
SignerManager signerManager = new SignerManager("dontcare", null, config,
UserGroupInformation.getCurrentUser());
signerManager.initCustomSigners();
- Signer s1 = SignerFactory.createSigner("testsigner1", null);
+ Signer s1 = SignerFactory.createSigner("testsigner1", config, null);
+ Configurable cs1 = (Configurable) s1;
+ Assertions.assertThat(cs1.getConf())
+ .describedAs("Configuration of %s", s1)
+ .isSameAs(config);
s1.sign(null, null);
Assertions.assertThat(SignerForTest1.initialized)
.as(SignerForTest1.class.getName() + " not initialized")
@@ -119,13 +125,13 @@ public void testMultipleCustomSignerInitialization()
throws IOException {
SignerManager signerManager = new SignerManager("dontcare", null, config,
UserGroupInformation.getCurrentUser());
signerManager.initCustomSigners();
- Signer s1 = SignerFactory.createSigner("testsigner1", null);
+ Signer s1 = SignerFactory.createSigner("testsigner1", config, null);
s1.sign(null, null);
Assertions.assertThat(SignerForTest1.initialized)
.as(SignerForTest1.class.getName() + " not initialized")
.isEqualTo(true);
- Signer s2 = SignerFactory.createSigner("testsigner2", null);
+ Signer s2 = SignerFactory.createSigner("testsigner2", config, null);
s2.sign(null, null);
Assertions.assertThat(SignerForTest2.initialized)
.as(SignerForTest2.class.getName() + " not initialized")
@@ -321,7 +327,7 @@ private void closeAndVerifyNull(Closeable closeable, String
bucketName,
* SignerForTest1.
*/
@Private
- public static class SignerForTest1 implements Signer {
+ public static final class SignerForTest1 extends Configured implements
Signer {
private static boolean initialized = false;
@@ -586,13 +592,13 @@ private SdkHttpFullRequest
constructSignableRequest(String bucketName) {
@Test
public void testV2SignerRejected() throws Throwable {
intercept(InstantiationIOException.class, "no longer supported",
- () -> SignerFactory.createSigner(S3_V2_SIGNER, "key"));
+ () -> SignerFactory.createSigner(S3_V2_SIGNER, new Configuration(),
"key"));
}
@Test
public void testUnknownSignerRejected() throws Throwable {
intercept(InstantiationIOException.class, "unknownSigner",
- () -> SignerFactory.createSigner("unknownSigner", "key"));
+ () -> SignerFactory.createSigner("unknownSigner", new Configuration(),
"key"));
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
