This is an automated email from the ASF dual-hosted git repository.
cnauroth pushed a commit to branch branch-3.4
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.4 by this push:
new b48e118b300 HADOOP-19648. Cos use token credential will lose token
field
b48e118b300 is described below
commit b48e118b3004df275a0ff7ec77238d5f2665da3b
Author: leosanqing <storm...@qq.com>
AuthorDate: Fri Aug 29 20:56:02 2025 +0000
HADOOP-19648. Cos use token credential will lose token field
Closes #7911
Signed-off-by: Chris Nauroth <cnaur...@apache.org>
---
hadoop-cloud-storage-project/hadoop-cos/pom.xml | 14 ++
.../hadoop/fs/cosn/CosNativeFileSystemStore.java | 9 +-
.../apache/hadoop/fs/cosn/TestCosCredentials.java | 55 +++++--
.../DynamicTemporaryCosnCredentialsProvider.java | 162 +++++++++++++++++++++
.../contract/TestCosNContractDynamicToken.java | 69 +++++++++
.../hadoop-cos/src/test/resources/core-site.xml | 2 +-
.../hadoop-cos/src/test/resources/log4j.properties | 2 +-
7 files changed, 293 insertions(+), 20 deletions(-)
diff --git a/hadoop-cloud-storage-project/hadoop-cos/pom.xml
b/hadoop-cloud-storage-project/hadoop-cos/pom.xml
index 2c2b9a55315..7eb111ba6fd 100644
--- a/hadoop-cloud-storage-project/hadoop-cos/pom.xml
+++ b/hadoop-cloud-storage-project/hadoop-cos/pom.xml
@@ -34,6 +34,7 @@
<properties>
<file.encoding>UTF-8</file.encoding>
<downloadSources>true</downloadSources>
+ <tencentcloud.verion>3.1.1322</tencentcloud.verion>
</properties>
<profiles>
@@ -113,6 +114,19 @@
<scope>compile</scope>
</dependency>
+ <dependency>
+ <groupId>org.assertj</groupId>
+ <artifactId>assertj-core</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>com.tencentcloudapi</groupId>
+ <artifactId>tencentcloud-sdk-java</artifactId>
+ <version>${tencentcloud.verion}</version>
+ <scope>test</scope>
+ </dependency>
+
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
diff --git
a/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/CosNativeFileSystemStore.java
b/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/CosNativeFileSystemStore.java
index d2484c0e47b..c145ce2ff9c 100644
---
a/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/CosNativeFileSystemStore.java
+++
b/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/CosNativeFileSystemStore.java
@@ -32,8 +32,6 @@
import com.qcloud.cos.COSClient;
import com.qcloud.cos.ClientConfig;
-import com.qcloud.cos.auth.BasicCOSCredentials;
-import com.qcloud.cos.auth.COSCredentials;
import com.qcloud.cos.endpoint.SuffixEndpointBuilder;
import com.qcloud.cos.exception.CosClientException;
import com.qcloud.cos.exception.CosServiceException;
@@ -103,11 +101,6 @@ private void initCOSClient(URI uri, Configuration conf)
throws IOException {
throw new IOException(exceptionMsg);
}
- COSCredentials cosCred;
- cosCred = new BasicCOSCredentials(
- credentialProviderList.getCredentials().getCOSAccessKeyId(),
- credentialProviderList.getCredentials().getCOSSecretKey());
-
boolean useHttps = conf.getBoolean(CosNConfigKeys.COSN_USE_HTTPS_KEY,
CosNConfigKeys.DEFAULT_USE_HTTPS);
@@ -133,7 +126,7 @@ private void initCOSClient(URI uri, Configuration conf)
throws IOException {
conf.getInt(CosNConfigKeys.MAX_CONNECTION_NUM,
CosNConfigKeys.DEFAULT_MAX_CONNECTION_NUM));
- this.cosClient = new COSClient(cosCred, config);
+ this.cosClient = new COSClient(credentialProviderList.getCredentials(),
config);
}
/**
diff --git
a/hadoop-cloud-storage-project/hadoop-cos/src/test/java/org/apache/hadoop/fs/cosn/TestCosCredentials.java
b/hadoop-cloud-storage-project/hadoop-cos/src/test/java/org/apache/hadoop/fs/cosn/TestCosCredentials.java
index 8b74f3639dd..88ede2fa5c5 100644
---
a/hadoop-cloud-storage-project/hadoop-cos/src/test/java/org/apache/hadoop/fs/cosn/TestCosCredentials.java
+++
b/hadoop-cloud-storage-project/hadoop-cos/src/test/java/org/apache/hadoop/fs/cosn/TestCosCredentials.java
@@ -17,9 +17,13 @@
*/
package org.apache.hadoop.fs.cosn;
+import com.qcloud.cos.auth.BasicSessionCredentials;
import com.qcloud.cos.auth.COSCredentials;
import com.qcloud.cos.auth.COSCredentialsProvider;
+
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
+
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -28,12 +32,14 @@
import java.net.URI;
import java.net.URISyntaxException;
+import static
org.apache.hadoop.fs.cosn.auth.DynamicTemporaryCosnCredentialsProvider.STS_SECRET_ID_KEY;
+import static
org.apache.hadoop.fs.cosn.auth.DynamicTemporaryCosnCredentialsProvider.STS_SECRET_KEY_KEY;
import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
public class TestCosCredentials {
- private static final Logger LOG =
- LoggerFactory.getLogger(TestCosCredentials.class);
+ private static final Logger LOG =
LoggerFactory.getLogger(TestCosCredentials.class);
private final URI fsUri;
@@ -50,10 +56,8 @@ public TestCosCredentials() throws URISyntaxException {
@Test
public void testSimpleCredentialsProvider() throws Throwable {
Configuration configuration = new Configuration();
- configuration.set(CosNConfigKeys.COSN_SECRET_ID_KEY,
- testCosNSecretId);
- configuration.set(CosNConfigKeys.COSN_SECRET_KEY_KEY,
- testCosNSecretKey);
+ configuration.set(CosNConfigKeys.COSN_SECRET_ID_KEY, testCosNSecretId);
+ configuration.set(CosNConfigKeys.COSN_SECRET_KEY_KEY, testCosNSecretKey);
validateCredentials(this.fsUri, configuration);
}
@@ -63,15 +67,14 @@ public void testEnvironmentCredentialsProvider() throws
Throwable {
// Set EnvironmentVariableCredentialsProvider as the CosCredentials
// Provider.
configuration.set(CosNConfigKeys.COSN_CREDENTIALS_PROVIDER,
- "org.apache.hadoop.fs.cosn.EnvironmentVariableCredentialsProvider");
+
"org.apache.hadoop.fs.cosn.auth.EnvironmentVariableCredentialsProvider");
// Set the environment variables storing the secret id and secret key.
System.setProperty(Constants.COSN_SECRET_ID_ENV, testCosNEnvSecretId);
System.setProperty(Constants.COSN_SECRET_KEY_ENV, testCosNEnvSecretKey);
validateCredentials(this.fsUri, configuration);
}
- private void validateCredentials(URI uri, Configuration configuration)
- throws IOException {
+ private void validateCredentials(URI uri, Configuration configuration)
throws IOException {
if (null != configuration) {
COSCredentialsProvider credentialsProvider =
CosNUtils.createCosCredentialsProviderSet(uri, configuration);
@@ -79,7 +82,7 @@ private void validateCredentials(URI uri, Configuration
configuration)
assertNotNull("The cos credentials obtained is null.", cosCredentials);
if (configuration.get(
CosNConfigKeys.COSN_CREDENTIALS_PROVIDER).compareToIgnoreCase(
- "org.apache.hadoop.fs.cosn.EnvironmentVariableCredentialsProvider")
+
"org.apache.hadoop.fs.cosn.auth.EnvironmentVariableCredentialsProvider")
== 0) {
if (null == cosCredentials.getCOSAccessKeyId()
|| cosCredentials.getCOSAccessKeyId().isEmpty()
@@ -131,4 +134,36 @@ private void validateCredentials(URI uri, Configuration
configuration)
}
}
}
+
+ @Test
+ public void testTmpTokenCredentialsProvider() throws Throwable {
+ Configuration configuration = new Configuration();
+ // Set DynamicTemporaryCosnCredentialsProvider as the CosCredentials
+ // Provider.
+ configuration.set(CosNConfigKeys.COSN_CREDENTIALS_PROVIDER,
+
"org.apache.hadoop.fs.cosn.auth.DynamicTemporaryCosnCredentialsProvider");
+
+ configuration.set(STS_SECRET_ID_KEY,
System.getProperty(STS_SECRET_ID_KEY));
+ configuration.set(STS_SECRET_KEY_KEY,
System.getProperty(STS_SECRET_KEY_KEY));
+ validateTmpTokenCredentials(this.fsUri, configuration);
+ }
+
+ private void validateTmpTokenCredentials(URI uri, Configuration
configuration)
+ throws IOException {
+ COSCredentialsProvider credentialsProvider =
+ CosNUtils.createCosCredentialsProviderSet(uri, configuration);
+ COSCredentials cosCredentials = credentialsProvider.getCredentials();
+ assertNotNull("The cos credentials obtained is null.", cosCredentials);
+ assertTrue("CredentialsProvider must be
DynamicTemporaryCosnCredentialsProvider",
+
StringUtils.equalsIgnoreCase(configuration.get(CosNConfigKeys.COSN_CREDENTIALS_PROVIDER),
+
"org.apache.hadoop.fs.cosn.auth.DynamicTemporaryCosnCredentialsProvider"));
+
+ assertTrue("cosCredentials must be instanceof BasicSessionCredentials",
+ cosCredentials instanceof BasicSessionCredentials);
+
+ assertNotNull("session access key id is null",
cosCredentials.getCOSAccessKeyId());
+ assertNotNull("session access key is null",
cosCredentials.getCOSSecretKey());
+ assertNotNull("access token is null",
+ ((BasicSessionCredentials) cosCredentials).getSessionToken());
+ }
}
diff --git
a/hadoop-cloud-storage-project/hadoop-cos/src/test/java/org/apache/hadoop/fs/cosn/auth/DynamicTemporaryCosnCredentialsProvider.java
b/hadoop-cloud-storage-project/hadoop-cos/src/test/java/org/apache/hadoop/fs/cosn/auth/DynamicTemporaryCosnCredentialsProvider.java
new file mode 100644
index 00000000000..8b46fa15929
--- /dev/null
+++
b/hadoop-cloud-storage-project/hadoop-cos/src/test/java/org/apache/hadoop/fs/cosn/auth/DynamicTemporaryCosnCredentialsProvider.java
@@ -0,0 +1,162 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.cosn.auth;
+
+import com.qcloud.cos.auth.BasicSessionCredentials;
+import com.qcloud.cos.auth.COSCredentials;
+import com.qcloud.cos.auth.COSCredentialsProvider;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.cosn.CosNConfigKeys;
+
+import com.tencentcloudapi.common.Credential;
+import com.tencentcloudapi.common.profile.ClientProfile;
+import com.tencentcloudapi.common.profile.HttpProfile;
+import com.tencentcloudapi.sts.v20180813.StsClient;
+import com.tencentcloudapi.sts.v20180813.models.GetFederationTokenRequest;
+import com.tencentcloudapi.sts.v20180813.models.GetFederationTokenResponse;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.concurrent.atomic.AtomicReference;
+
+/**
+ * A COSCredentialsProvider that generates temporary credentials from Tencent
Cloud STS.
+ * This provider requires a long-term secret ID and key with permission to call
+ * the STS GetFederationToken action.
+ */
+public class DynamicTemporaryCosnCredentialsProvider implements
COSCredentialsProvider {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(DynamicTemporaryCosnCredentialsProvider.class);
+
+ public static final String STS_SECRET_ID_KEY = "fs.cosn.auth.sts.secret.id";
+ public static final String STS_SECRET_KEY_KEY =
"fs.cosn.auth.sts.secret.key";
+ public static final String STS_ENDPOINT_KEY = "fs.cosn.auth.sts.endpoint";
+ public static final String DEFAULT_STS_ENDPOINT = "sts.tencentcloudapi.com";
+ public static final String TOKEN_DURATION_SECONDS_KEY =
"fs.cosn.auth.sts.token.duration.seconds";
+ public static final int DEFAULT_TOKEN_DURATION_SECONDS = 900; // 15 minutes
+
+ private final String longTermSecretId;
+ private final String longTermSecretKey;
+ private final String stsEndpoint;
+ private final String region;
+ private final String bucketName;
+ private final long durationSeconds;
+
+ private final AtomicReference<ExpiringCredentials> expiringCredentialsRef =
+ new AtomicReference<>();
+
+ public DynamicTemporaryCosnCredentialsProvider(Configuration conf) throws
IOException {
+ this.longTermSecretId = conf.get(STS_SECRET_ID_KEY);
+ this.longTermSecretKey = conf.get(STS_SECRET_KEY_KEY);
+ this.stsEndpoint = conf.get(STS_ENDPOINT_KEY, DEFAULT_STS_ENDPOINT);
+ this.region = conf.get(CosNConfigKeys.COSN_REGION_KEY);
+ this.bucketName = conf.get("fs.defaultFS").replace("cosn://", "");
+ this.durationSeconds = conf.getLong(TOKEN_DURATION_SECONDS_KEY,
DEFAULT_TOKEN_DURATION_SECONDS);
+
+ if (this.longTermSecretId == null || this.longTermSecretKey == null) {
+ throw new IOException(
+ "Long-term STS credentials not provided in configuration. Please set
" + STS_SECRET_ID_KEY
+ + " and " + STS_SECRET_KEY_KEY);
+ }
+ if (this.region == null || this.bucketName == null) {
+ throw new IOException("Bucket region or name not configured.");
+ }
+ }
+
+ @Override
+ public COSCredentials getCredentials() {
+ ExpiringCredentials current = expiringCredentialsRef.get();
+ // Refresh if credentials are not present, or are within 60 seconds of
expiry.
+ if (current == null
+ || System.currentTimeMillis() >= current.getExpirationTimeMillis() -
60000) {
+ LOG.info("STS credentials expired or not found, requesting new token.");
+ refresh();
+ }
+ return expiringCredentialsRef.get().getCredentials();
+ }
+
+ @Override
+ public void refresh() {
+ try {
+ Credential cred = new Credential(this.longTermSecretId,
this.longTermSecretKey);
+ HttpProfile httpProfile = new HttpProfile();
+ httpProfile.setEndpoint(this.stsEndpoint);
+ ClientProfile clientProfile = new ClientProfile();
+ clientProfile.setHttpProfile(httpProfile);
+
+ StsClient client = new StsClient(cred, this.region, clientProfile);
+ GetFederationTokenRequest req = new GetFederationTokenRequest();
+
+ String policyTemplate =
"{\"version\":\"2.0\",\"statement\":[{\"action\":[\"cos:*\"],"
+ +
"\"effect\":\"allow\",\"resource\":[\"qcs::cos:%s:uid/%s:%s/*\"]}]}";
+ String policy =
+ String.format(policyTemplate, this.region,
getAppIdFromBucket(this.bucketName),
+ this.bucketName);
+ req.setPolicy(policy);
+
+ req.setDurationSeconds(this.durationSeconds);
+ req.setName("HadoopCosNContractTest");
+
+ GetFederationTokenResponse resp = client.GetFederationToken(req);
+
+ long expirationTimeMillis = (resp.getExpiredTime() * 1000);
+ BasicSessionCredentials credentials =
+ new BasicSessionCredentials(resp.getCredentials().getTmpSecretId(),
+ resp.getCredentials().getTmpSecretKey(),
resp.getCredentials().getToken());
+
+ expiringCredentialsRef.set(new ExpiringCredentials(credentials,
expirationTimeMillis));
+ LOG.info("Successfully refreshed STS credentials. Expiration: {}",
+ new java.util.Date(expirationTimeMillis));
+
+ } catch (Exception e) {
+ LOG.error("Failed to get token from STS: {}", e.toString());
+ throw new RuntimeException("Failed to get token from STS", e);
+ }
+ }
+
+ private String getAppIdFromBucket(String bucket) {
+ int lastDash = bucket.lastIndexOf('-');
+ if (lastDash != -1 && lastDash < bucket.length() - 1) {
+ return bucket.substring(lastDash + 1);
+ }
+ throw new IllegalArgumentException("Could not determine AppID from bucket
name: " + bucket);
+ }
+
+ /**
+ * Helper class to hold credentials and their expiration time.
+ */
+ private static class ExpiringCredentials {
+ private final BasicSessionCredentials credentials;
+ private final long expirationTimeMillis;
+
+ ExpiringCredentials(BasicSessionCredentials credentials, long
expirationTimeMillis) {
+ this.credentials = credentials;
+ this.expirationTimeMillis = expirationTimeMillis;
+ }
+
+ BasicSessionCredentials getCredentials() {
+ return credentials;
+ }
+
+ long getExpirationTimeMillis() {
+ return expirationTimeMillis;
+ }
+ }
+}
\ No newline at end of file
diff --git
a/hadoop-cloud-storage-project/hadoop-cos/src/test/java/org/apache/hadoop/fs/cosn/contract/TestCosNContractDynamicToken.java
b/hadoop-cloud-storage-project/hadoop-cos/src/test/java/org/apache/hadoop/fs/cosn/contract/TestCosNContractDynamicToken.java
new file mode 100644
index 00000000000..3d53359ceff
--- /dev/null
+++
b/hadoop-cloud-storage-project/hadoop-cos/src/test/java/org/apache/hadoop/fs/cosn/contract/TestCosNContractDynamicToken.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.cosn.contract;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractCreateTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.apache.hadoop.fs.cosn.CosNConfigKeys;
+import org.apache.hadoop.fs.cosn.auth.DynamicTemporaryCosnCredentialsProvider;
+
+import static
org.apache.hadoop.fs.cosn.auth.DynamicTemporaryCosnCredentialsProvider.STS_SECRET_ID_KEY;
+import static
org.apache.hadoop.fs.cosn.auth.DynamicTemporaryCosnCredentialsProvider.STS_SECRET_KEY_KEY;
+
+/**
+ * Contract tests for CosN using a dynamic temporary token provider (STS).
+ * This test requires long-term credentials with STS access to be configured.
+ */
+public class TestCosNContractDynamicToken extends AbstractContractCreateTest {
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new CosNContract(conf);
+ }
+
+ @Override
+ protected Configuration createConfiguration() {
+ Configuration newConf = super.createConfiguration();
+
+ newConf.set(CosNConfigKeys.COSN_CREDENTIALS_PROVIDER,
+ DynamicTemporaryCosnCredentialsProvider.class.getName());
+ newConf.setBoolean("fs.cosn.impl.disable.cache", true);
+
+ String stsSecretId = System.getProperty(STS_SECRET_ID_KEY);
+ String stsSecretKey = System.getProperty(STS_SECRET_KEY_KEY);
+
+ if (stsSecretId == null || stsSecretKey == null) {
+ // Fallback to configuration for convenience, but log a warning.
+ System.err.println("WARN: Reading long-term STS credentials from
configuration file. "
+ + "It is recommended to use system properties for security.");
+ stsSecretId = newConf.get(STS_SECRET_ID_KEY);
+ stsSecretKey = newConf.get(STS_SECRET_KEY_KEY);
+ }
+
+ if (stsSecretId == null || stsSecretKey == null) {
+ throw new RuntimeException("STS credentials for tests are not provided. "
+ + "Please set them via system properties
(-Dfs.cosn.auth.sts.secret.id=... and "
+ + "-Dfs.cosn.auth.sts.secret.key=...)");
+ }
+
+ newConf.set(STS_SECRET_ID_KEY, stsSecretId);
+ newConf.set(STS_SECRET_KEY_KEY, stsSecretKey);
+
+ return newConf;
+ }
+}
\ No newline at end of file
diff --git
a/hadoop-cloud-storage-project/hadoop-cos/src/test/resources/core-site.xml
b/hadoop-cloud-storage-project/hadoop-cos/src/test/resources/core-site.xml
index fbd23bb326e..99f2c2db6d7 100644
--- a/hadoop-cloud-storage-project/hadoop-cos/src/test/resources/core-site.xml
+++ b/hadoop-cloud-storage-project/hadoop-cos/src/test/resources/core-site.xml
@@ -20,7 +20,7 @@
<property>
<name>fs.cosn.credentials.provider</name>
- <value>org.apache.hadoop.fs.cosn.auth.SimpleCredentialProvider</value>
+ <value>org.apache.hadoop.fs.cosn.auth.SimpleCredentialsProvider</value>
<description>
This option allows the user to specify how to get the credentials.
Comma-separated class names of credential provider classes which
diff --git
a/hadoop-cloud-storage-project/hadoop-cos/src/test/resources/log4j.properties
b/hadoop-cloud-storage-project/hadoop-cos/src/test/resources/log4j.properties
index 1a6baaec65a..ced0687caad 100644
---
a/hadoop-cloud-storage-project/hadoop-cos/src/test/resources/log4j.properties
+++
b/hadoop-cloud-storage-project/hadoop-cos/src/test/resources/log4j.properties
@@ -12,7 +12,7 @@
# log4j configuration used during build and unit tests
log4j.rootLogger=info,stdout
-log4j.threshhold=ALL
+log4j.threshold=ALL
log4j.appender.stdout=org.apache.log4j.ConsoleAppender
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %-5p %c{2}
(%F:%M(%L)) - %m%n
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
