This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 567661f0c06 [feature](maxcompute)support maxcompute ram_role_arn and
ecs_ram_role authenticate. (#60649)
567661f0c06 is described below
commit 567661f0c067e3c63331effdcdc682850ef9331c
Author: daidai <[email protected]>
AuthorDate: Fri Feb 27 15:29:54 2026 +0800
[feature](maxcompute)support maxcompute ram_role_arn and ecs_ram_role
authenticate. (#60649)
### What problem does this PR solve?
Problem Summary:
support maxcompute RAMRoleArn and ECSRAMRole authenticate, not only
ak-sk.
```
CREATE CATALOG mc PROPERTIES (
"type" = "max_compute",
"mc.default.project" = "xxx",
"mc.access_key" = "xx",
"mc.secret_key" = "xxx",
"mc.endpoint" = "xxx"
);
CREATE CATALOG mc PROPERTIES (
"type" = "max_compute",
"mc.auth.type" = "ram_role_arn",
"mc.ram_role_arn" = "xxxx",
"mc.default.project" = "xxx",
"mc.access_key" = "xx",
"mc.secret_key" = "xxx",
"mc.endpoint" = "xxx"
);
CREATE CATALOG mc PROPERTIES (
"type" = "max_compute",
"mc.auth.type" = "ecs_ram_role",
"mc.ecs_ram_role" = "xxxxxx",
"mc.default.project" = "xxx",
"mc.endpoint" = "xxx"
);
```
can ref :
https://www.alibabacloud.com/help/zh/maxcompute/user-guide/configure-access-credentials#599325c7000vc
method 3 & 4.
---
be/src/runtime/descriptors.cpp | 9 +++
be/src/runtime/descriptors.h | 10 +--
.../exec/format/table/max_compute_jni_reader.cpp | 34 +++++-----
.../sink/writer/maxcompute/vmc_table_writer.cpp | 4 +-
.../doris/maxcompute/MaxComputeJniScanner.java | 12 +---
.../doris/maxcompute/MaxComputeJniWriter.java | 12 ++--
fe/fe-common/pom.xml | 18 +++++
.../doris/common/maxcompute}/MCProperties.java | 19 +++---
.../apache/doris/common/maxcompute/MCUtils.java | 78 ++++++++++++++++++++++
.../org/apache/doris/common/util/PrintableMap.java | 2 +-
.../maxcompute/MaxComputeExternalCatalog.java | 33 +++------
.../maxcompute/MaxComputeExternalTable.java | 8 +--
.../maxcompute/source/MaxComputeScanNode.java | 2 +-
.../apache/doris/planner/MaxComputeTableSink.java | 3 +-
gensrc/thrift/DataSinks.thrift | 1 +
gensrc/thrift/Descriptors.thrift | 5 +-
16 files changed, 162 insertions(+), 88 deletions(-)
diff --git a/be/src/runtime/descriptors.cpp b/be/src/runtime/descriptors.cpp
index 8cce4c769fc..22fdf727799 100644
--- a/be/src/runtime/descriptors.cpp
+++ b/be/src/runtime/descriptors.cpp
@@ -310,6 +310,15 @@ MaxComputeTableDescriptor::MaxComputeTableDescriptor(const
TTableDescriptor& tde
_init_status =
Status::InvalidArgument("fail to init
MaxComputeTableDescriptor, missing quota.");
}
+
+ if (tdesc.mcTable.__isset.properties) [[likely]] {
+ _props = tdesc.mcTable.properties;
+ } else {
+ static const std::string MC_ACCESS_KEY = "mc.access_key";
+ static const std::string MC_SECRET_KEY = "mc.secret_key";
+ _props.insert({MC_ACCESS_KEY, _access_key});
+ _props.insert({MC_SECRET_KEY, _secret_key});
+ }
}
MaxComputeTableDescriptor::~MaxComputeTableDescriptor() = default;
diff --git a/be/src/runtime/descriptors.h b/be/src/runtime/descriptors.h
index bd6ff2195c7..a075126d8fd 100644
--- a/be/src/runtime/descriptors.h
+++ b/be/src/runtime/descriptors.h
@@ -245,18 +245,20 @@ public:
std::string endpoint() const { return _endpoint; }
std::string quota() const { return _quota; }
Status init_status() const { return _init_status; }
+ std::map<std::string, std::string> properties() const { return _props; }
private:
std::string _region; //deprecated
std::string _project;
std::string _table;
- std::string _odps_url; //deprecated
- std::string _tunnel_url; //deprecated
- std::string _access_key;
- std::string _secret_key;
+ std::string _odps_url; //deprecated
+ std::string _tunnel_url; //deprecated
+ std::string _access_key; //deprecated
+ std::string _secret_key; //deprecated
std::string _public_access; //deprecated
std::string _endpoint;
std::string _quota;
+ std::map<std::string, std::string> _props;
Status _init_status = Status::OK();
};
diff --git a/be/src/vec/exec/format/table/max_compute_jni_reader.cpp
b/be/src/vec/exec/format/table/max_compute_jni_reader.cpp
index 81999f89617..4e167dc9d40 100644
--- a/be/src/vec/exec/format/table/max_compute_jni_reader.cpp
+++ b/be/src/vec/exec/format/table/max_compute_jni_reader.cpp
@@ -62,27 +62,27 @@ MaxComputeJniReader::MaxComputeJniReader(const
MaxComputeTableDescriptor* mc_des
}
index++;
}
- std::map<String, String> params = {
- {"access_key", _table_desc->access_key()},
- {"secret_key", _table_desc->secret_key()},
- {"endpoint", _table_desc->endpoint()},
- {"quota", _table_desc->quota()},
- {"project", _table_desc->project()},
- {"table", _table_desc->table()},
- {"session_id", _max_compute_params.session_id},
- {"scan_serializer", _max_compute_params.table_batch_read_session},
+ auto properties = _table_desc->properties();
+ properties["endpoint"] = _table_desc->endpoint();
+ properties["quota"] = _table_desc->quota();
+ properties["project"] = _table_desc->project();
+ properties["table"] = _table_desc->table();
- {"start_offset", std::to_string(_range.start_offset)},
- {"split_size", std::to_string(_range.size)},
- {"required_fields", required_fields.str()},
- {"columns_types", columns_types.str()},
+ properties["session_id"] = _max_compute_params.session_id;
+ properties["scan_serializer"] =
_max_compute_params.table_batch_read_session;
+
+ properties["start_offset"] = std::to_string(_range.start_offset);
+ properties["split_size"] = std::to_string(_range.size);
+ properties["required_fields"] = required_fields.str();
+ properties["columns_types"] = columns_types.str();
+
+ properties["connect_timeout"] =
std::to_string(_max_compute_params.connect_timeout);
+ properties["read_timeout"] =
std::to_string(_max_compute_params.read_timeout);
+ properties["retry_count"] =
std::to_string(_max_compute_params.retry_times);
- {"connect_timeout",
std::to_string(_max_compute_params.connect_timeout)},
- {"read_timeout", std::to_string(_max_compute_params.read_timeout)},
- {"retry_count", std::to_string(_max_compute_params.retry_times)}};
_jni_connector = std::make_unique<JniConnector>(
- "org/apache/doris/maxcompute/MaxComputeJniScanner", params,
column_names);
+ "org/apache/doris/maxcompute/MaxComputeJniScanner", properties,
column_names);
}
Status MaxComputeJniReader::init_reader() {
diff --git a/be/src/vec/sink/writer/maxcompute/vmc_table_writer.cpp
b/be/src/vec/sink/writer/maxcompute/vmc_table_writer.cpp
index ff183b488fd..f9e3dbfa39b 100644
--- a/be/src/vec/sink/writer/maxcompute/vmc_table_writer.cpp
+++ b/be/src/vec/sink/writer/maxcompute/vmc_table_writer.cpp
@@ -102,9 +102,7 @@ Status VMCTableWriter::open(RuntimeState* state,
RuntimeProfile* profile) {
}
std::map<std::string, std::string> VMCTableWriter::_build_base_writer_params()
{
- std::map<std::string, std::string> params;
- if (_mc_sink.__isset.access_key) params["access_key"] =
_mc_sink.access_key;
- if (_mc_sink.__isset.secret_key) params["secret_key"] =
_mc_sink.secret_key;
+ auto params = _mc_sink.properties;
if (_mc_sink.__isset.endpoint) params["endpoint"] = _mc_sink.endpoint;
if (_mc_sink.__isset.project) params["project"] = _mc_sink.project;
if (_mc_sink.__isset.table_name) params["table"] = _mc_sink.table_name;
diff --git
a/fe/be-java-extensions/max-compute-connector/src/main/java/org/apache/doris/maxcompute/MaxComputeJniScanner.java
b/fe/be-java-extensions/max-compute-connector/src/main/java/org/apache/doris/maxcompute/MaxComputeJniScanner.java
index 4128d66f09d..ce1b3941dca 100644
---
a/fe/be-java-extensions/max-compute-connector/src/main/java/org/apache/doris/maxcompute/MaxComputeJniScanner.java
+++
b/fe/be-java-extensions/max-compute-connector/src/main/java/org/apache/doris/maxcompute/MaxComputeJniScanner.java
@@ -19,10 +19,9 @@ package org.apache.doris.maxcompute;
import org.apache.doris.common.jni.JniScanner;
import org.apache.doris.common.jni.vec.ColumnType;
+import org.apache.doris.common.maxcompute.MCUtils;
import com.aliyun.odps.Odps;
-import com.aliyun.odps.account.Account;
-import com.aliyun.odps.account.AliyunAccount;
import com.aliyun.odps.table.configuration.CompressionCodec;
import com.aliyun.odps.table.configuration.ReaderOptions;
import com.aliyun.odps.table.configuration.RestOptions;
@@ -87,7 +86,7 @@ public class MaxComputeJniScanner extends JniScanner {
private TableBatchReadSession scan;
public String sessionId;
- private String project; //final ???
+ private String project;
private String table;
private SplitReader<VectorSchemaRoot> currentSplitReader;
@@ -120,8 +119,6 @@ public class MaxComputeJniScanner extends JniScanner {
}
}
- String accessKey = Objects.requireNonNull(params.get(ACCESS_KEY),
"required property '" + ACCESS_KEY + "'.");
- String secretKey = Objects.requireNonNull(params.get(SECRET_KEY),
"required property '" + SECRET_KEY + "'.");
String endpoint = Objects.requireNonNull(params.get(ENDPOINT),
"required property '" + ENDPOINT + "'.");
String quota = Objects.requireNonNull(params.get(QUOTA), "required
property '" + QUOTA + "'.");
String scanSerializer =
Objects.requireNonNull(params.get(SCAN_SERIALIZER),
@@ -137,10 +134,7 @@ public class MaxComputeJniScanner extends JniScanner {
timeZone = ZoneId.systemDefault();
}
-
- Account account = new AliyunAccount(accessKey, secretKey);
- Odps odps = new Odps(account);
-
+ Odps odps = MCUtils.createMcClient(params);
odps.setDefaultProject(project);
odps.setEndpoint(endpoint);
diff --git
a/fe/be-java-extensions/max-compute-connector/src/main/java/org/apache/doris/maxcompute/MaxComputeJniWriter.java
b/fe/be-java-extensions/max-compute-connector/src/main/java/org/apache/doris/maxcompute/MaxComputeJniWriter.java
index daba96d3948..86b849cd627 100644
---
a/fe/be-java-extensions/max-compute-connector/src/main/java/org/apache/doris/maxcompute/MaxComputeJniWriter.java
+++
b/fe/be-java-extensions/max-compute-connector/src/main/java/org/apache/doris/maxcompute/MaxComputeJniWriter.java
@@ -19,11 +19,10 @@ package org.apache.doris.maxcompute;
import org.apache.doris.common.jni.JniWriter;
import org.apache.doris.common.jni.vec.VectorTable;
+import org.apache.doris.common.maxcompute.MCUtils;
import com.aliyun.odps.Odps;
import com.aliyun.odps.OdpsType;
-import com.aliyun.odps.account.Account;
-import com.aliyun.odps.account.AliyunAccount;
import com.aliyun.odps.table.configuration.ArrowOptions;
import com.aliyun.odps.table.configuration.ArrowOptions.TimestampUnit;
import com.aliyun.odps.table.configuration.RestOptions;
@@ -94,8 +93,7 @@ public class MaxComputeJniWriter extends JniWriter {
private static final String READ_TIMEOUT = "read_timeout";
private static final String RETRY_COUNT = "retry_count";
- private final String accessKey;
- private final String secretKey;
+ private final Map<String, String> params;
private final String endpoint;
private final String project;
private final String tableName;
@@ -121,8 +119,7 @@ public class MaxComputeJniWriter extends JniWriter {
public MaxComputeJniWriter(int batchSize, Map<String, String> params) {
super(batchSize, params);
- this.accessKey = Objects.requireNonNull(params.get(ACCESS_KEY),
"required property '" + ACCESS_KEY + "'.");
- this.secretKey = Objects.requireNonNull(params.get(SECRET_KEY),
"required property '" + SECRET_KEY + "'.");
+ this.params = params;
this.endpoint = Objects.requireNonNull(params.get(ENDPOINT), "required
property '" + ENDPOINT + "'.");
this.project = Objects.requireNonNull(params.get(PROJECT), "required
property '" + PROJECT + "'.");
this.tableName = Objects.requireNonNull(params.get(TABLE), "required
property '" + TABLE + "'.");
@@ -139,8 +136,7 @@ public class MaxComputeJniWriter extends JniWriter {
@Override
public void open() throws IOException {
try {
- Account account = new AliyunAccount(accessKey, secretKey);
- Odps odps = new Odps(account);
+ Odps odps = MCUtils.createMcClient(params);
odps.setDefaultProject(project);
odps.setEndpoint(endpoint);
diff --git a/fe/fe-common/pom.xml b/fe/fe-common/pom.xml
index 838b3e97881..e6a452fa760 100644
--- a/fe/fe-common/pom.xml
+++ b/fe/fe-common/pom.xml
@@ -108,6 +108,24 @@ under the License.
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-iostreams</artifactId>
</dependency>
+ <dependency>
+ <groupId>com.aliyun.odps</groupId>
+ <artifactId>odps-sdk-core</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.arrow</groupId>
+ <artifactId>arrow-vector</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.ini4j</groupId>
+ <artifactId>ini4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.bouncycastle</groupId>
+ <artifactId>bcprov-jdk18on</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
</dependencies>
<build>
<finalName>doris-fe-common</finalName>
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/MCProperties.java
b/fe/fe-common/src/main/java/org/apache/doris/common/maxcompute/MCProperties.java
similarity index 90%
rename from
fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/MCProperties.java
rename to
fe/fe-common/src/main/java/org/apache/doris/common/maxcompute/MCProperties.java
index 70feb48f2f7..627f3bc03e2 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/MCProperties.java
+++
b/fe/fe-common/src/main/java/org/apache/doris/common/maxcompute/MCProperties.java
@@ -15,16 +15,12 @@
// specific language governing permissions and limitations
// under the License.
-package org.apache.doris.datasource.property.constants;
-
-import org.apache.doris.common.credentials.CloudCredential;
-
-import java.util.Map;
+package org.apache.doris.common.maxcompute;
/**
* properties for aliyun max compute
*/
-public class MCProperties extends BaseProperties {
+public class MCProperties {
//To be compatible with previous versions of the catalog.
public static final String REGION = "mc.region";
@@ -99,7 +95,12 @@ public class MCProperties extends BaseProperties {
public static final String ENABLE_NAMESPACE_SCHEMA =
"mc.enable.namespace.schema";
public static final String DEFAULT_ENABLE_NAMESPACE_SCHEMA = "false";
- public static CloudCredential getCredential(Map<String, String> props) {
- return getCloudCredential(props, ACCESS_KEY, SECRET_KEY,
SESSION_TOKEN);
- }
+ public static final String AUTH_TYPE = "mc.auth.type";
+ public static final String AUTH_TYPE_AK_SK = "ak_sk";
+ public static final String AUTH_TYPE_RAM_ROLE_ARN = "ram_role_arn";
+ public static final String AUTH_TYPE_ECS_RAM_ROLE = "ecs_ram_role";
+ public static final String DEFAULT_AUTH_TYPE = AUTH_TYPE_AK_SK;
+
+ public static final String RAM_ROLE_ARN = "mc.ram_role_arn";
+ public static final String ECS_RAM_ROLE = "mc.ecs_ram_role";
}
diff --git
a/fe/fe-common/src/main/java/org/apache/doris/common/maxcompute/MCUtils.java
b/fe/fe-common/src/main/java/org/apache/doris/common/maxcompute/MCUtils.java
new file mode 100644
index 00000000000..fc7f47fc268
--- /dev/null
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/maxcompute/MCUtils.java
@@ -0,0 +1,78 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.common.maxcompute;
+
+import com.aliyun.auth.credentials.Credential;
+import com.aliyun.auth.credentials.provider.EcsRamRoleCredentialProvider;
+import com.aliyun.auth.credentials.provider.RamRoleArnCredentialProvider;
+import com.aliyun.odps.Odps;
+import com.aliyun.odps.account.Account;
+import com.aliyun.odps.account.AklessAccount;
+import com.aliyun.odps.account.AliyunAccount;
+
+import java.util.Map;
+
+public class MCUtils {
+ public static void checkAuthProperties(Map<String, String> properties) {
+ String authType = properties.getOrDefault(MCProperties.AUTH_TYPE,
MCProperties.DEFAULT_AUTH_TYPE);
+ if (authType.equalsIgnoreCase(MCProperties.AUTH_TYPE_AK_SK)) {
+ if (!properties.containsKey(MCProperties.ACCESS_KEY) ||
!properties.containsKey(MCProperties.SECRET_KEY)) {
+ throw new RuntimeException("Missing access key or secret key
for AK/SK auth type");
+ }
+ } else if
(authType.equalsIgnoreCase(MCProperties.AUTH_TYPE_RAM_ROLE_ARN)) {
+ if (!properties.containsKey(MCProperties.ACCESS_KEY) ||
!properties.containsKey(MCProperties.SECRET_KEY)
+ || !properties.containsKey(MCProperties.RAM_ROLE_ARN)) {
+ throw new RuntimeException("Missing access key, secret key or
role arn for RAM Role ARN auth type");
+ }
+ } else if
(authType.equalsIgnoreCase(MCProperties.AUTH_TYPE_ECS_RAM_ROLE)) {
+ if (!properties.containsKey(MCProperties.ECS_RAM_ROLE)) {
+ throw new RuntimeException("Missing role name for ECS RAM Role
auth type");
+ }
+ } else {
+ throw new RuntimeException("Unsupported auth type: " + authType);
+ }
+ }
+
+ public static Odps createMcClient(Map<String, String> properties) {
+ String authType = properties.getOrDefault(MCProperties.AUTH_TYPE,
MCProperties.DEFAULT_AUTH_TYPE);
+ if (authType.equalsIgnoreCase(MCProperties.AUTH_TYPE_AK_SK)) {
+ String accessKey = properties.get(MCProperties.ACCESS_KEY);
+ String secretKey = properties.get(MCProperties.SECRET_KEY);
+ Account account = new AliyunAccount(accessKey, secretKey);
+ return new Odps(account);
+ } else if
(authType.equalsIgnoreCase(MCProperties.AUTH_TYPE_RAM_ROLE_ARN)) {
+ String accessKey = properties.get(MCProperties.ACCESS_KEY);
+ String secretKey = properties.get(MCProperties.SECRET_KEY);
+ String roleArn = properties.get(MCProperties.RAM_ROLE_ARN);
+ RamRoleArnCredentialProvider ramRoleArnCredentialProvider =
+ RamRoleArnCredentialProvider.builder().credential(
+ Credential.builder().accessKeyId(accessKey)
+
.accessKeySecret(secretKey).build())
+ .roleArn(roleArn).build();
+ AklessAccount aklessAccount = new
AklessAccount(ramRoleArnCredentialProvider);
+ return new Odps(aklessAccount);
+ } else if
(authType.equalsIgnoreCase(MCProperties.AUTH_TYPE_ECS_RAM_ROLE)) {
+ String roleName = properties.get(MCProperties.ECS_RAM_ROLE);
+ EcsRamRoleCredentialProvider credentialProvider =
EcsRamRoleCredentialProvider.create(roleName);
+ AklessAccount aklessAccount = new
AklessAccount(credentialProvider);
+ return new Odps(aklessAccount);
+ } else {
+ throw new RuntimeException("Unsupported auth type: " + authType);
+ }
+ }
+}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/common/util/PrintableMap.java
b/fe/fe-core/src/main/java/org/apache/doris/common/util/PrintableMap.java
index 784432ba9fe..0f7d67c3454 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/util/PrintableMap.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/PrintableMap.java
@@ -17,8 +17,8 @@
package org.apache.doris.common.util;
+import org.apache.doris.common.maxcompute.MCProperties;
import org.apache.doris.datasource.property.ConnectorPropertiesUtils;
-import org.apache.doris.datasource.property.constants.MCProperties;
import
org.apache.doris.datasource.property.metastore.AWSGlueMetaStoreBaseProperties;
import org.apache.doris.datasource.property.metastore.AliyunDLFBaseProperties;
import org.apache.doris.datasource.property.storage.AzureProperties;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalCatalog.java
index dec483ef2ef..a0270c2d38d 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalCatalog.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalCatalog.java
@@ -19,20 +19,18 @@ package org.apache.doris.datasource.maxcompute;
import org.apache.doris.common.DdlException;
-import org.apache.doris.common.credentials.CloudCredential;
+import org.apache.doris.common.maxcompute.MCProperties;
+import org.apache.doris.common.maxcompute.MCUtils;
import org.apache.doris.datasource.CatalogProperty;
import org.apache.doris.datasource.ExternalCatalog;
import org.apache.doris.datasource.InitCatalogLog;
import org.apache.doris.datasource.SessionContext;
import org.apache.doris.datasource.operations.ExternalMetadataOperations;
-import org.apache.doris.datasource.property.constants.MCProperties;
import org.apache.doris.transaction.TransactionManagerFactory;
import com.aliyun.odps.Odps;
import com.aliyun.odps.Partition;
-import com.aliyun.odps.account.Account;
import com.aliyun.odps.account.AccountFormat;
-import com.aliyun.odps.account.AliyunAccount;
import com.aliyun.odps.table.TableIdentifier;
import com.aliyun.odps.table.configuration.RestOptions;
import com.aliyun.odps.table.configuration.SplitOptions;
@@ -56,9 +54,8 @@ public class MaxComputeExternalCatalog extends
ExternalCatalog {
// you can ref : https://help.aliyun.com/zh/maxcompute/user-guide/endpoints
private static final String endpointTemplate =
"http://service.{}.maxcompute.aliyun-inc.com/api";;
+ private Map<String, String> props;
private Odps odps;
- private String accessKey;
- private String secretKey;
private String endpoint;
private String defaultProject;
private String quota;
@@ -160,7 +157,7 @@ public class MaxComputeExternalCatalog extends
ExternalCatalog {
@Override
protected void initLocalObjectsImpl() {
- Map<String, String> props = catalogProperty.getProperties();
+ props = catalogProperty.getProperties();
generatorEndpoint();
@@ -200,16 +197,11 @@ public class MaxComputeExternalCatalog extends
ExternalCatalog {
.withReadTimeout(readTimeout)
.withRetryTimes(retryTimes).build();
- CloudCredential credential = MCProperties.getCredential(props);
- accessKey = credential.getAccessKey();
- secretKey = credential.getSecretKey();
-
dateTimePredicatePushDown = Boolean.parseBoolean(
props.getOrDefault(MCProperties.DATETIME_PREDICATE_PUSH_DOWN,
MCProperties.DEFAULT_DATETIME_PREDICATE_PUSH_DOWN));
- Account account = new AliyunAccount(accessKey, secretKey);
- this.odps = new Odps(account);
+ odps = MCUtils.createMcClient(props);
odps.setDefaultProject(defaultProject);
odps.setEndpoint(endpoint);
@@ -299,14 +291,9 @@ public class MaxComputeExternalCatalog extends
ExternalCatalog {
return mcStructureHelper.listTableNames(getClient(), dbName);
}
- public String getAccessKey() {
- makeSureInitialized();
- return accessKey;
- }
-
- public String getSecretKey() {
+ public Map<String, String> getProperties() {
makeSureInitialized();
- return secretKey;
+ return props;
}
public String getEndpoint() {
@@ -460,10 +447,6 @@ public class MaxComputeExternalCatalog extends
ExternalCatalog {
+ MCProperties.READ_TIMEOUT + "/" +
MCProperties.RETRY_COUNT + "must be an integer");
}
- CloudCredential credential = MCProperties.getCredential(props);
- if (!credential.isWhole()) {
- throw new DdlException("Max-Compute credential properties '"
- + MCProperties.ACCESS_KEY + "' and '" +
MCProperties.SECRET_KEY + "' are required.");
- }
+ MCUtils.checkAuthProperties(props);
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalTable.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalTable.java
index 09f052f2cea..d7724b1c7e0 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalTable.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalTable.java
@@ -322,17 +322,11 @@ public class MaxComputeExternalTable extends
ExternalTable {
TMCTable tMcTable = new TMCTable();
MaxComputeExternalCatalog mcCatalog = ((MaxComputeExternalCatalog)
catalog);
- tMcTable.setAccessKey(mcCatalog.getAccessKey());
- tMcTable.setSecretKey(mcCatalog.getSecretKey());
- tMcTable.setOdpsUrl("deprecated");
- tMcTable.setRegion("deprecated");
+ tMcTable.setProperties(mcCatalog.getProperties());
tMcTable.setEndpoint(mcCatalog.getEndpoint());
// use mc project as dbName
tMcTable.setProject(dbName);
tMcTable.setQuota(mcCatalog.getQuota());
-
- tMcTable.setTunnelUrl("deprecated");
- tMcTable.setProject("deprecated");
tMcTable.setTable(name);
TTableDescriptor tTableDescriptor = new TTableDescriptor(getId(),
TTableType.MAX_COMPUTE_TABLE,
schema.size(), 0, getName(), dbName);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/source/MaxComputeScanNode.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/source/MaxComputeScanNode.java
index 0e8865f8499..5f73f5cab8d 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/source/MaxComputeScanNode.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/source/MaxComputeScanNode.java
@@ -33,13 +33,13 @@ import org.apache.doris.catalog.ScalarType;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.UserException;
+import org.apache.doris.common.maxcompute.MCProperties;
import org.apache.doris.common.util.LocationPath;
import org.apache.doris.datasource.FileQueryScanNode;
import org.apache.doris.datasource.TableFormatType;
import org.apache.doris.datasource.maxcompute.MaxComputeExternalCatalog;
import org.apache.doris.datasource.maxcompute.MaxComputeExternalTable;
import org.apache.doris.datasource.maxcompute.source.MaxComputeSplit.SplitType;
-import org.apache.doris.datasource.property.constants.MCProperties;
import
org.apache.doris.nereids.trees.plans.logical.LogicalFileScan.SelectedPartitions;
import org.apache.doris.nereids.util.DateUtils;
import org.apache.doris.planner.PlanNodeId;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/planner/MaxComputeTableSink.java
b/fe/fe-core/src/main/java/org/apache/doris/planner/MaxComputeTableSink.java
index 91bfceb8ac9..fdb50245a8e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/MaxComputeTableSink.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/MaxComputeTableSink.java
@@ -66,8 +66,7 @@ public class MaxComputeTableSink extends
BaseExternalTableDataSink {
MaxComputeExternalCatalog catalog = (MaxComputeExternalCatalog)
targetTable.getCatalog();
- tSink.setAccessKey(catalog.getAccessKey());
- tSink.setSecretKey(catalog.getSecretKey());
+ tSink.setProperties(catalog.getProperties());
tSink.setEndpoint(catalog.getEndpoint());
tSink.setProject(catalog.getDefaultProject());
tSink.setTableName(targetTable.getName());
diff --git a/gensrc/thrift/DataSinks.thrift b/gensrc/thrift/DataSinks.thrift
index 29b4db2b189..8add59d47af 100644
--- a/gensrc/thrift/DataSinks.thrift
+++ b/gensrc/thrift/DataSinks.thrift
@@ -513,6 +513,7 @@ struct TMaxComputeTableSink {
13: optional i32 retry_count
14: optional list<string> partition_columns // partition column names for
dynamic partition
15: optional string write_session_id // Storage API write session
ID
+ 16: optional map<string, string> properties // contains authentication
properties
}
struct TDataSink {
diff --git a/gensrc/thrift/Descriptors.thrift b/gensrc/thrift/Descriptors.thrift
index 897ad4f377e..6f71a29f72a 100644
--- a/gensrc/thrift/Descriptors.thrift
+++ b/gensrc/thrift/Descriptors.thrift
@@ -444,13 +444,14 @@ struct TMCTable {
1: optional string region // deprecated
2: optional string project
3: optional string table
- 4: optional string access_key
- 5: optional string secret_key
+ 4: optional string access_key // deprecated
+ 5: optional string secret_key // deprecated
6: optional string public_access // deprecated
7: optional string odps_url // deprecated
8: optional string tunnel_url // deprecated
9: optional string endpoint
10: optional string quota
+ 11: optional map<string, string> properties // contains authentication
properties
}
struct TTrinoConnectorTable {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
