This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.1 by this push:
new 9782738437f branch-4.1:[fix](fe) Improve MaxCompute catalog validation
(#64119) (#66157)
9782738437f is described below
commit 9782738437f7d5d61f20cd215288bd48a78a75f9
Author: daidai <[email protected]>
AuthorDate: Thu Jul 30 08:54:04 2026 +0800
branch-4.1:[fix](fe) Improve MaxCompute catalog validation (#64119) (#66157)
### What problem does this PR solve?
bp #64119
Problem Summary:
Use the `test_connection` catalog parameter, to perform ak-sk、 project
、schema validation when creating a maxcompute catalog.
### Release note
None
### Check List (For Author)
- Test <!-- At least one of them must be included. -->
- [ ] Regression test
- [ ] Unit Test
- [ ] Manual test (add detailed scripts or steps below)
- [ ] No need to test or manual test. Explain why:
- [ ] This is a refactor/code format and no logic has been changed.
- [ ] Previous test can cover this change.
- [ ] No code files have been changed.
- [ ] Other reason <!-- Add your reason? -->
- Behavior changed:
- [ ] No.
- [ ] Yes. <!-- Explain the behavior change -->
- Does this need documentation?
- [ ] No.
- [ ] Yes. <!-- Add document PR link here. eg:
https://github.com/apache/doris-website/pull/1214 -->
### Check List (For Reviewer who merge this PR)
- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label <!-- Add branch pick label that this PR
should merge into -->
---
.../doris/datasource/maxcompute/MCTransaction.java | 4 +
.../maxcompute/MaxComputeExternalCatalog.java | 70 +++++++++-
.../maxcompute/MaxComputeExternalTable.java | 11 ++
.../maxcompute/source/MaxComputeScanNode.java | 5 +
.../datasource/maxcompute/MCTransactionTest.java | 54 ++++++++
.../maxcompute/MaxComputeExternalCatalogTest.java | 146 +++++++++++++++++++++
.../maxcompute/source/MaxComputeScanNodeTest.java | 79 +++++++++++
fe/pom.xml | 2 +-
.../test_external_catalog_maxcompute.groovy | 6 +-
.../maxcompute/test_max_compute_schema.groovy | 5 +-
.../test_max_compute_validate_connection.groovy | 102 ++++++++++++++
11 files changed, 477 insertions(+), 7 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MCTransaction.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MCTransaction.java
index 9f1c61ddf24..6d5a6c9112f 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MCTransaction.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MCTransaction.java
@@ -71,6 +71,10 @@ public class MCTransaction implements Transaction {
public void beginInsert(ExternalTable dorisTable,
Optional<InsertCommandContext> ctx) throws UserException {
this.table = (MaxComputeExternalTable) dorisTable;
+ if (table.isUnsupportedOdpsTable()) {
+ throw new UserException("Writing MaxCompute external table or
logical view is not supported: "
+ + table.getDbName() + "." + table.getName());
+ }
try {
TableIdentifier tableId =
catalog.getOdpsTableIdentifier(table.getDbName(), table.getName());
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalCatalog.java
index 1292746cc25..33de9d0e1f4 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalCatalog.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalCatalog.java
@@ -29,6 +29,7 @@ import
org.apache.doris.datasource.operations.ExternalMetadataOperations;
import org.apache.doris.transaction.TransactionManagerFactory;
import com.aliyun.odps.Odps;
+import com.aliyun.odps.OdpsException;
import com.aliyun.odps.Partition;
import com.aliyun.odps.account.AccountFormat;
import com.aliyun.odps.table.TableIdentifier;
@@ -53,7 +54,6 @@ public class MaxComputeExternalCatalog extends
ExternalCatalog {
// you can ref : https://help.aliyun.com/zh/maxcompute/user-guide/endpoints
private static final String endpointTemplate =
"http://service.{}.maxcompute.aliyun-inc.com/api";;
-
private Map<String, String> props;
private Odps odps;
private String endpoint;
@@ -207,6 +207,9 @@ public class MaxComputeExternalCatalog extends
ExternalCatalog {
odps = MCUtils.createMcClient(props);
odps.setDefaultProject(defaultProject);
odps.setEndpoint(endpoint);
+ odps.getRestClient().setConnectTimeout(connectTimeout);
+ odps.getRestClient().setReadTimeout(readTimeout);
+ odps.getRestClient().setRetryTimes(retryTimes);
String accountFormatProp =
props.getOrDefault(MCProperties.ACCOUNT_FORMAT,
MCProperties.DEFAULT_ACCOUNT_FORMAT);
if (accountFormatProp.equals(MCProperties.ACCOUNT_FORMAT_NAME)) {
@@ -234,6 +237,69 @@ public class MaxComputeExternalCatalog extends
ExternalCatalog {
transactionManager =
TransactionManagerFactory.createMCTransactionManager(this);
}
+ @Override
+ public void checkWhenCreating() throws DdlException {
+ boolean testConnection =
Boolean.parseBoolean(catalogProperty.getOrDefault(TEST_CONNECTION,
+ String.valueOf(DEFAULT_TEST_CONNECTION)));
+ if (!testConnection) {
+ return;
+ }
+ // MaxCompute has no MetastoreProperties-backed connectivity tester
yet,
+ // so run its catalog-specific test directly under the common
test_connection switch.
+ boolean enableNamespaceSchema = Boolean.parseBoolean(
+
catalogProperty.getOrDefault(MCProperties.ENABLE_NAMESPACE_SCHEMA,
+ MCProperties.DEFAULT_ENABLE_NAMESPACE_SCHEMA));
+ try {
+ initLocalObjects();
+ validateMaxComputeConnection(enableNamespaceSchema);
+ } catch (Exception e) {
+ throw new DdlException(e.getMessage(), e);
+ }
+ }
+
+ protected void validateMaxComputeConnection(boolean enableNamespaceSchema)
{
+ if (enableNamespaceSchema) {
+ validateMaxComputeProjectAndNamespaceSchema();
+ } else {
+ validateMaxComputeProject();
+ }
+ }
+
+ private void validateMaxComputeProject() {
+ boolean projectExists;
+ try {
+ projectExists = maxComputeProjectExists(defaultProject);
+ } catch (Exception e) {
+ throw new RuntimeException("Failed to validate MaxCompute project
'" + defaultProject
+ + "'. Check " + MCProperties.PROJECT + ", " +
MCProperties.ENDPOINT
+ + " and credentials. Cause: " + e.getMessage(), e);
+ }
+ if (!projectExists) {
+ throw new RuntimeException("Failed to validate MaxCompute project
'" + defaultProject
+ + "'. Check " + MCProperties.PROJECT + ", " +
MCProperties.ENDPOINT
+ + " and credentials. Cause: project does not exist or is
not accessible");
+ }
+ }
+
+ private void validateMaxComputeProjectAndNamespaceSchema() {
+ try {
+ validateMaxComputeNamespaceSchemaAccess(defaultProject);
+ } catch (Exception e) {
+ throw new RuntimeException("Failed to validate MaxCompute project
'" + defaultProject
+ + "' with namespace schema. Check " + MCProperties.PROJECT
+ ", " + MCProperties.ENDPOINT
+ + ", credentials, and whether the schema list is
accessible for the namespace schema "
+ + "configuration. Cause: " + e.getMessage(), e);
+ }
+ }
+
+ protected boolean maxComputeProjectExists(String projectName) throws
OdpsException {
+ return odps.projects().exists(projectName);
+ }
+
+ protected void validateMaxComputeNamespaceSchemaAccess(String projectName)
throws OdpsException {
+ odps.schemas().iterator(projectName).hasNext();
+ }
+
public Odps getClient() {
makeSureInitialized();
return odps;
@@ -402,7 +468,7 @@ public class MaxComputeExternalCatalog extends
ExternalCatalog {
MCProperties.DEFAULT_SPLIT_BYTE_SIZE));
if (splitByteSize < 10485760L) {
- throw new DdlException(MCProperties.SPLIT_ROW_COUNT + "
must be greater than or equal to 10485760");
+ throw new DdlException(MCProperties.SPLIT_BYTE_SIZE + "
must be greater than or equal to 10485760");
}
} else if
(splitStrategy.equals(MCProperties.SPLIT_BY_ROW_COUNT_STRATEGY)) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalTable.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalTable.java
index 85e592b6f3d..1ca1e42b0c7 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalTable.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalTable.java
@@ -54,6 +54,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
+import java.util.Objects;
import java.util.Optional;
import java.util.stream.Collectors;
@@ -347,6 +348,16 @@ public class MaxComputeExternalTable extends ExternalTable
{
.orElse(null);
}
+ public boolean isUnsupportedOdpsTable() {
+ Table odpsTable = getOdpsTable();
+ return isUnsupportedOdpsTable(odpsTable);
+ }
+
+ public static boolean isUnsupportedOdpsTable(Table odpsTable) {
+ Objects.requireNonNull(odpsTable, "MaxCompute table metadata is not
initialized");
+ return odpsTable.isExternalTable() || odpsTable.isVirtualView();
+ }
+
@Override
public boolean isPartitionedTable() {
makeSureInitialized();
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/source/MaxComputeScanNode.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/source/MaxComputeScanNode.java
index bf09288be98..3c71e1ad856 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/source/MaxComputeScanNode.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/source/MaxComputeScanNode.java
@@ -633,6 +633,11 @@ public class MaxComputeScanNode extends FileQueryScanNode {
public List<Split> getSplits(int numBackends) throws UserException {
List<Split> result = new ArrayList<>();
com.aliyun.odps.Table odpsTable = table.getOdpsTable();
+ if (MaxComputeExternalTable.isUnsupportedOdpsTable(odpsTable)) {
+ throw new UserException("Reading MaxCompute external table or
logical view is not supported: "
+ + table.getDbName() + "." + table.getName());
+ }
+
if (desc.getSlots().isEmpty() || odpsTable.getFileNum() <= 0) {
return result;
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/maxcompute/MCTransactionTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/maxcompute/MCTransactionTest.java
new file mode 100644
index 00000000000..e76f192a858
--- /dev/null
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/maxcompute/MCTransactionTest.java
@@ -0,0 +1,54 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.maxcompute;
+
+import org.apache.doris.common.UserException;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import java.util.Optional;
+
+public class MCTransactionTest {
+ @Test
+ public void testBeginInsertRejectsOdpsExternalTable() {
+ assertBeginInsertRejectsUnsupportedOdpsTable("mc_external_table");
+ }
+
+ @Test
+ public void testBeginInsertRejectsOdpsLogicalView() {
+ assertBeginInsertRejectsUnsupportedOdpsTable("mc_logical_view");
+ }
+
+ private void assertBeginInsertRejectsUnsupportedOdpsTable(String
tableName) {
+ MaxComputeExternalCatalog catalog =
Mockito.mock(MaxComputeExternalCatalog.class);
+ MaxComputeExternalTable table =
Mockito.mock(MaxComputeExternalTable.class);
+ Mockito.when(table.isUnsupportedOdpsTable()).thenReturn(true);
+ Mockito.when(table.getDbName()).thenReturn("default");
+ Mockito.when(table.getName()).thenReturn(tableName);
+
+ MCTransaction transaction = new MCTransaction(catalog);
+
+ UserException exception = Assert.assertThrows(UserException.class,
+ () -> transaction.beginInsert(table, Optional.empty()));
+ Assert.assertTrue(exception.getMessage().contains(
+ "Writing MaxCompute external table or logical view is not
supported: default." + tableName));
+ Mockito.verify(catalog,
Mockito.never()).getOdpsTableIdentifier(Mockito.anyString(),
Mockito.anyString());
+ }
+}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalCatalogTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalCatalogTest.java
new file mode 100644
index 00000000000..dfe22f136b5
--- /dev/null
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalCatalogTest.java
@@ -0,0 +1,146 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.maxcompute;
+
+import org.apache.doris.common.DdlException;
+import org.apache.doris.common.maxcompute.MCProperties;
+import org.apache.doris.datasource.ExternalCatalog;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class MaxComputeExternalCatalogTest {
+ @Test
+ public void testSplitByteSizeErrorMessage() {
+ Map<String, String> props = new HashMap<>();
+ addRequiredProperties(props);
+ props.put(MCProperties.SPLIT_STRATEGY,
MCProperties.SPLIT_BY_BYTE_SIZE_STRATEGY);
+ props.put(MCProperties.SPLIT_BYTE_SIZE, "1048576");
+
+ MaxComputeExternalCatalog catalog = new MaxComputeExternalCatalog(1L,
"mc_catalog", null, props, "");
+
+ DdlException exception = Assert.assertThrows(DdlException.class,
catalog::checkProperties);
+ Assert.assertTrue(exception.getMessage().contains(
+ MCProperties.SPLIT_BYTE_SIZE + " must be greater than or equal
to 10485760"));
+
Assert.assertFalse(exception.getMessage().contains(MCProperties.SPLIT_ROW_COUNT));
+ }
+
+ @Test
+ public void testCheckWhenCreatingSkipsValidationByDefault() throws
DdlException {
+ Map<String, String> props = createRequiredProperties(true);
+ TestMaxComputeExternalCatalog catalog = new
TestMaxComputeExternalCatalog(props);
+
+ catalog.checkWhenCreating();
+
+ Assert.assertNull(catalog.checkedProjectName);
+ Assert.assertNull(catalog.checkedNamespaceSchemaProjectName);
+ }
+
+ @Test
+ public void testCheckWhenCreatingValidatesProjectWhenValidationEnabled()
throws DdlException {
+ Map<String, String> props = createRequiredProperties(false);
+ props.put(ExternalCatalog.TEST_CONNECTION, "true");
+ TestMaxComputeExternalCatalog catalog = new
TestMaxComputeExternalCatalog(props);
+
+ catalog.checkWhenCreating();
+
+ Assert.assertEquals("mc_project", catalog.checkedProjectName);
+ Assert.assertNull(catalog.checkedNamespaceSchemaProjectName);
+ }
+
+ @Test
+ public void
testCheckWhenCreatingValidatesSchemaWhenNamespaceSchemaEnabled() throws
DdlException {
+ Map<String, String> props = createRequiredProperties(true);
+ props.put(ExternalCatalog.TEST_CONNECTION, "true");
+ TestMaxComputeExternalCatalog catalog = new
TestMaxComputeExternalCatalog(props);
+
+ catalog.checkWhenCreating();
+
+ Assert.assertNull(catalog.checkedProjectName);
+ Assert.assertEquals("mc_project",
catalog.checkedNamespaceSchemaProjectName);
+ }
+
+ @Test
+ public void testCheckWhenCreatingReportsInaccessibleProject() {
+ Map<String, String> props = createRequiredProperties(false);
+ props.put(ExternalCatalog.TEST_CONNECTION, "true");
+ TestMaxComputeExternalCatalog catalog = new
TestMaxComputeExternalCatalog(props);
+ catalog.projectExists = false;
+
+ DdlException exception = Assert.assertThrows(DdlException.class,
catalog::checkWhenCreating);
+
+ Assert.assertTrue(exception.getMessage().contains("Failed to validate
MaxCompute project 'mc_project'"));
+ Assert.assertTrue(exception.getMessage().contains("does not exist or
is not accessible"));
+ Assert.assertNull(catalog.checkedNamespaceSchemaProjectName);
+ }
+
+ @Test
+ public void testCheckWhenCreatingReportsInaccessibleNamespaceSchema() {
+ Map<String, String> props = createRequiredProperties(true);
+ props.put(ExternalCatalog.TEST_CONNECTION, "true");
+ TestMaxComputeExternalCatalog catalog = new
TestMaxComputeExternalCatalog(props);
+ catalog.threeTierModel = false;
+
+ DdlException exception = Assert.assertThrows(DdlException.class,
catalog::checkWhenCreating);
+
+ Assert.assertTrue(exception.getMessage().contains("Failed to validate
MaxCompute project 'mc_project'"));
+ Assert.assertTrue(exception.getMessage().contains("schema list is
accessible"));
+ }
+
+ private static Map<String, String> createRequiredProperties(boolean
enableNamespaceSchema) {
+ Map<String, String> props = new HashMap<>();
+ addRequiredProperties(props);
+ props.put(MCProperties.ENABLE_NAMESPACE_SCHEMA,
Boolean.toString(enableNamespaceSchema));
+ return props;
+ }
+
+ private static void addRequiredProperties(Map<String, String> props) {
+ props.put(MCProperties.PROJECT, "mc_project");
+ props.put(MCProperties.ENDPOINT,
"http://service.cn-beijing.maxcompute.aliyun-inc.com/api";);
+ props.put(MCProperties.ACCESS_KEY, "access_key");
+ props.put(MCProperties.SECRET_KEY, "secret_key");
+ }
+
+ private static class TestMaxComputeExternalCatalog extends
MaxComputeExternalCatalog {
+ private boolean projectExists = true;
+ private boolean threeTierModel = true;
+ private String checkedProjectName;
+ private String checkedNamespaceSchemaProjectName;
+
+ private TestMaxComputeExternalCatalog(Map<String, String> props) {
+ super(1L, "mc_catalog", null, props, "");
+ }
+
+ @Override
+ protected boolean maxComputeProjectExists(String projectName) {
+ checkedProjectName = projectName;
+ return projectExists;
+ }
+
+ @Override
+ protected void validateMaxComputeNamespaceSchemaAccess(String
projectName) {
+ checkedNamespaceSchemaProjectName = projectName;
+ if (!threeTierModel) {
+ throw new RuntimeException("schema list is not accessible");
+ }
+ }
+ }
+}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/maxcompute/source/MaxComputeScanNodeTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/maxcompute/source/MaxComputeScanNodeTest.java
new file mode 100644
index 00000000000..192bb301483
--- /dev/null
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/maxcompute/source/MaxComputeScanNodeTest.java
@@ -0,0 +1,79 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.maxcompute.source;
+
+import org.apache.doris.analysis.TupleDescriptor;
+import org.apache.doris.analysis.TupleId;
+import org.apache.doris.common.UserException;
+import org.apache.doris.datasource.maxcompute.MaxComputeExternalTable;
+import
org.apache.doris.nereids.trees.plans.logical.LogicalFileScan.SelectedPartitions;
+import org.apache.doris.planner.PlanNodeId;
+import org.apache.doris.planner.ScanContext;
+import org.apache.doris.qe.SessionVariable;
+
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.junit.MockitoJUnitRunner;
+
+@RunWith(MockitoJUnitRunner.class)
+public class MaxComputeScanNodeTest {
+ @Mock
+ private MaxComputeExternalTable table;
+ @Mock
+ private com.aliyun.odps.Table odpsTable;
+ @Mock
+ private TupleDescriptor desc;
+
+ private MaxComputeScanNode node;
+
+ @Before
+ public void setUp() {
+ Mockito.when(table.getOdpsTable()).thenReturn(odpsTable);
+ Mockito.when(desc.getTable()).thenReturn(table);
+ Mockito.when(desc.getId()).thenReturn(new TupleId(0));
+ node = new MaxComputeScanNode(new PlanNodeId(0), desc,
+ SelectedPartitions.NOT_PRUNED, false, new SessionVariable(),
ScanContext.EMPTY);
+ }
+
+ @Test
+ public void testGetSplitsRejectsOdpsExternalTable() {
+ assertGetSplitsRejectsUnsupportedOdpsTable(true, false,
"mc_external_table");
+ }
+
+ @Test
+ public void testGetSplitsRejectsOdpsLogicalView() {
+ assertGetSplitsRejectsUnsupportedOdpsTable(false, true,
"mc_logical_view");
+ }
+
+ private void assertGetSplitsRejectsUnsupportedOdpsTable(boolean
isExternalTable, boolean isVirtualView,
+ String tableName) {
+ Mockito.when(odpsTable.isExternalTable()).thenReturn(isExternalTable);
+ Mockito.when(odpsTable.isVirtualView()).thenReturn(isVirtualView);
+ Mockito.when(table.getDbName()).thenReturn("default");
+ Mockito.when(table.getName()).thenReturn(tableName);
+
+ UserException exception = Assert.assertThrows(UserException.class, ()
-> node.getSplits(1));
+ Assert.assertTrue(exception.getMessage().contains(
+ "Reading MaxCompute external table or logical view is not
supported: default." + tableName));
+ Mockito.verify(odpsTable, Mockito.never()).getFileNum();
+ }
+}
diff --git a/fe/pom.xml b/fe/pom.xml
index 0d0dbc0bf81..e8dc5655cec 100644
--- a/fe/pom.xml
+++ b/fe/pom.xml
@@ -1389,7 +1389,7 @@ under the License.
</exclusions>
</dependency>
<dependency>
- <!-- This is a dependency chain: odps-sdk-core:0.53.2-public →
credentials-java:0.3.12 → com.aliyun:tea:[1.1.14,2.0.0)-->
+ <!-- Keep tea pinned for Alibaba SDK dependency chains that
allow com.aliyun:tea:[1.1.14,2.0.0). -->
<groupId>com.aliyun</groupId>
<artifactId>tea</artifactId>
<version>1.4.1</version>
diff --git
a/regression-test/suites/external_table_p2/maxcompute/test_external_catalog_maxcompute.groovy
b/regression-test/suites/external_table_p2/maxcompute/test_external_catalog_maxcompute.groovy
index 827ed963873..474abf4b8ea 100644
---
a/regression-test/suites/external_table_p2/maxcompute/test_external_catalog_maxcompute.groovy
+++
b/regression-test/suites/external_table_p2/maxcompute/test_external_catalog_maxcompute.groovy
@@ -341,7 +341,8 @@ suite("test_external_catalog_maxcompute",
"p2,external,maxcompute,external_remot
"mc.default.project" = "${mc_db}",
"mc.access_key" = "${ak}",
"mc.secret_key" = "${sk}",
- "mc.endpoint" =
"http://service.cn-beijing-vpc.maxcompute.aliyun-inc.com/api";
+ "mc.endpoint" =
"http://service.cn-beijing-vpc.maxcompute.aliyun-inc.com/api";,
+ "test_connection" = "true"
);
"""
@@ -379,7 +380,8 @@ suite("test_external_catalog_maxcompute",
"p2,external,maxcompute,external_remot
"mc.default.project" = "${mc_db}",
"mc.access_key" = "${ak}",
"mc.secret_key" = "${sk}",
- "mc.endpoint" =
"http://service.cn-beijing-vpc.maxcompute.aliyun-inc.com/api";
+ "mc.endpoint" =
"http://service.cn-beijing-vpc.maxcompute.aliyun-inc.com/api";,
+ "test_connection" = "true"
);
"""
sql """ switch `${mc_catalog_name}`; """
diff --git
a/regression-test/suites/external_table_p2/maxcompute/test_max_compute_schema.groovy
b/regression-test/suites/external_table_p2/maxcompute/test_max_compute_schema.groovy
index d317c43da21..19580a7df13 100644
---
a/regression-test/suites/external_table_p2/maxcompute/test_max_compute_schema.groovy
+++
b/regression-test/suites/external_table_p2/maxcompute/test_max_compute_schema.groovy
@@ -106,7 +106,8 @@ suite("test_max_compute_schema",
"p2,external,maxcompute,external_remote,externa
"mc.access_key" = "${ak}",
"mc.secret_key" = "${sk}",
"mc.endpoint" =
"http://service.cn-beijing-vpc.maxcompute.aliyun-inc.com/api";,
- "mc.enable.namespace.schema" = "true"
+ "mc.enable.namespace.schema" = "true",
+ "test_connection" = "true"
);
"""
@@ -251,4 +252,4 @@ suite("test_max_compute_schema",
"p2,external,maxcompute,external_remote,externa
qt_mc_join_q34 """SELECT p.product_name, e.department, e.salary FROM
analytics.product_sales p JOIN iot.employee_salary e ON p.id=e.id ORDER BY
p.id;"""
qt_mc_join_q35 """SELECT o.order_id, p.product_name, e.emp_name FROM
`default`.order_detail o JOIN analytics.product_sales p ON o.id=p.id JOIN
iot.employee_salary e ON o.id=e.id WHERE o.ds1='202510' ORDER BY o.id;"""
}
-}
\ No newline at end of file
+}
diff --git
a/regression-test/suites/external_table_p2/maxcompute/test_max_compute_validate_connection.groovy
b/regression-test/suites/external_table_p2/maxcompute/test_max_compute_validate_connection.groovy
new file mode 100644
index 00000000000..d88e65a2340
--- /dev/null
+++
b/regression-test/suites/external_table_p2/maxcompute/test_max_compute_validate_connection.groovy
@@ -0,0 +1,102 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_max_compute_validate_connection", "p2,external") {
+ String enabled = context.config.otherConfigs.get("enableMaxComputeTest")
+ if (enabled == null || !enabled.equalsIgnoreCase("true")) {
+ logger.info("disable MaxCompute test.")
+ return
+ }
+
+ String ak = context.config.otherConfigs.get("ak")
+ String sk = context.config.otherConfigs.get("sk")
+ String defaultProject = "mc_datalake"
+ String invalidEndpoint = "http://127.0.0.1:1/api";
+ String catalogDefaultFalse = "test_mc_validate_default_false"
+ String catalogExplicitFalse = "test_mc_validate_explicit_false"
+ String catalogValidateProject = "test_mc_validate_project"
+ String catalogValidateSchema = "test_mc_validate_schema"
+
+ sql """drop catalog if exists ${catalogDefaultFalse}"""
+ sql """drop catalog if exists ${catalogExplicitFalse}"""
+ sql """drop catalog if exists ${catalogValidateProject}"""
+ sql """drop catalog if exists ${catalogValidateSchema}"""
+
+ sql """
+ create catalog ${catalogDefaultFalse} properties (
+ "type" = "max_compute",
+ "mc.default.project" = "${defaultProject}",
+ "mc.access_key" = "${ak}",
+ "mc.secret_key" = "${sk}",
+ "mc.endpoint" = "${invalidEndpoint}",
+ "mc.connect_timeout" = "1",
+ "mc.read_timeout" = "1",
+ "mc.retry_count" = "1"
+ );
+ """
+ sql """drop catalog if exists ${catalogDefaultFalse}"""
+
+ sql """
+ create catalog ${catalogExplicitFalse} properties (
+ "type" = "max_compute",
+ "mc.default.project" = "${defaultProject}",
+ "mc.access_key" = "${ak}",
+ "mc.secret_key" = "${sk}",
+ "mc.endpoint" = "${invalidEndpoint}",
+ "mc.connect_timeout" = "1",
+ "mc.read_timeout" = "1",
+ "mc.retry_count" = "1",
+ "test_connection" = "false"
+ );
+ """
+ sql """drop catalog if exists ${catalogExplicitFalse}"""
+
+ test {
+ sql """
+ create catalog ${catalogValidateProject} properties (
+ "type" = "max_compute",
+ "mc.default.project" = "${defaultProject}",
+ "mc.access_key" = "${ak}",
+ "mc.secret_key" = "${sk}",
+ "mc.endpoint" = "${invalidEndpoint}",
+ "mc.connect_timeout" = "1",
+ "mc.read_timeout" = "1",
+ "mc.retry_count" = "1",
+ "test_connection" = "true"
+ );
+ """
+ exception "Failed to validate MaxCompute project"
+ }
+
+ test {
+ sql """
+ create catalog ${catalogValidateSchema} properties (
+ "type" = "max_compute",
+ "mc.default.project" = "${defaultProject}",
+ "mc.access_key" = "${ak}",
+ "mc.secret_key" = "${sk}",
+ "mc.endpoint" = "${invalidEndpoint}",
+ "mc.connect_timeout" = "1",
+ "mc.read_timeout" = "1",
+ "mc.retry_count" = "1",
+ "mc.enable.namespace.schema" = "true",
+ "test_connection" = "true"
+ );
+ """
+ exception "with namespace schema"
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
