This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new ac2b060cd28 [fix](tvf) fix azure tvf: can not build s3() (#45872)
ac2b060cd28 is described below
commit ac2b060cd286688d70e992b974a1a8d6cbe43000
Author: Kaijie Chen <[email protected]>
AuthorDate: Fri Dec 27 19:18:03 2024 +0800
[fix](tvf) fix azure tvf: can not build s3() (#45872)
### What problem does this PR solve?
Issue Number: DORIS-17644
Related PR: #37240
Problem Summary:
`properties` is an immutable map, trying to modify it will cause
`UnsupportedOperationException` to be thrown.
```
2024-12-24 18:01:22,287 WARN (mysql-nio-pool-0|490)
[StmtExecutor.executeByNereids():836] Nereids plan query failed:
SELECT * FROM S3 ( ... )
org.apache.doris.nereids.exceptions.AnalysisException: Can not build s3():
null
at
org.apache.doris.nereids.trees.expressions.functions.table.S3.toCatalogFunction(S3.java:55)
~[doris-fe.jar:1.2-SNAPSHOT]
at
com.google.common.base.Suppliers$NonSerializableMemoizingSupplier.get(Suppliers.java:186)
~[guava-33.2.1-jre.jar:?]
at
org.apache.doris.nereids.trees.expressions.functions.table.TableValuedFunction.lambda$new$0(TableValuedFunction.java:55)
~[doris-fe.jar:1.2-SNAPSHOT]
at
com.google.common.base.Suppliers$NonSerializableMemoizingSupplier.get(Suppliers.java:186)
~[guava-33.2.1-jre.jar:?]
at
org.apache.doris.nereids.trees.expressions.functions.table.TableValuedFunction.getTable(TableValuedFunction.java:103)
~[doris-fe.jar:1.2-SNAPSHOT]
at
org.apache.doris.nereids.trees.plans.logical.LogicalTVFRelation.computeOutput(LogicalTVFRelation.java:105)
~[doris-fe.jar:1.2-SNAPSHOT]
at
com.google.common.base.Suppliers$NonSerializableMemoizingSupplier.get(Suppliers.java:186)
~[guava-33.2.1-jre.jar:?]
at
org.apache.doris.nereids.properties.LogicalProperties.getOutput(LogicalProperties.java:104)
~[doris-fe.jar:1.2-SNAPSHOT]
at
org.apache.doris.nereids.trees.plans.AbstractPlan.getOutput(AbstractPlan.java:169)
~[doris-fe.jar:1.2-SNAPSHOT]
at
org.apache.doris.nereids.util.PlanUtils.fastGetChildrenOutputs(PlanUtils.java:161)
~[doris-fe.jar:1.2-SNAPSHOT]
at
org.apache.doris.nereids.rules.analysis.BindExpression.buildSimpleExprAnalyzer(BindExpression.java:1241)
~[doris-fe.jar:1.2-SNAPSHOT]
at
org.apache.doris.nereids.rules.analysis.BindExpression.bindProject(BindExpression.java:626)
~[doris-fe.jar:1.2-SNAPSHOT]
at
org.apache.doris.nereids.pattern.PatternMatcher$1.transform(PatternMatcher.java:92)
~[doris-fe.jar:1.2-SNAPSHOT]
at
org.apache.doris.nereids.rules.AppliedAwareRule.transform(AppliedAwareRule.java:55)
~[doris-fe.jar:1.2-SNAPSHOT]
at
org.apache.doris.nereids.jobs.rewrite.PlanTreeRewriteJob.rewrite(PlanTreeRewriteJob.java:57)
~[doris-fe.jar:1.2-SNAPSHOT]
at
org.apache.doris.nereids.jobs.rewrite.PlanTreeRewriteBottomUpJob.rewriteThis(PlanTreeRewriteBottomUpJob.java:91)
~[doris-fe.jar:1.2-SNAPSHOT]
at
org.apache.doris.nereids.jobs.rewrite.PlanTreeRewriteBottomUpJob.execute(PlanTreeRewriteBottomUpJob.java:75)
~[doris-fe.jar:1.2-SNAPSHOT]
at
org.apache.doris.nereids.jobs.scheduler.SimpleJobScheduler.executeJobPool(SimpleJobScheduler.java:44)
~[doris-fe.jar:1.2-SNAPSHOT]
at
org.apache.doris.nereids.jobs.rewrite.RootPlanTreeRewriteJob.execute(RootPlanTreeRewriteJob.java:66)
~[doris-fe.jar:1.2-SNAPSHOT]
at
org.apache.doris.nereids.jobs.executor.AbstractBatchJobExecutor.execute(AbstractBatchJobExecutor.java:139)
~[doris-fe.jar:1.2-SNAPSHOT]
at
org.apache.doris.nereids.jobs.executor.Analyzer.analyze(Analyzer.java:87)
~[doris-fe.jar:1.2-SNAPSHOT]
at
org.apache.doris.nereids.NereidsPlanner.lambda$analyze$4(NereidsPlanner.java:361)
~[doris-fe.jar:1.2-SNAPSHOT]
at
org.apache.doris.nereids.NereidsPlanner.keepOrShowPlanProcess(NereidsPlanner.java:888)
~[doris-fe.jar:1.2-SNAPSHOT]
at
org.apache.doris.nereids.NereidsPlanner.analyze(NereidsPlanner.java:361)
~[doris-fe.jar:1.2-SNAPSHOT]
at
org.apache.doris.nereids.NereidsPlanner.planWithoutLock(NereidsPlanner.java:250)
~[doris-fe.jar:1.2-SNAPSHOT]
at
org.apache.doris.nereids.NereidsPlanner.planWithLock(NereidsPlanner.java:224)
~[doris-fe.jar:1.2-SNAPSHOT]
at
org.apache.doris.nereids.NereidsPlanner.plan(NereidsPlanner.java:145)
~[doris-fe.jar:1.2-SNAPSHOT]
at
org.apache.doris.qe.StmtExecutor.executeByNereids(StmtExecutor.java:830)
~[doris-fe.jar:1.2-SNAPSHOT]
at org.apache.doris.qe.StmtExecutor.execute(StmtExecutor.java:609)
~[doris-fe.jar:1.2-SNAPSHOT]
at
org.apache.doris.qe.StmtExecutor.queryRetry(StmtExecutor.java:572)
~[doris-fe.jar:1.2-SNAPSHOT]
at org.apache.doris.qe.StmtExecutor.execute(StmtExecutor.java:562)
~[doris-fe.jar:1.2-SNAPSHOT]
at
org.apache.doris.qe.ConnectProcessor.executeQuery(ConnectProcessor.java:347)
~[doris-fe.jar:1.2-SNAPSHOT]
at
org.apache.doris.qe.ConnectProcessor.handleQuery(ConnectProcessor.java:250)
~[doris-fe.jar:1.2-SNAPSHOT]
at
org.apache.doris.qe.MysqlConnectProcessor.handleQuery(MysqlConnectProcessor.java:209)
~[doris-fe.jar:1.2-SNAPSHOT]
at
org.apache.doris.qe.MysqlConnectProcessor.dispatch(MysqlConnectProcessor.java:237)
~[doris-fe.jar:1.2-SNAPSHOT]
at
org.apache.doris.qe.MysqlConnectProcessor.processOnce(MysqlConnectProcessor.java:417)
~[doris-fe.jar:1.2-SNAPSHOT]
at
org.apache.doris.mysql.ReadListener.lambda$handleEvent$0(ReadListener.java:52)
~[doris-fe.jar:1.2-SNAPSHOT]
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
~[?:?]
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
~[?:?]
at java.lang.Thread.run(Thread.java:833) ~[?:?]
Caused by: java.lang.UnsupportedOperationException
at
com.google.common.collect.ImmutableMap.put(ImmutableMap.java:814)
~[guava-33.2.1-jre.jar:?]
at
org.apache.doris.tablefunction.S3TableValuedFunction.<init>(S3TableValuedFunction.java:66)
~[doris-fe.jar:1.2-SNAPSHOT]
at
org.apache.doris.nereids.trees.expressions.functions.table.S3.toCatalogFunction(S3.java:53)
~[doris-fe.jar:1.2-SNAPSHOT]
... 39 more
```
---
.../doris/tablefunction/S3TableValuedFunction.java | 17 +++++++++--------
1 file changed, 9 insertions(+), 8 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/S3TableValuedFunction.java
b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/S3TableValuedFunction.java
index 7a9566c13e0..3defb171a9f 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/S3TableValuedFunction.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/S3TableValuedFunction.java
@@ -56,11 +56,6 @@ public class S3TableValuedFunction extends
ExternalFileTableValuedFunction {
"ACCESS_KEY", "SECRET_KEY", "SESSION_TOKEN", "REGION");
public S3TableValuedFunction(Map<String, String> properties) throws
AnalysisException {
- final boolean isAzureTvf =
AzureProperties.checkAzureProviderPropertyExist(properties);
- // Azure could run without region
- if (isAzureTvf) {
- properties.put(S3Properties.REGION, "DUMMY-REGION");
- }
// 1. analyze common properties
Map<String, String> otherProps =
super.parseCommonProperties(properties);
@@ -84,8 +79,14 @@ public class S3TableValuedFunction extends
ExternalFileTableValuedFunction {
// If endpoint is missing, exception will be thrown.
String endpoint = constructEndpoint(otherProps, s3uri);
if (!otherProps.containsKey(S3Properties.REGION)) {
- String region = s3uri.getRegion().orElseThrow(() ->
- new AnalysisException(String.format("Properties '%s' is
required.", S3Properties.REGION)));
+ String region;
+ if (AzureProperties.checkAzureProviderPropertyExist(properties)) {
+ // Azure could run without region
+ region = s3uri.getRegion().orElse("DUMMY-REGION");
+ } else {
+ region = s3uri.getRegion().orElseThrow(() -> new
AnalysisException(
+ String.format("Properties '%s' is required.",
S3Properties.REGION)));
+ }
otherProps.put(S3Properties.REGION, region);
}
checkNecessaryS3Properties(otherProps);
@@ -99,7 +100,7 @@ public class S3TableValuedFunction extends
ExternalFileTableValuedFunction {
locationProperties = S3Properties.credentialToMap(credential);
locationProperties.put(PropertyConverter.USE_PATH_STYLE, usePathStyle);
- if (isAzureTvf) {
+ if (AzureProperties.checkAzureProviderPropertyExist(properties)) {
// For Azure's compatibility, we need bucket to connect to the
blob storage's container
locationProperties.put(S3Properties.BUCKET, s3uri.getBucket());
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
