This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 19b63bb4374 [fix](load) fix load cast throw exception when plan
(#44968)
19b63bb4374 is described below
commit 19b63bb4374b37b5536534eb6d9bbd019d9a9c8b
Author: hui lai <[email protected]>
AuthorDate: Wed Dec 4 17:45:05 2024 +0800
[fix](load) fix load cast throw exception when plan (#44968)
### What problem does this PR solve?
Use the load command with the following columns mapping:
```
curl --location-trusted -u root: \
-H "Expect:100-continue" \
-H "column_separator:," \
-H
"columns:cache_key,result_cnt,result,result=bitmap_from_base64(result)" \
-T 1.txt \
-XPUT http://10.16.10.6:48733/api/mutong/task_result_dev/_stream_load
```
```
{
"TxnId": 37092,
"Label": "54b9e25b-853d-4ecc-aca9-82bd4ed91875",
"Comment": "",
"TwoPhaseCommit": "false",
"Status": "Fail",
"Message": "[ANALYSIS_ERROR]TStatus: errCode = 2, detailMessage = can
not cast from origin type bitmap to target type=varchar(65533)",
"NumberTotalRows": 0,
"NumberLoadedRows": 0,
"NumberFilteredRows": 0,
"NumberUnselectedRows": 0,
"LoadBytes": 0,
"LoadTimeMs": 0,
"BeginTxnTimeMs": 1,
"StreamLoadPutTimeMs": 2,
"ReadDataTimeMs": 0,
"WriteDataTimeMs": 0,
"CommitAndPublishTimeMs": 0
}
```
The pr aims to fix this issue.
### Release note
None
### Check List (For Author)
- Test <!-- At least one of them must be included. -->
- [ ] Regression test
- [ ] Unit Test
- [ ] Manual test (add detailed scripts or steps below)
- [ ] No need to test or manual test. Explain why:
- [ ] This is a refactor/code format and no logic has been changed.
- [ ] Previous test can cover this change.
- [ ] No code files have been changed.
- [ ] Other reason <!-- Add your reason? -->
- Behavior changed:
- [ ] No.
- [ ] Yes. <!-- Explain the behavior change -->
- Does this need documentation?
- [ ] No.
- [ ] Yes. <!-- Add document PR link here. eg:
https://github.com/apache/doris-website/pull/1214 -->
### Check List (For Reviewer who merge this PR)
- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label <!-- Add branch pick label that this PR
should merge into -->
---
.../main/java/org/apache/doris/datasource/FileQueryScanNode.java | 2 +-
.../src/main/java/org/apache/doris/datasource/FileScanNode.java | 8 ++++++++
.../src/main/java/org/apache/doris/planner/FileLoadScanNode.java | 3 ++-
3 files changed, 11 insertions(+), 2 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java
index bd312d0d612..fe8772e2341 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java
@@ -172,7 +172,7 @@ public abstract class FileQueryScanNode extends
FileScanNode {
slotInfo.setIsFileSlot(!partitionKeys.contains(slot.getColumn().getName()));
params.addToRequiredSlots(slotInfo);
}
- setDefaultValueExprs(getTargetTable(), destSlotDescByName, params,
false);
+ setDefaultValueExprs(getTargetTable(), destSlotDescByName, null,
params, false);
setColumnPositionMapping();
// For query, set src tuple id to -1.
params.setSrcTupleId(-1);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileScanNode.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileScanNode.java
index 73acb46e2de..457a580e520 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileScanNode.java
@@ -198,6 +198,7 @@ public abstract class FileScanNode extends ExternalScanNode
{
protected void setDefaultValueExprs(TableIf tbl,
Map<String, SlotDescriptor>
slotDescByName,
+ Map<String, Expr> exprByName,
TFileScanRangeParams params,
boolean useVarcharAsNull) throws
UserException {
Preconditions.checkNotNull(tbl);
@@ -225,6 +226,13 @@ public abstract class FileScanNode extends
ExternalScanNode {
expr = null;
}
}
+ // if there is already an expr , just skip it.
+ // eg:
+ // (a, b, c, c=to_bitmap(c)) in stream load
+ // c will be filled with to_bitmap(column c) , don't need to
specify it.
+ if (exprByName != null &&
exprByName.containsKey(column.getName())) {
+ continue;
+ }
SlotDescriptor slotDesc = slotDescByName.get(column.getName());
// if slot desc is null, which mean it is an unrelated slot, just
skip.
// eg:
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/planner/FileLoadScanNode.java
b/fe/fe-core/src/main/java/org/apache/doris/planner/FileLoadScanNode.java
index 16f388a058c..4047c8f30b3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/FileLoadScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/FileLoadScanNode.java
@@ -131,7 +131,8 @@ public class FileLoadScanNode extends FileScanNode {
// FIXME(cmy): we should support set different expr for different
file group.
initAndSetPrecedingFilter(context.fileGroup.getPrecedingFilterExpr(),
context.srcTupleDescriptor, analyzer);
initAndSetWhereExpr(context.fileGroup.getWhereExpr(),
context.destTupleDescriptor, analyzer);
- setDefaultValueExprs(scanProvider.getTargetTable(),
context.srcSlotDescByName, context.params, true);
+ setDefaultValueExprs(scanProvider.getTargetTable(),
context.srcSlotDescByName,
+ context.exprMap, context.params, true);
this.contexts.add(context);
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
