This is an automated email from the ASF dual-hosted git repository. morningman pushed a commit to branch branch-1.2-lts in repository https://gitbox.apache.org/repos/asf/doris.git
commit 8616b6f7a68e9995bd55cec7e59823a9b48356fa Author: Jibing-Li <[email protected]> AuthorDate: Thu Feb 23 13:54:36 2023 +0800 [Fix](load)Pass hidden column to load columns (#17004) The LoadScanProvider doesn't get Hidden Columns from stream load parameter. This may cause stream load delete operation fail. This pr is to pass the hidden columns to LoadScanProvider. --- .../main/java/org/apache/doris/planner/StreamLoadPlanner.java | 2 +- .../org/apache/doris/planner/external/ExternalFileScanNode.java | 5 +++-- .../java/org/apache/doris/planner/external/FileGroupInfo.java | 9 ++++++++- .../java/org/apache/doris/planner/external/LoadScanProvider.java | 3 ++- 4 files changed, 14 insertions(+), 5 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadPlanner.java index 81e9009f65..7af5ec7f2f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadPlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadPlanner.java @@ -191,7 +191,7 @@ public class StreamLoadPlanner { fileStatus.setSize(-1); // must set to -1, means stream. } fileScanNode.setLoadInfo(loadId, taskInfo.getTxnId(), destTable, BrokerDesc.createForStreamLoad(), - fileGroup, fileStatus, taskInfo.isStrictMode(), taskInfo.getFileType()); + fileGroup, fileStatus, taskInfo.isStrictMode(), taskInfo.getFileType(), taskInfo.getHiddenColumns()); scanNode = fileScanNode; } else { scanNode = new StreamLoadScanNode(loadId, new PlanNodeId(0), scanTupleDesc, destTable, taskInfo); diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/external/ExternalFileScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/external/ExternalFileScanNode.java index b0a0c162e6..a5a917bb17 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/external/ExternalFileScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/external/ExternalFileScanNode.java @@ -155,9 +155,10 @@ public class ExternalFileScanNode extends ExternalScanNode { // Only for stream load/routine load job. public void setLoadInfo(TUniqueId loadId, long txnId, Table targetTable, BrokerDesc brokerDesc, - BrokerFileGroup fileGroup, TBrokerFileStatus fileStatus, boolean strictMode, TFileType fileType) { + BrokerFileGroup fileGroup, TBrokerFileStatus fileStatus, boolean strictMode, TFileType fileType, + List<String> hiddenColumns) { FileGroupInfo fileGroupInfo = new FileGroupInfo(loadId, txnId, targetTable, brokerDesc, - fileGroup, fileStatus, strictMode, fileType); + fileGroup, fileStatus, strictMode, fileType, hiddenColumns); fileGroupInfos.add(fileGroupInfo); this.type = Type.LOAD; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/external/FileGroupInfo.java b/fe/fe-core/src/main/java/org/apache/doris/planner/external/FileGroupInfo.java index 90ecbb9ef0..507ce75910 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/external/FileGroupInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/external/FileGroupInfo.java @@ -84,6 +84,7 @@ public class FileGroupInfo { private long bytesPerInstance = 0; // used for stream load, FILE_LOCAL or FILE_STREAM private TFileType fileType; + private List<String> hiddenColumns = null; // for broker load public FileGroupInfo(long loadJobId, long txnId, Table targetTable, BrokerDesc brokerDesc, @@ -104,7 +105,8 @@ public class FileGroupInfo { // for stream load public FileGroupInfo(TUniqueId loadId, long txnId, Table targetTable, BrokerDesc brokerDesc, - BrokerFileGroup fileGroup, TBrokerFileStatus fileStatus, boolean strictMode, TFileType fileType) { + BrokerFileGroup fileGroup, TBrokerFileStatus fileStatus, boolean strictMode, + TFileType fileType, List<String> hiddenColumns) { this.jobType = JobType.STREAM_LOAD; this.loadId = loadId; this.txnId = txnId; @@ -116,6 +118,7 @@ public class FileGroupInfo { this.filesAdded = 1; this.strictMode = strictMode; this.fileType = fileType; + this.hiddenColumns = hiddenColumns; } public Table getTargetTable() { @@ -152,6 +155,10 @@ public class FileGroupInfo { return sb.toString(); } + public List<String> getHiddenColumns() { + return hiddenColumns; + } + public void getFileStatusAndCalcInstance(BackendPolicy backendPolicy) throws UserException { if (filesAdded == 0) { throw new UserException("No source file in this table(" + targetTable.getName() + ")."); diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/external/LoadScanProvider.java b/fe/fe-core/src/main/java/org/apache/doris/planner/external/LoadScanProvider.java index ae2bf5d4ff..faafaf2014 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/external/LoadScanProvider.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/external/LoadScanProvider.java @@ -205,7 +205,8 @@ public class LoadScanProvider implements FileScanProviderIf { List<Integer> srcSlotIds = Lists.newArrayList(); Load.initColumns(fileGroupInfo.getTargetTable(), columnDescs, context.fileGroup.getColumnToHadoopFunction(), context.exprMap, analyzer, context.srcTupleDescriptor, context.srcSlotDescByName, srcSlotIds, - formatType(context.fileGroup.getFileFormat(), ""), null, VectorizedUtil.isVectorized()); + formatType(context.fileGroup.getFileFormat(), ""), fileGroupInfo.getHiddenColumns(), + VectorizedUtil.isVectorized()); int columnCountFromPath = 0; if (context.fileGroup.getColumnNamesFromPath() != null) { --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
