This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new e9cd1d64ed (fix)[multi-catalog][nereids] Reset ExternalFileScanNode
required slots after Nereids planner do projection. #16549
e9cd1d64ed is described below
commit e9cd1d64edc8fef590d243765fc5ef85e5a1af5e
Author: Jibing-Li <[email protected]>
AuthorDate: Fri Feb 10 11:28:01 2023 +0800
(fix)[multi-catalog][nereids] Reset ExternalFileScanNode required slots
after Nereids planner do projection. #16549
The new Nereids planner do column projection after creating scan node. For
ExternalFileScanNode, this may cause the columns in required_slots mismatch
with the slots after projection. This pr is to reset the required_slots after
projection.
---
.../glue/translator/PhysicalPlanTranslator.java | 18 +++++++++++++----
.../java/org/apache/doris/planner/ScanNode.java | 9 +++++++++
.../planner/external/ExternalFileScanNode.java | 23 ++++++++++++++++++++++
3 files changed, 46 insertions(+), 4 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
index a72bba6b58..a55b9ef1d2 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
@@ -38,6 +38,8 @@ import org.apache.doris.catalog.Table;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.catalog.external.ExternalTable;
import org.apache.doris.common.Pair;
+import org.apache.doris.common.UserException;
+import org.apache.doris.common.util.Util;
import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.nereids.properties.DistributionSpecAny;
import org.apache.doris.nereids.properties.DistributionSpecGather;
@@ -128,8 +130,8 @@ import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
import java.util.ArrayList;
import java.util.Collection;
@@ -153,7 +155,7 @@ import java.util.stream.Stream;
* </STRONG>
*/
public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment,
PlanTranslatorContext> {
- private static final Logger LOG =
LoggerFactory.getLogger(PhysicalPlanTranslator.class);
+ private static final Logger LOG =
LogManager.getLogger(PhysicalPlanTranslator.class);
/**
* Translate Nereids Physical Plan tree to Stale Planner PlanFragment tree.
@@ -1098,7 +1100,7 @@ public class PhysicalPlanTranslator extends
DefaultPlanVisitor<PlanFragment, Pla
inputPlanNode.setProjectList(execExprList);
inputPlanNode.setOutputTupleDesc(tupleDescriptor);
- if (inputPlanNode instanceof OlapScanNode) {
+ if (inputPlanNode instanceof ScanNode) {
updateChildSlotsMaterialization(inputPlanNode, requiredSlotIdList,
context);
return inputFragment;
}
@@ -1126,6 +1128,14 @@ public class PhysicalPlanTranslator extends
DefaultPlanVisitor<PlanFragment, Pla
context.getDescTable()
.getTupleDesc(execPlan.getTupleIds().get(0)).getSlots().get(0).setIsMaterialized(true);
}
+ if (execPlan instanceof ScanNode) {
+ try {
+ ((ScanNode) execPlan).updateRequiredSlots();
+ } catch (UserException e) {
+ Util.logAndThrowRuntimeException(LOG,
+ "User Exception while reset external file scan node
contexts.", e);
+ }
+ }
}
@Override
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java
b/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java
index c7031283da..6e4d21cc81 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java
@@ -125,6 +125,15 @@ public abstract class ScanNode extends PlanNode {
*/
public abstract List<TScanRangeLocations> getScanRangeLocations(long
maxScanRangeLength);
+ /**
+ * Update required_slots in scan node contexts. This is called after
Nereids planner do the projection.
+ * In the projection process, some slots may be removed. So call this to
update the slots info.
+ * Currently, it is only used by ExternalFileScanNode, add the interface
here to keep the Nereids code clean.
+ */
+ public void updateRequiredSlots() throws UserException {
+ return;
+ }
+
// TODO(ML): move it into PrunerOptimizer
public void computeColumnFilter() {
for (Column column : desc.getTable().getBaseSchema()) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/planner/external/ExternalFileScanNode.java
b/fe/fe-core/src/main/java/org/apache/doris/planner/external/ExternalFileScanNode.java
index 7f5cbddb87..745da3d63e 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/planner/external/ExternalFileScanNode.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/planner/external/ExternalFileScanNode.java
@@ -241,6 +241,29 @@ public class ExternalFileScanNode extends ExternalScanNode
{
}
}
+ /**
+ * Reset required_slots in contexts. This is called after Nereids planner
do the projection.
+ * In the projection process, some slots may be removed. So call this to
update the slots info.
+ */
+ @Override
+ public void updateRequiredSlots() throws UserException {
+ for (int i = 0; i < contexts.size(); i++) {
+ ParamCreateContext context = contexts.get(i);
+ FileScanProviderIf scanProvider = scanProviders.get(i);
+ context.params.unsetRequiredSlots();
+ for (SlotDescriptor slot : desc.getSlots()) {
+ if (!slot.isMaterialized()) {
+ continue;
+ }
+
+ TFileScanSlotInfo slotInfo = new TFileScanSlotInfo();
+ slotInfo.setSlotId(slot.getId().asInt());
+
slotInfo.setIsFileSlot(!scanProvider.getPathPartitionKeys().contains(slot.getColumn().getName()));
+ context.params.addToRequiredSlots(slotInfo);
+ }
+ }
+ }
+
private void initHMSExternalTable(HMSExternalTable hmsTable) throws
UserException {
Preconditions.checkNotNull(hmsTable);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
