This is an automated email from the ASF dual-hosted git repository. yiguolei pushed a commit to branch branch-2.1 in repository https://gitbox.apache.org/repos/asf/doris.git
commit 779a9a1fbba2eb1e442171fa1b50af3296480b37 Author: morrySnow <[email protected]> AuthorDate: Mon Jan 29 14:09:15 2024 +0800 [opt](planner) use string for varchar in ctas if original table is not olap (#30323) --- .../apache/doris/datasource/InternalCatalog.java | 5 ++ .../rules/expression/rules/ElementAtToSlot.java | 2 +- .../rules/rewrite/DeferMaterializeTopNResult.java | 2 +- .../doris/nereids/trees/expressions/Alias.java | 3 ++ .../trees/expressions/ArrayItemReference.java | 2 +- .../nereids/trees/expressions/SlotReference.java | 61 ++++++++++++---------- .../trees/plans/commands/CreateTableCommand.java | 33 ++++++++---- .../trees/plans/commands/info/CreateTableInfo.java | 8 ++- .../trees/plans/logical/LogicalCTEConsumer.java | 1 + .../plans/logical/LogicalCatalogRelation.java | 2 +- .../trees/plans/logical/LogicalOlapScan.java | 12 +++-- .../trees/plans/logical/LogicalTVFRelation.java | 2 +- .../plans/physical/PhysicalCatalogRelation.java | 2 +- .../trees/plans/physical/PhysicalTVFRelation.java | 2 +- .../apache/doris/nereids/types/VarcharType.java | 8 ++- .../doris/nereids/util/TypeCoercionUtils.java | 5 +- .../nereids/jobs/cascades/DeriveStatsJobTest.java | 5 +- .../rules/expression/PredicatesSplitterTest.java | 2 +- .../doris/nereids/stats/StatsCalculatorTest.java | 6 +-- .../jdbc/test_doris_jdbc_catalog.out | 8 +-- .../jdbc/test_mysql_jdbc_catalog.out | 4 +- .../jdbc/test_pg_jdbc_catalog.out | 2 +- 22 files changed, 109 insertions(+), 68 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java index c977b61be2f..f5b703c8104 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java @@ -1228,6 +1228,11 @@ public class InternalCatalog implements CatalogIf<Database> { default: throw new DdlException("Unsupported string type for ctas"); } + if (resultExpr.getSrcSlotRef() != null + && resultExpr.getSrcSlotRef().getTable() != null + && !resultExpr.getSrcSlotRef().getTable().isManagedTable()) { + typeDef = new TypeDef(ScalarType.createStringType()); + } } else if (resultType.isDecimalV2() && resultType.equals(ScalarType.DECIMALV2)) { typeDef = new TypeDef(ScalarType.createDecimalType(27, 9)); } else if (resultType.isDecimalV3()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/ElementAtToSlot.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/ElementAtToSlot.java index 050c2bb3964..adc050c9871 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/ElementAtToSlot.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/ElementAtToSlot.java @@ -72,7 +72,7 @@ public class ElementAtToSlot extends DefaultExpressionRewriter<ExpressionRewrite } SlotReference slotRef = new SlotReference(StatementScopeIdGenerator.newExprId(), topColumnSlot.getName(), topColumnSlot.getDataType(), - topColumnSlot.nullable(), topColumnSlot.getQualifier(), + topColumnSlot.nullable(), topColumnSlot.getQualifier(), topColumnSlot.getTable().get(), topColumnSlot.getColumn().get(), Optional.of(topColumnSlot.getInternalName()), fullPaths); ctx.addPathSlotRef(topColumnSlot, fullPaths, slotRef, elementAt); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/DeferMaterializeTopNResult.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/DeferMaterializeTopNResult.java index c91945e029f..39803463841 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/DeferMaterializeTopNResult.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/DeferMaterializeTopNResult.java @@ -83,7 +83,7 @@ public class DeferMaterializeTopNResult implements RewriteRuleFactory { LogicalTopN<? extends Plan> logicalTopN, Optional<LogicalFilter<? extends Plan>> logicalFilter, LogicalOlapScan logicalOlapScan) { Column rowId = new Column(Column.ROWID_COL, Type.STRING, false, null, false, "", "rowid column"); - SlotReference columnId = SlotReference.fromColumn(rowId, + SlotReference columnId = SlotReference.fromColumn(logicalOlapScan.getTable(), rowId, logicalOlapScan.getQualifier(), logicalOlapScan); Set<ExprId> deferredMaterializedExprIds = Sets.newHashSet(logicalOlapScan.getOutputExprIdSet()); logicalFilter.ifPresent(filter -> filter.getConjuncts() diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Alias.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Alias.java index e8574b36e84..1aacb029491 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Alias.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Alias.java @@ -74,6 +74,9 @@ public class Alias extends NamedExpression implements UnaryExpression { SlotReference slotReference = child() instanceof SlotReference ? (SlotReference) child() : null; return new SlotReference(exprId, name, child().getDataType(), child().nullable(), qualifier, + slotReference != null + ? ((SlotReference) child()).getTable().orElse(null) + : null, slotReference != null ? slotReference.getColumn().orElse(null) : null, diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/ArrayItemReference.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/ArrayItemReference.java index 522779b508d..d2a78859e26 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/ArrayItemReference.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/ArrayItemReference.java @@ -142,7 +142,7 @@ public class ArrayItemReference extends NamedExpression implements ExpectsInputT * @param nullable true if nullable */ public ArrayItemSlot(ExprId exprId, String name, DataType dataType, boolean nullable) { - super(exprId, name, dataType, nullable, ImmutableList.of(), null, Optional.empty(), null); + super(exprId, name, dataType, nullable, ImmutableList.of(), null, null, Optional.empty(), null); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SlotReference.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SlotReference.java index afb31f71620..6917a5648f4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SlotReference.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SlotReference.java @@ -18,6 +18,7 @@ package org.apache.doris.nereids.trees.expressions; import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.TableIf; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.trees.plans.algebra.Relation; import org.apache.doris.nereids.types.DataType; @@ -50,34 +51,36 @@ public class SlotReference extends Slot { // TODO: remove this member variable after mv selection is refactored protected final Optional<String> internalName; + private final TableIf table; private final Column column; public SlotReference(String name, DataType dataType) { this(StatementScopeIdGenerator.newExprId(), name, dataType, true, ImmutableList.of(), - null, Optional.empty(), null); + null, null, Optional.empty(), null); } public SlotReference(String name, DataType dataType, boolean nullable) { this(StatementScopeIdGenerator.newExprId(), name, dataType, nullable, ImmutableList.of(), - null, Optional.empty(), null); + null, null, Optional.empty(), null); } public SlotReference(String name, DataType dataType, boolean nullable, List<String> qualifier) { - this(StatementScopeIdGenerator.newExprId(), name, dataType, nullable, qualifier, null, Optional.empty(), null); + this(StatementScopeIdGenerator.newExprId(), name, dataType, nullable, + qualifier, null, null, Optional.empty(), null); } public SlotReference(ExprId exprId, String name, DataType dataType, boolean nullable, List<String> qualifier) { - this(exprId, name, dataType, nullable, qualifier, null, Optional.empty(), null); + this(exprId, name, dataType, nullable, qualifier, null, null, Optional.empty(), null); } public SlotReference(ExprId exprId, String name, DataType dataType, boolean nullable, - List<String> qualifier, @Nullable Column column) { - this(exprId, name, dataType, nullable, qualifier, column, Optional.empty(), null); + List<String> qualifier, @Nullable TableIf table, @Nullable Column column) { + this(exprId, name, dataType, nullable, qualifier, table, column, Optional.empty(), null); } public SlotReference(ExprId exprId, String name, DataType dataType, boolean nullable, - List<String> qualifier, @Nullable Column column, Optional<String> internalName) { - this(exprId, name, dataType, nullable, qualifier, column, internalName, null); + List<String> qualifier, @Nullable TableIf table, @Nullable Column column, Optional<String> internalName) { + this(exprId, name, dataType, nullable, qualifier, table, column, internalName, null); } /** @@ -93,13 +96,14 @@ public class SlotReference extends Slot { * @param subColLabels subColumn access labels */ public SlotReference(ExprId exprId, String name, DataType dataType, boolean nullable, - List<String> qualifier, @Nullable Column column, Optional<String> internalName, - List<String> subColLabels) { + List<String> qualifier, @Nullable TableIf table, @Nullable Column column, + Optional<String> internalName, List<String> subColLabels) { this.exprId = exprId; this.name = name; this.dataType = dataType; this.qualifier = ImmutableList.copyOf(Objects.requireNonNull(qualifier, "qualifier can not be null")); this.nullable = nullable; + this.table = table; this.column = column; this.subColPath = subColLabels; if (subColLabels != null && !this.subColPath.isEmpty()) { @@ -122,10 +126,10 @@ public class SlotReference extends Slot { * @param qualifier the qualifier of SlotReference * @param relation the relation which column is from */ - public static SlotReference fromColumn(Column column, List<String> qualifier, Relation relation) { + public static SlotReference fromColumn(TableIf table, Column column, List<String> qualifier, Relation relation) { DataType dataType = DataType.fromCatalogType(column.getType()); SlotReference slot = new SlotReference(StatementScopeIdGenerator.newExprId(), column.getName(), dataType, - column.isAllowNull(), qualifier, column, Optional.empty(), null); + column.isAllowNull(), qualifier, table, column, Optional.empty(), null); if (relation != null && ConnectContext.get() != null && ConnectContext.get().getStatementContext() != null) { ConnectContext.get().getStatementContext().addSlotToRelation(slot, relation); @@ -133,16 +137,10 @@ public class SlotReference extends Slot { return slot; } - public static SlotReference fromColumn(Column column, String name, List<String> qualifier) { + public static SlotReference fromColumn(TableIf table, Column column, String name, List<String> qualifier) { DataType dataType = DataType.fromCatalogType(column.getType()); return new SlotReference(StatementScopeIdGenerator.newExprId(), name, dataType, - column.isAllowNull(), qualifier, column, Optional.empty(), null); - } - - public static boolean containsPathsSlotReference(Expression expression) { - return expression.collectToList(SlotReference.class::isInstance) - .stream().anyMatch(expr -> { - return ((SlotReference) expr).hasSubColPath(); }); + column.isAllowNull(), qualifier, table, column, Optional.empty(), null); } @Override @@ -175,6 +173,14 @@ public class SlotReference extends Slot { return internalName.get(); } + public Optional<Column> getColumn() { + return Optional.ofNullable(column); + } + + public Optional<TableIf> getTable() { + return Optional.ofNullable(table); + } + @Override public String toSql() { return name; @@ -223,10 +229,6 @@ public class SlotReference extends Slot { return exprId.asInt(); } - public Optional<Column> getColumn() { - return Optional.ofNullable(column); - } - @Override public <R, C> R accept(ExpressionVisitor<R, C> visitor, C context) { return visitor.visitSlotReference(this, context); @@ -234,7 +236,7 @@ public class SlotReference extends Slot { @Override public SlotReference withChildren(List<Expression> children) { - Preconditions.checkArgument(children.size() == 0); + Preconditions.checkArgument(children.isEmpty()); return this; } @@ -243,22 +245,23 @@ public class SlotReference extends Slot { if (this.nullable == newNullable) { return this; } - return new SlotReference(exprId, name, dataType, newNullable, qualifier, column, internalName, subColPath); + return new SlotReference(exprId, name, dataType, newNullable, + qualifier, table, column, internalName, subColPath); } @Override public SlotReference withQualifier(List<String> qualifier) { - return new SlotReference(exprId, name, dataType, nullable, qualifier, column, internalName, subColPath); + return new SlotReference(exprId, name, dataType, nullable, qualifier, table, column, internalName, subColPath); } @Override public SlotReference withName(String name) { - return new SlotReference(exprId, name, dataType, nullable, qualifier, column, internalName, subColPath); + return new SlotReference(exprId, name, dataType, nullable, qualifier, table, column, internalName, subColPath); } @Override public SlotReference withExprId(ExprId exprId) { - return new SlotReference(exprId, name, dataType, nullable, qualifier, column, internalName, subColPath); + return new SlotReference(exprId, name, dataType, nullable, qualifier, table, column, internalName, subColPath); } public boolean isVisible() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateTableCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateTableCommand.java index 2f4c1d55acc..df7d5ffc6aa 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateTableCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateTableCommand.java @@ -31,6 +31,7 @@ import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.properties.PhysicalProperties; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; import org.apache.doris.nereids.trees.plans.commands.ExplainCommand.ExplainLevel; @@ -38,11 +39,15 @@ import org.apache.doris.nereids.trees.plans.commands.info.ColumnDefinition; import org.apache.doris.nereids.trees.plans.commands.info.CreateTableInfo; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; +import org.apache.doris.nereids.types.CharType; import org.apache.doris.nereids.types.DataType; import org.apache.doris.nereids.types.DecimalV2Type; +import org.apache.doris.nereids.types.NullType; +import org.apache.doris.nereids.types.StringType; import org.apache.doris.nereids.types.TinyIntType; import org.apache.doris.nereids.types.VarcharType; import org.apache.doris.nereids.types.coercion.CharacterType; +import org.apache.doris.nereids.util.TypeCoercionUtils; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.QueryState.MysqlStateType; import org.apache.doris.qe.StmtExecutor; @@ -110,16 +115,24 @@ public class CreateTableCommand extends Command implements ForwardWithSync { for (int i = 0; i < slots.size(); i++) { Slot s = slots.get(i); DataType dataType = s.getDataType().conversion(); - if (dataType.isNullType()) { - dataType = TinyIntType.INSTANCE; - } else if (dataType.isDecimalV2Type()) { - dataType = DecimalV2Type.SYSTEM_DEFAULT; - } else if (i == 0 && dataType.isStringType()) { + if (i == 0 && dataType.isStringType()) { dataType = VarcharType.createVarcharType(ScalarType.MAX_VARCHAR_LENGTH); - } else if (dataType instanceof CharacterType) { - // if column is not come from table, we should set varchar length to max - if (!s.isColumnFromTable()) { - dataType = VarcharType.createVarcharType(ScalarType.MAX_VARCHAR_LENGTH); + } else { + dataType = TypeCoercionUtils.replaceSpecifiedType(dataType, + NullType.class, TinyIntType.INSTANCE); + dataType = TypeCoercionUtils.replaceSpecifiedType(dataType, + DecimalV2Type.class, DecimalV2Type.SYSTEM_DEFAULT); + if (s.isColumnFromTable()) { + if (!((SlotReference) s).getTable().isPresent() + || !((SlotReference) s).getTable().get().isManagedTable()) { + dataType = TypeCoercionUtils.replaceSpecifiedType(dataType, + CharacterType.class, StringType.INSTANCE); + } + } else { + dataType = TypeCoercionUtils.replaceSpecifiedType(dataType, + VarcharType.class, VarcharType.MAX_VARCHAR_TYPE); + dataType = TypeCoercionUtils.replaceSpecifiedType(dataType, + CharType.class, VarcharType.MAX_VARCHAR_TYPE); } } // if the column is an expression, we set it to nullable, otherwise according to the nullable of the slot. @@ -151,7 +164,7 @@ public class CreateTableCommand extends Command implements ForwardWithSync { void handleFallbackFailedCtas(ConnectContext ctx) { try { Env.getCurrentEnv().dropTable(new DropTableStmt(false, - new TableName(Env.getCurrentEnv().getCurrentCatalog().getName(), + new TableName(createTableInfo.getCtlName(), createTableInfo.getDbName(), createTableInfo.getTableName()), true)); } catch (Exception e) { // TODO: refactor it with normal error process. diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java index a1a5dd9836c..8353ccd308d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java @@ -198,7 +198,13 @@ public class CreateTableInfo { return tableName; } + /** + * full qualifier table name. + */ public List<String> getTableNameParts() { + if (ctlName != null && dbName != null) { + return ImmutableList.of(ctlName, dbName, tableName); + } if (dbName != null) { return ImmutableList.of(dbName, tableName); } @@ -860,7 +866,7 @@ public class CreateTableInfo { } return new CreateTableStmt(ifNotExists, isExternal, - new TableName(Env.getCurrentEnv().getCurrentCatalog().getName(), dbName, tableName), + new TableName(ctlName, dbName, tableName), catalogColumns, catalogIndexes, engineName, new KeysDesc(keysType, keys, clusterKeysColumnNames, clusterKeysColumnIds), partitionDesc, distributionDesc, Maps.newHashMap(properties), extProperties, diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCTEConsumer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCTEConsumer.java index 9c0996a2c3c..e098f79d843 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCTEConsumer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCTEConsumer.java @@ -99,6 +99,7 @@ public class LogicalCTEConsumer extends LogicalRelation implements BlockFuncDeps Slot consumerSlot = new SlotReference(StatementScopeIdGenerator.newExprId(), producerOutputSlot.getName(), producerOutputSlot.getDataType(), producerOutputSlot.nullable(), ImmutableList.of(name), + slotRef != null ? (slotRef.getTable().isPresent() ? slotRef.getTable().get() : null) : null, slotRef != null ? (slotRef.getColumn().isPresent() ? slotRef.getColumn().get() : null) : null, slotRef != null ? Optional.of(slotRef.getInternalName()) : Optional.empty()); producerToConsumerOutputMap.put(producerOutputSlot, consumerSlot); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCatalogRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCatalogRelation.java index 743365f782d..f0ff94c4c60 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCatalogRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCatalogRelation.java @@ -93,7 +93,7 @@ public abstract class LogicalCatalogRelation extends LogicalRelation implements public List<Slot> computeOutput() { return table.getBaseSchema() .stream() - .map(col -> SlotReference.fromColumn(col, qualified(), this)) + .map(col -> SlotReference.fromColumn(table, col, qualified(), this)) .collect(ImmutableList.toImmutableList()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java index 4e9174097cb..5f80823041b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java @@ -334,7 +334,7 @@ public class LogicalOlapScan extends LogicalCatalogRelation implements OlapScan if (cacheSlotWithSlotName.containsKey(Pair.of(selectedIndexId, col.getName()))) { return cacheSlotWithSlotName.get(Pair.of(selectedIndexId, col.getName())); } - Slot slot = SlotReference.fromColumn(col, qualified(), this); + Slot slot = SlotReference.fromColumn(table, col, qualified(), this); cacheSlotWithSlotName.put(Pair.of(selectedIndexId, col.getName()), slot); return slot; }).collect(ImmutableList.toImmutableList()); @@ -357,22 +357,24 @@ public class LogicalOlapScan extends LogicalCatalogRelation implements OlapScan // when we have a partitioned table without any partition, visible index is empty if (-1 == indexId || olapTable.getIndexMetaByIndexId(indexId) == null) { return olapTable.getIndexMetaByIndexId(indexId).getSchema().stream() - .map(s -> generateUniqueSlot(s, indexId == ((OlapTable) table).getBaseIndexId(), indexId)) + .map(c -> generateUniqueSlot(olapTable, c, + indexId == ((OlapTable) table).getBaseIndexId(), indexId)) .collect(Collectors.toList()); } return olapTable.getIndexMetaByIndexId(indexId).getSchema().stream() - .map(s -> generateUniqueSlot(s, indexId == ((OlapTable) table).getBaseIndexId(), indexId)) + .map(s -> generateUniqueSlot(olapTable, s, + indexId == ((OlapTable) table).getBaseIndexId(), indexId)) .collect(ImmutableList.toImmutableList()); } - private Slot generateUniqueSlot(Column column, boolean isBaseIndex, long indexId) { + private Slot generateUniqueSlot(OlapTable table, Column column, boolean isBaseIndex, long indexId) { String name = isBaseIndex ? column.getName() : AbstractSelectMaterializedIndexRule.parseMvColumnToMvName(column.getName(), column.isAggregated() ? Optional.of(column.getAggregationType().toSql()) : Optional.empty()); if (cacheSlotWithSlotName.containsKey(Pair.of(indexId, name))) { return cacheSlotWithSlotName.get(Pair.of(indexId, name)); } - Slot slot = SlotReference.fromColumn(column, name, qualified()); + Slot slot = SlotReference.fromColumn(table, column, name, qualified()); cacheSlotWithSlotName.put(Pair.of(indexId, name), slot); return slot; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalTVFRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalTVFRelation.java index fa48a27d4df..8302c9345dc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalTVFRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalTVFRelation.java @@ -99,7 +99,7 @@ public class LogicalTVFRelation extends LogicalRelation implements TVFRelation, public List<Slot> computeOutput() { return function.getTable().getBaseSchema() .stream() - .map(col -> SlotReference.fromColumn(col, qualifier, this)) + .map(col -> SlotReference.fromColumn(function.getTable(), col, qualifier, this)) .collect(ImmutableList.toImmutableList()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalCatalogRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalCatalogRelation.java index c1c98b61393..70ac9aaa645 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalCatalogRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalCatalogRelation.java @@ -101,7 +101,7 @@ public abstract class PhysicalCatalogRelation extends PhysicalRelation implement public List<Slot> computeOutput() { return table.getBaseSchema() .stream() - .map(col -> SlotReference.fromColumn(col, qualified(), this)) + .map(col -> SlotReference.fromColumn(table, col, qualified(), this)) .collect(ImmutableList.toImmutableList()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalTVFRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalTVFRelation.java index 07cb6c44d24..1f409921c6c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalTVFRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalTVFRelation.java @@ -107,7 +107,7 @@ public class PhysicalTVFRelation extends PhysicalRelation implements TVFRelation public List<Slot> computeOutput() { return function.getTable().getBaseSchema() .stream() - .map(col -> SlotReference.fromColumn(col, ImmutableList.of(), this)) + .map(col -> SlotReference.fromColumn(function.getTable(), col, ImmutableList.of(), this)) .collect(ImmutableList.toImmutableList()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/VarcharType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/VarcharType.java index b2bcc594ea1..620d0de29c9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/VarcharType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/VarcharType.java @@ -28,8 +28,9 @@ import java.util.Objects; */ public class VarcharType extends CharacterType { - public static final VarcharType SYSTEM_DEFAULT = new VarcharType(-1); public static final int MAX_VARCHAR_LENGTH = ScalarType.MAX_VARCHAR_LENGTH; + public static final VarcharType SYSTEM_DEFAULT = new VarcharType(-1); + public static final VarcharType MAX_VARCHAR_TYPE = new VarcharType(MAX_VARCHAR_LENGTH); public VarcharType(int len) { super(len); @@ -40,9 +41,14 @@ public class VarcharType extends CharacterType { return len; } + /** + * create varchar type from length. + */ public static VarcharType createVarcharType(int len) { if (len == SYSTEM_DEFAULT.len) { return SYSTEM_DEFAULT; + } else if (len == MAX_VARCHAR_LENGTH) { + return MAX_VARCHAR_TYPE; } return new VarcharType(len); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java index 310f8ca6369..4aa2d8ae095 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java @@ -332,7 +332,10 @@ public class TypeCoercionUtils { return replaceSpecifiedType(dataType, DateTimeV2Type.class, DateTimeV2Type.MAX); } - private static DataType replaceSpecifiedType(DataType dataType, + /** + * replace specifiedType in dataType to newType. + */ + public static DataType replaceSpecifiedType(DataType dataType, Class<? extends DataType> specifiedType, DataType newType) { if (dataType instanceof ArrayType) { return ArrayType.of(replaceSpecifiedType(((ArrayType) dataType).getItemType(), specifiedType, newType)); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/cascades/DeriveStatsJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/cascades/DeriveStatsJobTest.java index 0b63dacf782..4034bfd6481 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/cascades/DeriveStatsJobTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/cascades/DeriveStatsJobTest.java @@ -76,16 +76,15 @@ public class DeriveStatsJobTest { private LogicalOlapScan constructOlapSCan() { long tableId1 = 0; - + OlapTable table1 = PlanConstructor.newOlapTable(tableId1, "t1", 0); List<String> qualifier = ImmutableList.of("test", "t"); - slot1 = new SlotReference(new ExprId(1), "c1", IntegerType.INSTANCE, true, qualifier, + slot1 = new SlotReference(new ExprId(1), "c1", IntegerType.INSTANCE, true, qualifier, table1, new Column("e", PrimitiveType.INT)); new Expectations() {{ ConnectContext.get(); result = context; }}; - OlapTable table1 = PlanConstructor.newOlapTable(tableId1, "t1", 0); return (LogicalOlapScan) new LogicalOlapScan(StatementScopeIdGenerator.newRelationId(), table1, Collections.emptyList()).withGroupExprLogicalPropChildren(Optional.empty(), Optional.of(new LogicalProperties(() -> ImmutableList.of(slot1), () -> FunctionalDependencies.EMPTY_FUNC_DEPS)), ImmutableList.of()); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/PredicatesSplitterTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/PredicatesSplitterTest.java index 5f24770ca1c..cab2c2f5a64 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/PredicatesSplitterTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/PredicatesSplitterTest.java @@ -100,7 +100,7 @@ public class PredicatesSplitterTest extends ExpressionRewriteTestHelper { } if (expression instanceof UnboundSlot) { String name = ((UnboundSlot) expression).getName(); - mem.putIfAbsent(name, SlotReference.fromColumn( + mem.putIfAbsent(name, SlotReference.fromColumn(null, new Column(name, getType(name.charAt(0)).toCatalogDataType()), Lists.newArrayList("table"), null)); return mem.get(name); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/StatsCalculatorTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/StatsCalculatorTest.java index 28591ab8cc3..839339e04ec 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/StatsCalculatorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/StatsCalculatorTest.java @@ -244,11 +244,11 @@ public class StatsCalculatorTest { @Test public void testOlapScan() { long tableId1 = 0; + OlapTable table1 = PlanConstructor.newOlapTable(tableId1, "t1", 0); List<String> qualifier = ImmutableList.of("test", "t"); SlotReference slot1 = new SlotReference(new ExprId(0), - "c1", IntegerType.INSTANCE, true, qualifier, new Column("c1", PrimitiveType.INT)); + "c1", IntegerType.INSTANCE, true, qualifier, table1, new Column("c1", PrimitiveType.INT)); - OlapTable table1 = PlanConstructor.newOlapTable(tableId1, "t1", 0); LogicalOlapScan logicalOlapScan1 = (LogicalOlapScan) new LogicalOlapScan( StatementScopeIdGenerator.newRelationId(), table1, Collections.emptyList()).withGroupExprLogicalPropChildren(Optional.empty(), @@ -266,7 +266,7 @@ public class StatsCalculatorTest { public void testLimit() { List<String> qualifier = ImmutableList.of("test", "t"); SlotReference slot1 = new SlotReference(new ExprId(0), - "c1", IntegerType.INSTANCE, true, qualifier, new Column("c1", PrimitiveType.INT)); + "c1", IntegerType.INSTANCE, true, qualifier, null, new Column("c1", PrimitiveType.INT)); ColumnStatisticBuilder columnStat1 = new ColumnStatisticBuilder(); columnStat1.setNdv(10); columnStat1.setNumNulls(5); diff --git a/regression-test/data/external_table_p0/jdbc/test_doris_jdbc_catalog.out b/regression-test/data/external_table_p0/jdbc/test_doris_jdbc_catalog.out index 25793a482d5..04812eb79ba 100644 --- a/regression-test/data/external_table_p0/jdbc/test_doris_jdbc_catalog.out +++ b/regression-test/data/external_table_p0/jdbc/test_doris_jdbc_catalog.out @@ -90,8 +90,8 @@ decimal_col DECIMAL(10, 5) Yes false \N NONE decimal_col2 DECIMAL(30, 10) Yes false \N NONE date_col DATE Yes false \N NONE datetime_col DATETIME(3) Yes false \N NONE -char_col CHAR(10) Yes false \N NONE -varchar_col VARCHAR(10) Yes false \N NONE +char_col TEXT Yes false \N NONE +varchar_col TEXT Yes false \N NONE json_col TEXT Yes false \N NONE -- !desc_ctas_arr -- @@ -108,8 +108,8 @@ arr_decimal1_col ARRAY<DECIMALV3(10, 5)> Yes false \N NONE arr_decimal2_col ARRAY<DECIMALV3(30, 10)> Yes false \N NONE arr_date_col ARRAY<DATEV2> Yes false \N NONE arr_datetime_col ARRAY<DATETIMEV2(3)> Yes false \N NONE -arr_char_col ARRAY<CHAR(10)> Yes false \N NONE -arr_varchar_col ARRAY<VARCHAR(10)> Yes false \N NONE +arr_char_col ARRAY<TEXT> Yes false \N NONE +arr_varchar_col ARRAY<TEXT> Yes false \N NONE arr_string_col ARRAY<TEXT> Yes false \N NONE -- !query_ctas_base -- diff --git a/regression-test/data/external_table_p0/jdbc/test_mysql_jdbc_catalog.out b/regression-test/data/external_table_p0/jdbc/test_mysql_jdbc_catalog.out index bff117916a4..555665f08a1 100644 --- a/regression-test/data/external_table_p0/jdbc/test_mysql_jdbc_catalog.out +++ b/regression-test/data/external_table_p0/jdbc/test_mysql_jdbc_catalog.out @@ -315,7 +315,7 @@ binary TEXT Yes false \N NONE bit TEXT Yes false \N NONE blob TEXT Yes false \N NONE boolean TINYINT Yes false \N NONE -char VARCHAR(65533) Yes false \N NONE +char TEXT Yes false \N NONE date DATE Yes false \N NONE datetime DATETIME Yes false \N NONE decimal DECIMAL(12, 4) Yes false \N NONE @@ -339,7 +339,7 @@ timestamp DATETIME(4) Yes false \N NONE tinyint TINYINT Yes false \N NONE tinyint_u SMALLINT Yes true \N varbinary TEXT Yes false \N NONE -varchar VARCHAR(10) Yes false \N NONE +varchar TEXT Yes false \N NONE year SMALLINT Yes false \N NONE -- !mysql_view -- diff --git a/regression-test/data/external_table_p0/jdbc/test_pg_jdbc_catalog.out b/regression-test/data/external_table_p0/jdbc/test_pg_jdbc_catalog.out index ce9e24dd177..8faf5a4e4df 100644 --- a/regression-test/data/external_table_p0/jdbc/test_pg_jdbc_catalog.out +++ b/regression-test/data/external_table_p0/jdbc/test_pg_jdbc_catalog.out @@ -2239,7 +2239,7 @@ bit_value BOOLEAN Yes false \N NONE bitn_value TEXT Yes false \N NONE bitnv_value TEXT Yes false \N NONE box_value TEXT Yes false \N NONE -char_value VARCHAR(65533) Yes true \N +char_value TEXT Yes false \N NONE cidr_value TEXT Yes false \N NONE circle_value TEXT Yes false \N NONE date_value DATE Yes false \N NONE --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
