This is an automated email from the ASF dual-hosted git repository.
panxiaolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 9c1aad06ea [Improve](query) improve column match performance by
introducing a column name map in `MaterializedIndexMeta` (#18203)
9c1aad06ea is described below
commit 9c1aad06eac9b921db0e9dc63c4b5092da4c33c4
Author: lihangyu <[email protected]>
AuthorDate: Thu Mar 30 11:24:51 2023 +0800
[Improve](query) improve column match performance by introducing a column
name map in `MaterializedIndexMeta` (#18203)
improve column match performance by introducing a column name map in
`MaterializedIndexMeta`
`getColumnByName` is slow due to the linear search process, using a map to
speed up search.
---
.../doris/catalog/MaterializedIndexMeta.java | 29 +++++++++++++++++-----
.../java/org/apache/doris/catalog/OlapTable.java | 9 ++++---
2 files changed, 28 insertions(+), 10 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndexMeta.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndexMeta.java
index 37ea259091..975bee6bab 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndexMeta.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndexMeta.java
@@ -33,6 +33,7 @@ import org.apache.doris.thrift.TStorageType;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
import com.google.gson.annotations.SerializedName;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -68,6 +69,8 @@ public class MaterializedIndexMeta implements Writable,
GsonPostProcessable {
private int maxColUniqueId = Column.COLUMN_UNIQUE_ID_INIT_VALUE;
private Expr whereClause;
+ private Map<String, Column> nameToColumn;
+ private Map<String, Column> definedNameToColumn;
private static final Logger LOG =
LogManager.getLogger(MaterializedIndexMeta.class);
@@ -86,6 +89,7 @@ public class MaterializedIndexMeta implements Writable,
GsonPostProcessable {
Preconditions.checkState(keysType != null);
this.keysType = keysType;
this.defineStmt = defineStmt;
+ initColumnNameMap();
}
public void setWhereClause(Expr whereClause) {
@@ -126,6 +130,7 @@ public class MaterializedIndexMeta implements Writable,
GsonPostProcessable {
public void setSchema(List<Column> newSchema) {
this.schema = newSchema;
+ initColumnNameMap();
}
public void setSchemaHash(int newSchemaHash) {
@@ -215,13 +220,14 @@ public class MaterializedIndexMeta implements Writable,
GsonPostProcessable {
return normalizeName(lhs).equalsIgnoreCase(normalizeName(rhs));
}
+ public Column getColumnByDefineName(String colDefineName) {
+ String normalizedName = normalizeName(colDefineName);
+ return definedNameToColumn.getOrDefault(normalizedName, null);
+ }
+
public Column getColumnByName(String columnName) {
- for (Column column : schema) {
- if (matchColumnName(column.getName(), columnName)) {
- return column;
- }
- }
- return null;
+ String normalizedName = normalizeName(columnName);
+ return nameToColumn.getOrDefault(normalizedName, null);
}
public OriginStatement getDefineStmt() {
@@ -258,6 +264,7 @@ public class MaterializedIndexMeta implements Writable,
GsonPostProcessable {
@Override
public void gsonPostProcess() throws IOException {
+ initColumnNameMap();
// analyze define stmt
if (defineStmt == null) {
return;
@@ -300,4 +307,14 @@ public class MaterializedIndexMeta implements Writable,
GsonPostProcessable {
indexId, column, column.getUniqueId());
});
}
+
+ public void initColumnNameMap() {
+ // case insensitive
+ nameToColumn = Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER);
+ definedNameToColumn = Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER);
+ for (Column column : schema) {
+ nameToColumn.put(normalizeName(column.getName()), column);
+ definedNameToColumn.put(normalizeName(column.getDefineName()),
column);
+ }
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
index 76ddddbb1f..701d9faece 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
@@ -343,6 +343,8 @@ public class OlapTable extends Table {
nameToColumn.put(column.getDefineName(), column);
}
}
+ // Column maybe renamed, rebuild the column name map
+ indexMeta.initColumnNameMap();
}
LOG.debug("after rebuild full schema. table {}, schema size: {}", id,
fullSchema.size());
}
@@ -407,10 +409,9 @@ public class OlapTable extends Table {
public Column getVisibleColumn(String columnName) {
for (MaterializedIndexMeta meta : getVisibleIndexIdToMeta().values()) {
- for (Column column : meta.getSchema()) {
- if
(MaterializedIndexMeta.matchColumnName(column.getDefineName(), columnName)) {
- return column;
- }
+ Column target = meta.getColumnByDefineName(columnName);
+ if (target != null) {
+ return target;
}
}
return null;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
