This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.0 by this push:
new e2814381b1d branch-4.0: [fix](maxcompute)fix maxcompute push down
predicates null pointer. #57474 (#57568)
e2814381b1d is described below
commit e2814381b1de676ae9216cfdf3186c44c82857e1
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Mon Nov 3 12:01:55 2025 +0800
branch-4.0: [fix](maxcompute)fix maxcompute push down predicates null
pointer. #57474 (#57568)
Cherry-picked from #57474
Co-authored-by: daidai <[email protected]>
---
.../doris/datasource/maxcompute/MaxComputeExternalTable.java | 11 ++++++-----
.../datasource/maxcompute/MaxComputeSchemaCacheValue.java | 9 ++++++++-
.../datasource/maxcompute/source/MaxComputeScanNode.java | 10 ++++++++++
3 files changed, 24 insertions(+), 6 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalTable.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalTable.java
index d9d167e786b..09f052f2cea 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalTable.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalTable.java
@@ -66,8 +66,6 @@ public class MaxComputeExternalTable extends ExternalTable {
super(id, name, remoteName, catalog, db,
TableType.MAX_COMPUTE_EXTERNAL_TABLE);
}
- private Map<String, com.aliyun.odps.Column> columnNameToOdpsColumn = new
HashMap();
-
@Override
protected synchronized void makeSureInitialized() {
super.makeSureInitialized();
@@ -166,7 +164,10 @@ public class MaxComputeExternalTable extends ExternalTable
{
}
public Map<String, com.aliyun.odps.Column> getColumnNameToOdpsColumn() {
- return columnNameToOdpsColumn;
+ makeSureInitialized();
+ Optional<SchemaCacheValue> schemaCacheValue = getSchemaCacheValue();
+ return schemaCacheValue.map(value -> ((MaxComputeSchemaCacheValue)
value).getColumnNameToOdpsColumn())
+ .orElse(Collections.emptyMap());
}
@Override
@@ -179,7 +180,7 @@ public class MaxComputeExternalTable extends ExternalTable {
TableIdentifier tableIdentifier =
mcCatalog.getOdpsTableIdentifier(dbName, name);
List<com.aliyun.odps.Column> columns =
odpsTable.getSchema().getColumns();
-
+ Map<String, com.aliyun.odps.Column> columnNameToOdpsColumn = new
HashMap<>();
for (com.aliyun.odps.Column column : columns) {
columnNameToOdpsColumn.put(column.getName(), column);
}
@@ -218,7 +219,7 @@ public class MaxComputeExternalTable extends ExternalTable {
}
return Optional.of(new MaxComputeSchemaCacheValue(schema, odpsTable,
tableIdentifier,
- partitionColumnNames, partitionSpecs, partitionDorisColumns,
partitionTypes));
+ partitionColumnNames, partitionSpecs, partitionDorisColumns,
partitionTypes, columnNameToOdpsColumn));
}
private Type mcTypeToDorisType(TypeInfo typeInfo) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeSchemaCacheValue.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeSchemaCacheValue.java
index cf87725c680..cd734985e6e 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeSchemaCacheValue.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeSchemaCacheValue.java
@@ -27,6 +27,7 @@ import lombok.Getter;
import lombok.Setter;
import java.util.List;
+import java.util.Map;
@Getter
@Setter
@@ -37,10 +38,11 @@ public class MaxComputeSchemaCacheValue extends
SchemaCacheValue {
private List<String> partitionSpecs;
private List<Column> partitionColumns;
private List<Type> partitionTypes;
+ private Map<String, com.aliyun.odps.Column> columnNameToOdpsColumn;
public MaxComputeSchemaCacheValue(List<Column> schema, Table odpsTable,
TableIdentifier tableIdentifier,
List<String> partitionColumnNames, List<String> partitionSpecs,
List<Column> partitionColumns,
- List<Type> partitionTypes) {
+ List<Type> partitionTypes, Map<String, com.aliyun.odps.Column>
columnNameToOdpsColumn) {
super(schema);
this.odpsTable = odpsTable;
this.tableIdentifier = tableIdentifier;
@@ -48,6 +50,7 @@ public class MaxComputeSchemaCacheValue extends
SchemaCacheValue {
this.partitionColumnNames = partitionColumnNames;
this.partitionColumns = partitionColumns;
this.partitionTypes = partitionTypes;
+ this.columnNameToOdpsColumn = columnNameToOdpsColumn;
}
public List<Column> getPartitionColumns() {
@@ -57,4 +60,8 @@ public class MaxComputeSchemaCacheValue extends
SchemaCacheValue {
public List<String> getPartitionColumnNames() {
return partitionColumnNames;
}
+
+ public Map<String, com.aliyun.odps.Column> getColumnNameToOdpsColumn() {
+ return columnNameToOdpsColumn;
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/source/MaxComputeScanNode.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/source/MaxComputeScanNode.java
index e923b600d23..dc6ced6ee01 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/source/MaxComputeScanNode.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/source/MaxComputeScanNode.java
@@ -62,6 +62,8 @@ import com.aliyun.odps.table.read.split.InputSplitAssigner;
import com.aliyun.odps.table.read.split.impl.IndexedInputSplit;
import jline.internal.Log;
import lombok.Setter;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
@@ -88,6 +90,8 @@ public class MaxComputeScanNode extends FileQueryScanNode {
static final DateTimeFormatter dateTime3Formatter =
DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSS");
static final DateTimeFormatter dateTime6Formatter =
DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSSSSS");
+ private static final Logger LOG =
LogManager.getLogger(MaxComputeScanNode.class);
+
private final MaxComputeExternalTable table;
private Predicate filterPredicate;
List<String> requiredPartitionColumns = new ArrayList<>();
@@ -355,6 +359,9 @@ public class MaxComputeScanNode extends FileQueryScanNode {
String columnName = convertSlotRefToColumnName(expr.getChild(0));
if (!table.getColumnNameToOdpsColumn().containsKey(columnName)) {
+ Map<String, com.aliyun.odps.Column> columnMap =
table.getColumnNameToOdpsColumn();
+ LOG.warn("ColumnNameToOdpsColumn size=" + columnMap.size()
+ + ", keys=[" + String.join(", ", columnMap.keySet()) +
"]");
throw new AnalysisException("Column " + columnName + " not
found in table, can not push "
+ "down predicate to MaxCompute " + table.getName());
}
@@ -415,6 +422,9 @@ public class MaxComputeScanNode extends FileQueryScanNode {
if (odpsOp != null) {
String columnName =
convertSlotRefToColumnName(expr.getChild(0));
if
(!table.getColumnNameToOdpsColumn().containsKey(columnName)) {
+ Map<String, com.aliyun.odps.Column> columnMap =
table.getColumnNameToOdpsColumn();
+ LOG.warn("ColumnNameToOdpsColumn size=" + columnMap.size()
+ + ", keys=[" + String.join(", ",
columnMap.keySet()) + "]");
throw new AnalysisException("Column " + columnName + " not
found in table, can not push "
+ "down predicate to MaxCompute " +
table.getName());
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
