This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-1.2-lts by this push:
new 666d3f5b06 [fix](catalog) wrong required slot info causing BE crash.
(#21702)
666d3f5b06 is described below
commit 666d3f5b06751bc3cd811f82dc4246acf7833fcb
Author: Qi Chen <[email protected]>
AuthorDate: Wed Jul 12 09:33:17 2023 +0800
[fix](catalog) wrong required slot info causing BE crash. (#21702)
Merge #21598 to branch-1.2-lts.
Co-Authored-By: morningman [email protected]
---
.../apache/doris/datasource/hive/HiveMetaStoreCache.java | 2 +-
.../doris/planner/external/ExternalFileScanNode.java | 1 +
.../apache/doris/planner/external/HiveScanProvider.java | 15 ++++++++++++++-
.../hive/test_external_catalog_hive.out | 3 +++
.../hive/test_external_catalog_hive.groovy | 2 ++
5 files changed, 21 insertions(+), 2 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java
index 778fb10262..169ed3727b 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java
@@ -701,7 +701,7 @@ public class HiveMetaStoreCache {
return dummyKey.equals(((FileCacheKey) obj).dummyKey);
}
return location.equals(((FileCacheKey) obj).location)
- && partitionValues.equals(((FileCacheKey)
obj).partitionValues);
+ && Objects.equals(partitionValues, ((FileCacheKey)
obj).partitionValues);
}
@Override
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/planner/external/ExternalFileScanNode.java
b/fe/fe-core/src/main/java/org/apache/doris/planner/external/ExternalFileScanNode.java
index 1aadb1e5c6..c94305233f 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/planner/external/ExternalFileScanNode.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/planner/external/ExternalFileScanNode.java
@@ -353,6 +353,7 @@ public class ExternalFileScanNode extends ExternalScanNode {
if (scanProvider instanceof HiveScanProvider) {
this.totalPartitionNum = ((HiveScanProvider)
scanProvider).getTotalPartitionNum();
this.readPartitionNum = ((HiveScanProvider)
scanProvider).getReadPartitionNum();
+ ((HiveScanProvider) scanProvider).updateRequiredSlots(context);
}
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/planner/external/HiveScanProvider.java
b/fe/fe-core/src/main/java/org/apache/doris/planner/external/HiveScanProvider.java
index 845d62f92c..94d50ebd45 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/planner/external/HiveScanProvider.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/planner/external/HiveScanProvider.java
@@ -258,6 +258,19 @@ public class HiveScanProvider extends HMSTableScanProvider
{
List<String> partitionKeys = getPathPartitionKeys();
List<Column> columns = hmsTable.getBaseSchema(false);
context.params.setNumOfColumnsFromFile(columns.size() -
partitionKeys.size());
+ updateRequiredSlots(context);
+ return context;
+ }
+
+ public void updateRequiredSlots(ParamCreateContext context) throws
UserException {
+ updateRequiredSlots(context, null);
+ }
+
+ public void updateRequiredSlots(ParamCreateContext context, List<String>
partitionKeys) throws UserException {
+ context.params.unsetRequiredSlots();
+ if (partitionKeys == null) {
+ partitionKeys = getPathPartitionKeys();
+ }
for (SlotDescriptor slot : desc.getSlots()) {
if (!slot.isMaterialized()) {
continue;
@@ -268,9 +281,9 @@ public class HiveScanProvider extends HMSTableScanProvider {
slotInfo.setIsFileSlot(!partitionKeys.contains(slot.getColumn().getName()));
context.params.addToRequiredSlots(slotInfo);
}
- return context;
}
+
@Override
public TFileAttributes getFileAttributes() throws UserException {
TFileTextScanRangeParams textParams = new TFileTextScanRangeParams();
diff --git
a/regression-test/data/external_table_emr_p2/hive/test_external_catalog_hive.out
b/regression-test/data/external_table_emr_p2/hive/test_external_catalog_hive.out
index b7a66d44d4..54dc2d9a40 100644
---
a/regression-test/data/external_table_emr_p2/hive/test_external_catalog_hive.out
+++
b/regression-test/data/external_table_emr_p2/hive/test_external_catalog_hive.out
@@ -93,3 +93,6 @@ moccasin steel bisque cornsilk lace
-- !q25 --
Z6n2t4XA2n7CXTECJ,PE,iBbsCh0RE1Dd2A,z48
+-- !pr21598 --
+5
+
diff --git
a/regression-test/suites/external_table_emr_p2/hive/test_external_catalog_hive.groovy
b/regression-test/suites/external_table_emr_p2/hive/test_external_catalog_hive.groovy
index 10bcea354e..ca445dbcda 100644
---
a/regression-test/suites/external_table_emr_p2/hive/test_external_catalog_hive.groovy
+++
b/regression-test/suites/external_table_emr_p2/hive/test_external_catalog_hive.groovy
@@ -87,6 +87,8 @@ suite("test_external_catalog_hive", "p2") {
sql """ use tpch_1000_orc; """
q03()
+ // test #21598
+ qt_pr21598 """select count(*) from( (SELECT r_regionkey AS key1,
r_name AS name, pday AS pday FROM (SELECT r_regionkey, r_name,
replace(r_comment, ' ', 'aaaa') AS pday FROM
${catalog_name}.tpch_1000_parquet.region) t2))x;"""
// test remember last used database after switch / rename catalog
sql """switch ${catalog_name};"""
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
