This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 07336980f99 [fix](meta) show partitions with Limit for external HMS
tables (27835) (#27835)
07336980f99 is described below
commit 07336980f99b50b9a73fa91790cc047192e8cd2d
Author: Nitin-Kashyap <[email protected]>
AuthorDate: Fri Dec 8 23:14:45 2023 +0530
[fix](meta) show partitions with Limit for external HMS tables (27835)
(#27835)
This enhancement shall extend existing logic for SHOW PARTITIONS FROM to
include: -
Limit/Offset
Where [partition name only] [equal operator and like operator]
Order by [partition name only]
Issue Number: close #27834
---
.../apache/doris/analysis/ShowPartitionsStmt.java | 11 ++++-
.../doris/common/proc/PartitionsProcDir.java | 5 ++-
.../java/org/apache/doris/qe/ShowExecutor.java | 36 +++++++++++++---
.../hive/test_hive_partitions.out | 49 ++++++++++++++++++++++
.../hive/test_hive_partitions.groovy | 33 +++++++++++++++
5 files changed, 126 insertions(+), 8 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowPartitionsStmt.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowPartitionsStmt.java
index c9faaf42174..3ba1db23d52 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowPartitionsStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowPartitionsStmt.java
@@ -58,7 +58,7 @@ public class ShowPartitionsStmt extends ShowStmt {
private static final Logger LOG =
LogManager.getLogger(ShowPartitionsStmt.class);
private static final String FILTER_PARTITION_ID = "PartitionId";
- private static final String FILTER_PARTITION_NAME = "PartitionName";
+ public static final String FILTER_PARTITION_NAME = "PartitionName";
private static final String FILTER_STATE = "State";
private static final String FILTER_BUCKETS = "Buckets";
private static final String FILTER_REPLICATION_NUM = "ReplicationNum";
@@ -198,6 +198,11 @@ public class ShowPartitionsStmt extends ShowStmt {
throw new AnalysisException("Should order by column");
}
SlotRef slotRef = (SlotRef) orderByElement.getExpr();
+ if (catalog instanceof HMSExternalCatalog
+ &&
!slotRef.getColumnName().equalsIgnoreCase(FILTER_PARTITION_NAME)) {
+ throw new AnalysisException("External table only support
Order By on PartitionName");
+ }
+
int index =
PartitionsProcDir.analyzeColumn(slotRef.getColumnName());
OrderByPair orderByPair = new OrderByPair(index,
!orderByElement.getIsAsc());
orderByPairs.add(orderByPair);
@@ -228,6 +233,10 @@ public class ShowPartitionsStmt extends ShowStmt {
}
String leftKey = ((SlotRef) subExpr.getChild(0)).getColumnName();
+ if (catalog instanceof HMSExternalCatalog &&
!leftKey.equalsIgnoreCase(FILTER_PARTITION_NAME)) {
+ throw new AnalysisException(String.format("Only %s column
supported in where clause for this catalog",
+ FILTER_PARTITION_NAME));
+ }
if (subExpr instanceof BinaryPredicate) {
BinaryPredicate binaryPredicate = (BinaryPredicate) subExpr;
if (leftKey.equalsIgnoreCase(FILTER_PARTITION_NAME) ||
leftKey.equalsIgnoreCase(FILTER_STATE)) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/common/proc/PartitionsProcDir.java
b/fe/fe-core/src/main/java/org/apache/doris/common/proc/PartitionsProcDir.java
index 11dd8d4c188..9319c96e040 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/common/proc/PartitionsProcDir.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/common/proc/PartitionsProcDir.java
@@ -81,7 +81,8 @@ public class PartitionsProcDir implements ProcDirInterface {
this.isTempPartition = isTempPartition;
}
- public boolean filter(String columnName, Comparable element, Map<String,
Expr> filterMap) throws AnalysisException {
+ public static boolean filter(String columnName, Comparable element,
Map<String, Expr> filterMap)
+ throws AnalysisException {
if (filterMap == null) {
return true;
}
@@ -142,7 +143,7 @@ public class PartitionsProcDir implements ProcDirInterface {
return true;
}
- public boolean like(String str, String expr) {
+ public static boolean like(String str, String expr) {
expr = expr.toLowerCase();
expr = expr.replace(".", "\\.");
expr = expr.replace("?", ".");
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
index ffc8e2652f3..df3ddc3151d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
@@ -24,6 +24,7 @@ import
org.apache.doris.analysis.AdminShowReplicaDistributionStmt;
import org.apache.doris.analysis.AdminShowReplicaStatusStmt;
import org.apache.doris.analysis.AdminShowTabletStorageFormatStmt;
import org.apache.doris.analysis.DescribeStmt;
+import org.apache.doris.analysis.Expr;
import org.apache.doris.analysis.HelpStmt;
import org.apache.doris.analysis.LimitElement;
import org.apache.doris.analysis.PartitionNames;
@@ -1655,28 +1656,53 @@ public class ShowExecutor {
resultSet = new ShowResultSet(showStmt.getMetaData(), rows);
}
- private void handleShowHMSTablePartitions(ShowPartitionsStmt showStmt) {
+ private void handleShowHMSTablePartitions(ShowPartitionsStmt showStmt)
throws AnalysisException {
HMSExternalCatalog catalog = (HMSExternalCatalog)
(showStmt.getCatalog());
List<List<String>> rows = new ArrayList<>();
String dbName =
ClusterNamespace.getNameFromFullName(showStmt.getTableName().getDb());
List<String> partitionNames;
LimitElement limit = showStmt.getLimitElement();
- if (limit != null && limit.hasLimit()) {
- // only limit is valid on Hive
+ Map<String, Expr> filterMap = showStmt.getFilterMap();
+ List<OrderByPair> orderByPairs = showStmt.getOrderByPairs();
+
+ if (limit != null && limit.hasLimit() && limit.getOffset() == 0
+ && (orderByPairs == null || !orderByPairs.get(0).isDesc())) {
+ // hmsClient returns unordered partition list, hence if offset > 0
cannot pass limit
partitionNames = catalog.getClient()
- .listPartitionNames(dbName,
showStmt.getTableName().getTbl(), limit.getLimit());
+ .listPartitionNames(dbName, showStmt.getTableName().getTbl(),
limit.getLimit());
} else {
partitionNames = catalog.getClient().listPartitionNames(dbName,
showStmt.getTableName().getTbl());
}
+
+ /* Filter add rows */
for (String partition : partitionNames) {
List<String> list = new ArrayList<>();
+
+ if (filterMap != null && !filterMap.isEmpty()) {
+ if
(!PartitionsProcDir.filter(ShowPartitionsStmt.FILTER_PARTITION_NAME, partition,
filterMap)) {
+ continue;
+ }
+ }
list.add(partition);
rows.add(list);
}
// sort by partition name
- rows.sort(Comparator.comparing(x -> x.get(0)));
+ if (orderByPairs != null && orderByPairs.get(0).isDesc()) {
+ rows.sort(Comparator.comparing(x -> x.get(0),
Comparator.reverseOrder()));
+ } else {
+ rows.sort(Comparator.comparing(x -> x.get(0)));
+ }
+
+ if (limit != null && limit.hasLimit()) {
+ int beginIndex = (int) limit.getOffset();
+ int endIndex = (int) (beginIndex + limit.getLimit());
+ if (endIndex > rows.size()) {
+ endIndex = rows.size();
+ }
+ rows = rows.subList(beginIndex, endIndex);
+ }
resultSet = new ShowResultSet(showStmt.getMetaData(), rows);
}
diff --git
a/regression-test/data/external_table_p0/hive/test_hive_partitions.out
b/regression-test/data/external_table_p0/hive/test_hive_partitions.out
index f5f44b2368a..646cc560fb5 100644
--- a/regression-test/data/external_table_p0/hive/test_hive_partitions.out
+++ b/regression-test/data/external_table_p0/hive/test_hive_partitions.out
@@ -71,3 +71,52 @@
55 1.22zxy
56 1.22ZXY
+-- !q11 --
+nation=cn/city=beijing
+nation=cn/city=shanghai
+nation=jp/city=tokyo
+nation=rus/city=moscow
+nation=us/city=chicago
+nation=us/city=washington
+
+-- !q12 --
+nation=cn/city=beijing
+
+-- !q13 --
+nation=us/city=chicago
+nation=us/city=washington
+
+-- !q14 --
+nation=rus/city=moscow
+nation=us/city=chicago
+nation=us/city=washington
+
+-- !q16 --
+nation=cn/city=beijing
+nation=cn/city=shanghai
+nation=jp/city=tokyo
+
+-- !q17 --
+nation=jp/city=tokyo
+nation=rus/city=moscow
+nation=us/city=chicago
+
+-- !q18 --
+nation=us/city=chicago
+nation=us/city=washington
+
+-- !q19 --
+nation=rus/city=moscow
+nation=jp/city=tokyo
+nation=cn/city=shanghai
+
+-- !q20 --
+nation=cn/city=beijing
+nation=cn/city=shanghai
+nation=jp/city=tokyo
+nation=rus/city=moscow
+nation=us/city=chicago
+nation=us/city=washington
+
+-- !q21 --
+
diff --git
a/regression-test/suites/external_table_p0/hive/test_hive_partitions.groovy
b/regression-test/suites/external_table_p0/hive/test_hive_partitions.groovy
index 8eae4e82e63..09bcc1b8608 100644
--- a/regression-test/suites/external_table_p0/hive/test_hive_partitions.groovy
+++ b/regression-test/suites/external_table_p0/hive/test_hive_partitions.groovy
@@ -35,6 +35,39 @@ suite("test_hive_partitions",
"p0,external,hive,external_docker,external_docker_
select id, data from table_with_pars where dt_par = '2023-02-01' and
time_par = '2023-02-01 01:30:00'
and decimal_par1 = '1' and decimal_par2 = '1.2' and decimal_par3 =
'1.22' order by id;
"""
+ qt_q11 """
+ show partitions from partition_table;
+ """
+ qt_q12 """
+ show partitions from partition_table WHERE
partitionName='nation=cn/city=beijing';
+ """
+ qt_q13 """
+ show partitions from partition_table WHERE partitionName like
'nation=us/%';
+ """
+ qt_q14 """
+ show partitions from partition_table WHERE partitionName like
'nation=%us%';
+ """
+ qt_q16 """
+ show partitions from partition_table LIMIT 3;
+ """
+ qt_q17 """
+ show partitions from partition_table LIMIT 3 OFFSET 2;
+ """
+ qt_q18 """
+ show partitions from partition_table LIMIT 3 OFFSET 4;
+ """
+ qt_q19 """
+ show partitions from partition_table ORDER BY partitionName desc
LIMIT 3 OFFSET 2;
+ """
+ qt_q20 """
+ show partitions from partition_table ORDER BY partitionName asc;
+ """
+ qt_q21 """
+ show partitions from partition_table
+ WHERE partitionName like '%X%'
+ ORDER BY partitionName DESC
+ LIMIT 1;
+ """
}
String enabled = context.config.otherConfigs.get("enableHiveTest")
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
