This is an automated email from the ASF dual-hosted git repository.
gsaihemanth pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new d06fb43b617 HIVE-27827: Improve performance of direct SQL implement
for getPartitionsByFilter (#4831) (Wechar Yu, Reviewed by Sai Hemanth Gantasala)
d06fb43b617 is described below
commit d06fb43b617b3c52de39cf7a078e44108451b10c
Author: Wechar Yu <[email protected]>
AuthorDate: Sat Jan 20 01:46:10 2024 +0800
HIVE-27827: Improve performance of direct SQL implement for
getPartitionsByFilter (#4831) (Wechar Yu, Reviewed by Sai Hemanth Gantasala)
---
.../hadoop/hive/metastore/MetaStoreDirectSql.java | 46 +++++++++++++++++++---
.../hive/metastore/parser/ExpressionTree.java | 12 +++++-
.../sql/derby/hive-schema-4.0.0-beta-2.derby.sql | 2 +-
.../upgrade-4.0.0-beta-1-to-4.0.0-beta-2.derby.sql | 4 ++
.../sql/mssql/hive-schema-4.0.0-beta-2.mssql.sql | 4 +-
.../upgrade-4.0.0-beta-1-to-4.0.0-beta-2.mssql.sql | 5 +++
.../sql/mysql/hive-schema-4.0.0-beta-2.mysql.sql | 3 +-
.../upgrade-4.0.0-beta-1-to-4.0.0-beta-2.mysql.sql | 5 +++
.../sql/oracle/hive-schema-4.0.0-beta-2.oracle.sql | 4 +-
...upgrade-4.0.0-beta-1-to-4.0.0-beta-2.oracle.sql | 5 +++
.../postgres/hive-schema-4.0.0-beta-2.postgres.sql | 9 +----
...grade-4.0.0-beta-1-to-4.0.0-beta-2.postgres.sql | 5 +++
.../hadoop/hive/metastore/tools/BenchmarkTool.java | 5 +++
.../hadoop/hive/metastore/tools/HMSBenchmarks.java | 21 ++++++++++
.../hadoop/hive/metastore/tools/HMSClient.java | 5 +++
15 files changed, 111 insertions(+), 24 deletions(-)
diff --git
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
index 18bffce36f7..515721791bb 100644
---
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
+++
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
@@ -30,6 +30,7 @@ import static
org.apache.hadoop.hive.metastore.ColumnType.STRING_TYPE_NAME;
import static org.apache.hadoop.hive.metastore.ColumnType.TIMESTAMP_TYPE_NAME;
import static org.apache.hadoop.hive.metastore.ColumnType.TINYINT_TYPE_NAME;
import static org.apache.hadoop.hive.metastore.ColumnType.VARCHAR_TYPE_NAME;
+import static
org.apache.hadoop.hive.metastore.utils.FileUtils.unescapePathName;
import java.sql.Connection;
import java.sql.Date;
@@ -1409,7 +1410,8 @@ class MetaStoreDirectSql {
return;
}
- String colTypeStr =
ColumnType.getTypeName(partitionKeys.get(partColIndex).getType());
+ FieldSchema partCol = partitionKeys.get(partColIndex);
+ String colTypeStr = ColumnType.getTypeName(partCol.getType());
FilterType colType = FilterType.fromType(colTypeStr);
if (colType == FilterType.Invalid) {
filterBuffer.setError("Filter pushdown not supported for type " +
colTypeStr);
@@ -1434,7 +1436,8 @@ class MetaStoreDirectSql {
}
if (colType == FilterType.Timestamp && valType == FilterType.String) {
- nodeValue = MetaStoreUtils.convertStringToTimestamp((String)nodeValue);
+ // timestamp value may be escaped in client side, so we need unescape
it here.
+ nodeValue =
MetaStoreUtils.convertStringToTimestamp(unescapePathName((String) nodeValue));
valType = FilterType.Timestamp;
}
@@ -1520,10 +1523,43 @@ class MetaStoreDirectSql {
if (node.operator == Operator.LIKE) {
nodeValue0 = nodeValue0 + " ESCAPE '\\' ";
}
+ String filter = node.isReverseOrder
+ ? nodeValue0 + " " + node.operator.getSqlOp() + " " + tableValue
+ : tableValue + " " + node.operator.getSqlOp() + " " + nodeValue0;
+ // For equals and not-equals filter, we can add partition name filter to
improve performance.
+ boolean isOpEquals = Operator.isEqualOperator(node.operator);
+ if (isOpEquals || Operator.isNotEqualOperator(node.operator)) {
+ Map<String, String> partKeyToVal = new HashMap<>();
+ partKeyToVal.put(partCol.getName(), nodeValue.toString());
+ String escapedNameFragment = Warehouse.makePartName(partKeyToVal,
false);
+ if (colType == FilterType.Date) {
+ // Some engines like Pig will record both date and time values, in
which case we need
+ // match PART_NAME by like clause.
+ escapedNameFragment += "%";
+ }
+ if (colType != FilterType.Date && partColCount == 1) {
+ // Case where partition column type is not date and there is no
other partition columns
+ params.add(escapedNameFragment);
+ filter += " and " + PARTITIONS + ".\"PART_NAME\"" + (isOpEquals ? "
=? " : " !=? ");
+ } else {
+ if (partColCount == 1) {
+ // Case where partition column type is date and there is no other
partition columns
+ params.add(escapedNameFragment);
+ } else if (partColIndex + 1 == partColCount) {
+ // Case where the partition column is at the end of the name.
+ params.add("%/" + escapedNameFragment);
+ } else if (partColIndex == 0) {
+ // Case where the partition column is at the beginning of the name.
+ params.add(escapedNameFragment + "/%");
+ } else {
+ // Case where the partition column is in the middle of the name.
+ params.add("%/" + escapedNameFragment + "/%");
+ }
+ filter += " and " + PARTITIONS + ".\"PART_NAME\"" + (isOpEquals ? "
like ? " : " not like ? ");
+ }
+ }
- filterBuffer.append(node.isReverseOrder
- ? "(" + nodeValue0 + " " + node.operator.getSqlOp() + " " +
tableValue + ")"
- : "(" + tableValue + " " + node.operator.getSqlOp() + " " +
nodeValue0 + ")");
+ filterBuffer.append("(" + filter + ")");
}
}
diff --git
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/parser/ExpressionTree.java
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/parser/ExpressionTree.java
index f2f91cbedfb..2e325a4e431 100644
---
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/parser/ExpressionTree.java
+++
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/parser/ExpressionTree.java
@@ -101,6 +101,14 @@ public class ExpressionTree {
" for " + Operator.class.getSimpleName());
}
+ public static boolean isEqualOperator(Operator op) {
+ return op == EQUALS;
+ }
+
+ public static boolean isNotEqualOperator(Operator op) {
+ return op == NOTEQUALS || op == NOTEQUALS2;
+ }
+
@Override
public String toString() {
return op;
@@ -376,8 +384,8 @@ public class ExpressionTree {
params.put(paramName, valueAsString);
}
- boolean isOpEquals = operator == Operator.EQUALS;
- if (isOpEquals || operator == Operator.NOTEQUALS || operator ==
Operator.NOTEQUALS2) {
+ boolean isOpEquals = Operator.isEqualOperator(operator);
+ if (isOpEquals || Operator.isNotEqualOperator(operator)) {
String partitionKey =
partitionKeys.get(partitionColumnIndex).getName();
makeFilterForEquals(partitionKey, valueAsString, paramName, params,
partitionColumnIndex, partitionColumnCount, isOpEquals,
filterBuilder);
diff --git
a/standalone-metastore/metastore-server/src/main/sql/derby/hive-schema-4.0.0-beta-2.derby.sql
b/standalone-metastore/metastore-server/src/main/sql/derby/hive-schema-4.0.0-beta-2.derby.sql
index 9feaa6a0c3f..3b9d46700e4 100644
---
a/standalone-metastore/metastore-server/src/main/sql/derby/hive-schema-4.0.0-beta-2.derby.sql
+++
b/standalone-metastore/metastore-server/src/main/sql/derby/hive-schema-4.0.0-beta-2.derby.sql
@@ -288,7 +288,7 @@ CREATE UNIQUE INDEX "APP"."UNIQUE_TYPE" ON "APP"."TYPES"
("TYPE_NAME");
CREATE INDEX "APP"."PARTITIONCOLUMNPRIVILEGEINDEX" ON "APP"."PART_COL_PRIVS"
("AUTHORIZER", "PART_ID", "COLUMN_NAME", "PRINCIPAL_NAME", "PRINCIPAL_TYPE",
"PART_COL_PRIV", "GRANTOR", "GRANTOR_TYPE");
-CREATE UNIQUE INDEX "APP"."UNIQUEPARTITION" ON "APP"."PARTITIONS"
("PART_NAME", "TBL_ID");
+CREATE UNIQUE INDEX "APP"."UNIQUEPARTITION" ON "APP"."PARTITIONS" ("TBL_ID",
"PART_NAME");
CREATE UNIQUE INDEX "APP"."UNIQUEFUNCTION" ON "APP"."FUNCS" ("FUNC_NAME",
"DB_ID");
diff --git
a/standalone-metastore/metastore-server/src/main/sql/derby/upgrade-4.0.0-beta-1-to-4.0.0-beta-2.derby.sql
b/standalone-metastore/metastore-server/src/main/sql/derby/upgrade-4.0.0-beta-1-to-4.0.0-beta-2.derby.sql
index 2695e10742c..525a7d5215e 100644
---
a/standalone-metastore/metastore-server/src/main/sql/derby/upgrade-4.0.0-beta-1-to-4.0.0-beta-2.derby.sql
+++
b/standalone-metastore/metastore-server/src/main/sql/derby/upgrade-4.0.0-beta-1-to-4.0.0-beta-2.derby.sql
@@ -2,5 +2,9 @@
DROP TABLE "APP"."INDEX_PARAMS";
DROP TABLE "APP"."IDXS";
+-- HIVE-27827
+DROP INDEX "APP"."UNIQUEPARTITION";
+CREATE UNIQUE INDEX "APP"."UNIQUEPARTITION" ON "APP"."PARTITIONS" ("TBL_ID",
"PART_NAME");
+
-- This needs to be the last thing done. Insert any changes above this line.
UPDATE "APP".VERSION SET SCHEMA_VERSION='4.0.0-beta-2', VERSION_COMMENT='Hive
release version 4.0.0-beta-2' where VER_ID=1;
diff --git
a/standalone-metastore/metastore-server/src/main/sql/mssql/hive-schema-4.0.0-beta-2.mssql.sql
b/standalone-metastore/metastore-server/src/main/sql/mssql/hive-schema-4.0.0-beta-2.mssql.sql
index 190a245c2b6..2bbf62d5f1c 100644
---
a/standalone-metastore/metastore-server/src/main/sql/mssql/hive-schema-4.0.0-beta-2.mssql.sql
+++
b/standalone-metastore/metastore-server/src/main/sql/mssql/hive-schema-4.0.0-beta-2.mssql.sql
@@ -747,9 +747,7 @@ ALTER TABLE PARTITIONS ADD CONSTRAINT PARTITIONS_FK2
FOREIGN KEY (SD_ID) REFEREN
CREATE INDEX PARTITIONS_N49 ON PARTITIONS (SD_ID);
-CREATE INDEX PARTITIONS_N50 ON PARTITIONS (TBL_ID);
-
-CREATE UNIQUE INDEX UNIQUEPARTITION ON PARTITIONS (PART_NAME,TBL_ID);
+CREATE UNIQUE INDEX UNIQUEPARTITION ON PARTITIONS (TBL_ID,PART_NAME);
-- Constraints for table CDS for class(es)
[org.apache.hadoop.hive.metastore.model.MColumnDescriptor]
diff --git
a/standalone-metastore/metastore-server/src/main/sql/mssql/upgrade-4.0.0-beta-1-to-4.0.0-beta-2.mssql.sql
b/standalone-metastore/metastore-server/src/main/sql/mssql/upgrade-4.0.0-beta-1-to-4.0.0-beta-2.mssql.sql
index a7afd05ceb3..f7ded61aef9 100644
---
a/standalone-metastore/metastore-server/src/main/sql/mssql/upgrade-4.0.0-beta-1-to-4.0.0-beta-2.mssql.sql
+++
b/standalone-metastore/metastore-server/src/main/sql/mssql/upgrade-4.0.0-beta-1-to-4.0.0-beta-2.mssql.sql
@@ -4,6 +4,11 @@ SELECT 'Upgrading MetaStore schema from 4.0.0-beta-1 to
4.0.0-beta-2' AS MESSAG
DROP TABLE INDEX_PARAMS;
DROP TABLE IDXS;
+-- HIVE-27827
+DROP INDEX UNIQUEPARTITION ON PARTITIONS;
+CREATE UNIQUE INDEX UNIQUEPARTITION ON PARTITIONS (TBL_ID,PART_NAME);
+DROP INDEX PARTITIONS_N50 ON PARTITIONS;
+
-- These lines need to be last. Insert any changes above.
UPDATE VERSION SET SCHEMA_VERSION='4.0.0-beta-2', VERSION_COMMENT='Hive
release version 4.0.0-beta-2' where VER_ID=1;
SELECT 'Finished upgrading MetaStore schema from 4.0.0-beta-1 to 4.0.0-beta-2'
AS MESSAGE;
diff --git
a/standalone-metastore/metastore-server/src/main/sql/mysql/hive-schema-4.0.0-beta-2.mysql.sql
b/standalone-metastore/metastore-server/src/main/sql/mysql/hive-schema-4.0.0-beta-2.mysql.sql
index c1140eacf44..f99632267a8 100644
---
a/standalone-metastore/metastore-server/src/main/sql/mysql/hive-schema-4.0.0-beta-2.mysql.sql
+++
b/standalone-metastore/metastore-server/src/main/sql/mysql/hive-schema-4.0.0-beta-2.mysql.sql
@@ -191,8 +191,7 @@ CREATE TABLE IF NOT EXISTS `PARTITIONS` (
`TBL_ID` bigint(20) DEFAULT NULL,
`WRITE_ID` bigint(20) DEFAULT 0,
PRIMARY KEY (`PART_ID`),
- UNIQUE KEY `UNIQUEPARTITION` (`PART_NAME`,`TBL_ID`),
- KEY `PARTITIONS_N49` (`TBL_ID`),
+ UNIQUE KEY `UNIQUEPARTITION` (`TBL_ID`, `PART_NAME`),
KEY `PARTITIONS_N50` (`SD_ID`),
CONSTRAINT `PARTITIONS_FK1` FOREIGN KEY (`TBL_ID`) REFERENCES `TBLS`
(`TBL_ID`),
CONSTRAINT `PARTITIONS_FK2` FOREIGN KEY (`SD_ID`) REFERENCES `SDS` (`SD_ID`)
diff --git
a/standalone-metastore/metastore-server/src/main/sql/mysql/upgrade-4.0.0-beta-1-to-4.0.0-beta-2.mysql.sql
b/standalone-metastore/metastore-server/src/main/sql/mysql/upgrade-4.0.0-beta-1-to-4.0.0-beta-2.mysql.sql
index 9d3bcb028ab..a57b58b2f7b 100644
---
a/standalone-metastore/metastore-server/src/main/sql/mysql/upgrade-4.0.0-beta-1-to-4.0.0-beta-2.mysql.sql
+++
b/standalone-metastore/metastore-server/src/main/sql/mysql/upgrade-4.0.0-beta-1-to-4.0.0-beta-2.mysql.sql
@@ -4,6 +4,11 @@ SELECT 'Upgrading MetaStore schema from 4.0.0-beta-1 to
4.0.0-beta-2' AS MESSAGE
DROP TABLE `INDEX_PARAMS`;
DROP TABLE `IDXS`;
+-- HIVE-27827
+DROP INDEX UNIQUEPARTITION ON PARTITIONS;
+CREATE UNIQUE INDEX UNIQUEPARTITION ON PARTITIONS (TBL_ID, PART_NAME);
+DROP INDEX PARTITIONS_N49 on PARTITIONS;
+
-- These lines need to be last. Insert any changes above.
UPDATE VERSION SET SCHEMA_VERSION='4.0.0-beta-2', VERSION_COMMENT='Hive
release version 4.0.0-beta-2' where VER_ID=1;
SELECT 'Finished upgrading MetaStore schema from 4.0.0-beta-1 to 4.0.0-beta-2'
AS MESSAGE;
diff --git
a/standalone-metastore/metastore-server/src/main/sql/oracle/hive-schema-4.0.0-beta-2.oracle.sql
b/standalone-metastore/metastore-server/src/main/sql/oracle/hive-schema-4.0.0-beta-2.oracle.sql
index a2fb02c8911..d4026910343 100644
---
a/standalone-metastore/metastore-server/src/main/sql/oracle/hive-schema-4.0.0-beta-2.oracle.sql
+++
b/standalone-metastore/metastore-server/src/main/sql/oracle/hive-schema-4.0.0-beta-2.oracle.sql
@@ -763,9 +763,7 @@ ALTER TABLE PARTITIONS ADD CONSTRAINT PARTITIONS_FK2
FOREIGN KEY (SD_ID) REFEREN
CREATE INDEX PARTITIONS_N49 ON PARTITIONS (SD_ID);
-CREATE INDEX PARTITIONS_N50 ON PARTITIONS (TBL_ID);
-
-CREATE UNIQUE INDEX UNIQUEPARTITION ON PARTITIONS (PART_NAME,TBL_ID);
+CREATE UNIQUE INDEX UNIQUEPARTITION ON PARTITIONS (TBL_ID, PART_NAME);
-- Constraints for table TBL_COL_PRIVS for class(es)
[org.apache.hadoop.hive.metastore.model.MTableColumnPrivilege]
ALTER TABLE TBL_COL_PRIVS ADD CONSTRAINT TBL_COL_PRIVS_FK1 FOREIGN KEY
(TBL_ID) REFERENCES TBLS (TBL_ID) INITIALLY DEFERRED ;
diff --git
a/standalone-metastore/metastore-server/src/main/sql/oracle/upgrade-4.0.0-beta-1-to-4.0.0-beta-2.oracle.sql
b/standalone-metastore/metastore-server/src/main/sql/oracle/upgrade-4.0.0-beta-1-to-4.0.0-beta-2.oracle.sql
index 53c7cb140d7..ab163b6cd00 100644
---
a/standalone-metastore/metastore-server/src/main/sql/oracle/upgrade-4.0.0-beta-1-to-4.0.0-beta-2.oracle.sql
+++
b/standalone-metastore/metastore-server/src/main/sql/oracle/upgrade-4.0.0-beta-1-to-4.0.0-beta-2.oracle.sql
@@ -5,6 +5,11 @@ SELECT 'Upgrading MetaStore schema from 4.0.0-beta-1 to
4.0.0-beta-2' AS Status
DROP TABLE INDEX_PARAMS;
DROP TABLE IDXS;
+-- HIVE-27827
+DROP INDEX UNIQUEPARTITION;
+CREATE UNIQUE INDEX UNIQUEPARTITION ON PARTITIONS (TBL_ID, PART_NAME);
+DROP INDEX PARTITIONS_N50;
+
-- These lines need to be last. Insert any changes above.
UPDATE VERSION SET SCHEMA_VERSION='4.0.0-beta-2', VERSION_COMMENT='Hive
release version 4.0.0-beta-2' where VER_ID=1;
SELECT 'Finished upgrading MetaStore schema from 4.0.0-beta-1 to 4.0.0-beta-2'
AS Status from dual;
diff --git
a/standalone-metastore/metastore-server/src/main/sql/postgres/hive-schema-4.0.0-beta-2.postgres.sql
b/standalone-metastore/metastore-server/src/main/sql/postgres/hive-schema-4.0.0-beta-2.postgres.sql
index b247bc3fda0..292bc200fca 100644
---
a/standalone-metastore/metastore-server/src/main/sql/postgres/hive-schema-4.0.0-beta-2.postgres.sql
+++
b/standalone-metastore/metastore-server/src/main/sql/postgres/hive-schema-4.0.0-beta-2.postgres.sql
@@ -1027,7 +1027,7 @@ ALTER TABLE ONLY "PART_COL_STATS" ADD CONSTRAINT
"PART_COL_STATS_pkey" PRIMARY K
--
ALTER TABLE ONLY "PARTITIONS"
- ADD CONSTRAINT "UNIQUEPARTITION" UNIQUE ("PART_NAME", "TBL_ID");
+ ADD CONSTRAINT "UNIQUEPARTITION" UNIQUE ("TBL_ID", "PART_NAME");
--
@@ -1135,13 +1135,6 @@ CREATE INDEX "PARTITIONCOLUMNPRIVILEGEINDEX" ON
"PART_COL_PRIVS" USING btree ("A
CREATE INDEX "PARTITIONEVENTINDEX" ON "PARTITION_EVENTS" USING btree
("PARTITION_NAME");
---
--- Name: PARTITIONS_N49; Type: INDEX; Schema: public; Owner: hiveuser;
Tablespace:
---
-
-CREATE INDEX "PARTITIONS_N49" ON "PARTITIONS" USING btree ("TBL_ID");
-
-
--
-- Name: PARTITIONS_N50; Type: INDEX; Schema: public; Owner: hiveuser;
Tablespace:
--
diff --git
a/standalone-metastore/metastore-server/src/main/sql/postgres/upgrade-4.0.0-beta-1-to-4.0.0-beta-2.postgres.sql
b/standalone-metastore/metastore-server/src/main/sql/postgres/upgrade-4.0.0-beta-1-to-4.0.0-beta-2.postgres.sql
index 1d285bbe842..af17c7d9524 100644
---
a/standalone-metastore/metastore-server/src/main/sql/postgres/upgrade-4.0.0-beta-1-to-4.0.0-beta-2.postgres.sql
+++
b/standalone-metastore/metastore-server/src/main/sql/postgres/upgrade-4.0.0-beta-1-to-4.0.0-beta-2.postgres.sql
@@ -4,6 +4,11 @@ SELECT 'Upgrading MetaStore schema from 4.0.0-beta-1 to
4.0.0-beta-2';
DROP TABLE IF EXISTS "INDEX_PARAMS";
DROP TABLE IF EXISTS "IDXS";
+-- HIVE-27827
+ALTER TABLE ONLY "PARTITIONS" DROP CONSTRAINT "UNIQUEPARTITION";
+ALTER TABLE ONLY "PARTITIONS" ADD CONSTRAINT "UNIQUEPARTITION" UNIQUE
("TBL_ID", "PART_NAME");
+DROP INDEX "PARTITIONS_N49";
+
-- These lines need to be last. Insert any changes above.
UPDATE "VERSION" SET "SCHEMA_VERSION"='4.0.0-beta-2', "VERSION_COMMENT"='Hive
release version 4.0.0-beta-2' where "VER_ID"=1;
SELECT 'Finished upgrading MetaStore schema from 4.0.0-beta-1 to 4.0.0-beta-2';
diff --git
a/standalone-metastore/metastore-tools/metastore-benchmarks/src/main/java/org/apache/hadoop/hive/metastore/tools/BenchmarkTool.java
b/standalone-metastore/metastore-tools/metastore-benchmarks/src/main/java/org/apache/hadoop/hive/metastore/tools/BenchmarkTool.java
index f19576d6940..93556e7f0fa 100644
---
a/standalone-metastore/metastore-tools/metastore-benchmarks/src/main/java/org/apache/hadoop/hive/metastore/tools/BenchmarkTool.java
+++
b/standalone-metastore/metastore-tools/metastore-benchmarks/src/main/java/org/apache/hadoop/hive/metastore/tools/BenchmarkTool.java
@@ -57,6 +57,7 @@ import static
org.apache.hadoop.hive.metastore.tools.HMSBenchmarks.benchmarkDrop
import static
org.apache.hadoop.hive.metastore.tools.HMSBenchmarks.benchmarkGetNotificationId;
import static
org.apache.hadoop.hive.metastore.tools.HMSBenchmarks.benchmarkGetPartitionNames;
import static
org.apache.hadoop.hive.metastore.tools.HMSBenchmarks.benchmarkGetPartitions;
+import static
org.apache.hadoop.hive.metastore.tools.HMSBenchmarks.benchmarkGetPartitionsByFilter;
import static
org.apache.hadoop.hive.metastore.tools.HMSBenchmarks.benchmarkGetPartitionsByName;
import static
org.apache.hadoop.hive.metastore.tools.HMSBenchmarks.benchmarkGetTable;
import static
org.apache.hadoop.hive.metastore.tools.HMSBenchmarks.benchmarkListAllTables;
@@ -284,6 +285,8 @@ public class BenchmarkTool implements Runnable {
() -> benchmarkGetPartitionNames(bench, bData, 1))
.add("getPartitionsByNames",
() -> benchmarkGetPartitionsByName(bench, bData, 1))
+ .add("getPartitionsByFilter",
+ () -> benchmarkGetPartitionsByFilter(bench, bData, 1))
.add("renameTable",
() -> benchmarkRenameTable(bench, bData, 1))
.add("dropDatabase",
@@ -308,6 +311,8 @@ public class BenchmarkTool implements Runnable {
() -> benchmarkGetPartitionNames(bench, bData, howMany))
.add("getPartitionsByNames" + '.' + howMany,
() -> benchmarkGetPartitionsByName(bench, bData, howMany))
+ .add("getPartitionsByFilter" + '.' + howMany,
+ () -> benchmarkGetPartitionsByFilter(bench, bData, howMany))
.add("addPartitions" + '.' + howMany,
() -> benchmarkCreatePartitions(bench, bData, howMany))
.add("dropPartitions" + '.' + howMany,
diff --git
a/standalone-metastore/metastore-tools/metastore-benchmarks/src/main/java/org/apache/hadoop/hive/metastore/tools/HMSBenchmarks.java
b/standalone-metastore/metastore-tools/metastore-benchmarks/src/main/java/org/apache/hadoop/hive/metastore/tools/HMSBenchmarks.java
index a2f97eb3170..fdab0717835 100644
---
a/standalone-metastore/metastore-tools/metastore-benchmarks/src/main/java/org/apache/hadoop/hive/metastore/tools/HMSBenchmarks.java
+++
b/standalone-metastore/metastore-tools/metastore-benchmarks/src/main/java/org/apache/hadoop/hive/metastore/tools/HMSBenchmarks.java
@@ -410,6 +410,27 @@ final class HMSBenchmarks {
}
}
+ static DescriptiveStatistics benchmarkGetPartitionsByFilter(@NotNull
MicroBenchmark bench,
+ @NotNull
BenchData data,
+ int count) {
+ final HMSClient client = data.getClient();
+ String dbName = data.dbName;
+ String tableName = data.tableName;
+
+ BenchmarkUtils.createPartitionedTable(client, dbName, tableName);
+ try {
+ addManyPartitionsNoException(client, dbName, tableName, null,
+ Collections.singletonList("d"), count);
+ return bench.measure(
+ () ->
+ throwingSupplierWrapper(() ->
+ client.getPartitionsByFilter(dbName, tableName,
"`date`='d0'"))
+ );
+ } finally {
+ throwingSupplierWrapper(() -> client.dropTable(dbName, tableName));
+ }
+ }
+
static DescriptiveStatistics benchmarkRenameTable(@NotNull MicroBenchmark
bench,
@NotNull BenchData data,
int count) {
diff --git
a/standalone-metastore/metastore-tools/tools-common/src/main/java/org/apache/hadoop/hive/metastore/tools/HMSClient.java
b/standalone-metastore/metastore-tools/tools-common/src/main/java/org/apache/hadoop/hive/metastore/tools/HMSClient.java
index 6b7e2a450b1..61580aa7b22 100644
---
a/standalone-metastore/metastore-tools/tools-common/src/main/java/org/apache/hadoop/hive/metastore/tools/HMSClient.java
+++
b/standalone-metastore/metastore-tools/tools-common/src/main/java/org/apache/hadoop/hive/metastore/tools/HMSClient.java
@@ -329,6 +329,11 @@ final class HMSClient implements AutoCloseable {
return client.get_partitions_by_names(dbName, tableName, names);
}
+ List<Partition> getPartitionsByFilter(@NotNull String dbName, @NotNull
String tableName,
+ @NotNull String filter) throws
TException {
+ return client.get_partitions_by_filter(dbName, tableName, filter, (short)
-1);
+ }
+
boolean alterTable(@NotNull String dbName, @NotNull String tableName,
@NotNull Table newTable)
throws TException {
client.alter_table(dbName, tableName, newTable);
