This is an automated email from the ASF dual-hosted git repository.
dengzh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 3a99fc26796 HIVE-28200: Improve get_partitions_by_filter/expr when
partition limit enabled (#5198) (Wechar Yu, reviewed by Zhihua Deng)
3a99fc26796 is described below
commit 3a99fc267967be7bf31277d9b23fa4329c21476a
Author: Wechar Yu <[email protected]>
AuthorDate: Wed May 8 12:11:04 2024 +0800
HIVE-28200: Improve get_partitions_by_filter/expr when partition limit
enabled (#5198) (Wechar Yu, reviewed by Zhihua Deng)
---
.../hcatalog/listener/DummyRawStoreFailEvent.java | 8 +-
.../ql/TestMetaStoreLimitPartitionRequest.java | 8 +-
.../hadoop/hive/metastore/conf/MetastoreConf.java | 2 +-
.../apache/hadoop/hive/metastore/HMSHandler.java | 81 ++++++++++---------
.../hadoop/hive/metastore/MetaStoreDirectSql.java | 36 ++++-----
.../apache/hadoop/hive/metastore/ObjectStore.java | 92 ++++++++++++++++------
.../org/apache/hadoop/hive/metastore/RawStore.java | 16 +++-
.../hadoop/hive/metastore/cache/CachedStore.java | 8 +-
.../metastore/DummyRawStoreControlledCommit.java | 8 +-
.../metastore/DummyRawStoreForJdoConnection.java | 8 +-
.../hadoop/hive/metastore/TestObjectStore.java | 60 ++++++++++++++
.../hive/metastore/tools/BenchmarkUtils.java | 6 +-
.../hadoop/hive/metastore/tools/HMSBenchmarks.java | 13 ++-
13 files changed, 252 insertions(+), 94 deletions(-)
diff --git
a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java
b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java
index 31569d6793f..bd3816b8f28 100644
---
a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java
+++
b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java
@@ -472,12 +472,18 @@ public class DummyRawStoreFailEvent implements RawStore,
Configurable {
@Override
public List<String> listPartitionNames(String catName, String dbName, String
tblName,
String defaultPartName, byte[] exprBytes, String order,
- short maxParts) throws MetaException, NoSuchObjectException {
+ int maxParts) throws MetaException, NoSuchObjectException {
return objectStore.listPartitionNames(catName, dbName, tblName,
defaultPartName, exprBytes, order, maxParts);
}
+ @Override
+ public List<String> listPartitionNamesByFilter(String catName, String
dbName, String tblName,
+ GetPartitionsArgs args) throws MetaException, NoSuchObjectException {
+ return objectStore.listPartitionNamesByFilter(catName, dbName, tblName,
args);
+ }
+
@Override
public PartitionValuesResponse listPartitionValues(String catName, String
db_name,
String tbl_name,
List<FieldSchema> cols,
diff --git
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestMetaStoreLimitPartitionRequest.java
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestMetaStoreLimitPartitionRequest.java
index 8ea7ff9e1b2..a68f89cf50a 100644
---
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestMetaStoreLimitPartitionRequest.java
+++
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestMetaStoreLimitPartitionRequest.java
@@ -160,7 +160,7 @@ public class TestMetaStoreLimitPartitionRequest {
@Test
public void testSimpleQueryWithDirectSqlTooManyPartitions() throws Exception
{
String queryString = "select value from %s where ds>'2008-04-20'";
- executeQueryExceedPartitionLimit(queryString, 8);
+ executeQueryExceedPartitionLimit(queryString, 5);
}
@Test
@@ -244,20 +244,20 @@ public class TestMetaStoreLimitPartitionRequest {
public void testQueryWithInWithFallbackToORMTooManyPartitions() throws
Exception {
setupNumTmpTable();
String queryString = "select value from %s a where a.num in (select value
from " + TABLE_NAME + "_num_tmp)";
- executeQueryExceedPartitionLimit(queryString, 12);
+ executeQueryExceedPartitionLimit(queryString, 5);
}
@Test
public void testQueryWithInWithFallbackToORMTooManyPartitions2() throws
Exception {
setupNumTmpTable();
String queryString = "select value from %s a where a.num in (select value
from " + TABLE_NAME + "_num_tmp where value='25')";
- executeQueryExceedPartitionLimit(queryString, 12);
+ executeQueryExceedPartitionLimit(queryString, 5);
}
@Test
public void testQueryWithLikeWithFallbackToORMTooManyPartitions() throws
Exception {
String queryString = "select value from %s where num like '3%%'";
- executeQueryExceedPartitionLimit(queryString, 6);
+ executeQueryExceedPartitionLimit(queryString, 5);
}
private void setupNumTmpTable() throws SQLException {
diff --git
a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
index b2713144bdb..713a45aa6f7 100644
---
a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
+++
b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
@@ -1052,7 +1052,7 @@ public class MetastoreConf {
"get_partitions_by_names, \n" +
"get_partitions_with_auth, \n" +
"get_partitions_by_filter, \n" +
- "get_partitions_spec_by_filter, \n" +
+ "get_partitions_spec_by_expr, \n" +
"get_partitions_by_expr,\n" +
"get_partitions_ps,\n" +
"get_partitions_ps_with_auth.\n" +
diff --git
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java
index a4c476e48fc..c77daf96fac 100644
---
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java
+++
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java
@@ -5569,15 +5569,6 @@ public class HMSHandler extends FacebookBase implements
IHMSHandler {
}
}
- private void checkLimitNumberOfPartitionsByExpr(String catName, String
dbName, String tblName,
- byte[] filterExpr, int
maxParts)
- throws TException {
- if (isPartitionLimitEnabled()) {
- checkLimitNumberOfPartitions(tblName,
get_num_partitions_by_expr(catName, dbName, tblName,
- filterExpr), maxParts);
- }
- }
-
private void checkLimitNumberOfPartitionsByPs(String catName, String dbName,
String tblName,
List<String> partVals, int
maxParts)
throws TException {
@@ -5592,6 +5583,14 @@ public class HMSHandler extends FacebookBase implements
IHMSHandler {
return partitionLimit > -1;
}
+ // Check request partition limit iff:
+ // 1. partition limit is enabled.
+ // 2. request size is greater than the limit.
+ private boolean needCheckPartitionLimit(int requestSize) {
+ int partitionLimit = MetastoreConf.getIntVar(conf,
ConfVars.LIMIT_PARTITION_REQUEST);
+ return partitionLimit > -1 && (requestSize < 0 || requestSize >
partitionLimit);
+ }
+
private void checkLimitNumberOfPartitions(String tblName, int numPartitions,
int maxToFetch) throws MetaException {
if (isPartitionLimitEnabled()) {
int partitionLimit = MetastoreConf.getIntVar(conf,
ConfVars.LIMIT_PARTITION_REQUEST);
@@ -7223,13 +7222,22 @@ public class HMSHandler extends FacebookBase implements
IHMSHandler {
fireReadTablePreEvent(catName, dbName, tblName);
List<Partition> ret = null;
Exception ex = null;
+ RawStore rs = getMS();
try {
- checkLimitNumberOfPartitionsByFilter(catName, dbName,
- tblName, args.getFilter(), args.getMax());
-
authorizeTableForPartitionMetadata(catName, dbName, tblName);
+ if (needCheckPartitionLimit(args.getMax())) {
+ // Since partition limit is configured, we need fetch at most (limit +
1) partition names
+ int requestMax = args.getMax();
+ int max = MetastoreConf.getIntVar(conf,
ConfVars.LIMIT_PARTITION_REQUEST) + 1;
+ args = new
GetPartitionsArgs.GetPartitionsArgsBuilder(args).max(max).build();
+ List<String> partNames = rs.listPartitionNamesByFilter(catName,
dbName, tblName, args);
+ checkLimitNumberOfPartitions(tblName, partNames.size(), requestMax);
+ ret = rs.getPartitionsByNames(catName, dbName, tblName,
+ new
GetPartitionsArgs.GetPartitionsArgsBuilder(args).partNames(partNames).build());
+ } else {
+ ret = rs.getPartitionsByFilter(catName, dbName, tblName, args);
+ }
- ret = getMS().getPartitionsByFilter(catName, dbName, tblName, args);
ret = FilterUtils.filterPartitionsIfEnabled(isServerFilterEnabled,
filterHook, ret);
} catch (Exception e) {
ex = e;
@@ -7296,9 +7304,7 @@ public class HMSHandler extends FacebookBase implements
IHMSHandler {
PartitionsSpecByExprResult ret = null;
Exception ex = null;
try {
- checkLimitNumberOfPartitionsByExpr(catName, dbName, tblName,
req.getExpr(), UNLIMITED_MAX_PARTITIONS);
- List<Partition> partitions = new LinkedList<>();
- boolean hasUnknownPartitions = getMS().getPartitionsByExpr(catName,
dbName, tblName, partitions,
+ PartitionsByExprResult result = get_partitions_by_expr_internal(catName,
dbName, tblName,
new GetPartitionsArgs.GetPartitionsArgsBuilder()
.expr(req.getExpr()).max(req.getMaxParts()).defaultPartName(req.getDefaultPartitionName())
.skipColumnSchemaForPartition(req.isSkipColumnSchemaForPartition())
@@ -7307,8 +7313,8 @@ public class HMSHandler extends FacebookBase implements
IHMSHandler {
.build());
Table table = get_table_core(catName, dbName, tblName);
List<PartitionSpec> partitionSpecs =
-
MetaStoreServerUtils.getPartitionspecsGroupedByStorageDescriptor(table,
partitions);
- ret = new PartitionsSpecByExprResult(partitionSpecs,
hasUnknownPartitions);
+
MetaStoreServerUtils.getPartitionspecsGroupedByStorageDescriptor(table,
result.getPartitions());
+ ret = new PartitionsSpecByExprResult(partitionSpecs,
result.isHasUnknownPartitions());
} catch (Exception e) {
ex = e;
rethrowException(e);
@@ -7331,16 +7337,13 @@ public class HMSHandler extends FacebookBase implements
IHMSHandler {
PartitionsByExprResult ret = null;
Exception ex = null;
try {
- checkLimitNumberOfPartitionsByExpr(catName, dbName, tblName,
req.getExpr(), UNLIMITED_MAX_PARTITIONS);
- List<Partition> partitions = new LinkedList<>();
- boolean hasUnknownPartitions = getMS().getPartitionsByExpr(catName,
dbName, tblName, partitions,
+ ret = get_partitions_by_expr_internal(catName, dbName, tblName,
new GetPartitionsArgs.GetPartitionsArgsBuilder()
.expr(req.getExpr()).defaultPartName(req.getDefaultPartitionName()).max(req.getMaxParts())
.skipColumnSchemaForPartition(req.isSkipColumnSchemaForPartition())
.excludeParamKeyPattern(req.getExcludeParamKeyPattern())
.includeParamKeyPattern(req.getIncludeParamKeyPattern())
.build());
- ret = new PartitionsByExprResult(partitions, hasUnknownPartitions);
} catch (Exception e) {
ex = e;
rethrowException(e);
@@ -7350,6 +7353,24 @@ public class HMSHandler extends FacebookBase implements
IHMSHandler {
return ret;
}
+ private PartitionsByExprResult get_partitions_by_expr_internal(
+ String catName, String dbName, String tblName, GetPartitionsArgs args)
throws TException {
+ List<Partition> partitions = new LinkedList<>();
+ boolean hasUnknownPartitions = false;
+ RawStore rs = getMS();
+ if (needCheckPartitionLimit(args.getMax())) {
+ // Since partition limit is configured, we need fetch at most (limit +
1) partition names
+ int max = MetastoreConf.getIntVar(conf,
ConfVars.LIMIT_PARTITION_REQUEST) + 1;
+ List<String> partNames = rs.listPartitionNames(catName, dbName, tblName,
args.getDefaultPartName(), args.getExpr(), null, max);
+ checkLimitNumberOfPartitions(tblName, partNames.size(), args.getMax());
+ partitions = rs.getPartitionsByNames(catName, dbName, tblName,
+ new
GetPartitionsArgs.GetPartitionsArgsBuilder(args).partNames(partNames).build());
+ } else {
+ hasUnknownPartitions = rs.getPartitionsByExpr(catName, dbName, tblName,
partitions, args);
+ }
+ return new PartitionsByExprResult(partitions, hasUnknownPartitions);
+ }
+
@Override
@Deprecated
public int get_num_partitions_by_filter(final String dbName,
@@ -7377,22 +7398,6 @@ public class HMSHandler extends FacebookBase implements
IHMSHandler {
return ret;
}
- private int get_num_partitions_by_expr(final String catName, final String
dbName,
- final String tblName, final byte[]
expr)
- throws TException {
- int ret = -1;
- Exception ex = null;
- try {
- ret = getMS().getNumPartitionsByExpr(catName, dbName, tblName, expr);
- } catch (Exception e) {
- ex = e;
- rethrowException(e);
- } finally {
- endFunction("get_num_partitions_by_expr", ret != -1, ex, tblName);
- }
- return ret;
- }
-
private int getNumPartitionsByPs(final String catName, final String dbName,
final String tblName, List<String> partVals)
throws TException {
diff --git
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
index ce04b9b8054..272b388a40d 100644
---
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
+++
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
@@ -758,15 +758,8 @@ class MetaStoreDirectSql {
List<Long> partitionIds = getPartitionIdsViaSqlFilter(catName,
dbName, tableName, filter.filter, filter.params,
filter.joins, args.getMax());
- if (partitionIds.isEmpty()) {
- return Collections.emptyList(); // no partitions, bail early.
- }
- return Batchable.runBatched(batchSize, partitionIds, new Batchable<Long,
Partition>() {
- @Override
- public List<Partition> run(List<Long> input) throws MetaException {
- return getPartitionsByPartitionIds(catName, dbName, tableName, input,
isAcidTable, args);
- }
- });
+
+ return getPartitionsByPartitionIdsInBatch(catName, dbName, tableName,
partitionIds, isAcidTable, args);
}
/**
@@ -910,18 +903,9 @@ class MetaStoreDirectSql {
String dbName, String tblName, GetPartitionsArgs args) throws
MetaException {
List<Long> partitionIds = getPartitionIdsViaSqlFilter(catName, dbName,
tblName, null, Collections.<String>emptyList(),
Collections.<String>emptyList(), args.getMax());
- if (partitionIds.isEmpty()) {
- return Collections.emptyList(); // no partitions, bail early.
- }
// Get full objects. For Oracle/etc. do it in batches.
- List<Partition> result = Batchable.runBatched(batchSize, partitionIds, new
Batchable<Long, Partition>() {
- @Override
- public List<Partition> run(List<Long> input) throws MetaException {
- return getPartitionsByPartitionIds(catName, dbName, tblName, input,
false, args);
- }
- });
- return result;
+ return getPartitionsByPartitionIdsInBatch(catName, dbName, tblName,
partitionIds, false, args);
}
private static Boolean isViewTable(Table t) {
@@ -1047,6 +1031,20 @@ class MetaStoreDirectSql {
return getPartitionsByQuery(catName, dbName, tblName, queryText, params,
isAcidTable, args);
}
+ private List<Partition> getPartitionsByPartitionIdsInBatch(String catName,
String dbName,
+ String tblName, List<Long> partIdList, boolean isAcidTable,
GetPartitionsArgs args)
+ throws MetaException {
+ if (partIdList.isEmpty()) {
+ return Collections.emptyList(); // no partitions, bail early.
+ }
+ return Batchable.runBatched(batchSize, partIdList, new Batchable<Long,
Partition>() {
+ @Override
+ public List<Partition> run(List<Long> input) throws MetaException {
+ return getPartitionsByPartitionIds(catName, dbName, tblName, input,
isAcidTable, args);
+ }
+ });
+ }
+
/** Should be called with the list short enough to not trip up Oracle/etc. */
private List<Partition> getPartitionsByPartitionIds(String catName, String
dbName, String tblName,
List<Long> partIdList, boolean isAcidTable, GetPartitionsArgs args)
throws MetaException {
diff --git
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
index 8398352a0ae..a0d548943d1 100644
---
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
+++
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
@@ -3425,7 +3425,7 @@ public class ObjectStore implements RawStore,
Configurable {
@Override
public List<String> listPartitionNames(final String catName, final String
dbName, final String tblName,
final String defaultPartName, final byte[] exprBytes,
- final String order, final short maxParts) throws MetaException,
NoSuchObjectException {
+ final String order, final int maxParts) throws MetaException,
NoSuchObjectException {
final String defaultPartitionName =
getDefaultPartitionName(defaultPartName);
final boolean isEmptyFilter = exprBytes.length == 1 && exprBytes[0] == -1;
ExpressionTree tmp = null;
@@ -3439,17 +3439,17 @@ public class ObjectStore implements RawStore,
Configurable {
int max = isEmptyFilter ? maxParts : -1;
List<String> result;
if (isJdoQuery) {
- result = getPartitionNamesViaOrm(table, ExpressionTree.EMPTY_TREE,
order, max, true);
+ result = getPartitionNamesViaOrm(catName, dbName, tblName,
ExpressionTree.EMPTY_TREE,
+ order, max, true, table.getPartitionKeys());
} else {
SqlFilterForPushdown filter = new SqlFilterForPushdown(table, false);
result = directSql.getPartitionNamesViaSql(filter,
table.getPartitionKeys(),
defaultPartitionName, order, max);
}
if (!isEmptyFilter) {
- expressionProxy.filterPartitionsByExpr(table.getPartitionKeys(),
exprBytes, defaultPartitionName, result);
- if (maxParts >= 0 && result.size() > maxParts) {
- result = result.subList(0, maxParts);
- }
+ prunePartitionNamesByExpr(catName, dbName, tblName, result,
+ new GetPartitionsArgs.GetPartitionsArgsBuilder()
+
.expr(exprBytes).defaultPartName(defaultPartName).max(maxParts).build());
}
return result;
}
@@ -3476,7 +3476,8 @@ public class ObjectStore implements RawStore,
Configurable {
List<String> result = null;
if (exprTree != null) {
try {
- result = getPartitionNamesViaOrm(ctx.getTable(), exprTree, order,
(int)maxParts, true);
+ result = getPartitionNamesViaOrm(catName, dbName, tblName,
exprTree, order,
+ maxParts, true, ctx.getTable().getPartitionKeys());
} catch (MetaException e) {
result = null;
}
@@ -3489,14 +3490,52 @@ public class ObjectStore implements RawStore,
Configurable {
}.run(true);
}
- private List<String> getPartitionNamesViaOrm(Table table, ExpressionTree
tree, String order,
- Integer maxParts, boolean isValidatedFilter) throws MetaException {
+ @Override
+ public List<String> listPartitionNamesByFilter(String catName, String
dbName, String tblName,
+ GetPartitionsArgs args) throws MetaException, NoSuchObjectException {
+
+ catName = normalizeIdentifier(catName);
+ dbName = normalizeIdentifier(dbName);
+ tblName = normalizeIdentifier(tblName);
+
+ MTable mTable = ensureGetMTable(catName, dbName, tblName);
+ List<FieldSchema> partitionKeys =
convertToFieldSchemas(mTable.getPartitionKeys());
+ String filter = args.getFilter();
+ final ExpressionTree tree = (filter != null && !filter.isEmpty())
+ ? PartFilterExprUtil.parseFilterTree(filter) :
ExpressionTree.EMPTY_TREE;
+ return new GetListHelper<String>(catName, dbName, tblName, true, true) {
+ private final SqlFilterForPushdown filter = new SqlFilterForPushdown();
+
+ @Override
+ protected boolean canUseDirectSql(GetHelper<List<String>> ctx) throws
MetaException {
+ return directSql.generateSqlFilterForPushdown(catName, dbName, tblName,
+ partitionKeys, tree, null, filter);
+ }
+
+ @Override
+ protected List<String> getSqlResult(GetHelper<List<String>> ctx) throws
MetaException {
+ return directSql.getPartitionNamesViaSql(filter, partitionKeys,
+ getDefaultPartitionName(args.getDefaultPartName()), null,
args.getMax());
+ }
+
+ @Override
+ protected List<String> getJdoResult(GetHelper<List<String>> ctx)
+ throws MetaException, NoSuchObjectException, InvalidObjectException {
+ return getPartitionNamesViaOrm(catName, dbName, tblName, tree, null,
+ args.getMax(), true, partitionKeys);
+ }
+ }.run(false);
+ }
+
+ private List<String> getPartitionNamesViaOrm(String catName, String dbName,
String tblName,
+ ExpressionTree tree, String order, Integer maxParts, boolean
isValidatedFilter,
+ List<FieldSchema> partitionKeys) throws MetaException {
Map<String, Object> params = new HashMap<String, Object>();
- String jdoFilter = makeQueryFilterString(table.getCatName(),
table.getDbName(), table, tree,
- params, isValidatedFilter);
+ String jdoFilter = makeQueryFilterString(catName, dbName, tblName, tree,
+ params, isValidatedFilter, partitionKeys);
if (jdoFilter == null) {
assert !isValidatedFilter;
- return null;
+ throw new MetaException("Failed to generate filter.");
}
try (QueryWrapper query = new QueryWrapper(pm.newQuery(
@@ -4095,6 +4134,21 @@ public class ObjectStore implements RawStore,
Configurable {
return getPartitionsByExprInternal(catName, dbName, tblName, result, true,
true, args);
}
+ private boolean prunePartitionNamesByExpr(String catName, String dbName,
String tblName,
+ List<String> result, GetPartitionsArgs args) throws MetaException {
+ MTable mTable = getMTable(catName, dbName, tblName);
+ List<FieldSchema> partitionKeys =
convertToFieldSchemas(mTable.getPartitionKeys());
+ boolean hasUnknownPartitions = expressionProxy.filterPartitionsByExpr(
+ partitionKeys,
+ args.getExpr(),
+ getDefaultPartitionName(args.getDefaultPartName()),
+ result);
+ if (args.getMax() >= 0 && result.size() > args.getMax()) {
+ result = result.subList(0, args.getMax());
+ }
+ return hasUnknownPartitions;
+ }
+
protected boolean getPartitionsByExprInternal(String catName, String dbName,
String tblName,
List<Partition> result, boolean allowSql, boolean allowJdo,
GetPartitionsArgs args) throws TException {
assert result != null;
@@ -4179,16 +4233,10 @@ public class ObjectStore implements RawStore,
Configurable {
*/
private boolean getPartitionNamesPrunedByExprNoTxn(String catName, String
dbName, String tblName, List<FieldSchema> partColumns, byte[] expr,
String defaultPartName,
short maxParts, List<String> result) throws MetaException {
- result.addAll(getPartitionNamesNoTxn(
- catName,
- dbName,
- tblName,
- maxParts));
- return expressionProxy.filterPartitionsByExpr(
- partColumns,
- expr,
- getDefaultPartitionName(defaultPartName),
- result);
+ result.addAll(getPartitionNamesNoTxn(catName, dbName, tblName, (short)
-1));
+ return prunePartitionNamesByExpr(catName, dbName, tblName, result,
+ new GetPartitionsArgs.GetPartitionsArgsBuilder()
+
.expr(expr).defaultPartName(defaultPartName).max(maxParts).build());
}
/**
diff --git
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java
index 5ab2907f5ba..6684d3b09b1 100644
---
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java
+++
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java
@@ -697,7 +697,20 @@ public interface RawStore extends Configurable {
*/
List<String> listPartitionNames(String catName, String dbName, String
tblName,
String defaultPartName, byte[] exprBytes, String order,
- short maxParts) throws MetaException, NoSuchObjectException;
+ int maxParts) throws MetaException, NoSuchObjectException;
+
+ /**
+ * Get partition names with a filter. This is a portion of the SQL where
clause.
+ * @param catName catalog name
+ * @param dbName database name
+ * @param tblName table name
+ * @param args additional arguments for getting partition names
+ * @return list of partition names matching the criteria
+ * @throws MetaException Error accessing the RDBMS or processing the filter.
+ * @throws NoSuchObjectException no such table.
+ */
+ List<String> listPartitionNamesByFilter(String catName, String dbName,
String tblName,
+ GetPartitionsArgs args) throws MetaException, NoSuchObjectException;
/**
* Get a list of partition values as one big struct.
@@ -878,6 +891,7 @@ public interface RawStore extends Configurable {
* @throws MetaException error accessing the RDBMS or working with the
expression.
* @throws NoSuchObjectException no such table.
*/
+ @Deprecated
int getNumPartitionsByExpr(String catName, String dbName, String tblName,
byte[] expr)
throws MetaException, NoSuchObjectException;
diff --git
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java
index f7c630c5c28..8037168fe41 100644
---
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java
+++
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java
@@ -1643,7 +1643,13 @@ public class CachedStore implements RawStore,
Configurable {
@Override
public List<String> listPartitionNames(String catName, String dbName, String
tblName, String defaultPartName,
- byte[] exprBytes, String order, short maxParts) throws MetaException,
NoSuchObjectException {
+ byte[] exprBytes, String order, int maxParts) throws MetaException,
NoSuchObjectException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public List<String> listPartitionNamesByFilter(String catName, String
dbName, String tblName,
+ GetPartitionsArgs args) throws MetaException, NoSuchObjectException {
throw new UnsupportedOperationException();
}
diff --git
a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java
b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java
index 4c574d68c73..179b479a163 100644
---
a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java
+++
b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java
@@ -438,11 +438,17 @@ public class DummyRawStoreControlledCommit implements
RawStore, Configurable {
@Override
public List<String> listPartitionNames(String catName, String dbName, String
tblName, String defaultPartName,
- byte[] exprBytes, String order, short maxParts) throws MetaException,
NoSuchObjectException {
+ byte[] exprBytes, String order, int maxParts) throws MetaException,
NoSuchObjectException {
return objectStore.listPartitionNames(catName, dbName, tblName,
defaultPartName, exprBytes, order, maxParts);
}
+ @Override
+ public List<String> listPartitionNamesByFilter(String catName, String
dbName, String tblName,
+ GetPartitionsArgs args) throws MetaException, NoSuchObjectException {
+ return objectStore.listPartitionNamesByFilter(catName, dbName, tblName,
args);
+ }
+
@Override
public PartitionValuesResponse listPartitionValues(String catName, String
db_name,
String tbl_name, List<FieldSchema> cols, boolean applyDistinct, String
filter,
diff --git
a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java
b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java
index 53889e2b6f0..3eeaabda408 100644
---
a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java
+++
b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java
@@ -453,11 +453,17 @@ public class DummyRawStoreForJdoConnection implements
RawStore {
@Override
public List<String> listPartitionNames(String catName, String dbName, String
tblName, String defaultPartName,
- byte[] exprBytes, String order, short maxParts) throws MetaException,
NoSuchObjectException {
+ byte[] exprBytes, String order, int maxParts) throws MetaException,
NoSuchObjectException {
return Collections.emptyList();
}
+ @Override
+ public List<String> listPartitionNamesByFilter(String catName, String
dbName, String tblName,
+ GetPartitionsArgs args) throws MetaException, NoSuchObjectException {
+ return Collections.emptyList();
+ }
+
@Override
public PartitionValuesResponse listPartitionValues(String catName, String
db_name,
String tbl_name,
List<FieldSchema> cols,
diff --git
a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestObjectStore.java
b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestObjectStore.java
index 40674b9691f..4ee768b163a 100644
---
a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestObjectStore.java
+++
b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestObjectStore.java
@@ -508,6 +508,66 @@ public class TestObjectStore {
}
}
+ @Test
+ public void testListPartitionNamesByFilter() throws Exception {
+ Database db1 = new DatabaseBuilder()
+ .setName(DB1)
+ .setDescription("description")
+ .setLocation("locationurl")
+ .build(conf);
+ try (AutoCloseable c = deadline()) {
+ objectStore.createDatabase(db1);
+ }
+ StorageDescriptor sd = createFakeSd("location");
+ HashMap<String, String> tableParams = new HashMap<>();
+ tableParams.put("EXTERNAL", "false");
+ FieldSchema partitionKey1 = new FieldSchema("Country",
ColumnType.STRING_TYPE_NAME, "");
+ FieldSchema partitionKey2 = new FieldSchema("State",
ColumnType.STRING_TYPE_NAME, "");
+ Table tbl1 =
+ new Table(TABLE1, DB1, "owner", 1, 2, 3, sd,
Arrays.asList(partitionKey1, partitionKey2),
+ tableParams, null, null, "MANAGED_TABLE");
+ try (AutoCloseable c = deadline()) {
+ objectStore.createTable(tbl1);
+ }
+ HashMap<String, String> partitionParams = new HashMap<>();
+ partitionParams.put("PARTITION_LEVEL_PRIVILEGE", "true");
+ List<String> value1 = Arrays.asList("US", "CA");
+ Partition part1 = new Partition(value1, DB1, TABLE1, 111, 111, sd,
partitionParams);
+ part1.setCatName(DEFAULT_CATALOG_NAME);
+ try (AutoCloseable c = deadline()) {
+ objectStore.addPartition(part1);
+ }
+ List<String> value2 = Arrays.asList("US", "MA");
+ Partition part2 = new Partition(value2, DB1, TABLE1, 222, 222, sd,
partitionParams);
+ part2.setCatName(DEFAULT_CATALOG_NAME);
+ try (AutoCloseable c = deadline()) {
+ objectStore.addPartition(part2);
+ }
+
+ List<String> partNames;
+ try (AutoCloseable c = deadline()) {
+ partNames = objectStore.listPartitionNamesByFilter(DEFAULT_CATALOG_NAME,
DB1, TABLE1,
+ new GetPartitionsArgs.GetPartitionsArgsBuilder().filter("Country =
'US'").build());
+ }
+ Assert.assertEquals(2, partNames.size());
+ Assert.assertEquals("country=US/state=CA", partNames.get(0));
+ Assert.assertEquals("country=US/state=MA", partNames.get(1));
+
+ try (AutoCloseable c = deadline()) {
+ partNames = objectStore.listPartitionNamesByFilter(DEFAULT_CATALOG_NAME,
DB1, TABLE1,
+ new GetPartitionsArgs.GetPartitionsArgsBuilder().filter("State =
'MA'").build());
+ }
+ Assert.assertEquals(1, partNames.size());
+ Assert.assertEquals("country=US/state=MA", partNames.get(0));
+
+ try (AutoCloseable c = deadline()) {
+ partNames = objectStore.listPartitionNamesByFilter(DEFAULT_CATALOG_NAME,
DB1, TABLE1,
+ new GetPartitionsArgs.GetPartitionsArgsBuilder().filter("Country =
'US' and State = 'MA'").build());
+ }
+ Assert.assertEquals(1, partNames.size());
+ Assert.assertEquals("country=US/state=MA", partNames.get(0));
+ }
+
@Test
public void testDropPartitionByName() throws Exception {
Database db1 = new DatabaseBuilder()
diff --git
a/standalone-metastore/metastore-tools/metastore-benchmarks/src/main/java/org/apache/hadoop/hive/metastore/tools/BenchmarkUtils.java
b/standalone-metastore/metastore-tools/metastore-benchmarks/src/main/java/org/apache/hadoop/hive/metastore/tools/BenchmarkUtils.java
index 8d071262559..2cb9887a9c3 100644
---
a/standalone-metastore/metastore-tools/metastore-benchmarks/src/main/java/org/apache/hadoop/hive/metastore/tools/BenchmarkUtils.java
+++
b/standalone-metastore/metastore-tools/metastore-benchmarks/src/main/java/org/apache/hadoop/hive/metastore/tools/BenchmarkUtils.java
@@ -58,11 +58,15 @@ public class BenchmarkUtils {
// Create a simple table with a single column and single partition
static void createPartitionedTable(HMSClient client, String dbName, String
tableName) {
+ createPartitionedTable(client, dbName, tableName,
createSchema(Collections.singletonList("date")));
+ }
+
+ static void createPartitionedTable(HMSClient client, String dbName, String
tableName, List<FieldSchema> partitionKeys) {
throwingSupplierWrapper(() -> client.createTable(
new Util.TableBuilder(dbName, tableName)
.withType(TableType.MANAGED_TABLE)
.withColumns(createSchema(Collections.singletonList("name:string")))
- .withPartitionKeys(createSchema(Collections.singletonList("date")))
+ .withPartitionKeys(partitionKeys)
.build()));
}
diff --git
a/standalone-metastore/metastore-tools/metastore-benchmarks/src/main/java/org/apache/hadoop/hive/metastore/tools/HMSBenchmarks.java
b/standalone-metastore/metastore-tools/metastore-benchmarks/src/main/java/org/apache/hadoop/hive/metastore/tools/HMSBenchmarks.java
index f3f4c74662e..1f7a0d4ad35 100644
---
a/standalone-metastore/metastore-tools/metastore-benchmarks/src/main/java/org/apache/hadoop/hive/metastore/tools/HMSBenchmarks.java
+++
b/standalone-metastore/metastore-tools/metastore-benchmarks/src/main/java/org/apache/hadoop/hive/metastore/tools/HMSBenchmarks.java
@@ -419,14 +419,19 @@ final class HMSBenchmarks {
String dbName = data.dbName;
String tableName = data.tableName;
- BenchmarkUtils.createPartitionedTable(client, dbName, tableName);
+ BenchmarkUtils.createPartitionedTable(client, dbName, tableName,
createSchema(Arrays.asList("p_a", "p_b", "p_c")));
try {
addManyPartitionsNoException(client, dbName, tableName, null,
- Collections.singletonList("d"), count);
+ Arrays.asList("a", "b", "c"), count);
return bench.measure(
() ->
- throwingSupplierWrapper(() ->
- client.getPartitionsByFilter(dbName, tableName,
"`date`='d0'"))
+ throwingSupplierWrapper(() -> {
+ // test multiple cases for get_partitions_by_filter
+ client.getPartitionsByFilter(dbName, tableName,
"`p_a`='a0'");
+ client.getPartitionsByFilter(dbName, tableName,
+ "`p_a`='a0' or `p_b`='b0' or `p_c`='c0' or `p_a`='a1' or
`p_b`='b1' or `p_c`='c1'");
+ return null;
+ })
);
} finally {
throwingSupplierWrapper(() -> client.dropTable(dbName, tableName));
