This is an automated email from the ASF dual-hosted git repository.
dkuzmenko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 6dc569da2eb HIVE-28239: Fix bug on
HMSHandler#checkLimitNumberOfPartitions (Wechar Yu, reviewed by Denys Kuzmenko)
6dc569da2eb is described below
commit 6dc569da2eb570fda76d2edb95704c6ba15a7924
Author: Wechar Yu <[email protected]>
AuthorDate: Tue May 21 21:54:03 2024 +0800
HIVE-28239: Fix bug on HMSHandler#checkLimitNumberOfPartitions (Wechar Yu,
reviewed by Denys Kuzmenko)
Closes #5246
---
.../apache/hadoop/hive/metastore/HMSHandler.java | 49 +++++++++-------------
.../hive/metastore/client/TestListPartitions.java | 28 ++++++++++---
2 files changed, 42 insertions(+), 35 deletions(-)
diff --git
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java
index c77daf96fac..486830fd0bb 100644
---
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java
+++
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java
@@ -5562,44 +5562,36 @@ public class HMSHandler extends FacebookBase implements
IHMSHandler {
private void checkLimitNumberOfPartitionsByFilter(String catName, String
dbName,
String tblName, String
filterString,
- int maxParts) throws
TException {
- if (isPartitionLimitEnabled()) {
+ int requestMax) throws
TException {
+ if (exceedsPartitionFetchLimit(requestMax)) {
checkLimitNumberOfPartitions(tblName,
get_num_partitions_by_filter(prependCatalogToDbName(
- catName, dbName, conf), tblName, filterString), maxParts);
+ catName, dbName, conf), tblName, filterString));
}
}
private void checkLimitNumberOfPartitionsByPs(String catName, String dbName,
String tblName,
- List<String> partVals, int
maxParts)
+ List<String> partVals, int
requestMax)
throws TException {
- if (isPartitionLimitEnabled()) {
+ if (exceedsPartitionFetchLimit(requestMax)) {
checkLimitNumberOfPartitions(tblName, getNumPartitionsByPs(catName,
dbName, tblName,
- partVals), maxParts);
+ partVals));
}
}
- private boolean isPartitionLimitEnabled() {
- int partitionLimit = MetastoreConf.getIntVar(conf,
ConfVars.LIMIT_PARTITION_REQUEST);
- return partitionLimit > -1;
- }
-
- // Check request partition limit iff:
+ // Check input count exceeding partition limit iff:
// 1. partition limit is enabled.
- // 2. request size is greater than the limit.
- private boolean needCheckPartitionLimit(int requestSize) {
+ // 2. input count is greater than the limit.
+ private boolean exceedsPartitionFetchLimit(int count) {
int partitionLimit = MetastoreConf.getIntVar(conf,
ConfVars.LIMIT_PARTITION_REQUEST);
- return partitionLimit > -1 && (requestSize < 0 || requestSize >
partitionLimit);
+ return partitionLimit > -1 && (count < 0 || count > partitionLimit);
}
- private void checkLimitNumberOfPartitions(String tblName, int numPartitions,
int maxToFetch) throws MetaException {
- if (isPartitionLimitEnabled()) {
+ private void checkLimitNumberOfPartitions(String tblName, int numPartitions)
throws MetaException {
+ if (exceedsPartitionFetchLimit(numPartitions)) {
int partitionLimit = MetastoreConf.getIntVar(conf,
ConfVars.LIMIT_PARTITION_REQUEST);
- int partitionRequest = (maxToFetch < 0) ? numPartitions : maxToFetch;
- if (partitionRequest > partitionLimit) {
- String configName = ConfVars.LIMIT_PARTITION_REQUEST.toString();
- throw new
MetaException(String.format(PARTITION_NUMBER_EXCEED_LIMIT_MSG, partitionRequest,
- tblName, partitionLimit, configName));
- }
+ String configName = ConfVars.LIMIT_PARTITION_REQUEST.toString();
+ throw new MetaException(String.format(PARTITION_NUMBER_EXCEED_LIMIT_MSG,
numPartitions,
+ tblName, partitionLimit, configName));
}
}
@@ -7225,13 +7217,12 @@ public class HMSHandler extends FacebookBase implements
IHMSHandler {
RawStore rs = getMS();
try {
authorizeTableForPartitionMetadata(catName, dbName, tblName);
- if (needCheckPartitionLimit(args.getMax())) {
+ if (exceedsPartitionFetchLimit(args.getMax())) {
// Since partition limit is configured, we need fetch at most (limit +
1) partition names
- int requestMax = args.getMax();
int max = MetastoreConf.getIntVar(conf,
ConfVars.LIMIT_PARTITION_REQUEST) + 1;
args = new
GetPartitionsArgs.GetPartitionsArgsBuilder(args).max(max).build();
List<String> partNames = rs.listPartitionNamesByFilter(catName,
dbName, tblName, args);
- checkLimitNumberOfPartitions(tblName, partNames.size(), requestMax);
+ checkLimitNumberOfPartitions(tblName, partNames.size());
ret = rs.getPartitionsByNames(catName, dbName, tblName,
new
GetPartitionsArgs.GetPartitionsArgsBuilder(args).partNames(partNames).build());
} else {
@@ -7358,11 +7349,11 @@ public class HMSHandler extends FacebookBase implements
IHMSHandler {
List<Partition> partitions = new LinkedList<>();
boolean hasUnknownPartitions = false;
RawStore rs = getMS();
- if (needCheckPartitionLimit(args.getMax())) {
+ if (exceedsPartitionFetchLimit(args.getMax())) {
// Since partition limit is configured, we need fetch at most (limit +
1) partition names
int max = MetastoreConf.getIntVar(conf,
ConfVars.LIMIT_PARTITION_REQUEST) + 1;
List<String> partNames = rs.listPartitionNames(catName, dbName, tblName,
args.getDefaultPartName(), args.getExpr(), null, max);
- checkLimitNumberOfPartitions(tblName, partNames.size(), args.getMax());
+ checkLimitNumberOfPartitions(tblName, partNames.size());
partitions = rs.getPartitionsByNames(catName, dbName, tblName,
new
GetPartitionsArgs.GetPartitionsArgsBuilder(args).partNames(partNames).build());
} else {
@@ -7462,7 +7453,7 @@ public class HMSHandler extends FacebookBase implements
IHMSHandler {
fireReadTablePreEvent(parsedCatName, parsedDbName, tblName);
- checkLimitNumberOfPartitions(tblName, args.getPartNames().size(), -1);
+ checkLimitNumberOfPartitions(tblName, args.getPartNames().size());
ret = getMS().getPartitionsByNames(parsedCatName, parsedDbName, tblName,
args);
ret = FilterUtils.filterPartitionsIfEnabled(isServerFilterEnabled,
filterHook, ret);
table = getTable(parsedCatName, parsedDbName, tblName);
diff --git
a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/TestListPartitions.java
b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/TestListPartitions.java
index 91ba1c311f3..abaf28217ca 100644
---
a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/TestListPartitions.java
+++
b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/TestListPartitions.java
@@ -203,6 +203,22 @@ public class TestListPartitions extends
MetaStoreClientTest {
return new ReturnTable(t, testValues);
}
+ private ReturnTable createTableNParts(IMetaStoreClient client, boolean
authOn, int nParts)
+ throws Exception {
+ Table t = createTestTable(client, DB_NAME, TABLE_NAME,
Lists.newArrayList("yyyy", "mm", "dd"),
+ authOn);
+ List<List<String>> testValues = new ArrayList<>(nParts);
+ for (int i = 0; i < nParts; i++) {
+ testValues.add(Lists.newArrayList("2017", "01", String.valueOf(i)));
+ }
+
+ for(List<String> vals : testValues) {
+ addPartition(client, t, vals);
+ }
+
+ return new ReturnTable(t, testValues);
+ }
+
protected ReturnTable createTable4PartColsParts(IMetaStoreClient client)
throws
Exception {
return createTable4PartColsPartsGeneric(client, false);
@@ -324,7 +340,7 @@ public class TestListPartitions extends MetaStoreClientTest
{
@Test(expected = MetaException.class)
@ConditionalIgnoreOnSessionHiveMetastoreClient
public void testListPartitionsAllHighMaxParts() throws Exception {
- createTable3PartCols1Part(client);
+ createTableNParts(client, false, 101);
List<Partition> partitions = client.listPartitions(DB_NAME, TABLE_NAME,
(short)101);
assertTrue(partitions.isEmpty());
}
@@ -496,7 +512,7 @@ public class TestListPartitions extends MetaStoreClientTest
{
@Test(expected = MetaException.class)
@ConditionalIgnoreOnSessionHiveMetastoreClient
public void testListPartitionSpecsHighMaxParts() throws Exception {
- createTable4PartColsParts(client);
+ createTableNParts(client, false, 101);
client.listPartitionSpecs(DB_NAME, TABLE_NAME, 101);
}
@@ -562,7 +578,7 @@ public class TestListPartitions extends MetaStoreClientTest
{
@Test(expected = MetaException.class)
@ConditionalIgnoreOnSessionHiveMetastoreClient
public void testListPartitionsWithAuthHighMaxParts() throws Exception {
- createTable4PartColsParts(client);
+ createTableNParts(client, false, 101);
client.listPartitionsWithAuthInfo(DB_NAME, TABLE_NAME, (short)101, "",
Lists.newArrayList());
}
@@ -754,7 +770,7 @@ public class TestListPartitions extends MetaStoreClientTest
{
@Test(expected = MetaException.class)
@ConditionalIgnoreOnSessionHiveMetastoreClient
public void testListPartitionsWithAuthByValuesHighMaxParts() throws
Exception {
- createTable4PartColsParts(client);
+ createTableNParts(client, false, 101);
client.listPartitionsWithAuthInfo(DB_NAME, TABLE_NAME, Lists
.newArrayList("2017"), (short) 101, "", Lists.newArrayList());
}
@@ -982,7 +998,7 @@ public class TestListPartitions extends MetaStoreClientTest
{
@Test(expected = MetaException.class)
@ConditionalIgnoreOnSessionHiveMetastoreClient
public void testListPartitionsByFilterHighMaxParts() throws Exception {
- createTable4PartColsParts(client);
+ createTableNParts(client, false, 101);
client.listPartitionsByFilter(DB_NAME, TABLE_NAME, "yyyy=\"2017\"",
(short)101);
}
@@ -1095,7 +1111,7 @@ public class TestListPartitions extends
MetaStoreClientTest {
@Test(expected = MetaException.class)
@ConditionalIgnoreOnSessionHiveMetastoreClient
public void testListPartitionSpecsByFilterHighMaxParts() throws Exception {
- createTable4PartColsParts(client);
+ createTableNParts(client, false, 101);
client.listPartitionSpecsByFilter(DB_NAME, TABLE_NAME, "yyyy=\"2017\"",
101);
}
