This is an automated email from the ASF dual-hosted git repository.
gsaihemanth pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new f88d77ba0a0 HIVE-27676: Reuse the add_partitions logic for
add_partition in ObjetStore (#4678) (Wechar Yu, Reviewed by Sai Hemanth
Gantasala)
f88d77ba0a0 is described below
commit f88d77ba0a050c25303c0c7100758b069c07a783
Author: Wechar Yu <[email protected]>
AuthorDate: Wed Oct 18 01:59:10 2023 +0800
HIVE-27676: Reuse the add_partitions logic for add_partition in ObjetStore
(#4678) (Wechar Yu, Reviewed by Sai Hemanth Gantasala)
---
.../apache/hadoop/hive/metastore/ObjectStore.java | 202 +++++++++------------
.../hadoop/hive/metastore/tools/BenchmarkTool.java | 4 +-
.../hadoop/hive/metastore/tools/HMSBenchmarks.java | 16 +-
3 files changed, 94 insertions(+), 128 deletions(-)
diff --git
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
index ddacabe0fa5..e7df10673b5 100644
---
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
+++
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.hive.metastore;
import static org.apache.commons.lang3.StringUtils.join;
import static
org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars.COMPACTOR_USE_CUSTOM_POOL;
import static
org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.getDefaultCatalog;
+import static
org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.newMetaException;
import static
org.apache.hadoop.hive.metastore.utils.StringUtils.normalizeIdentifier;
import java.io.IOException;
@@ -2660,88 +2661,93 @@ public class ObjectStore implements RawStore,
Configurable {
boolean success = false;
openTransaction();
try {
- List<MTablePrivilege> tabGrants = null;
- List<MTableColumnPrivilege> tabColumnGrants = null;
- MTable table = this.getMTable(catName, dbName, tblName);
- if (table == null) {
- throw new InvalidObjectException("Unable to add partitions because "
- + TableName.getQualified(catName, dbName, tblName) +
- " does not exist");
+ addPartitionsInternal(catName, dbName, tblName, parts);
+ success = commitTransaction();
+ } finally {
+ if (!success) {
+ rollbackTransaction();
}
- if
("TRUE".equalsIgnoreCase(table.getParameters().get("PARTITION_LEVEL_PRIVILEGE")))
{
- tabGrants = this.listAllTableGrants(catName, dbName, tblName);
- tabColumnGrants = this.listTableAllColumnGrants(catName, dbName,
tblName);
+ }
+ return success;
+ }
+
+ private void addPartitionsInternal(String catName, String dbName,
+ String tblName, List<Partition> parts)
+ throws MetaException, InvalidObjectException {
+ List<MTablePrivilege> tabGrants = null;
+ List<MTableColumnPrivilege> tabColumnGrants = null;
+ MTable table = this.getMTable(catName, dbName, tblName);
+ if (table == null) {
+ throw new InvalidObjectException("Unable to add partitions because "
+ + TableName.getQualified(catName, dbName, tblName) +
+ " does not exist");
+ }
+ if
("TRUE".equalsIgnoreCase(table.getParameters().get("PARTITION_LEVEL_PRIVILEGE")))
{
+ tabGrants = this.listAllTableGrants(catName, dbName, tblName);
+ tabColumnGrants = this.listTableAllColumnGrants(catName, dbName,
tblName);
+ }
+ List<MPartition> mParts = new ArrayList<>();
+ List<List<MPartitionPrivilege>> mPartPrivilegesList = new ArrayList<>();
+ List<List<MPartitionColumnPrivilege>> mPartColPrivilegesList = new
ArrayList<>();
+ for (Partition part : parts) {
+ if (!part.getTableName().equals(tblName) ||
!part.getDbName().equals(dbName)) {
+ throw new MetaException("Partition does not belong to target table "
+ + dbName + "." + tblName + ": " + part);
}
- List<MPartition> mParts = new ArrayList<>();
- List<List<MPartitionPrivilege>> mPartPrivilegesList = new ArrayList<>();
- List<List<MPartitionColumnPrivilege>> mPartColPrivilegesList = new
ArrayList<>();
- for (Partition part : parts) {
- if (!part.getTableName().equals(tblName) ||
!part.getDbName().equals(dbName)) {
- throw new MetaException("Partition does not belong to target table "
- + dbName + "." + tblName + ": " + part);
- }
- MPartition mpart = convertToMPart(part, table, true);
- mParts.add(mpart);
- int now = (int) (System.currentTimeMillis() / 1000);
- List<MPartitionPrivilege> mPartPrivileges = new ArrayList<>();
- if (tabGrants != null) {
- for (MTablePrivilege tab: tabGrants) {
- MPartitionPrivilege mPartPrivilege = new
MPartitionPrivilege(tab.getPrincipalName(), tab.getPrincipalType(),
- mpart, tab.getPrivilege(), now, tab.getGrantor(),
tab.getGrantorType(), tab.getGrantOption(),
- tab.getAuthorizer());
- mPartPrivileges.add(mPartPrivilege);
- }
+ MPartition mpart = convertToMPart(part, table, true);
+ mParts.add(mpart);
+ int now = (int) (System.currentTimeMillis() / 1000);
+ List<MPartitionPrivilege> mPartPrivileges = new ArrayList<>();
+ if (tabGrants != null) {
+ for (MTablePrivilege tab: tabGrants) {
+ MPartitionPrivilege mPartPrivilege = new
MPartitionPrivilege(tab.getPrincipalName(), tab.getPrincipalType(),
+ mpart, tab.getPrivilege(), now, tab.getGrantor(),
tab.getGrantorType(), tab.getGrantOption(),
+ tab.getAuthorizer());
+ mPartPrivileges.add(mPartPrivilege);
}
+ }
- List<MPartitionColumnPrivilege> mPartColumnPrivileges = new
ArrayList<>();
- if (tabColumnGrants != null) {
- for (MTableColumnPrivilege col : tabColumnGrants) {
- MPartitionColumnPrivilege mPartColumnPrivilege = new
MPartitionColumnPrivilege(col.getPrincipalName(),
- col.getPrincipalType(), mpart, col.getColumnName(),
col.getPrivilege(), now, col.getGrantor(),
- col.getGrantorType(), col.getGrantOption(),
col.getAuthorizer());
- mPartColumnPrivileges.add(mPartColumnPrivilege);
- }
+ List<MPartitionColumnPrivilege> mPartColumnPrivileges = new
ArrayList<>();
+ if (tabColumnGrants != null) {
+ for (MTableColumnPrivilege col : tabColumnGrants) {
+ MPartitionColumnPrivilege mPartColumnPrivilege = new
MPartitionColumnPrivilege(col.getPrincipalName(),
+ col.getPrincipalType(), mpart, col.getColumnName(),
col.getPrivilege(), now, col.getGrantor(),
+ col.getGrantorType(), col.getGrantOption(), col.getAuthorizer());
+ mPartColumnPrivileges.add(mPartColumnPrivilege);
}
- mPartPrivilegesList.add(mPartPrivileges);
- mPartColPrivilegesList.add(mPartColumnPrivileges);
}
- if (CollectionUtils.isNotEmpty(mParts)) {
- GetHelper<Void> helper = new GetHelper<Void>(null, null, null, true,
- true) {
- @Override
- protected Void getSqlResult(GetHelper<Void> ctx) throws
MetaException {
- directSql.addPartitions(mParts, mPartPrivilegesList,
mPartColPrivilegesList);
- return null;
- }
+ mPartPrivilegesList.add(mPartPrivileges);
+ mPartColPrivilegesList.add(mPartColumnPrivileges);
+ }
+ if (CollectionUtils.isNotEmpty(mParts)) {
+ GetHelper<Void> helper = new GetHelper<Void>(null, null, null, true,
true) {
+ @Override
+ protected Void getSqlResult(GetHelper<Void> ctx) throws MetaException {
+ directSql.addPartitions(mParts, mPartPrivilegesList,
mPartColPrivilegesList);
+ return null;
+ }
- @Override
- protected Void getJdoResult(GetHelper<Void> ctx) {
- List<Object> toPersist = new ArrayList<>(mParts);
- mPartPrivilegesList.forEach(toPersist::addAll);
- mPartColPrivilegesList.forEach(toPersist::addAll);
- pm.makePersistentAll(toPersist);
- pm.flush();
- return null;
- }
+ @Override
+ protected Void getJdoResult(GetHelper<Void> ctx) {
+ List<Object> toPersist = new ArrayList<>(mParts);
+ mPartPrivilegesList.forEach(toPersist::addAll);
+ mPartColPrivilegesList.forEach(toPersist::addAll);
+ pm.makePersistentAll(toPersist);
+ pm.flush();
+ return null;
+ }
- @Override
- protected String describeResult() {
- return "add partitions";
- }
- };
- try {
- helper.run(false);
- } catch (NoSuchObjectException e) {
- throw new MetaException(e.getMessage());
+ @Override
+ protected String describeResult() {
+ return "add partitions";
}
- }
- success = commitTransaction();
- } finally {
- if (!success) {
- rollbackTransaction();
+ };
+ try {
+ helper.run(false);
+ } catch (NoSuchObjectException e) {
+ throw newMetaException(e);
}
}
- return success;
}
private boolean isValidPartition(
@@ -2824,62 +2830,18 @@ public class ObjectStore implements RawStore,
Configurable {
@Override
public boolean addPartition(Partition part) throws InvalidObjectException,
MetaException {
- boolean success = false;
- boolean commited = false;
-
+ boolean committed = false;
try {
openTransaction();
String catName = part.isSetCatName() ? part.getCatName() :
getDefaultCatalog(conf);
- MTable table = this.getMTable(catName, part.getDbName(),
part.getTableName());
- if (table == null) {
- throw new InvalidObjectException("Unable to add partition because "
- + TableName.getQualified(catName, part.getDbName(),
part.getTableName()) +
- " does not exist");
- }
- List<MTablePrivilege> tabGrants = null;
- List<MTableColumnPrivilege> tabColumnGrants = null;
- if
("TRUE".equalsIgnoreCase(table.getParameters().get("PARTITION_LEVEL_PRIVILEGE")))
{
- tabGrants = this.listAllTableGrants(catName, part.getDbName(),
part.getTableName());
- tabColumnGrants = this.listTableAllColumnGrants(
- catName, part.getDbName(), part.getTableName());
- }
- MPartition mpart = convertToMPart(part, table, true);
- pm.makePersistent(mpart);
-
- int now = (int) (System.currentTimeMillis() / 1000);
- List<Object> toPersist = new ArrayList<>();
- if (tabGrants != null) {
- for (MTablePrivilege tab: tabGrants) {
- MPartitionPrivilege partGrant = new MPartitionPrivilege(tab
- .getPrincipalName(), tab.getPrincipalType(),
- mpart, tab.getPrivilege(), now, tab.getGrantor(), tab
- .getGrantorType(), tab.getGrantOption(),
tab.getAuthorizer());
- toPersist.add(partGrant);
- }
- }
-
- if (tabColumnGrants != null) {
- for (MTableColumnPrivilege col : tabColumnGrants) {
- MPartitionColumnPrivilege partColumn = new
MPartitionColumnPrivilege(col
- .getPrincipalName(), col.getPrincipalType(), mpart, col
- .getColumnName(), col.getPrivilege(), now, col.getGrantor(), col
- .getGrantorType(), col.getGrantOption(), col.getAuthorizer());
- toPersist.add(partColumn);
- }
-
- if (CollectionUtils.isNotEmpty(toPersist)) {
- pm.makePersistentAll(toPersist);
- }
- }
-
- commited = commitTransaction();
- success = true;
+ addPartitionsInternal(catName, part.getDbName(), part.getTableName(),
Arrays.asList(part));
+ committed = commitTransaction();
} finally {
- if (!commited) {
+ if (!committed) {
rollbackTransaction();
}
}
- return success;
+ return committed;
}
@Override
diff --git
a/standalone-metastore/metastore-tools/metastore-benchmarks/src/main/java/org/apache/hadoop/hive/metastore/tools/BenchmarkTool.java
b/standalone-metastore/metastore-tools/metastore-benchmarks/src/main/java/org/apache/hadoop/hive/metastore/tools/BenchmarkTool.java
index 025b39339bc..90672bf483d 100644
---
a/standalone-metastore/metastore-tools/metastore-benchmarks/src/main/java/org/apache/hadoop/hive/metastore/tools/BenchmarkTool.java
+++
b/standalone-metastore/metastore-tools/metastore-benchmarks/src/main/java/org/apache/hadoop/hive/metastore/tools/BenchmarkTool.java
@@ -274,7 +274,7 @@ public class BenchmarkTool implements Runnable {
() -> benchmarkDeleteWithPartitions(bench, bData, 1,
nParameters[0]))
.add("dropTableMetadataWithPartitions",
() -> benchmarkDeleteMetaOnlyWithPartitions(bench, bData, 1,
nParameters[0]))
- .add("addPartition", () -> benchmarkCreatePartition(bench, bData))
+ .add("addPartition", () -> benchmarkCreatePartition(bench, bData, 1))
.add("dropPartition", () -> benchmarkDropPartition(bench, bData, 1))
.add("listPartition", () -> benchmarkListPartition(bench, bData))
.add("getPartition",
@@ -315,6 +315,8 @@ public class BenchmarkTool implements Runnable {
() -> benchmarkRenameTable(bench, bData, howMany))
.add("dropDatabase" + '.' + howMany,
() -> benchmarkDropDatabase(bench, bData, howMany))
+ .add("addPartition" + '.' + howMany,
+ () -> benchmarkCreatePartition(bench, bData, howMany))
.add("dropPartition" + '.' + howMany,
() -> benchmarkDropPartition(bench, bData, howMany))
.add("openTxns" + '.' + howMany,
diff --git
a/standalone-metastore/metastore-tools/metastore-benchmarks/src/main/java/org/apache/hadoop/hive/metastore/tools/HMSBenchmarks.java
b/standalone-metastore/metastore-tools/metastore-benchmarks/src/main/java/org/apache/hadoop/hive/metastore/tools/HMSBenchmarks.java
index c48ab7d2b17..214e9e1cd6b 100644
---
a/standalone-metastore/metastore-tools/metastore-benchmarks/src/main/java/org/apache/hadoop/hive/metastore/tools/HMSBenchmarks.java
+++
b/standalone-metastore/metastore-tools/metastore-benchmarks/src/main/java/org/apache/hadoop/hive/metastore/tools/HMSBenchmarks.java
@@ -44,6 +44,7 @@ import java.util.concurrent.atomic.AtomicLong;
import static org.apache.hadoop.hive.metastore.tools.Util.addManyPartitions;
import static
org.apache.hadoop.hive.metastore.tools.Util.addManyPartitionsNoException;
+import static org.apache.hadoop.hive.metastore.tools.Util.createManyPartitions;
import static org.apache.hadoop.hive.metastore.tools.Util.createSchema;
import static
org.apache.hadoop.hive.metastore.tools.Util.throwingSupplierWrapper;
@@ -181,22 +182,23 @@ final class HMSBenchmarks {
}
static DescriptiveStatistics benchmarkCreatePartition(@NotNull
MicroBenchmark bench,
- @NotNull BenchData
data) {
+ @NotNull BenchData
data,
+ int howMany) {
final HMSClient client = data.getClient();
String dbName = data.dbName;
String tableName = data.tableName;
BenchmarkUtils.createPartitionedTable(client, dbName, tableName);
- final List<String> values = Collections.singletonList("d1");
try {
Table t = client.getTable(dbName, tableName);
- Partition partition = new Util.PartitionBuilder(t)
- .withValues(values)
- .build();
+ List<Partition> parts = createManyPartitions(t, null,
Collections.singletonList("d"), howMany);
return bench.measure(null,
- () -> throwingSupplierWrapper(() -> client.addPartition(partition)),
- () -> throwingSupplierWrapper(() -> client.dropPartition(dbName,
tableName, values)));
+ () -> throwingSupplierWrapper(() -> {
+ parts.forEach(part -> throwingSupplierWrapper(() ->
client.addPartition(part)));
+ return null;
+ }),
+ () -> throwingSupplierWrapper(() -> client.dropPartitions(dbName,
tableName, null)));
} catch (TException e) {
e.printStackTrace();
return new DescriptiveStatistics();
