This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 45fc1c71820 [opt](hive) save hive table schema in transaction for 2.1
(#37127)
45fc1c71820 is described below
commit 45fc1c71820841377626a4c37835f25febbbf1de
Author: wuwenchi <[email protected]>
AuthorDate: Wed Jul 3 17:32:58 2024 +0800
[opt](hive) save hive table schema in transaction for 2.1 (#37127)
## Proposed changes
pick #37008
---
.../org/apache/doris/datasource/hive/HMSTransaction.java | 15 +++++++++++----
1 file changed, 11 insertions(+), 4 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSTransaction.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSTransaction.java
index d883b9dc786..bd0d2315c1e 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSTransaction.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSTransaction.java
@@ -48,6 +48,7 @@ import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import io.airlift.concurrent.MoreFutures;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.Partition;
import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
import org.apache.hadoop.hive.metastore.api.Table;
@@ -88,6 +89,7 @@ public class HMSTransaction implements Transaction {
private final Map<DatabaseTableName, Action<TableAndMore>> tableActions =
new HashMap<>();
private final Map<DatabaseTableName, Map<List<String>,
Action<PartitionAndMore>>>
partitionActions = new HashMap<>();
+ private final Map<DatabaseTableName, List<FieldSchema>> tableColumns = new
HashMap<>();
private final Executor fileSystemExecutor;
private HmsCommitter hmsCommitter;
@@ -123,7 +125,7 @@ public class HMSTransaction implements Transaction {
}
}
- private Set<UncompletedMpuPendingUpload> uncompletedMpuPendingUploads =
new HashSet<>();
+ private final Set<UncompletedMpuPendingUpload>
uncompletedMpuPendingUploads = new HashSet<>();
public HMSTransaction(HiveMetadataOps hiveOps, FileSystemProvider
fileSystemProvider, Executor fileSystemExecutor) {
this.hiveOps = hiveOps;
@@ -241,7 +243,7 @@ public class HMSTransaction implements Transaction {
Maps.newHashMap(),
sd.getOutputFormat(),
sd.getSerdeInfo().getSerializationLib(),
- hiveOps.getClient().getSchema(dbName, tbName)
+ getTableColumns(dbName, tbName)
);
if (updateMode == TUpdateMode.OVERWRITE) {
dropPartition(dbName, tbName,
hivePartition.getPartitionValues(), true);
@@ -396,7 +398,7 @@ public class HMSTransaction implements Transaction {
partition.getParameters(),
sd.getOutputFormat(),
sd.getSerdeInfo().getSerializationLib(),
- hiveOps.getClient().getSchema(dbName, tbName)
+ getTableColumns(dbName, tbName)
);
partitionActionsForTable.put(
@@ -913,6 +915,11 @@ public class HMSTransaction implements Transaction {
throw new RuntimeException("Not Found table: " + databaseName + "." +
tableName);
}
+ public synchronized List<FieldSchema> getTableColumns(String databaseName,
String tableName) {
+ return tableColumns.computeIfAbsent(new
DatabaseTableName(databaseName, tableName),
+ key -> hiveOps.getClient().getSchema(dbName, tbName));
+ }
+
public synchronized void finishChangingExistingTable(
ActionType actionType,
String databaseName,
@@ -1276,7 +1283,7 @@ public class HMSTransaction implements Transaction {
Maps.newHashMap(),
sd.getOutputFormat(),
sd.getSerdeInfo().getSerializationLib(),
- hiveOps.getClient().getSchema(dbName, tbName)
+ getTableColumns(dbName, tbName)
);
HivePartitionWithStatistics partitionWithStats =
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
