This is an automated email from the ASF dual-hosted git repository.
ayushsaxena pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new a65c53322dc HIVE-29451: Optimize MapWork to configure JobConf once per
table (#6317)
a65c53322dc is described below
commit a65c53322dcca482539b4b2e1754eadd9ca08e5d
Author: Hemanth Umashankar
<[email protected]>
AuthorDate: Mon Feb 23 21:31:03 2026 +0530
HIVE-29451: Optimize MapWork to configure JobConf once per table (#6317)
---
ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java | 13 ++++++++++++-
1 file changed, 12 insertions(+), 1 deletion(-)
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java
b/ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java
index e705af31dea..fa0f1e7b7f5 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java
@@ -658,8 +658,19 @@ public String getSamplingTypeString() {
@Override
public void configureJobConf(JobConf job) {
super.configureJobConf(job);
+ // Configure each table only once, even if we read thousands of its
partitions.
+ // This avoids repeating expensive work (like loading storage drivers) for
every single partition.
+ Set<String> processedTables = new HashSet<>();
+
for (PartitionDesc partition : aliasToPartnInfo.values()) {
- PlanUtils.configureJobConf(partition.getTableDesc(), job);
+ TableDesc tableDesc = partition.getTableDesc();
+
+ // If we haven't seen this table before, configure it and remember it.
+ // If we have seen it, skip it.
+ if (tableDesc != null &&
!processedTables.contains(tableDesc.getTableName())) {
+ processedTables.add(tableDesc.getTableName());
+ PlanUtils.configureJobConf(tableDesc, job);
+ }
}
Collection<Operator<?>> mappers = aliasToWork.values();
for (IConfigureJobConf icjc : OperatorUtils.findOperators(mappers,
IConfigureJobConf.class)) {
