[hive] branch master updated: HIVE-25581: Iceberg storage handler should set common projection pruning config (Marton Bod, reviewed by Peter Vary and Adam Szita)

[mbod]

This is an automated email from the ASF dual-hosted git repository.

mbod pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git


The following commit(s) were added to refs/heads/master by this push:
     new 3f2067f  HIVE-25581: Iceberg storage handler should set common 
projection pruning config (Marton Bod, reviewed by Peter Vary and Adam Szita)
3f2067f is described below

commit 3f2067f02c86460030948d930379c168c40eca87
Author: Marton Bod <marton....@gmail.com>
AuthorDate: Sat Oct 2 19:29:44 2021 +0200

    HIVE-25581: Iceberg storage handler should set common projection pruning 
config (Marton Bod, reviewed by Peter Vary and Adam Szita)
    
    Iceberg queries always need "tez.mrreader.config.update.properties" to be 
set in order for projection pruning to work. Currently it's only set as part of 
the TestHiveShell setup for unit tests. We should ensure it's set by the 
Iceberg storage
    handler by default for all Iceberg queries.
---
 data/conf/iceberg/llap/tez-site.xml                                  | 4 ----
 data/conf/iceberg/tez/tez-site.xml                                   | 4 ----
 .../java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java   | 5 +++++
 .../src/test/java/org/apache/iceberg/mr/hive/TestHiveShell.java      | 3 ---
 4 files changed, 5 insertions(+), 11 deletions(-)

diff --git a/data/conf/iceberg/llap/tez-site.xml 
b/data/conf/iceberg/llap/tez-site.xml
index 779886f..7ad5ad4 100644
--- a/data/conf/iceberg/llap/tez-site.xml
+++ b/data/conf/iceberg/llap/tez-site.xml
@@ -8,10 +8,6 @@
     
<value>org.apache.tez.dag.app.dag.impl.DAGSchedulerNaturalOrderControlled</value>
   </property>
   <property>
-    <name>tez.mrreader.config.update.properties</name>
-    <value>hive.io.file.readcolumn.names,hive.io.file.readcolumn.ids</value>
-  </property>
-  <property>
     <name>tez.am.resource.memory.mb</name>
     <value>256</value>
   </property>
diff --git a/data/conf/iceberg/tez/tez-site.xml 
b/data/conf/iceberg/tez/tez-site.xml
index 779886f..7ad5ad4 100644
--- a/data/conf/iceberg/tez/tez-site.xml
+++ b/data/conf/iceberg/tez/tez-site.xml
@@ -8,10 +8,6 @@
     
<value>org.apache.tez.dag.app.dag.impl.DAGSchedulerNaturalOrderControlled</value>
   </property>
   <property>
-    <name>tez.mrreader.config.update.properties</name>
-    <value>hive.io.file.readcolumn.names,hive.io.file.readcolumn.ids</value>
-  </property>
-  <property>
     <name>tez.am.resource.memory.mb</name>
     <value>256</value>
   </property>
diff --git 
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java
 
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java
index 5df64bf..7a357af 100644
--- 
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java
+++ 
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java
@@ -164,6 +164,7 @@ public class HiveIcebergStorageHandler implements 
HiveStoragePredicateHandler, H
 
   @Override
   public void configureJobConf(TableDesc tableDesc, JobConf jobConf) {
+    setCommonJobConf(jobConf);
     if (tableDesc != null && tableDesc.getProperties() != null &&
         tableDesc.getProperties().get(WRITE_KEY) != null) {
       String tableName = tableDesc.getTableName();
@@ -359,6 +360,10 @@ public class HiveIcebergStorageHandler implements 
HiveStoragePredicateHandler, H
     return IcebergMetadataTables.isValidMetaTable(metaTableName);
   }
 
+  private void setCommonJobConf(JobConf jobConf) {
+    jobConf.set("tez.mrreader.config.update.properties", 
"hive.io.file.readcolumn.names,hive.io.file.readcolumn.ids");
+  }
+
   public boolean 
addDynamicSplitPruningEdge(org.apache.hadoop.hive.ql.metadata.Table table,
                                             ExprNodeDesc 
syntheticFilterPredicate) {
     try {
diff --git 
a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveShell.java
 
b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveShell.java
index 3d39889..8538cb1 100644
--- 
a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveShell.java
+++ 
b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveShell.java
@@ -214,9 +214,6 @@ public class TestHiveShell {
     // set to true so that the Tez session will create an empty jar for 
localization
     hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_IN_TEST_IDE, true);
 
-    // enables vectorization on Tez
-    hiveConf.set("tez.mrreader.config.update.properties", 
"hive.io.file.readcolumn.names,hive.io.file.readcolumn.ids");
-
     // set lifecycle hooks
     hiveConf.setVar(HiveConf.ConfVars.HIVE_QUERY_LIFETIME_HOOKS, 
HiveIcebergQueryLifeTimeHook.class.getName());