[iceberg] branch master updated: Hive: Run StorageHandler tests in both MR and Tez (#1695)

[blue]

This is an automated email from the ASF dual-hosted git repository.

blue pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iceberg.git


The following commit(s) were added to refs/heads/master by this push:
     new 4f2b82e  Hive: Run StorageHandler tests in both MR and Tez (#1695)
4f2b82e is described below

commit 4f2b82e53aca0b5d4bee50353396e53a3f55cd10
Author: Marton Bod <marton....@gmail.com>
AuthorDate: Tue Nov 3 01:52:58 2020 +0100

    Hive: Run StorageHandler tests in both MR and Tez (#1695)
    
    Co-authored-by: Marton Bod <m...@cloudera.com>
---
 build.gradle                                       |  3 +++
 .../mr/hive/HiveIcebergStorageHandlerBaseTest.java | 28 ++++++++++++++++++----
 .../org/apache/iceberg/mr/hive/TestHiveShell.java  |  6 +++++
 versions.props                                     |  2 ++
 4 files changed, 35 insertions(+), 4 deletions(-)

diff --git a/build.gradle b/build.gradle
index 60ff14e..eb4a7cb 100644
--- a/build.gradle
+++ b/build.gradle
@@ -473,6 +473,8 @@ project(':iceberg-mr') {
     testCompile("org.apache.hive:hive-service") {
       exclude group: 'org.apache.hive', module: 'hive-exec'
     }
+    testCompile("org.apache.tez:tez-dag")
+    testCompile("org.apache.tez:tez-mapreduce")
   }
 
   test {
@@ -538,6 +540,7 @@ if (jdkVersion == '8') {
         exclude group: 'org.apache.hive', module: 'hive-exec'
       }
       testCompile("org.apache.tez:tez-dag:0.9.1")
+      testCompile("org.apache.tez:tez-mapreduce:0.9.1")
     }
 
     test {
diff --git 
a/mr/src/test/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandlerBaseTest.java
 
b/mr/src/test/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandlerBaseTest.java
index 2912a64..8922c98 100644
--- 
a/mr/src/test/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandlerBaseTest.java
+++ 
b/mr/src/test/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandlerBaseTest.java
@@ -20,6 +20,8 @@
 package org.apache.iceberg.mr.hive;
 
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
@@ -109,14 +111,29 @@ public abstract class HiveIcebergStorageHandlerBaseTest {
 
   public abstract TestTables testTables(Configuration conf, TemporaryFolder 
tmp) throws IOException;
 
-  @Parameters(name = "fileFormat={0}")
-  public static Iterable<FileFormat> fileFormats() {
-    return ImmutableList.of(FileFormat.PARQUET, FileFormat.ORC, 
FileFormat.AVRO);
+  @Parameters(name = "fileFormat={0}, engine={1}")
+  public static Collection<Object[]> parameters() {
+    Collection<Object[]> testParams = new ArrayList<>();
+    testParams.add(new Object[] { FileFormat.PARQUET, "mr" });
+    testParams.add(new Object[] { FileFormat.ORC, "mr" });
+    testParams.add(new Object[] { FileFormat.AVRO, "mr" });
+
+    // include Tez tests only for Java 8
+    String javaVersion = System.getProperty("java.specification.version");
+    if (javaVersion.equals("1.8")) {
+      testParams.add(new Object[] { FileFormat.PARQUET, "tez" });
+      testParams.add(new Object[] { FileFormat.ORC, "tez" });
+      testParams.add(new Object[] { FileFormat.AVRO, "tez" });
+    }
+    return testParams;
   }
 
-  @Parameter
+  @Parameter(0)
   public FileFormat fileFormat;
 
+  @Parameter(1)
+  public String executionEngine;
+
   @BeforeClass
   public static void beforeClass() {
     shell = new TestHiveShell();
@@ -136,6 +153,9 @@ public abstract class HiveIcebergStorageHandlerBaseTest {
     for (Map.Entry<String, String> property : 
testTables.properties().entrySet()) {
       shell.setHiveSessionValue(property.getKey(), property.getValue());
     }
+    shell.setHiveSessionValue("hive.execution.engine", executionEngine);
+    shell.setHiveSessionValue("hive.jar.directory", 
temp.getRoot().getAbsolutePath());
+    shell.setHiveSessionValue("tez.staging-dir", 
temp.getRoot().getAbsolutePath());
   }
 
   @After
diff --git a/mr/src/test/java/org/apache/iceberg/mr/hive/TestHiveShell.java 
b/mr/src/test/java/org/apache/iceberg/mr/hive/TestHiveShell.java
index 7be1518..0f8314c 100644
--- a/mr/src/test/java/org/apache/iceberg/mr/hive/TestHiveShell.java
+++ b/mr/src/test/java/org/apache/iceberg/mr/hive/TestHiveShell.java
@@ -171,6 +171,12 @@ public class TestHiveShell {
     hiveConf.setLongVar(HiveConf.ConfVars.HIVECOUNTERSPULLINTERVAL, 1L);
     hiveConf.setBoolVar(HiveConf.ConfVars.HIVESTATSAUTOGATHER, false);
 
+    // Resource configuration
+    hiveConf.setInt("mapreduce.map.memory.mb", 1024);
+
+    // Tez configuration
+    hiveConf.setBoolean("tez.local.mode", true);
+
     return hiveConf;
   }
 }
diff --git a/versions.props b/versions.props
index 273ae64..2d587c0 100644
--- a/versions.props
+++ b/versions.props
@@ -23,3 +23,5 @@ junit:junit = 4.12
 org.mockito:mockito-core = 1.10.19
 org.apache.hive:hive-exec = 2.3.7
 org.apache.hive:hive-service = 2.3.7
+org.apache.tez:tez-dag = 0.8.4
+org.apache.tez:tez-mapreduce = 0.8.4