Repository: hive Updated Branches: refs/heads/master 249c4ef13 -> 3cde8882e
HIVE-12388 : GetTables cannot get external tables when TABLE type argument is given (Navis and Szehon, via Aihua) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/3cde8882 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/3cde8882 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/3cde8882 Branch: refs/heads/master Commit: 3cde8882ee3c442111f89cec70c0332c3ceba59d Parents: 249c4ef Author: Szehon Ho <[email protected]> Authored: Wed Nov 18 11:02:58 2015 -0800 Committer: Szehon Ho <[email protected]> Committed: Wed Nov 18 11:03:24 2015 -0800 ---------------------------------------------------------------------- .../org/apache/hive/jdbc/TestJdbcDriver2.java | 89 +++++++++++++++++--- .../cli/operation/ClassicTableTypeMapping.java | 48 ++++++----- .../cli/operation/GetTablesOperation.java | 3 +- .../cli/operation/HiveTableTypeMapping.java | 19 ++++- .../service/cli/operation/TableTypeMapping.java | 6 +- 5 files changed, 127 insertions(+), 38 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/3cde8882/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcDriver2.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcDriver2.java b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcDriver2.java index 5450eaa..7bbacd5 100644 --- a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcDriver2.java +++ b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcDriver2.java @@ -18,6 +18,8 @@ package org.apache.hive.jdbc; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Sets; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.common.type.HiveIntervalDayTime; import org.apache.hadoop.hive.common.type.HiveIntervalYearMonth; @@ -41,6 +43,7 @@ import org.junit.rules.ExpectedException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.File; import java.io.InputStream; import java.lang.Exception; import java.lang.Object; @@ -97,6 +100,12 @@ public class TestJdbcDriver2 { private static final String partitionedTableComment = "Partitioned table"; private static final String dataTypeTableName = "testdatatypetable"; private static final String dataTypeTableComment = "Table with many column data types"; + + private static File workDir = new File(System.getProperty("test.tmp.dir")); + private static final String externalTable = "testHiveJdbcDriver_External"; + private static final String externalTableComment = "An external table"; + + private final HiveConf conf; public static String dataFileDir; private final Path dataFilePath; @@ -156,6 +165,10 @@ public class TestJdbcDriver2 { createTestTables(stmt, "", true); createTestTables(stmt, "testdb.", false); + + stmt.execute("drop table " + externalTable); + stmt.execute("create external table " + externalTable + " (a int) comment '" + externalTableComment + + "' location '" + dataFileDir + "'"); } private void createTestTables(Statement stmt, String prefix, boolean loadData) @@ -165,6 +178,7 @@ public class TestJdbcDriver2 { dropTestTables(stmt, prefix); String tableName = prefix + this.tableName; + String externalTable = prefix + this.tableName; String partitionedTableName = prefix + this.partitionedTableName; String dataTypeTableName = prefix + this.dataTypeTableName; String viewName = prefix + this.viewName; @@ -1116,7 +1130,7 @@ public class TestJdbcDriver2 { @Test public void testMetaDataGetTables() throws SQLException { - getTablesTest(ClassicTableTypes.TABLE.toString(), ClassicTableTypes.VIEW.toString()); + getTablesTest(ImmutableSet.of(ClassicTableTypes.TABLE.toString()), ClassicTableTypes.VIEW.toString()); } @Test @@ -1124,7 +1138,9 @@ public class TestJdbcDriver2 { Statement stmt = con.createStatement(); stmt.execute("set " + HiveConf.ConfVars.HIVE_SERVER2_TABLE_TYPE_MAPPING.varname + " = " + TableTypeMappings.HIVE.toString()); - getTablesTest(TableType.MANAGED_TABLE.toString(), TableType.VIRTUAL_VIEW.toString()); + getTablesTest(ImmutableSet.of(TableType.MANAGED_TABLE.toString(), + TableType.EXTERNAL_TABLE.toString()), + TableType.VIRTUAL_VIEW.toString()); } @Test @@ -1133,21 +1149,56 @@ public class TestJdbcDriver2 { stmt.execute("set " + HiveConf.ConfVars.HIVE_SERVER2_TABLE_TYPE_MAPPING.varname + " = " + TableTypeMappings.CLASSIC.toString()); stmt.close(); - getTablesTest(ClassicTableTypes.TABLE.toString(), ClassicTableTypes.VIEW.toString()); + getTablesTest(ImmutableSet.of(ClassicTableTypes.TABLE.toString()), ClassicTableTypes.VIEW.toString()); + } + + @Test + public void testMetaDataGetExternalTables() throws SQLException { + Statement stmt = con.createStatement(); + stmt.execute("set " + HiveConf.ConfVars.HIVE_SERVER2_TABLE_TYPE_MAPPING.varname + + " = " + TableTypeMappings.HIVE.toString()); + stmt.close(); + ResultSet rs = con.getMetaData().getTables(null, + null, null, new String[] { TableType.EXTERNAL_TABLE.toString() }); + ResultSetMetaData resMeta = rs.getMetaData(); + assertEquals(10, resMeta.getColumnCount()); + assertEquals("TABLE_CAT", resMeta.getColumnName(1)); + assertEquals("TABLE_SCHEM", resMeta.getColumnName(2)); + assertEquals("TABLE_NAME", resMeta.getColumnName(3)); + assertEquals("TABLE_TYPE", resMeta.getColumnName(4)); + assertEquals("REMARKS", resMeta.getColumnName(5)); + + rs.next(); + + String resultDbName = rs.getString("TABLE_SCHEM"); + assertEquals(resultDbName, "default"); + String resultTableName = rs.getString("TABLE_NAME"); + assertEquals(resultTableName, externalTable.toLowerCase()); + + String resultTableComment = rs.getString("REMARKS"); + assertTrue("Missing comment on the table.", resultTableComment.length()>0); + String tableType = rs.getString("TABLE_TYPE"); + assertEquals(TableType.EXTERNAL_TABLE.toString(), tableType); + + assertFalse("Unexpected table", rs.next()); } /** * Test the type returned for pre-created table type table and view type * table - * @param tableTypeName expected table type + * @param tableTypeNames expected table types * @param viewTypeName expected view type * @throws SQLException */ - private void getTablesTest(String tableTypeName, String viewTypeName) throws SQLException { + private void getTablesTest(Set<String> tableTypeNames, String viewTypeName) throws SQLException { String[] ALL = null; String[] VIEW_ONLY = {viewTypeName}; - String[] TABLE_ONLY = {tableTypeName}; - String[] VIEWORTABLE = {tableTypeName, viewTypeName}; + String[] TABLE_ONLY = tableTypeNames.toArray(new String[tableTypeNames.size()]); + + Set<String> viewOrTableArray = new HashSet<String>(); + viewOrTableArray.addAll(tableTypeNames); + viewOrTableArray.add(viewTypeName); + String[] VIEWORTABLE = viewOrTableArray.toArray(new String[viewOrTableArray.size()]); Map<Object[], String[]> tests = new IdentityHashMap<Object[], String[]>(); tests.put(new Object[] { null, "test%jdbc%", ALL}, new String[]{ @@ -1156,7 +1207,8 @@ public class TestJdbcDriver2 { "default.testhivejdbcdriverview", "testdb.testhivejdbcdriver_table", "testdb.testhivejdbcdriverpartitionedtable", - "testdb.testhivejdbcdriverview"}); + "testdb.testhivejdbcdriverview", + "default.testhivejdbcdriver_external"}); tests.put(new Object[] { "test%", "test%jdbc%", ALL}, new String[]{ "testdb.testhivejdbcdriver_table", "testdb.testhivejdbcdriverpartitionedtable", @@ -1199,12 +1251,25 @@ public class TestJdbcDriver2 { "default.testhivejdbcdriverview", "testdb.testhivejdbcdriver_table", "testdb.testhivejdbcdriverpartitionedtable", - "testdb.testhivejdbcdriverview"}); + "testdb.testhivejdbcdriverview", + "default.testhivejdbcdriver_external"}); tests.put(new Object[] { "%", "%jdbc%", VIEW_ONLY}, new String[]{ "default.testhivejdbcdriverview", "testdb.testhivejdbcdriverview"}); + + tests.put(new Object[] { "%", "%jdbc%", VIEW_ONLY}, new String[]{ + "default.testhivejdbcdriverview", + "testdb.testhivejdbcdriverview"}); tests.put(new Object[] { null, "", ALL}, new String[]{}); + tests.put(new Object[] { null, "%jdbc%", TABLE_ONLY}, new String[]{ + "default.testhivejdbcdriver_table", + "default.testhivejdbcdriverpartitionedtable", + "testdb.testhivejdbcdriver_table", + "testdb.testhivejdbcdriverpartitionedtable", + "default.testhivejdbcdriver_external" + }); + for (Map.Entry<Object[], String[]> entry : tests.entrySet()) { Object[] checkPattern = entry.getKey(); String debugString = checkPattern[0] + ", " + checkPattern[1] + ", " + @@ -1234,7 +1299,7 @@ public class TestJdbcDriver2 { if (resultTableName.endsWith("view")) { assertEquals("Expected a tabletype view but got something else.", viewTypeName, tableType); } else { - assertEquals("Expected a tabletype table but got something else.", tableTypeName, tableType); + assertTrue("Expected one of " + tableTypeNames + " table but got something else: " + tableType, tableTypeNames.contains(tableType)); } cnt++; } @@ -1326,8 +1391,8 @@ public class TestJdbcDriver2 { public void testMetaDataGetColumns() throws SQLException { Map<String[], Integer> tests = new HashMap<String[], Integer>(); tests.put(new String[]{"testhivejdbcdriver\\_table", null}, 2); - tests.put(new String[]{"testhivejdbc%", null}, 7); - tests.put(new String[]{"testhiveJDBC%", null}, 7); + tests.put(new String[]{"testhivejdbc%", null}, 8); + tests.put(new String[]{"testhiveJDBC%", null}, 8); tests.put(new String[]{"%jdbcdriver\\_table", null}, 2); tests.put(new String[]{"%jdbcdriver\\_table%", "under\\_col"}, 1); // tests.put(new String[]{"%jdbcdriver\\_table%", "under\\_COL"}, 1); http://git-wip-us.apache.org/repos/asf/hive/blob/3cde8882/service/src/java/org/apache/hive/service/cli/operation/ClassicTableTypeMapping.java ---------------------------------------------------------------------- diff --git a/service/src/java/org/apache/hive/service/cli/operation/ClassicTableTypeMapping.java b/service/src/java/org/apache/hive/service/cli/operation/ClassicTableTypeMapping.java index 87ac39b..e5a186c 100644 --- a/service/src/java/org/apache/hive/service/cli/operation/ClassicTableTypeMapping.java +++ b/service/src/java/org/apache/hive/service/cli/operation/ClassicTableTypeMapping.java @@ -18,6 +18,14 @@ package org.apache.hive.service.cli.operation; +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.Iterables; +import com.google.common.collect.Multimap; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Arrays; +import java.util.Collection; import java.util.HashMap; import java.util.HashSet; import java.util.Map; @@ -34,51 +42,51 @@ import org.apache.hadoop.hive.metastore.TableType; */ public class ClassicTableTypeMapping implements TableTypeMapping { + private static final Logger LOG = LoggerFactory.getLogger(ClassicTableTypeMapping.class); + public enum ClassicTableTypes { TABLE, VIEW, } private final Map<String, String> hiveToClientMap = new HashMap<String, String>(); - private final Map<String, String> clientToHiveMap = new HashMap<String, String>(); + private final Multimap<String, String> clientToHiveMap = ArrayListMultimap.create(); public ClassicTableTypeMapping () { - hiveToClientMap.put(TableType.MANAGED_TABLE.toString(), - ClassicTableTypes.TABLE.toString()); - hiveToClientMap.put(TableType.EXTERNAL_TABLE.toString(), - ClassicTableTypes.TABLE.toString()); - hiveToClientMap.put(TableType.VIRTUAL_VIEW.toString(), - ClassicTableTypes.VIEW.toString()); + hiveToClientMap.put(TableType.MANAGED_TABLE.name(), ClassicTableTypes.TABLE.name()); + hiveToClientMap.put(TableType.EXTERNAL_TABLE.name(), ClassicTableTypes.TABLE.name()); + hiveToClientMap.put(TableType.VIRTUAL_VIEW.name(), ClassicTableTypes.VIEW.name()); - clientToHiveMap.put(ClassicTableTypes.TABLE.toString(), - TableType.MANAGED_TABLE.toString()); - clientToHiveMap.put(ClassicTableTypes.VIEW.toString(), - TableType.VIRTUAL_VIEW.toString()); + clientToHiveMap.putAll(ClassicTableTypes.TABLE.name(), Arrays.asList( + TableType.MANAGED_TABLE.name(), TableType.EXTERNAL_TABLE.name())); + clientToHiveMap.put(ClassicTableTypes.VIEW.name(), TableType.VIRTUAL_VIEW.name()); } @Override - public String mapToHiveType(String clientTypeName) { - if (clientToHiveMap.containsKey(clientTypeName)) { - return clientToHiveMap.get(clientTypeName); - } else { - return clientTypeName; + public String[] mapToHiveType(String clientTypeName) { + Collection<String> hiveTableType = clientToHiveMap.get(clientTypeName.toUpperCase()); + if (hiveTableType == null) { + LOG.warn("Not supported client table type " + clientTypeName); + return new String[] {clientTypeName}; } + return Iterables.toArray(hiveTableType, String.class); } @Override public String mapToClientType(String hiveTypeName) { - if (hiveToClientMap.containsKey(hiveTypeName)) { - return hiveToClientMap.get(hiveTypeName); - } else { + String clientTypeName = hiveToClientMap.get(hiveTypeName); + if (clientTypeName == null) { + LOG.warn("Invalid hive table type " + hiveTypeName); return hiveTypeName; } + return clientTypeName; } @Override public Set<String> getTableTypeNames() { Set<String> typeNameSet = new HashSet<String>(); for (ClassicTableTypes typeNames : ClassicTableTypes.values()) { - typeNameSet.add(typeNames.toString()); + typeNameSet.add(typeNames.name()); } return typeNameSet; } http://git-wip-us.apache.org/repos/asf/hive/blob/3cde8882/service/src/java/org/apache/hive/service/cli/operation/GetTablesOperation.java ---------------------------------------------------------------------- diff --git a/service/src/java/org/apache/hive/service/cli/operation/GetTablesOperation.java b/service/src/java/org/apache/hive/service/cli/operation/GetTablesOperation.java index 65bbc1c..740b851 100644 --- a/service/src/java/org/apache/hive/service/cli/operation/GetTablesOperation.java +++ b/service/src/java/org/apache/hive/service/cli/operation/GetTablesOperation.java @@ -19,6 +19,7 @@ package org.apache.hive.service.cli.operation; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import org.apache.hadoop.hive.conf.HiveConf; @@ -78,7 +79,7 @@ public class GetTablesOperation extends MetadataOperation { if (tableTypes != null) { tableTypeList = new ArrayList<String>(); for (String tableType : tableTypes) { - tableTypeList.add(tableTypeMapping.mapToHiveType(tableType.trim())); + tableTypeList.addAll(Arrays.asList(tableTypeMapping.mapToHiveType(tableType.trim()))); } } else { tableTypeList = null; http://git-wip-us.apache.org/repos/asf/hive/blob/3cde8882/service/src/java/org/apache/hive/service/cli/operation/HiveTableTypeMapping.java ---------------------------------------------------------------------- diff --git a/service/src/java/org/apache/hive/service/cli/operation/HiveTableTypeMapping.java b/service/src/java/org/apache/hive/service/cli/operation/HiveTableTypeMapping.java index b530f21..d45c441 100644 --- a/service/src/java/org/apache/hive/service/cli/operation/HiveTableTypeMapping.java +++ b/service/src/java/org/apache/hive/service/cli/operation/HiveTableTypeMapping.java @@ -18,6 +18,9 @@ package org.apache.hive.service.cli.operation; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import java.util.HashSet; import java.util.Set; @@ -30,21 +33,29 @@ import org.apache.hadoop.hive.metastore.TableType; */ public class HiveTableTypeMapping implements TableTypeMapping { + private static final Logger LOG = LoggerFactory.getLogger(HiveTableTypeMapping.class); + @Override - public String mapToHiveType(String clientTypeName) { - return clientTypeName; + public String[] mapToHiveType(String clientTypeName) { + return new String[] {mapToClientType(clientTypeName)}; } @Override public String mapToClientType(String hiveTypeName) { - return hiveTypeName; + try { + TableType hiveType = TableType.valueOf(hiveTypeName.toUpperCase()); + return hiveType.name(); + } catch (IllegalArgumentException e) { + LOG.warn("Invalid hive table type " + hiveTypeName); + return hiveTypeName; + } } @Override public Set<String> getTableTypeNames() { Set<String> typeNameSet = new HashSet<String>(); for (TableType typeNames : TableType.values()) { - typeNameSet.add(typeNames.toString()); + typeNameSet.add(typeNames.name()); } return typeNameSet; } http://git-wip-us.apache.org/repos/asf/hive/blob/3cde8882/service/src/java/org/apache/hive/service/cli/operation/TableTypeMapping.java ---------------------------------------------------------------------- diff --git a/service/src/java/org/apache/hive/service/cli/operation/TableTypeMapping.java b/service/src/java/org/apache/hive/service/cli/operation/TableTypeMapping.java index 3a8a07f..8f531f7 100644 --- a/service/src/java/org/apache/hive/service/cli/operation/TableTypeMapping.java +++ b/service/src/java/org/apache/hive/service/cli/operation/TableTypeMapping.java @@ -20,14 +20,18 @@ package org.apache.hive.service.cli.operation; import java.util.Set; +import org.apache.hadoop.hive.common.classification.InterfaceAudience; +import org.apache.hadoop.hive.common.classification.InterfaceStability; [email protected] [email protected] public interface TableTypeMapping { /** * Map client's table type name to hive's table type * @param clientTypeName * @return */ - public String mapToHiveType (String clientTypeName); + public String[] mapToHiveType (String clientTypeName); /** * Map hive's table type name to client's table type
