Repository: incubator-impala Updated Branches: refs/heads/master 874a11589 -> 999a444e2
IMPALA-2636: HS2 GetTables() returns TABLE_TYPE as TABLE for VIEW Added code to read the table type from metastore table but defaults to "TABLE" if the metastore table is not loaded. After the change, GetTabletypes also returns "VIEW" apart from "TABLE" Changed unit and jdbc testcases for GetTableTypes. Added new Frontend test for reading views. Change-Id: I90616388e6181cf342b3de389af940214ed46428 Reviewed-on: http://gerrit.cloudera.org:8080/7353 Reviewed-by: Alex Behm <[email protected]> Tested-by: Impala Public Jenkins Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/999a444e Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/999a444e Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/999a444e Branch: refs/heads/master Commit: 999a444e246fcfc7aca2f48541428886dd994b63 Parents: 874a115 Author: Sandeep Akinapelli <[email protected]> Authored: Fri Jun 30 13:05:51 2017 -0700 Committer: Impala Public Jenkins <[email protected]> Committed: Sat Oct 7 03:16:06 2017 +0000 ---------------------------------------------------------------------- .../org/apache/impala/service/MetadataOp.java | 74 ++++++++++++---- .../org/apache/impala/service/FrontendTest.java | 88 +++++++++++++++++++- .../org/apache/impala/service/JdbcTest.java | 3 +- 3 files changed, 142 insertions(+), 23 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/999a444e/fe/src/main/java/org/apache/impala/service/MetadataOp.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/org/apache/impala/service/MetadataOp.java b/fe/src/main/java/org/apache/impala/service/MetadataOp.java index 8887908..290720e 100644 --- a/fe/src/main/java/org/apache/impala/service/MetadataOp.java +++ b/fe/src/main/java/org/apache/impala/service/MetadataOp.java @@ -24,7 +24,7 @@ import java.util.Set; import org.slf4j.Logger; import org.slf4j.LoggerFactory; - +import org.apache.hadoop.hive.metastore.TableType; import org.apache.impala.analysis.TableName; import org.apache.impala.authorization.User; import org.apache.impala.catalog.Column; @@ -57,7 +57,8 @@ public class MetadataOp { // Static column values private static final TColumnValue NULL_COL_VAL = new TColumnValue(); private static final TColumnValue EMPTY_COL_VAL = createTColumnValue(""); - private static final TColumnValue TABLE_TYPE_COL_VAL = createTColumnValue("TABLE"); + private static final String TABLE_TYPE_TABLE = "TABLE"; + private static final String TABLE_TYPE_VIEW = "VIEW"; // Result set schema for each of the metadata operations. private final static TResultSetMetadata GET_CATALOGS_MD = new TResultSetMetadata(); @@ -71,7 +72,7 @@ public class MetadataOp { // GetTypeInfo contains all primitive types supported by Impala. private static final List<TResultRow> GET_TYPEINFO_RESULTS = Lists.newArrayList(); - // GetTableTypes only returns a single value: "TABLE". + // GetTableTypes returns values: "TABLE", "VIEW" private static final List<TResultRow> GET_TABLE_TYPES_RESULTS = Lists.newArrayList(); // Initialize result set schemas and static result set @@ -215,6 +216,9 @@ public class MetadataOp { // tableNames[i] are the tables within dbs[i] public List<List<String>> tableNames = Lists.newArrayList(); + // tableTypes[i] are the type of tables within dbs[i] + public List<List<String>> tableTypes = Lists.newArrayList(); + // comments[i][j] is the comment of tableNames[j] in dbs[i]. public List<List<String>> comments = Lists.newArrayList(); @@ -271,6 +275,7 @@ public class MetadataOp { List<String> tableList = Lists.newArrayList(); List<List<Column>> tablesColumnsList = Lists.newArrayList(); List<String> tableComments = Lists.newArrayList(); + List<String> tableTypes = Lists.newArrayList(); for (String tabName: fe.getTableNames(db.getName(), tablePatternMatcher, user)) { Table table = null; try { @@ -284,25 +289,53 @@ public class MetadataOp { List<Column> columns = Lists.newArrayList(); // If the table is not yet loaded, the columns will be unknown. Add it // to the set of missing tables. + String tableType = TABLE_TYPE_TABLE; if (!table.isLoaded()) { result.missingTbls.add(new TableName(db.getName(), tabName)); } else { - comment = table.getMetaStoreTable().getParameters().get("comment"); + if (table.getMetaStoreTable() != null) { + comment = table.getMetaStoreTable().getParameters().get("comment"); + tableType = mapToInternalTableType(table.getMetaStoreTable().getTableType()); + } columns.addAll(fe.getColumns(table, columnPatternMatcher, user)); } tableList.add(tabName); tablesColumnsList.add(columns); tableComments.add(Strings.nullToEmpty(comment)); + tableTypes.add(tableType); } result.dbs.add(db.getName()); result.tableNames.add(tableList); result.comments.add(tableComments); result.columns.add(tablesColumnsList); + result.tableTypes.add(tableTypes); } } return result; } + private static String mapToInternalTableType(String typeStr) { + String defaultTableType = TABLE_TYPE_TABLE; + TableType tType; + + if (typeStr == null) return defaultTableType; + try { + tType = TableType.valueOf(typeStr.toUpperCase()); + } catch (Exception e) { + return defaultTableType; + } + switch (tType) { + case EXTERNAL_TABLE: + case MANAGED_TABLE: + case INDEX_TABLE: + return TABLE_TYPE_TABLE; + case VIRTUAL_VIEW: + return TABLE_TYPE_VIEW; + default: + return defaultTableType; + } + } + /** * Executes the GetCatalogs HiveServer2 operation and returns TResultSet. * Hive does not have a catalog concept. It always returns an empty result set. @@ -455,19 +488,17 @@ public class MetadataOp { throws ImpalaException{ TResultSet result = createEmptyResultSet(GET_TABLES_MD); - // Impala catalog only contains TABLE. Returns an empty set if the search does not - // include TABLE. + List<String> upperCaseTableTypes = null; if (tableTypes != null && !tableTypes.isEmpty()) { - boolean hasTableType = false; - for (String tableType: tableTypes) { - if (tableType.toLowerCase().equals("table")) { - hasTableType = true; - break; - } - } - if (!hasTableType) { - return result; + boolean hasValidTableType = false; + upperCaseTableTypes = Lists.newArrayList(); + for (String tableType : tableTypes) { + tableType = tableType.toUpperCase(); + upperCaseTableTypes.add(tableType); + if (tableType.equals(TABLE_TYPE_TABLE)) hasValidTableType = true; + if (tableType.equals(TABLE_TYPE_VIEW)) hasValidTableType = true; } + if (!hasValidTableType) return result; } // Get the list of schemas, tables that satisfy the search conditions. @@ -481,12 +512,15 @@ public class MetadataOp { String dbName = dbsMetadata.dbs.get(i); for (int j = 0; j < dbsMetadata.tableNames.get(i).size(); ++j) { String tabName = dbsMetadata.tableNames.get(i).get(j); + String tableType = dbsMetadata.tableTypes.get(i).get(j); + if (upperCaseTableTypes != null && !upperCaseTableTypes.contains(tableType)) continue; + TResultRow row = new TResultRow(); row.colVals = Lists.newArrayList(); row.colVals.add(EMPTY_COL_VAL); row.colVals.add(createTColumnValue(dbName)); row.colVals.add(createTColumnValue(tabName)); - row.colVals.add(TABLE_TYPE_COL_VAL); + row.colVals.add(createTColumnValue(tableType)); row.colVals.add(createTColumnValue(dbsMetadata.comments.get(i).get(j))); result.rows.add(row); } @@ -599,12 +633,16 @@ public class MetadataOp { } /** - * Fills the GET_TYPEINFO_RESULTS with "TABLE". + * Fills the GET_TYPEINFO_RESULTS with "TABLE", "VIEW". */ private static void createGetTableTypesResults() { TResultRow row = new TResultRow(); row.colVals = Lists.newArrayList(); - row.colVals.add(createTColumnValue("TABLE")); + row.colVals.add(createTColumnValue(TABLE_TYPE_TABLE)); + GET_TABLE_TYPES_RESULTS.add(row); + row = new TResultRow(); + row.colVals = Lists.newArrayList(); + row.colVals.add(createTColumnValue(TABLE_TYPE_VIEW)); GET_TABLE_TYPES_RESULTS.add(row); } http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/999a444e/fe/src/test/java/org/apache/impala/service/FrontendTest.java ---------------------------------------------------------------------- diff --git a/fe/src/test/java/org/apache/impala/service/FrontendTest.java b/fe/src/test/java/org/apache/impala/service/FrontendTest.java index 938b00e..54fa302 100644 --- a/fe/src/test/java/org/apache/impala/service/FrontendTest.java +++ b/fe/src/test/java/org/apache/impala/service/FrontendTest.java @@ -22,6 +22,7 @@ import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import java.util.Arrays; import java.util.List; import java.util.Set; @@ -44,11 +45,9 @@ import org.apache.impala.common.FrontendTestBase; import org.apache.impala.common.ImpalaException; import org.apache.impala.testutil.ImpaladTestCatalog; import org.apache.impala.testutil.TestUtils; -import org.apache.impala.thrift.TExecRequest; import org.apache.impala.thrift.TMetadataOpRequest; import org.apache.impala.thrift.TMetadataOpcode; import org.apache.impala.thrift.TQueryCtx; -import org.apache.impala.thrift.TQueryOptions; import org.apache.impala.thrift.TResultRow; import org.apache.impala.thrift.TResultSet; @@ -178,6 +177,47 @@ public class FrontendTest extends FrontendTestBase { } @Test + public void TestGetTablesTypeTable() throws ImpalaException { + TMetadataOpRequest req = new TMetadataOpRequest(); + req.opcode = TMetadataOpcode.GET_TABLES; + req.get_tables_req = new TGetTablesReq(); + req.get_tables_req.setSchemaName("functional"); + req.get_tables_req.setTableName("alltypes\\_%"); + req.get_tables_req.setTableTypes(Arrays.asList("TABLE")); + TResultSet resp = execMetadataOp(req); + // HiveServer2 GetTables has 5 columns. + assertEquals(5, resp.schema.columns.size()); + assertEquals(5, resp.rows.get(0).colVals.size()); + assertEquals(1, resp.rows.size()); + assertEquals("alltypes_datasource", resp.rows.get(0).colVals.get(2).string_val.toLowerCase()); + } + + @Test + public void TestGetTablesTypeView() throws ImpalaException { + // Issue the statements to make sure all the views are loaded + AnalyzesOk("select * from functional.alltypes_hive_view"); + AnalyzesOk("select * from functional.alltypes_parens"); + AnalyzesOk("select * from functional.alltypes_view"); + AnalyzesOk("select * from functional.alltypes_view_sub"); + + TMetadataOpRequest req = new TMetadataOpRequest(); + req.opcode = TMetadataOpcode.GET_TABLES; + req.get_tables_req = new TGetTablesReq(); + req.get_tables_req.setSchemaName("functional"); + req.get_tables_req.setTableName("alltypes%"); + req.get_tables_req.setTableTypes(Arrays.asList("VIEW")); + TResultSet resp = execMetadataOp(req); + // HiveServer2 GetTables has 5 columns. + assertEquals(5, resp.schema.columns.size()); + assertEquals(5, resp.rows.get(0).colVals.size()); + assertEquals(4, resp.rows.size()); + assertEquals("alltypes_hive_view", resp.rows.get(0).colVals.get(2).string_val.toLowerCase()); + assertEquals("alltypes_parens", resp.rows.get(1).colVals.get(2).string_val.toLowerCase()); + assertEquals("alltypes_view", resp.rows.get(2).colVals.get(2).string_val.toLowerCase()); + assertEquals("alltypes_view_sub", resp.rows.get(3).colVals.get(2).string_val.toLowerCase()); + } + + @Test public void TestGetTablesWithComments() throws ImpalaException { // Add test db and test tables with comments final String dbName = "tbls_with_comments_test_db"; @@ -220,6 +260,46 @@ public class FrontendTest extends FrontendTestBase { } @Test + public void TestUnloadedView() throws ImpalaException { + final String dbName = "tbls_for_views_test_db"; + Db testDb = addTestDb(dbName, "Stores views"); + assertNotNull(testDb); + Table view = addTestView(String.format( + "create view %s.test_view as select * from functional.alltypes", dbName)); + assertNotNull(view); + + // Prepare and perform the GetTables request + TMetadataOpRequest req = new TMetadataOpRequest(); + req.opcode = TMetadataOpcode.GET_TABLES; + req.get_tables_req = new TGetTablesReq(); + req.get_tables_req.setSchemaName(dbName); + TResultSet resp = execMetadataOp(req); + assertEquals(1, resp.rows.size()); + for (TResultRow row : resp.rows) { + assertEquals(row.colVals.get(2).string_val.toLowerCase(), "test_view"); + assertEquals("table", row.colVals.get(3).string_val.toLowerCase()); + } + } + + @Test + public void TestLoadedView() throws ImpalaException { + // Issue the statement to make sure the view is loaded + AnalyzesOk("select * from functional.alltypes_view"); + // Prepare and perform the GetTables request + TMetadataOpRequest req = new TMetadataOpRequest(); + req.opcode = TMetadataOpcode.GET_TABLES; + req.get_tables_req = new TGetTablesReq(); + req.get_tables_req.setSchemaName("functional"); + req.get_tables_req.setTableName("alltypes_view"); + + TResultSet resp = execMetadataOp(req); + assertEquals(1, resp.rows.size()); + for (TResultRow row : resp.rows) { + assertEquals("VIEW", row.colVals.get(3).string_val); + } + } + + @Test public void TestGetColumns() throws ImpalaException { // It should return one column: default.alltypes.string_col TMetadataOpRequest req = new TMetadataOpRequest(); @@ -257,15 +337,15 @@ public class FrontendTest extends FrontendTestBase { @Test public void TestGetTableTypes() throws ImpalaException { - // Impala should only return TABLE as the only table type. TMetadataOpRequest req = new TMetadataOpRequest(); req.opcode = TMetadataOpcode.GET_TABLE_TYPES; TResultSet resp = execMetadataOp(req); // HiveServer2 GetTableTypes() has 1 column. + assertEquals(2, resp.rows.size()); assertEquals(1, resp.schema.columns.size()); assertEquals(1, resp.rows.get(0).colVals.size()); - assertEquals(1, resp.rows.size()); assertEquals("TABLE", resp.rows.get(0).getColVals().get(0).string_val); + assertEquals("VIEW", resp.rows.get(1).getColVals().get(0).string_val); } @Test http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/999a444e/fe/src/test/java/org/apache/impala/service/JdbcTest.java ---------------------------------------------------------------------- diff --git a/fe/src/test/java/org/apache/impala/service/JdbcTest.java b/fe/src/test/java/org/apache/impala/service/JdbcTest.java index 6309e8e..4c06cb4 100644 --- a/fe/src/test/java/org/apache/impala/service/JdbcTest.java +++ b/fe/src/test/java/org/apache/impala/service/JdbcTest.java @@ -190,10 +190,11 @@ public class JdbcTest { @Test public void testMetaDataGetTableTypes() throws SQLException { - // There is only one table type: "table". ResultSet rs = con_.getMetaData().getTableTypes(); assertTrue(rs.next()); assertEquals(rs.getString(1).toLowerCase(), "table"); + assertTrue(rs.next()); + assertEquals(rs.getString(1).toLowerCase(), "view"); assertFalse(rs.next()); rs.close(); }
