IMPALA-7140 (part 8): support views in LocalCatalog This adds basic support for loading views in LocalCatalog. Tested with a small unit test and also verified from the shell that I can select from a view.
Change-Id: Ib3516b9ceff6dce12ded68d93afde09728627e08 Reviewed-on: http://gerrit.cloudera.org:8080/10805 Tested-by: Impala Public Jenkins <impala-public-jenk...@cloudera.com> Reviewed-by: Todd Lipcon <t...@apache.org> Project: http://git-wip-us.apache.org/repos/asf/impala/repo Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/f14ae3fc Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/f14ae3fc Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/f14ae3fc Branch: refs/heads/master Commit: f14ae3fc3a16bf707f5c23ec60816c4df0996682 Parents: 7f8a4d3 Author: Todd Lipcon <t...@cloudera.com> Authored: Fri Jun 22 10:53:23 2018 -0700 Committer: Todd Lipcon <t...@apache.org> Committed: Fri Jul 13 17:39:06 2018 +0000 ---------------------------------------------------------------------- .../java/org/apache/impala/catalog/View.java | 54 ++++++------- .../apache/impala/catalog/local/LocalTable.java | 3 +- .../apache/impala/catalog/local/LocalView.java | 83 ++++++++++++++++++++ .../impala/catalog/local/LocalCatalogTest.java | 9 +++ 4 files changed, 116 insertions(+), 33 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/impala/blob/f14ae3fc/fe/src/main/java/org/apache/impala/catalog/View.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/org/apache/impala/catalog/View.java b/fe/src/main/java/org/apache/impala/catalog/View.java index 66d942a..4026d6a 100644 --- a/fe/src/main/java/org/apache/impala/catalog/View.java +++ b/fe/src/main/java/org/apache/impala/catalog/View.java @@ -45,24 +45,6 @@ import com.google.common.collect.Lists; */ public class View extends Table implements FeView { - // The original SQL-string given as view definition. Set during analysis. - // Corresponds to Hive's viewOriginalText. - private String originalViewDef_; - - // Query statement (as SQL string) that defines the View for view substitution. - // It is a transformation of the original view definition, e.g., to enforce the - // explicit column definitions even if the original view definition has explicit - // column aliases. - // If column definitions were given, then this "expanded" view definition - // wraps the original view definition in a select stmt as follows. - // - // SELECT viewName.origCol1 AS colDesc1, viewName.origCol2 AS colDesc2, ... - // FROM (originalViewDef) AS viewName - // - // Corresponds to Hive's viewExpandedText, but is not identical to the SQL - // Hive would produce in view creation. - private String inlineViewDef_; - // View definition created by parsing inlineViewDef_ into a QueryStmt. private QueryStmt queryStmt_; @@ -117,7 +99,7 @@ public class View extends Table implements FeView { numClusteringCols_ = 0; tableStats_ = new TTableStats(-1); tableStats_.setTotal_file_bytes(-1); - init(); + queryStmt_ = parseViewDef(this); } catch (TableLoadingException e) { throw e; } catch (Exception e) { @@ -128,22 +110,32 @@ public class View extends Table implements FeView { @Override protected void loadFromThrift(TTable t) throws TableLoadingException { super.loadFromThrift(t); - init(); + queryStmt_ = parseViewDef(this); } /** - * Initializes the originalViewDef_, inlineViewDef_, and queryStmt_ members - * by parsing the expanded view definition SQL-string. + * Parse the expanded view definition SQL-string. * Throws a TableLoadingException if there was any error parsing the * the SQL or if the view definition did not parse into a QueryStmt. */ - private void init() throws TableLoadingException { - // Set view-definition SQL strings. - originalViewDef_ = getMetaStoreTable().getViewOriginalText(); - inlineViewDef_ = getMetaStoreTable().getViewExpandedText(); + public static QueryStmt parseViewDef(FeView view) throws TableLoadingException { + // Query statement (as SQL string) that defines the View for view substitution. + // It is a transformation of the original view definition, e.g., to enforce the + // explicit column definitions even if the original view definition has explicit + // column aliases. + // If column definitions were given, then this "expanded" view definition + // wraps the original view definition in a select stmt as follows. + // + // SELECT viewName.origCol1 AS colDesc1, viewName.origCol2 AS colDesc2, ... + // FROM (originalViewDef) AS viewName + // + // Corresponds to Hive's viewExpandedText, but is not identical to the SQL + // Hive would produce in view creation. + String inlineViewDef = view.getMetaStoreTable().getViewExpandedText(); + // Parse the expanded view definition SQL-string into a QueryStmt and // populate a view definition. - SqlScanner input = new SqlScanner(new StringReader(inlineViewDef_)); + SqlScanner input = new SqlScanner(new StringReader(inlineViewDef)); SqlParser parser = new SqlParser(input); ParseNode node = null; try { @@ -153,14 +145,14 @@ public class View extends Table implements FeView { // of tables that the user triggering this load may not have privileges on. throw new TableLoadingException( String.format("Failed to parse view-definition statement of view: " + - "%s.%s", db_.getName(), name_)); + "%s", view.getFullName())); } // Make sure the view definition parses to a query statement. if (!(node instanceof QueryStmt)) { - throw new TableLoadingException(String.format("View definition of %s.%s " + - "is not a query statement", db_.getName(), name_)); + throw new TableLoadingException(String.format("View definition of %s " + + "is not a query statement", view.getFullName())); } - queryStmt_ = (QueryStmt) node; + return (QueryStmt) node; } @Override http://git-wip-us.apache.org/repos/asf/impala/blob/f14ae3fc/fe/src/main/java/org/apache/impala/catalog/local/LocalTable.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/org/apache/impala/catalog/local/LocalTable.java b/fe/src/main/java/org/apache/impala/catalog/local/LocalTable.java index 4b57cb8..d128ca2 100644 --- a/fe/src/main/java/org/apache/impala/catalog/local/LocalTable.java +++ b/fe/src/main/java/org/apache/impala/catalog/local/LocalTable.java @@ -40,7 +40,6 @@ import org.apache.impala.catalog.KuduTable; import org.apache.impala.catalog.StructField; import org.apache.impala.catalog.StructType; import org.apache.impala.catalog.TableLoadingException; -import org.apache.impala.catalog.View; import org.apache.impala.thrift.TCatalogObjectType; import org.apache.impala.thrift.TTableStats; import org.apache.log4j.Logger; @@ -73,7 +72,7 @@ abstract class LocalTable implements FeTable { LocalTable t = null; Table msTbl = schemaInfo.msTable_; if (TableType.valueOf(msTbl.getTableType()) == TableType.VIRTUAL_VIEW) { - // TODO(todd) support View + t = new LocalView(db, tblName, schemaInfo); } else if (HBaseTable.isHBaseTable(msTbl)) { // TODO(todd) support HBase table } else if (KuduTable.isKuduTable(msTbl)) { http://git-wip-us.apache.org/repos/asf/impala/blob/f14ae3fc/fe/src/main/java/org/apache/impala/catalog/local/LocalView.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/org/apache/impala/catalog/local/LocalView.java b/fe/src/main/java/org/apache/impala/catalog/local/LocalView.java new file mode 100644 index 0000000..c380eb1 --- /dev/null +++ b/fe/src/main/java/org/apache/impala/catalog/local/LocalView.java @@ -0,0 +1,83 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.impala.catalog.local; + +import java.util.List; +import java.util.Set; + +import org.apache.impala.analysis.QueryStmt; +import org.apache.impala.catalog.FeView; +import org.apache.impala.catalog.TableLoadingException; +import org.apache.impala.catalog.View; +import org.apache.impala.thrift.TCatalogObjectType; +import org.apache.impala.thrift.TTableDescriptor; + +/** + * View implementation corresponding to the LocalCatalog. + * + * NOTE: this does _not_ implement "local views" in the sense of a view defined + * in the scope of a query using a WITH expression. + */ +public class LocalView extends LocalTable implements FeView { + private final QueryStmt queryStmt_; + + public LocalView(LocalDb db, String tblName, SchemaInfo schemaInfo) { + super(db, tblName, schemaInfo); + + try { + queryStmt_ = View.parseViewDef(this); + } catch (TableLoadingException e) { + throw new LocalCatalogException(e); + } + } + + @Override + public TTableDescriptor toThriftDescriptor(int tableId, + Set<Long> referencedPartitions) { + // Views are always resolved into underlying tables before being serialized. + throw new IllegalStateException("Cannot call toThriftDescriptor() on a view."); + } + + // NOTE: "local view" in this context means "a view defined local to a query" + // whereas "local" in the class name "LocalView" means "view from the + // LocalCatalog catalog implementation". + @Override + public boolean isLocalView() { + // TODO: the org.apache.impala.catalog.View class is still used for local views + // even in the LocalCatalog implementation. + return false; + } + + @Override + public QueryStmt getQueryStmt() { + return queryStmt_; + } + + @Override + public List<String> getColLabels() { + // Explicit column labels are only used by local views. + // Returning null indicates that the column labels are derived + // from the underlying statement. + return null; + } + + @Override + public TCatalogObjectType getCatalogObjectType() { + return TCatalogObjectType.VIEW; + } +} http://git-wip-us.apache.org/repos/asf/impala/blob/f14ae3fc/fe/src/test/java/org/apache/impala/catalog/local/LocalCatalogTest.java ---------------------------------------------------------------------- diff --git a/fe/src/test/java/org/apache/impala/catalog/local/LocalCatalogTest.java b/fe/src/test/java/org/apache/impala/catalog/local/LocalCatalogTest.java index e334d20..d85b8e3 100644 --- a/fe/src/test/java/org/apache/impala/catalog/local/LocalCatalogTest.java +++ b/fe/src/test/java/org/apache/impala/catalog/local/LocalCatalogTest.java @@ -29,9 +29,11 @@ import org.apache.impala.catalog.FeDb; import org.apache.impala.catalog.FeFsPartition; import org.apache.impala.catalog.FeFsTable; import org.apache.impala.catalog.FeTable; +import org.apache.impala.catalog.FeView; import org.apache.impala.catalog.HdfsPartition.FileDescriptor; import org.apache.impala.catalog.Type; import org.apache.impala.service.FeSupport; +import org.apache.impala.thrift.TCatalogObjectType; import org.apache.impala.thrift.TResultSet; import org.apache.impala.util.MetaStoreUtil; import org.apache.impala.util.PatternMatcher; @@ -167,4 +169,11 @@ public class LocalCatalogTest { assertEquals(10210, stats.getNumDistinctValues()); assertEquals(-1, stats.getNumNulls()); } + + @Test + public void testView() throws Exception { + FeView v = (FeView) catalog_.getTable("functional", "alltypes_view"); + assertEquals(TCatalogObjectType.VIEW, v.getCatalogObjectType()); + assertEquals("SELECT * FROM functional.alltypes", v.getQueryStmt().toSql()); + } }