DRILL-858: Fix bugs in handling Hive views in INFORMATION_SCHEMA generation.
Also: - Added a new interface DrillViewInfoProvider which provides view related info to components such as InfoSchema. Drill or Hive view table implement this interface. - Throw UnsupportedOperationException if any queries use Hive views as querying Hive views is not supported in current version. Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/60a429fb Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/60a429fb Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/60a429fb Branch: refs/heads/master Commit: 60a429fb7815c655234055ded237dbda1af2d769 Parents: b328d7b Author: vkorukanti <[email protected]> Authored: Wed Jun 11 17:13:59 2014 -0700 Committer: Jacques Nadeau <[email protected]> Committed: Wed Jun 11 21:12:44 2014 -0700 ---------------------------------------------------------------------- .../planner/logical/DrillViewInfoProvider.java | 28 ++++++++++++++ .../exec/planner/logical/DrillViewTable.java | 3 +- .../drill/exec/store/hive/HiveReadEntry.java | 9 +++++ .../exec/store/hive/HiveStoragePlugin.java | 5 +++ .../exec/store/hive/schema/DrillHiveTable.java | 2 +- .../store/hive/schema/DrillHiveViewTable.java | 40 ++++++++++++++++++++ .../store/hive/schema/HiveSchemaFactory.java | 7 +++- .../exec/store/ischema/RecordGenerator.java | 4 +- .../exec/store/hive/HiveTestDataGenerator.java | 3 ++ .../apache/drill/jdbc/test/TestMetadataDDL.java | 2 + .../org/apache/drill/jdbc/test/TestViews.java | 8 ++++ 11 files changed, 106 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/60a429fb/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillViewInfoProvider.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillViewInfoProvider.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillViewInfoProvider.java new file mode 100644 index 0000000..50e1d8f --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillViewInfoProvider.java @@ -0,0 +1,28 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.planner.logical; + +/** + * Interface used by Drill components such as InformationSchema generator to get view info. + * All view tables need to implement this interface. + */ +public interface DrillViewInfoProvider { + + /** Get the query part of the view. */ + String getViewSql(); +} http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/60a429fb/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillViewTable.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillViewTable.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillViewTable.java index aaf32ad..19fd7ba 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillViewTable.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillViewTable.java @@ -32,7 +32,7 @@ import org.eigenbase.relopt.RelOptUtil; import org.eigenbase.reltype.RelDataType; import org.eigenbase.reltype.RelDataTypeFactory; -public class DrillViewTable implements TranslatableTable{ +public class DrillViewTable implements TranslatableTable, DrillViewInfoProvider { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillViewTable.class); private final View view; @@ -71,6 +71,7 @@ public class DrillViewTable implements TranslatableTable{ return TableType.VIEW; } + @Override public String getViewSql() { return view.getSql(); } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/60a429fb/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveReadEntry.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveReadEntry.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveReadEntry.java index f330a1e..855b05f 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveReadEntry.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveReadEntry.java @@ -19,6 +19,7 @@ package org.apache.drill.exec.store.hive; import java.util.List; +import net.hydromatic.optiq.Schema.TableType; import org.apache.drill.exec.physical.OperatorCost; import org.apache.drill.exec.physical.base.Size; import org.apache.hadoop.hive.metastore.api.Partition; @@ -60,5 +61,13 @@ public class HiveReadEntry { return partitionsUnwrapped; } + @JsonIgnore + public TableType getJdbcTableType() { + if (table.getTable().getTableType().equals(org.apache.hadoop.hive.metastore.TableType.VIRTUAL_VIEW.toString())) { + return TableType.VIEW; + } + + return TableType.TABLE; + } } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/60a429fb/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveStoragePlugin.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveStoragePlugin.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveStoragePlugin.java index 6d5f6d2..c5a6e2c 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveStoragePlugin.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveStoragePlugin.java @@ -21,6 +21,7 @@ import java.io.IOException; import java.util.List; import net.hydromatic.optiq.Schema; +import net.hydromatic.optiq.Schema.TableType; import net.hydromatic.optiq.SchemaPlus; import org.apache.drill.common.JSONOptions; @@ -67,6 +68,10 @@ public class HiveStoragePlugin extends AbstractStoragePlugin { public HiveScan getPhysicalScan(JSONOptions selection, List<SchemaPath> columns) throws IOException { HiveReadEntry hiveReadEntry = selection.getListWith(new ObjectMapper(), new TypeReference<HiveReadEntry>(){}); try { + if (hiveReadEntry.getJdbcTableType() == TableType.VIEW) { + throw new UnsupportedOperationException("Querying Hive views from Drill is not supported in current version."); + } + return new HiveScan(hiveReadEntry, this, null); } catch (ExecutionSetupException e) { throw new IOException(e); http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/60a429fb/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/schema/DrillHiveTable.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/schema/DrillHiveTable.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/schema/DrillHiveTable.java index d1a5659..547c0bb 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/schema/DrillHiveTable.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/schema/DrillHiveTable.java @@ -41,7 +41,7 @@ import org.eigenbase.sql.type.SqlTypeName; public class DrillHiveTable extends DrillTable{ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillHiveTable.class); - private final Table hiveTable; + protected final Table hiveTable; public DrillHiveTable(String storageEngineName, HiveStoragePlugin plugin, HiveReadEntry readEntry) { super(storageEngineName, plugin, readEntry); http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/60a429fb/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/schema/DrillHiveViewTable.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/schema/DrillHiveViewTable.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/schema/DrillHiveViewTable.java new file mode 100644 index 0000000..b575972 --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/schema/DrillHiveViewTable.java @@ -0,0 +1,40 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.hive.schema; + +import net.hydromatic.optiq.Schema.TableType; +import org.apache.drill.exec.planner.logical.DrillViewInfoProvider; +import org.apache.drill.exec.store.hive.HiveReadEntry; +import org.apache.drill.exec.store.hive.HiveStoragePlugin; + +public class DrillHiveViewTable extends DrillHiveTable implements DrillViewInfoProvider { + + public DrillHiveViewTable(String storageEngineName, HiveStoragePlugin plugin, HiveReadEntry readEntry) { + super(storageEngineName, plugin, readEntry); + } + + @Override + public TableType getJdbcTableType() { + return TableType.VIEW; + } + + @Override + public String getViewSql() { + return hiveTable.getViewExpandedText(); + } +} http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/60a429fb/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/schema/HiveSchemaFactory.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/schema/HiveSchemaFactory.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/schema/HiveSchemaFactory.java index fb672d4..7e6b92b 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/schema/HiveSchemaFactory.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/schema/HiveSchemaFactory.java @@ -258,7 +258,12 @@ public class HiveSchemaFactory implements SchemaFactory { DrillTable getDrillTable(String dbName, String t){ HiveReadEntry entry = getSelectionBaseOnName(dbName, t); if(entry == null) return null; - return new DrillHiveTable(schemaName, plugin, entry); + + if (entry.getJdbcTableType() == TableType.VIEW) { + return new DrillHiveViewTable(schemaName, plugin, entry); + } else { + return new DrillHiveTable(schemaName, plugin, entry); + } } HiveReadEntry getSelectionBaseOnName(String dbName, String t) { http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/60a429fb/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/RecordGenerator.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/RecordGenerator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/RecordGenerator.java index a8792fe..df67da1 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/RecordGenerator.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/RecordGenerator.java @@ -23,7 +23,7 @@ import net.hydromatic.optiq.Schema.TableType; import net.hydromatic.optiq.SchemaPlus; import net.hydromatic.optiq.Table; import net.hydromatic.optiq.jdbc.JavaTypeFactoryImpl; -import org.apache.drill.exec.planner.logical.DrillViewTable; +import org.apache.drill.exec.planner.logical.DrillViewInfoProvider; import org.apache.drill.exec.store.AbstractSchema; import org.apache.drill.exec.store.RecordReader; import org.apache.drill.exec.store.pojo.PojoRecordReader; @@ -148,7 +148,7 @@ public abstract class RecordGenerator { @Override public boolean visitTable(String schemaName, String tableName, Table table) { if (table.getJdbcTableType() == TableType.VIEW) { - records.add(new Records.View("DRILL", schemaName, tableName, ((DrillViewTable)table).getViewSql())); + records.add(new Records.View("DRILL", schemaName, tableName, ((DrillViewInfoProvider) table).getViewSql())); } return false; } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/60a429fb/exec/java-exec/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java index 8433931..f914661 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java @@ -118,6 +118,9 @@ public class HiveTestDataGenerator { "uniontypeType UNIONTYPE<int, double, array<string>>)" ); + // create a Hive view to test how its metadata is populated in Drill's INFORMATION_SCHEMA + executeQuery("CREATE VIEW IF NOT EXISTS hiveview AS SELECT * FROM kv"); + ss.close(); } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/60a429fb/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestMetadataDDL.java ---------------------------------------------------------------------- diff --git a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestMetadataDDL.java b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestMetadataDDL.java index 501556a..123c160 100644 --- a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestMetadataDDL.java +++ b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestMetadataDDL.java @@ -51,6 +51,7 @@ public class TestMetadataDDL extends TestJdbcQuery { .returns( "TABLE_SCHEMA=hive.default; TABLE_NAME=empty_table\n" + "TABLE_SCHEMA=hive.default; TABLE_NAME=allhivedatatypes\n" + + "TABLE_SCHEMA=hive.default; TABLE_NAME=hiveview\n" + "TABLE_SCHEMA=hive.default; TABLE_NAME=allreadsupportedhivedatatypes\n" + "TABLE_SCHEMA=hive.default; TABLE_NAME=kv\n" + "TABLE_SCHEMA=hive.default; TABLE_NAME=foodate\n" @@ -74,6 +75,7 @@ public class TestMetadataDDL extends TestJdbcQuery { .returns( "TABLE_SCHEMA=hive.default; TABLE_NAME=empty_table\n" + "TABLE_SCHEMA=hive.default; TABLE_NAME=allhivedatatypes\n" + + "TABLE_SCHEMA=hive.default; TABLE_NAME=hiveview\n" + "TABLE_SCHEMA=hive.default; TABLE_NAME=allreadsupportedhivedatatypes\n" + "TABLE_SCHEMA=hive.default; TABLE_NAME=kv\n" + "TABLE_SCHEMA=hive.default; TABLE_NAME=foodate\n"); http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/60a429fb/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestViews.java ---------------------------------------------------------------------- diff --git a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestViews.java b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestViews.java index 7dc9c30..93626d5 100644 --- a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestViews.java +++ b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestViews.java @@ -302,6 +302,14 @@ public class TestViews extends TestJdbcQuery { } @Test + public void testInfoSchemaWithHiveView() throws Exception { + JdbcAssert.withFull("hive.default") + .sql("SELECT * FROM INFORMATION_SCHEMA.VIEWS WHERE TABLE_NAME = 'hiveview'") + .returns("TABLE_CATALOG=DRILL; TABLE_SCHEMA=hive.default; TABLE_NAME=hiveview; " + + "VIEW_DEFINITION=SELECT `kv`.`key`, `kv`.`value` FROM `default`.`kv`"); + } + + @Test public void testViewWithFullSchemaIdentifier() throws Exception{ JdbcAssert.withNoDefaultSchema().withConnection(new Function<Connection, Void>() { public Void apply(Connection connection) {
