Repository: drill Updated Branches: refs/heads/master 5705d4509 -> b8f6ebc65
DRILL-4577: Construct a specific path for querying all the tables from a hive database Project: http://git-wip-us.apache.org/repos/asf/drill/repo Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/b8f6ebc6 Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/b8f6ebc6 Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/b8f6ebc6 Branch: refs/heads/master Commit: b8f6ebc651445ccecd3e393250f6cd2781fc07e3 Parents: 5705d45 Author: Hsuan-Yi Chu <[email protected]> Authored: Mon Apr 4 16:05:04 2016 -0700 Committer: hsuanyi <[email protected]> Committed: Mon May 2 21:41:32 2016 -0700 ---------------------------------------------------------------------- .../store/hive/schema/HiveDatabaseSchema.java | 71 ++++++++++- .../hive/BaseTestHiveImpersonation.java | 21 ++++ .../hive/TestStorageBasedHiveAuthorization.java | 125 +++++++++++++++++++ .../org/apache/drill/exec/ExecConstants.java | 3 + .../server/options/SystemOptionManager.java | 3 +- .../apache/drill/exec/store/AbstractSchema.java | 39 +++++- .../store/ischema/InfoSchemaBatchCreator.java | 2 +- .../exec/store/ischema/InfoSchemaTable.java | 23 ++-- .../exec/store/ischema/RecordGenerator.java | 85 ++++++++++--- .../drill/exec/store/ischema/SelectedTable.java | 5 +- 10 files changed, 344 insertions(+), 33 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/drill/blob/b8f6ebc6/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/schema/HiveDatabaseSchema.java ---------------------------------------------------------------------- diff --git a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/schema/HiveDatabaseSchema.java b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/schema/HiveDatabaseSchema.java index 6f43639..ff61f8d 100644 --- a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/schema/HiveDatabaseSchema.java +++ b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/schema/HiveDatabaseSchema.java @@ -17,17 +17,25 @@ */ package org.apache.drill.exec.store.hive.schema; +import java.util.List; import java.util.Set; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; + +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.schema.Schema; +import org.apache.calcite.schema.Statistic; import org.apache.calcite.schema.Table; +import org.apache.commons.lang3.tuple.Pair; import org.apache.drill.exec.store.AbstractSchema; import org.apache.drill.exec.store.SchemaConfig; import org.apache.drill.exec.store.hive.DrillHiveMetaStoreClient; import org.apache.drill.exec.store.hive.HiveStoragePluginConfig; import org.apache.drill.exec.store.hive.schema.HiveSchemaFactory.HiveSchema; -import com.google.common.collect.Sets; import org.apache.thrift.TException; public class HiveDatabaseSchema extends AbstractSchema{ @@ -72,4 +80,65 @@ public class HiveDatabaseSchema extends AbstractSchema{ return HiveStoragePluginConfig.NAME; } + @Override + public List<Pair<String, ? extends Table>> getTablesByNamesByBulkLoad(final List<String> tableNames) { + final String schemaName = getName(); + final List<Pair<String, ? extends Table>> tableNameToTable = Lists.newArrayList(); + List<org.apache.hadoop.hive.metastore.api.Table> tables; + // Retries once if the first call to fetch the metadata fails + synchronized(mClient) { + try { + tables = mClient.getTableObjectsByName(schemaName, tableNames); + } catch(TException tException) { + try { + mClient.reconnect(); + tables = mClient.getTableObjectsByName(schemaName, tableNames); + } catch(Exception e) { + logger.warn("Exception occurred while trying to read tables from {}: {}", schemaName, e.getCause()); + return tableNameToTable; + } + } + } + + for(final org.apache.hadoop.hive.metastore.api.Table table : tables) { + if(table == null) { + continue; + } + + final String tableName = table.getTableName(); + final TableType tableType; + if(table.getTableType().equals(org.apache.hadoop.hive.metastore.TableType.VIRTUAL_VIEW.toString())) { + tableType = TableType.VIEW; + } else { + tableType = TableType.TABLE; + } + tableNameToTable.add(Pair.of( + tableName, + new HiveTableWithoutStatisticAndRowType(tableType))); + } + return tableNameToTable; + } + + private static class HiveTableWithoutStatisticAndRowType implements Table { + private final TableType tableType; + + public HiveTableWithoutStatisticAndRowType(final TableType tableType) { + this.tableType = tableType; + } + + @Override + public RelDataType getRowType(RelDataTypeFactory typeFactory) { + throw new UnsupportedOperationException("RowType was not retrieved when this table had been being requested"); + } + + @Override + public Statistic getStatistic() { + throw new UnsupportedOperationException("Statistic was not retrieved when this table had been being requested"); + } + + @Override + public Schema.TableType getJdbcTableType() { + return tableType; + } + } } http://git-wip-us.apache.org/repos/asf/drill/blob/b8f6ebc6/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/impersonation/hive/BaseTestHiveImpersonation.java ---------------------------------------------------------------------- diff --git a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/impersonation/hive/BaseTestHiveImpersonation.java b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/impersonation/hive/BaseTestHiveImpersonation.java index 35c9d64..66c595f 100644 --- a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/impersonation/hive/BaseTestHiveImpersonation.java +++ b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/impersonation/hive/BaseTestHiveImpersonation.java @@ -17,6 +17,7 @@ */ package org.apache.drill.exec.impersonation.hive; +import org.apache.calcite.schema.Schema.TableType; import org.apache.drill.TestBuilder; import org.apache.drill.exec.ExecConstants; import org.apache.drill.exec.dotdrill.DotDrillType; @@ -124,6 +125,26 @@ public class BaseTestHiveImpersonation extends BaseTestImpersonation { testBuilder.go(); } + protected void fromInfoSchemaHelper(final String pluginName, final String db, List<String> expectedTables, List<TableType> expectedTableTypes) throws Exception { + final String dbQualified = pluginName + "." + db; + final TestBuilder testBuilder = testBuilder() + .sqlQuery("SELECT TABLE_SCHEMA, TABLE_NAME, TABLE_TYPE \n" + + "FROM INFORMATION_SCHEMA.`TABLES` \n" + + "WHERE TABLE_SCHEMA = '" + dbQualified + "'") + .unOrdered() + .baselineColumns("TABLE_SCHEMA", "TABLE_NAME", "TABLE_TYPE"); + + if (expectedTables.size() == 0) { + testBuilder.expectsEmptyResultSet(); + } else { + for (int i = 0; i < expectedTables.size(); ++i) { + testBuilder.baselineValues(dbQualified, expectedTables.get(i), expectedTableTypes.get(i).toString()); + } + } + + testBuilder.go(); + } + protected static void createView(final String viewOwner, final String viewGroup, final String viewName, final String viewDef) throws Exception { updateClient(viewOwner); http://git-wip-us.apache.org/repos/asf/drill/blob/b8f6ebc6/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/impersonation/hive/TestStorageBasedHiveAuthorization.java ---------------------------------------------------------------------- diff --git a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/impersonation/hive/TestStorageBasedHiveAuthorization.java b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/impersonation/hive/TestStorageBasedHiveAuthorization.java index 21559c9..978c34e 100644 --- a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/impersonation/hive/TestStorageBasedHiveAuthorization.java +++ b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/impersonation/hive/TestStorageBasedHiveAuthorization.java @@ -19,6 +19,8 @@ package org.apache.drill.exec.impersonation.hive; import com.google.common.collect.ImmutableList; import com.google.common.collect.Maps; + +import org.apache.calcite.schema.Schema.TableType; import org.apache.drill.exec.store.dfs.WorkspaceConfig; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; @@ -269,6 +271,47 @@ public class TestStorageBasedHiveAuthorization extends BaseTestHiveImpersonation } @Test + public void fromInfoSchemaUser0() throws Exception { + updateClient(org1Users[0]); + + fromInfoSchemaHelper( + hivePluginName, + db_general, + ImmutableList.of( + g_student_u0_700, + g_student_u0g0_750, + g_student_all_755, + g_voter_all_755, + g_partitioned_student_u0_700 + ), + ImmutableList.of( + TableType.TABLE, + TableType.TABLE, + TableType.TABLE, + TableType.TABLE, + TableType.TABLE + )); + + fromInfoSchemaHelper( + hivePluginName, + db_u0_only, + ImmutableList.of( + u0_student_all_755, + u0_voter_all_755 + ), + ImmutableList.of( + TableType.TABLE, + TableType.TABLE + )); + + fromInfoSchemaHelper( + hivePluginName, + db_u1g1_only, + Collections.<String>emptyList(), + Collections.<TableType>emptyList()); + } + + @Test public void showTablesUser1() throws Exception { updateClient(org1Users[1]); @@ -293,6 +336,51 @@ public class TestStorageBasedHiveAuthorization extends BaseTestHiveImpersonation } @Test + public void fromInfoSchemaUser1() throws Exception { + updateClient(org1Users[1]); + + fromInfoSchemaHelper( + hivePluginName, + db_general, + ImmutableList.of( + g_student_u0g0_750, + g_student_all_755, + g_voter_u1_700, + g_voter_u2g1_750, + g_voter_all_755 + ), + ImmutableList.of( + TableType.TABLE, + TableType.TABLE, + TableType.TABLE, + TableType.TABLE, + TableType.TABLE + )); + + fromInfoSchemaHelper( + hivePluginName, + db_u1g1_only, + ImmutableList.of( + u1g1_student_all_755, + u1g1_student_u1_700, + u1g1_voter_all_755, + u1g1_voter_u1_700 + ), + ImmutableList.of( + TableType.TABLE, + TableType.TABLE, + TableType.TABLE, + TableType.TABLE + )); + + fromInfoSchemaHelper( + hivePluginName, + db_u0_only, + Collections.<String>emptyList(), + Collections.<TableType>emptyList()); + } + + @Test public void showTablesUser2() throws Exception { updateClient(org1Users[2]); @@ -312,6 +400,43 @@ public class TestStorageBasedHiveAuthorization extends BaseTestHiveImpersonation showTablesHelper(db_u0_only, Collections.<String>emptyList()); } + @Test + public void fromInfoSchemaUser2() throws Exception { + updateClient(org1Users[2]); + + fromInfoSchemaHelper( + hivePluginName, + db_general, + ImmutableList.of( + g_student_all_755, + g_voter_u2g1_750, + g_voter_all_755 + ), + ImmutableList.of( + TableType.TABLE, + TableType.TABLE, + TableType.TABLE + )); + + fromInfoSchemaHelper( + hivePluginName, + db_u1g1_only, + ImmutableList.of( + u1g1_student_all_755, + u1g1_voter_all_755 + ), + ImmutableList.of( + TableType.TABLE, + TableType.TABLE + )); + + fromInfoSchemaHelper( + hivePluginName, + db_u0_only, + Collections.<String>emptyList(), + Collections.<TableType>emptyList()); + } + // Try to read the tables "user0" has access to read in db_general. @Test public void selectUser0_db_general() throws Exception { http://git-wip-us.apache.org/repos/asf/drill/blob/b8f6ebc6/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java index 6a0889d..7f216f0 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java @@ -270,6 +270,9 @@ public interface ExecConstants { String CTAS_PARTITIONING_HASH_DISTRIBUTE = "store.partition.hash_distribute"; BooleanValidator CTAS_PARTITIONING_HASH_DISTRIBUTE_VALIDATOR = new BooleanValidator(CTAS_PARTITIONING_HASH_DISTRIBUTE, false); + String ENABLE_BULK_LOAD_TABLE_LIST_KEY = "exec.enable_bulk_load_table_list"; + BooleanValidator ENABLE_BULK_LOAD_TABLE_LIST = new BooleanValidator(ENABLE_BULK_LOAD_TABLE_LIST_KEY, false); + /** * Option whose value is a comma separated list of admin usernames. Admin users are users who have special privileges * such as changing system options. http://git-wip-us.apache.org/repos/asf/drill/blob/b8f6ebc6/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java index 0abdb76..db78108 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java @@ -134,7 +134,8 @@ public class SystemOptionManager extends BaseOptionManager implements AutoClosea ExecConstants.ENABLE_VERBOSE_ERRORS, ExecConstants.ENABLE_WINDOW_FUNCTIONS_VALIDATOR, ClassTransformer.SCALAR_REPLACEMENT_VALIDATOR, - ExecConstants.ENABLE_NEW_TEXT_READER + ExecConstants.ENABLE_NEW_TEXT_READER, + ExecConstants.ENABLE_BULK_LOAD_TABLE_LIST }; final Map<String, OptionValidator> tmp = new HashMap<>(); for (final OptionValidator validator : validators) { http://git-wip-us.apache.org/repos/asf/drill/blob/b8f6ebc6/exec/java-exec/src/main/java/org/apache/drill/exec/store/AbstractSchema.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/AbstractSchema.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/AbstractSchema.java index 2403252..f7ec3fe 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/AbstractSchema.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/AbstractSchema.java @@ -25,10 +25,12 @@ import java.util.Set; import org.apache.calcite.linq4j.tree.DefaultExpression; import org.apache.calcite.linq4j.tree.Expression; +import org.apache.calcite.rel.type.RelDataTypeFactory; import org.apache.calcite.schema.Function; import org.apache.calcite.schema.Schema; import org.apache.calcite.schema.SchemaPlus; import org.apache.calcite.schema.Table; +import org.apache.commons.lang3.tuple.Pair; import org.apache.drill.common.exceptions.UserException; import org.apache.drill.exec.dotdrill.View; import org.apache.drill.exec.planner.logical.CreateTableEntry; @@ -194,4 +196,39 @@ public abstract class AbstractSchema implements Schema, SchemaPartitionExplorer, .message("Dropping tables is not supported in schema [%s]", getSchemaPath()) .build(logger); } -} + + /** + * Get the collection of {@link Table} tables specified in the tableNames with bulk-load (if the underlying storage + * plugin supports). + * It is not guaranteed that the retrieved tables would have RowType and Statistic being fully populated. + * + * Specifically, calling {@link Table#getRowType(RelDataTypeFactory)} or {@link Table#getStatistic()} might incur + * {@link UnsupportedOperationException} being thrown. + * + * @param tableNames the requested tables, specified by the table names + * @return the collection of requested tables + */ + public List<Pair<String, ? extends Table>> getTablesByNamesByBulkLoad(final List<String> tableNames) { + return getTablesByNames(tableNames); + } + + /** + * Get the collection of {@link Table} tables specified in the tableNames. + * + * @param tableNames the requested tables, specified by the table names + * @return the collection of requested tables + */ + public List<Pair<String, ? extends Table>> getTablesByNames(final List<String> tableNames) { + final List<Pair<String, ? extends Table>> tables = Lists.newArrayList(); + for (String tableName : tableNames) { + final Table table = getTable(tableName); + if (table == null) { + // Schema may return NULL for table if the query user doesn't have permissions to load the table. Ignore such + // tables as INFO SCHEMA is about showing tables which the use has access to query. + continue; + } + tables.add(Pair.of(tableName, table)); + } + return tables; + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/drill/blob/b8f6ebc6/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaBatchCreator.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaBatchCreator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaBatchCreator.java index 2ef2333..199119d 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaBatchCreator.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaBatchCreator.java @@ -33,7 +33,7 @@ public class InfoSchemaBatchCreator implements BatchCreator<InfoSchemaSubScan>{ @Override public ScanBatch getBatch(FragmentContext context, InfoSchemaSubScan config, List<RecordBatch> children) throws ExecutionSetupException { - RecordReader rr = config.getTable().getRecordReader(context.getRootSchema(), config.getFilter()); + RecordReader rr = config.getTable().getRecordReader(context.getRootSchema(), config.getFilter(), context.getOptions()); return new ScanBatch(config, context, Collections.singleton(rr).iterator()); } } http://git-wip-us.apache.org/repos/asf/drill/blob/b8f6ebc6/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaTable.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaTable.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaTable.java index 3f8d35f..a6d56b3 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaTable.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaTable.java @@ -29,6 +29,7 @@ import org.apache.calcite.sql.type.SqlTypeName; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; +import org.apache.drill.exec.server.options.OptionManager; /** * Base class for tables in INFORMATION_SCHEMA. Defines the table (fields and @@ -85,7 +86,7 @@ public abstract class InfoSchemaTable { return typeFactory.createStructType(relTypes, fieldNames); } - public abstract RecordGenerator getRecordGenerator(); + public abstract RecordGenerator getRecordGenerator(OptionManager optionManager); /** Layout for the CATALOGS table. */ static public class Catalogs extends InfoSchemaTable { @@ -101,8 +102,8 @@ public abstract class InfoSchemaTable { } @Override - public RecordGenerator getRecordGenerator() { - return new RecordGenerator.Catalogs(); + public RecordGenerator getRecordGenerator(OptionManager optionManager) { + return new RecordGenerator.Catalogs(optionManager); } } @@ -122,8 +123,8 @@ public abstract class InfoSchemaTable { } @Override - public RecordGenerator getRecordGenerator() { - return new RecordGenerator.Schemata(); + public RecordGenerator getRecordGenerator(OptionManager optionManager) { + return new RecordGenerator.Schemata(optionManager); } } @@ -142,8 +143,8 @@ public abstract class InfoSchemaTable { } @Override - public RecordGenerator getRecordGenerator() { - return new RecordGenerator.Tables(); + public RecordGenerator getRecordGenerator(OptionManager optionManager) { + return new RecordGenerator.Tables(optionManager); } } @@ -162,8 +163,8 @@ public abstract class InfoSchemaTable { } @Override - public RecordGenerator getRecordGenerator() { - return new RecordGenerator.Views(); + public RecordGenerator getRecordGenerator(OptionManager optionManager) { + return new RecordGenerator.Views(optionManager); } } @@ -214,8 +215,8 @@ public abstract class InfoSchemaTable { } @Override - public RecordGenerator getRecordGenerator() { - return new RecordGenerator.Columns(); + public RecordGenerator getRecordGenerator(OptionManager optionManager) { + return new RecordGenerator.Columns(optionManager); } } } http://git-wip-us.apache.org/repos/asf/drill/blob/b8f6ebc6/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/RecordGenerator.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/RecordGenerator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/RecordGenerator.java index f464727..29ccbce 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/RecordGenerator.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/RecordGenerator.java @@ -31,7 +31,10 @@ import org.apache.calcite.rel.type.RelDataTypeField; import org.apache.calcite.schema.Schema.TableType; import org.apache.calcite.schema.SchemaPlus; import org.apache.calcite.schema.Table; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.drill.exec.ExecConstants; import org.apache.drill.exec.planner.logical.DrillViewInfoProvider; +import org.apache.drill.exec.server.options.OptionManager; import org.apache.drill.exec.store.AbstractSchema; import org.apache.drill.exec.store.RecordReader; import org.apache.drill.exec.store.ischema.InfoSchemaFilter.Result; @@ -48,6 +51,11 @@ import com.google.common.collect.Lists; public abstract class RecordGenerator { protected InfoSchemaFilter filter; + protected OptionManager optionManager; + public RecordGenerator(OptionManager optionManager) { + this.optionManager = optionManager; + } + public void setInfoSchemaFilter(InfoSchemaFilter filter) { this.filter = filter; } @@ -124,29 +132,37 @@ public abstract class RecordGenerator { // Visit this schema and if requested ... if (shouldVisitSchema(schemaPath, schema) && visitSchema(schemaPath, schema)) { - // ... do for each of the schema's tables. - for (String tableName: schema.getTableNames()) { - Table table = schema.getTable(tableName); - - if (table == null) { - // Schema may return NULL for table if the query user doesn't have permissions to load the table. Ignore such - // tables as INFO SCHEMA is about showing tables which the use has access to query. - continue; - } + visitTables(schemaPath, schema); + } + } - // Visit the table, and if requested ... - if (shouldVisitTable(schemaPath, tableName) && visitTable(schemaPath, tableName, table)) { - // ... do for each of the table's fields. - RelDataType tableRow = table.getRowType(new JavaTypeFactoryImpl()); - for (RelDataTypeField field: tableRow.getFieldList()) { - visitField(schemaPath, tableName, field); - } + /** + * Visit the tables in the given schema. The + * @param schemaPath the path to the given schema + * @param schema the given schema + */ + public void visitTables(String schemaPath, SchemaPlus schema) { + final AbstractSchema drillSchema = schema.unwrap(AbstractSchema.class); + final List<String> tableNames = Lists.newArrayList(schema.getTableNames()); + for(Pair<String, ? extends Table> tableNameToTable : drillSchema.getTablesByNames(tableNames)) { + final String tableName = tableNameToTable.getKey(); + final Table table = tableNameToTable.getValue(); + // Visit the table, and if requested ... + if(shouldVisitTable(schemaPath, tableName) && visitTable(schemaPath, tableName, table)) { + // ... do for each of the table's fields. + final RelDataType tableRow = table.getRowType(new JavaTypeFactoryImpl()); + for (RelDataTypeField field: tableRow.getFieldList()) { + visitField(schemaPath, tableName, field); } } } } public static class Catalogs extends RecordGenerator { + public Catalogs(OptionManager optionManager) { + super(optionManager); + } + @Override public RecordReader getRecordReader() { Records.Catalog catalogRecord = @@ -159,6 +175,10 @@ public abstract class RecordGenerator { public static class Schemata extends RecordGenerator { List<Records.Schema> records = Lists.newArrayList(); + public Schemata(OptionManager optionManager) { + super(optionManager); + } + @Override public RecordReader getRecordReader() { return new PojoRecordReader<>(Records.Schema.class, records.iterator()); @@ -176,12 +196,38 @@ public abstract class RecordGenerator { public static class Tables extends RecordGenerator { List<Records.Table> records = Lists.newArrayList(); + public Tables(OptionManager optionManager) { + super(optionManager); + } + @Override public RecordReader getRecordReader() { return new PojoRecordReader<>(Records.Table.class, records.iterator()); } @Override + public void visitTables(String schemaPath, SchemaPlus schema) { + final AbstractSchema drillSchema = schema.unwrap(AbstractSchema.class); + + final List<String> tableNames = Lists.newArrayList(schema.getTableNames()); + final List<Pair<String, ? extends Table>> tableNameToTables; + if(optionManager.getOption(ExecConstants.ENABLE_BULK_LOAD_TABLE_LIST)) { + tableNameToTables = drillSchema.getTablesByNamesByBulkLoad(tableNames); + } else { + tableNameToTables = drillSchema.getTablesByNames(tableNames); + } + + for(Pair<String, ? extends Table> tableNameToTable : tableNameToTables) { + final String tableName = tableNameToTable.getKey(); + final Table table = tableNameToTable.getValue(); + // Visit the table, and if requested ... + if(shouldVisitTable(schemaPath, tableName)) { + visitTable(schemaPath, tableName, table); + } + } + } + + @Override public boolean visitTable(String schemaName, String tableName, Table table) { Preconditions.checkNotNull(table, "Error. Table %s.%s provided is null.", schemaName, tableName); @@ -198,6 +244,10 @@ public abstract class RecordGenerator { public static class Views extends RecordGenerator { List<Records.View> records = Lists.newArrayList(); + public Views(OptionManager optionManager) { + super(optionManager); + } + @Override public RecordReader getRecordReader() { return new PojoRecordReader<>(Records.View.class, records.iterator()); @@ -215,6 +265,9 @@ public abstract class RecordGenerator { public static class Columns extends RecordGenerator { List<Records.Column> records = Lists.newArrayList(); + public Columns(OptionManager optionManager) { + super(optionManager); + } @Override public RecordReader getRecordReader() { http://git-wip-us.apache.org/repos/asf/drill/blob/b8f6ebc6/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/SelectedTable.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/SelectedTable.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/SelectedTable.java index 79e7fd2..e2a2b2f 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/SelectedTable.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/SelectedTable.java @@ -19,6 +19,7 @@ package org.apache.drill.exec.store.ischema; import org.apache.calcite.schema.SchemaPlus; +import org.apache.drill.exec.server.options.OptionManager; import org.apache.drill.exec.store.RecordReader; import org.apache.drill.exec.store.ischema.InfoSchemaTable.Catalogs; import org.apache.drill.exec.store.ischema.InfoSchemaTable.Columns; @@ -51,8 +52,8 @@ public enum SelectedTable{ this.tableDef = tableDef; } - public RecordReader getRecordReader(SchemaPlus rootSchema, InfoSchemaFilter filter) { - RecordGenerator recordGenerator = tableDef.getRecordGenerator(); + public RecordReader getRecordReader(SchemaPlus rootSchema, InfoSchemaFilter filter, OptionManager optionManager) { + RecordGenerator recordGenerator = tableDef.getRecordGenerator(optionManager); recordGenerator.setInfoSchemaFilter(filter); recordGenerator.scanSchema(rootSchema); return recordGenerator.getRecordReader();
