http://git-wip-us.apache.org/repos/asf/impala/blob/ef15da08/fe/src/main/java/org/apache/impala/catalog/local/LocalFsTable.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/org/apache/impala/catalog/local/LocalFsTable.java b/fe/src/main/java/org/apache/impala/catalog/local/LocalFsTable.java index f19adfd..da784d6 100644 --- a/fe/src/main/java/org/apache/impala/catalog/local/LocalFsTable.java +++ b/fe/src/main/java/org/apache/impala/catalog/local/LocalFsTable.java @@ -44,6 +44,9 @@ import org.apache.impala.catalog.FeFsTable; import org.apache.impala.catalog.HdfsFileFormat; import org.apache.impala.catalog.HdfsTable; import org.apache.impala.catalog.PrunablePartition; +import org.apache.impala.catalog.local.MetaProvider.PartitionMetadata; +import org.apache.impala.catalog.local.MetaProvider.PartitionRef; +import org.apache.impala.catalog.local.MetaProvider.TableMetaRef; import org.apache.impala.common.AnalysisException; import org.apache.impala.thrift.CatalogObjectsConstants; import org.apache.impala.thrift.THdfsPartition; @@ -108,7 +111,7 @@ public class LocalFsTable extends LocalTable implements FeFsTable { */ private final String avroSchema_; - public static LocalFsTable load(LocalDb db, Table msTbl) { + public static LocalFsTable load(LocalDb db, Table msTbl, TableMetaRef ref) { String fullName = msTbl.getDbName() + "." + msTbl.getTableName(); // Set Avro schema if necessary. @@ -141,16 +144,16 @@ public class LocalFsTable extends LocalTable implements FeFsTable { cmap = ColumnMap.fromMsTable(msTbl); } - return new LocalFsTable(db, msTbl, cmap, avroSchema); + return new LocalFsTable(db, msTbl, ref, cmap, avroSchema); } catch (AnalysisException e) { throw new LocalCatalogException("Failed to load Avro schema for table " + fullName); } } - private LocalFsTable(LocalDb db, Table msTbl, ColumnMap cmap, + private LocalFsTable(LocalDb db, Table msTbl, TableMetaRef ref, ColumnMap cmap, String explicitAvroSchema) { - super(db, msTbl, cmap); + super(db, msTbl, ref, cmap); // set NULL indicator string from table properties String tableNullFormat = @@ -178,11 +181,7 @@ public class LocalFsTable extends LocalTable implements FeFsTable { */ public static LocalFsTable createCtasTarget(LocalDb db, Table msTbl) throws CatalogException { - // TODO(todd): set a member variable indicating this is a CTAS target - // so we can checkState() against it in various other methods and make - // sure we don't try to do something like load partitions for a not-yet-created - // table. - return new LocalFsTable(db, msTbl, ColumnMap.fromMsTable(msTbl), + return new LocalFsTable(db, msTbl, /*ref=*/null, ColumnMap.fromMsTable(msTbl), /*explicitAvroSchema=*/null); } @@ -240,7 +239,13 @@ public class LocalFsTable extends LocalTable implements FeFsTable { public Set<HdfsFileFormat> getFileFormats() { // TODO(todd): can we avoid loading all partitions here? this is called // for any INSERT query, even if the partition is specified. - Collection<? extends FeFsPartition> parts = FeCatalogUtils.loadAllPartitions(this); + Collection<? extends FeFsPartition> parts; + if (ref_ != null) { + parts = FeCatalogUtils.loadAllPartitions(this); + } else { + // If this is a CTAS target, we don't want to try to load the partition list. + parts = Collections.emptyList(); + } // In the case that we have no partitions added to the table yet, it's // important to add the "prototype" partition as a fallback. Iterable<FeFsPartition> partitionsToConsider = Iterables.concat( @@ -329,9 +334,9 @@ public class LocalFsTable extends LocalTable implements FeFsTable { protoMsPartition.setParameters(Collections.<String, String>emptyMap()); LocalPartitionSpec spec = new LocalPartitionSpec( - this, "", CatalogObjectsConstants.PROTOTYPE_PARTITION_ID); + this, CatalogObjectsConstants.PROTOTYPE_PARTITION_ID); LocalFsPartition prototypePartition = new LocalFsPartition( - this, spec, protoMsPartition); + this, spec, protoMsPartition, /*fileDescriptors=*/null); return prototypePartition; } @@ -374,29 +379,17 @@ public class LocalFsTable extends LocalTable implements FeFsTable { // Possible in the case that all partitions were pruned. if (ids.isEmpty()) return Collections.emptyList(); - List<String> names = Lists.newArrayList(); + List<PartitionRef> refs = Lists.newArrayList(); for (Long id : ids) { LocalPartitionSpec spec = partitionSpecs_.get(id); Preconditions.checkArgument(spec != null, "Invalid partition ID for table %s: %s", getFullName(), id); - String name = spec.getName(); - if (name.isEmpty()) { - // Unpartitioned tables don't need to fetch partitions from the metadata - // provider. Rather, we just create a partition on the fly. - Preconditions.checkState(getNumClusteringCols() == 0, - "Cannot fetch empty partition name from a partitioned table"); - Preconditions.checkArgument(ids.size() == 1, - "Expected to only fetch one partition for unpartitioned table %s", - getFullName()); - return Lists.newArrayList(createUnpartitionedPartition(spec)); - } else { - names.add(name); - } + refs.add(Preconditions.checkNotNull(spec.getRef())); } - Map<String, Partition> partsByName; + Map<String, PartitionMetadata> partsByName; try { - partsByName = db_.getCatalog().getMetaProvider().loadPartitionsByNames( - db_.getName(), name_, getClusteringColumnNames(), names); + partsByName = db_.getCatalog().getMetaProvider().loadPartitionsByRefs( + ref_, getClusteringColumnNames(), hostIndex_, refs); } catch (TException e) { throw new LocalCatalogException( "Could not load partitions for table " + getFullName(), e); @@ -404,16 +397,19 @@ public class LocalFsTable extends LocalTable implements FeFsTable { List<FeFsPartition> ret = Lists.newArrayListWithCapacity(ids.size()); for (Long id : ids) { LocalPartitionSpec spec = partitionSpecs_.get(id); - Partition p = partsByName.get(spec.getName()); + PartitionMetadata p = partsByName.get(spec.getRef().getName()); if (p == null) { // TODO(todd): concurrent drop partition could result in this error. // Should we recover in a more graceful way from such an unexpected event? throw new LocalCatalogException( "Could not load expected partitions for table " + getFullName() + - ": missing expected partition with name '" + spec.getName() + + ": missing expected partition with name '" + spec.getRef().getName() + "' (perhaps it was concurrently dropped by another process)"); } - ret.add(new LocalFsPartition(this, spec, p)); + + LocalFsPartition part = new LocalFsPartition(this, spec, p.getHmsPartition(), + p.getFileDescriptors()); + ret.add(part); } return ret; } @@ -426,23 +422,6 @@ public class LocalFsTable extends LocalTable implements FeFsTable { return names; } - /** - * Create a partition which represents the main partition of an unpartitioned - * table. - */ - private LocalFsPartition createUnpartitionedPartition(LocalPartitionSpec spec) { - Preconditions.checkArgument(spec.getName().isEmpty()); - Partition msp = new Partition(); - msp.setSd(getMetaStoreTable().getSd()); - msp.setParameters(getMetaStoreTable().getParameters()); - msp.setValues(Collections.<String>emptyList()); - return new LocalFsPartition(this, spec, msp); - } - - private LocalPartitionSpec createUnpartitionedPartitionSpec() { - return new LocalPartitionSpec(this, "", /*id=*/0); - } - private void loadPartitionValueMap() { if (partitionValueMap_ != null) return; @@ -478,28 +457,23 @@ public class LocalFsTable extends LocalTable implements FeFsTable { private void loadPartitionSpecs() { if (partitionSpecs_ != null) return; - - if (getNumClusteringCols() == 0) { - // Unpartitioned table. - // This table has no partition key, which means it has no declared partitions. - // We model partitions slightly differently to Hive - every file must exist in a - // partition, so add a single partition with no keys which will get all the - // files in the table's root directory. - partitionSpecs_ = ImmutableMap.of(0L, createUnpartitionedPartitionSpec()); + if (ref_ == null) { + // This is a CTAS target. Don't try to load metadata. + partitionSpecs_ = ImmutableMap.of(); return; } - List<String> partNames; + + List<PartitionRef> partList; try { - partNames = db_.getCatalog().getMetaProvider().loadPartitionNames( - db_.getName(), name_); + partList = db_.getCatalog().getMetaProvider().loadPartitionList(ref_); } catch (TException e) { throw new LocalCatalogException("Could not load partition names for table " + getFullName(), e); } ImmutableMap.Builder<Long, LocalPartitionSpec> b = new ImmutableMap.Builder<>(); long id = 0; - for (String partName : partNames) { - b.put(id, new LocalPartitionSpec(this, partName, id)); + for (PartitionRef part: partList) { + b.put(id, new LocalPartitionSpec(this, part, id)); id++; } partitionSpecs_ = b.build();
http://git-wip-us.apache.org/repos/asf/impala/blob/ef15da08/fe/src/main/java/org/apache/impala/catalog/local/LocalHbaseTable.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/org/apache/impala/catalog/local/LocalHbaseTable.java b/fe/src/main/java/org/apache/impala/catalog/local/LocalHbaseTable.java index 4ad2c14..8480500 100644 --- a/fe/src/main/java/org/apache/impala/catalog/local/LocalHbaseTable.java +++ b/fe/src/main/java/org/apache/impala/catalog/local/LocalHbaseTable.java @@ -24,6 +24,7 @@ import org.apache.hadoop.hive.serde2.SerDeException; import org.apache.impala.catalog.Column; import org.apache.impala.catalog.FeCatalogUtils; import org.apache.impala.catalog.FeHBaseTable; +import org.apache.impala.catalog.local.MetaProvider.TableMetaRef; import org.apache.impala.common.Pair; import org.apache.impala.thrift.TResultSet; import org.apache.impala.thrift.TTableDescriptor; @@ -44,19 +45,20 @@ public class LocalHbaseTable extends LocalTable implements FeHBaseTable { // TODO: revisit after caching is implemented for local catalog private HColumnDescriptor[] columnFamilies_ = null; - private LocalHbaseTable(LocalDb db, Table msTbl, ColumnMap cols) { - super(db, msTbl, cols); + private LocalHbaseTable(LocalDb db, Table msTbl, TableMetaRef ref, ColumnMap cols) { + super(db, msTbl, ref, cols); hbaseTableName_ = Util.getHBaseTableName(msTbl); } - static LocalHbaseTable loadFromHbase(LocalDb db, Table msTable) { + static LocalHbaseTable loadFromHbase(LocalDb db, Table msTable, TableMetaRef ref) { try { // Warm up the connection and verify the table exists. Util.getHBaseTable(Util.getHBaseTableName(msTable)).close(); // since we don't support composite hbase rowkeys yet, all hbase tables have a // single clustering col - return new LocalHbaseTable(db, msTable, new ColumnMap(Util.loadColumns(msTable), 1, - msTable.getDbName() + "." + msTable.getTableName())); + ColumnMap cmap = new ColumnMap(Util.loadColumns(msTable), 1, + msTable.getDbName() + "." + msTable.getTableName()); + return new LocalHbaseTable(db, msTable, ref, cmap); } catch (IOException | MetaException | SerDeException e) { throw new LocalCatalogException(e); } http://git-wip-us.apache.org/repos/asf/impala/blob/ef15da08/fe/src/main/java/org/apache/impala/catalog/local/LocalKuduTable.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/org/apache/impala/catalog/local/LocalKuduTable.java b/fe/src/main/java/org/apache/impala/catalog/local/LocalKuduTable.java index 2095449..fc48ca1 100644 --- a/fe/src/main/java/org/apache/impala/catalog/local/LocalKuduTable.java +++ b/fe/src/main/java/org/apache/impala/catalog/local/LocalKuduTable.java @@ -31,6 +31,7 @@ import org.apache.impala.catalog.FeCatalogUtils; import org.apache.impala.catalog.FeKuduTable; import org.apache.impala.catalog.KuduColumn; import org.apache.impala.catalog.KuduTable; +import org.apache.impala.catalog.local.MetaProvider.TableMetaRef; import org.apache.impala.common.ImpalaRuntimeException; import org.apache.impala.thrift.TKuduTable; import org.apache.impala.thrift.TTableDescriptor; @@ -55,7 +56,7 @@ public class LocalKuduTable extends LocalTable implements FeKuduTable { * Create a new instance based on the table metadata 'msTable' stored * in the metastore. */ - static LocalTable loadFromKudu(LocalDb db, Table msTable) { + static LocalTable loadFromKudu(LocalDb db, Table msTable, TableMetaRef ref) { Preconditions.checkNotNull(db); Preconditions.checkNotNull(msTable); String fullTableName = msTable.getDbName() + "." + msTable.getTableName(); @@ -82,7 +83,7 @@ public class LocalKuduTable extends LocalTable implements FeKuduTable { List<KuduPartitionParam> partitionBy = Utils.loadPartitionByParams(kuduTable); ColumnMap cmap = new ColumnMap(cols, /*numClusteringCols=*/0, fullTableName); - return new LocalKuduTable(db, msTable, cmap, pkNames, partitionBy); + return new LocalKuduTable(db, msTable, ref, cmap, pkNames, partitionBy); } @@ -104,7 +105,9 @@ public class LocalKuduTable extends LocalTable implements FeKuduTable { } ColumnMap cmap = new ColumnMap(columns, /*numClusteringCols=*/0, fullTableName); - return new LocalKuduTable(db, msTable, cmap, pkNames, kuduPartitionParams); + + return new LocalKuduTable(db, msTable, /*ref=*/null, cmap, pkNames, + kuduPartitionParams); } private static void convertColsFromKudu(Schema schema, List<Column> cols, @@ -128,10 +131,10 @@ public class LocalKuduTable extends LocalTable implements FeKuduTable { } } - private LocalKuduTable(LocalDb db, Table msTable, ColumnMap cmap, + private LocalKuduTable(LocalDb db, Table msTable, TableMetaRef ref, ColumnMap cmap, List<String> primaryKeyColumnNames, List<KuduPartitionParam> partitionBy) { - super(db, msTable, cmap); + super(db, msTable, ref, cmap); tableParams_ = new TableParams(msTable); partitionBy_ = ImmutableList.copyOf(partitionBy); primaryKeyColumnNames_ = ImmutableList.copyOf(primaryKeyColumnNames); http://git-wip-us.apache.org/repos/asf/impala/blob/ef15da08/fe/src/main/java/org/apache/impala/catalog/local/LocalPartitionSpec.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/org/apache/impala/catalog/local/LocalPartitionSpec.java b/fe/src/main/java/org/apache/impala/catalog/local/LocalPartitionSpec.java index 690a4bf..c635158 100644 --- a/fe/src/main/java/org/apache/impala/catalog/local/LocalPartitionSpec.java +++ b/fe/src/main/java/org/apache/impala/catalog/local/LocalPartitionSpec.java @@ -19,11 +19,15 @@ package org.apache.impala.catalog.local; import java.util.List; +import javax.annotation.Nullable; + import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.impala.analysis.LiteralExpr; import org.apache.impala.catalog.CatalogException; import org.apache.impala.catalog.FeCatalogUtils; import org.apache.impala.catalog.PrunablePartition; +import org.apache.impala.catalog.local.MetaProvider.PartitionRef; +import org.apache.impala.thrift.CatalogObjectsConstants; import org.apache.impala.util.MetaStoreUtil; import com.google.common.base.Preconditions; @@ -36,31 +40,43 @@ import com.google.errorprone.annotations.Immutable; */ @Immutable class LocalPartitionSpec implements PrunablePartition { + static final long UNPARTITIONED_ID = 0; private final long id_; - private final String name_; + + @Nullable + private final PartitionRef ref_; // LiteralExprs are technically mutable prior to analysis. @SuppressWarnings("Immutable") private final ImmutableList<LiteralExpr> partitionValues_; - LocalPartitionSpec(LocalFsTable table, String partName, long id) { + LocalPartitionSpec(LocalFsTable table, PartitionRef ref, long id) { id_ = id; - name_ = Preconditions.checkNotNull(partName); - if (!partName.isEmpty()) { - try { - List<String> partValues = MetaStoreUtil.getPartValsFromName( - table.getMetaStoreTable(), partName); - partitionValues_ = ImmutableList.copyOf(FeCatalogUtils.parsePartitionKeyValues( - table, partValues)); - } catch (CatalogException | MetaException e) { - throw new LocalCatalogException(String.format( - "Failed to parse partition name '%s' for table %s", - partName, table.getFullName()), e); - } - } else { - // Unpartitioned tables have a single partition with empty name. - partitionValues_= ImmutableList.of(); + ref_ = Preconditions.checkNotNull(ref); + if (ref.getName().isEmpty()) { + // "unpartitioned" partition + partitionValues_ = ImmutableList.of(); + return; } + try { + List<String> partValues = MetaStoreUtil.getPartValsFromName( + table.getMetaStoreTable(), ref_.getName()); + partitionValues_ = ImmutableList.copyOf(FeCatalogUtils.parsePartitionKeyValues( + table, partValues)); + } catch (CatalogException | MetaException e) { + throw new LocalCatalogException(String.format( + "Failed to parse partition name '%s' for table %s", + ref.getName(), table.getFullName()), e); + } + } + + LocalPartitionSpec(LocalFsTable table, long id) { + // Unpartitioned tables have a single partition with empty name. + Preconditions.checkArgument(id == CatalogObjectsConstants.PROTOTYPE_PARTITION_ID || + id == UNPARTITIONED_ID); + this.id_ = id; + this.ref_ = null; + partitionValues_= ImmutableList.of(); } @Override @@ -69,5 +85,17 @@ class LocalPartitionSpec implements PrunablePartition { @Override public List<LiteralExpr> getPartitionValues() { return partitionValues_; } - String getName() { return name_; } + PartitionRef getRef() { return ref_; } + + @Override + public String toString() { + if (ref_ != null) { + return ref_.getName(); + } else if (id_ == CatalogObjectsConstants.PROTOTYPE_PARTITION_ID) { + return "<prototype>"; + } else { + return "<default>"; + } + + } } http://git-wip-us.apache.org/repos/asf/impala/blob/ef15da08/fe/src/main/java/org/apache/impala/catalog/local/LocalTable.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/org/apache/impala/catalog/local/LocalTable.java b/fe/src/main/java/org/apache/impala/catalog/local/LocalTable.java index 1a14831..81a0741 100644 --- a/fe/src/main/java/org/apache/impala/catalog/local/LocalTable.java +++ b/fe/src/main/java/org/apache/impala/catalog/local/LocalTable.java @@ -20,6 +20,8 @@ package org.apache.impala.catalog.local; import java.util.ArrayList; import java.util.List; +import javax.annotation.Nullable; + import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; import org.apache.hadoop.hive.metastore.api.Table; @@ -36,6 +38,8 @@ import org.apache.impala.catalog.KuduTable; import org.apache.impala.catalog.StructField; import org.apache.impala.catalog.StructType; import org.apache.impala.catalog.TableLoadingException; +import org.apache.impala.catalog.local.MetaProvider.TableMetaRef; +import org.apache.impala.common.Pair; import org.apache.impala.thrift.TCatalogObjectType; import org.apache.impala.thrift.TTableStats; import org.apache.log4j.Logger; @@ -66,22 +70,36 @@ abstract class LocalTable implements FeTable { private final TTableStats tableStats_; + /** + * Table reference as provided by the initial call to the metadata provider. + * This must be passed back to any further calls to the metadata provider + * in order to verify consistency. + * + * In the case of CTAS target tables, this may be null. Since the tables don't + * exist yet in any metadata storage, it would be invalid to try to load any metadata + * about them. + */ + @Nullable + protected final TableMetaRef ref_; + public static LocalTable load(LocalDb db, String tblName) { // In order to know which kind of table subclass to instantiate, we need // to eagerly grab and parse the top-level Table object from the HMS. LocalTable t = null; - Table msTbl = loadMsTable(db, tblName); + Pair<Table, TableMetaRef> tableMeta = loadTableMetadata(db, tblName); + Table msTbl = tableMeta.first; + TableMetaRef ref = tableMeta.second; if (TableType.valueOf(msTbl.getTableType()) == TableType.VIRTUAL_VIEW) { - t = new LocalView(db, msTbl); + t = new LocalView(db, msTbl, ref); } else if (HBaseTable.isHBaseTable(msTbl)) { - t = LocalHbaseTable.loadFromHbase(db, msTbl); + t = LocalHbaseTable.loadFromHbase(db, msTbl, ref); } else if (KuduTable.isKuduTable(msTbl)) { - t = LocalKuduTable.loadFromKudu(db, msTbl); + t = LocalKuduTable.loadFromKudu(db, msTbl, ref); } else if (DataSourceTable.isDataSourceTable(msTbl)) { // TODO(todd) support datasource table } else if (HdfsFileFormat.isHdfsInputFormatClass( msTbl.getSd().getInputFormat())) { - t = LocalFsTable.load(db, msTbl); + t = LocalFsTable.load(db, msTbl, ref); } if (t == null) { @@ -101,7 +119,7 @@ abstract class LocalTable implements FeTable { /** * Load the Table instance from the metastore. */ - private static Table loadMsTable(LocalDb db, String tblName) { + private static Pair<Table, TableMetaRef> loadTableMetadata(LocalDb db, String tblName) { Preconditions.checkArgument(tblName.toLowerCase().equals(tblName)); try { @@ -113,11 +131,11 @@ abstract class LocalTable implements FeTable { } } - public LocalTable(LocalDb db, Table msTbl, ColumnMap cols) { + public LocalTable(LocalDb db, Table msTbl, TableMetaRef ref, ColumnMap cols) { this.db_ = Preconditions.checkNotNull(db); this.name_ = msTbl.getTableName(); this.cols_ = cols; - + this.ref_ = ref; this.msTable_ = msTbl; tableStats_ = new TTableStats( @@ -126,8 +144,8 @@ abstract class LocalTable implements FeTable { FeCatalogUtils.getTotalSize(msTable_.getParameters())); } - public LocalTable(LocalDb db, Table msTbl) { - this(db, msTbl, ColumnMap.fromMsTable(msTbl)); + public LocalTable(LocalDb db, Table msTbl, TableMetaRef ref) { + this(db, msTbl, ref, ColumnMap.fromMsTable(msTbl)); } @Override @@ -232,7 +250,7 @@ abstract class LocalTable implements FeTable { protected void loadColumnStats() { try { List<ColumnStatisticsObj> stats = db_.getCatalog().getMetaProvider() - .loadTableColumnStatistics(db_.getName(), getName(), getColumnNames()); + .loadTableColumnStatistics(ref_, getColumnNames()); FeCatalogUtils.injectColumnStats(stats, this); } catch (TException e) { LOG.warn("Could not load column statistics for: " + getFullName(), e); http://git-wip-us.apache.org/repos/asf/impala/blob/ef15da08/fe/src/main/java/org/apache/impala/catalog/local/LocalView.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/org/apache/impala/catalog/local/LocalView.java b/fe/src/main/java/org/apache/impala/catalog/local/LocalView.java index 1aecdd2..d5b0796 100644 --- a/fe/src/main/java/org/apache/impala/catalog/local/LocalView.java +++ b/fe/src/main/java/org/apache/impala/catalog/local/LocalView.java @@ -25,6 +25,7 @@ import org.apache.impala.analysis.QueryStmt; import org.apache.impala.catalog.FeView; import org.apache.impala.catalog.TableLoadingException; import org.apache.impala.catalog.View; +import org.apache.impala.catalog.local.MetaProvider.TableMetaRef; import org.apache.impala.thrift.TCatalogObjectType; import org.apache.impala.thrift.TTableDescriptor; @@ -37,8 +38,8 @@ import org.apache.impala.thrift.TTableDescriptor; public class LocalView extends LocalTable implements FeView { private final QueryStmt queryStmt_; - public LocalView(LocalDb db, Table msTbl) { - super(db, msTbl); + public LocalView(LocalDb db, Table msTbl, TableMetaRef ref) { + super(db, msTbl, ref); try { queryStmt_ = View.parseViewDef(this); http://git-wip-us.apache.org/repos/asf/impala/blob/ef15da08/fe/src/main/java/org/apache/impala/catalog/local/MetaProvider.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/org/apache/impala/catalog/local/MetaProvider.java b/fe/src/main/java/org/apache/impala/catalog/local/MetaProvider.java index 75d389e..0a217da 100644 --- a/fe/src/main/java/org/apache/impala/catalog/local/MetaProvider.java +++ b/fe/src/main/java/org/apache/impala/catalog/local/MetaProvider.java @@ -17,12 +17,9 @@ package org.apache.impala.catalog.local; -import java.io.IOException; import java.util.List; import java.util.Map; -import org.apache.hadoop.fs.LocatedFileStatus; -import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.Function; @@ -31,9 +28,14 @@ import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hadoop.hive.metastore.api.Table; import org.apache.hadoop.hive.metastore.api.UnknownDBException; +import org.apache.impala.catalog.HdfsPartition.FileDescriptor; +import org.apache.impala.common.Pair; +import org.apache.impala.thrift.TNetworkAddress; +import org.apache.impala.util.ListMap; import org.apache.thrift.TException; import com.google.common.collect.ImmutableList; +import com.google.errorprone.annotations.Immutable; /** * Interface for loading metadata. See {@link LocalCatalog} for an example. @@ -52,13 +54,13 @@ interface MetaProvider { ImmutableList<String> loadTableNames(String dbName) throws MetaException, UnknownDBException, TException; - Table loadTable(String dbName, String tableName) + Pair<Table, TableMetaRef> loadTable(String dbName, String tableName) throws NoSuchObjectException, MetaException, TException; String loadNullPartitionKeyValue() throws MetaException, TException; - List<String> loadPartitionNames(String dbName, String tableName) + List<PartitionRef> loadPartitionList(TableMetaRef table) throws MetaException, TException; /** @@ -77,19 +79,40 @@ interface MetaProvider { * If a requested partition does not exist, no exception will be thrown. * Instead, the resulting map will contain no entry for that partition. */ - Map<String, Partition> loadPartitionsByNames(String dbName, String tableName, - List<String> partitionColumnNames, List<String> partitionNames) + Map<String, PartitionMetadata> loadPartitionsByRefs(TableMetaRef table, + List<String> partitionColumnNames, ListMap<TNetworkAddress> hostIndex, + List<PartitionRef> partitionRefs) throws MetaException, TException; /** * Load statistics for the given columns from the given table. */ - List<ColumnStatisticsObj> loadTableColumnStatistics(String dbName, - String tblName, List<String> colNames) throws TException; + List<ColumnStatisticsObj> loadTableColumnStatistics(TableMetaRef table, + List<String> colNames) throws TException; /** - * Load file metadata and block locations for the files in the given - * partition directory. + * Reference to a table as returned by loadTable(). This reference must be passed + * back to other functions to fetch more details about the table. Implementations + * may use this reference to store internal information such as version numbers + * in order to perform concurrency control checks, etc. */ - List<LocatedFileStatus> loadFileMetadata(Path dir) throws IOException; + interface TableMetaRef { + } + + /** + * Reference to a partition as returned from loadPartitionList(). These references + * may be passed back into loadPartitionsByRefs() to load detailed partition metadata. + */ + @Immutable + interface PartitionRef { + String getName(); + } + + /** + * Partition metadata as returned by loadPartitionsByRefs(). + */ + interface PartitionMetadata { + Partition getHmsPartition(); + ImmutableList<FileDescriptor> getFileDescriptors(); + } } http://git-wip-us.apache.org/repos/asf/impala/blob/ef15da08/fe/src/main/java/org/apache/impala/common/RuntimeEnv.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/org/apache/impala/common/RuntimeEnv.java b/fe/src/main/java/org/apache/impala/common/RuntimeEnv.java index 3c33bf1..6299dd4 100644 --- a/fe/src/main/java/org/apache/impala/common/RuntimeEnv.java +++ b/fe/src/main/java/org/apache/impala/common/RuntimeEnv.java @@ -43,6 +43,7 @@ public class RuntimeEnv { */ public void reset() { numCores_ = Runtime.getRuntime().availableProcessors(); + isTestEnv_ = false; } public int getNumCores() { return numCores_; } http://git-wip-us.apache.org/repos/asf/impala/blob/ef15da08/fe/src/main/java/org/apache/impala/planner/HdfsScanNode.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/org/apache/impala/planner/HdfsScanNode.java b/fe/src/main/java/org/apache/impala/planner/HdfsScanNode.java index cb33556..8c10e5f 100644 --- a/fe/src/main/java/org/apache/impala/planner/HdfsScanNode.java +++ b/fe/src/main/java/org/apache/impala/planner/HdfsScanNode.java @@ -49,7 +49,6 @@ import org.apache.impala.analysis.TupleDescriptor; import org.apache.impala.analysis.TupleId; import org.apache.impala.catalog.Column; import org.apache.impala.catalog.ColumnStats; -import org.apache.impala.catalog.FeCatalogUtils; import org.apache.impala.catalog.HdfsCompression; import org.apache.impala.catalog.FeFsPartition; import org.apache.impala.catalog.FeFsTable; http://git-wip-us.apache.org/repos/asf/impala/blob/ef15da08/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java b/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java index 20d47bf..ad3add6 100644 --- a/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java +++ b/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java @@ -58,6 +58,7 @@ import org.apache.impala.catalog.CatalogException; import org.apache.impala.catalog.CatalogServiceCatalog; import org.apache.impala.catalog.Column; import org.apache.impala.catalog.ColumnNotFoundException; +import org.apache.impala.catalog.ColumnStats; import org.apache.impala.catalog.DataSource; import org.apache.impala.catalog.Db; import org.apache.impala.catalog.FeCatalogUtils; @@ -901,8 +902,10 @@ public class CatalogOpExecutor { Column tableCol = table.getColumn(entry.getKey()); // Ignore columns that were dropped in the meantime. if (tableCol == null) continue; - ColumnStatisticsData colStatsData = - createHiveColStatsData(params, entry.getValue(), tableCol.getType()); + // If we know the number of rows in the table, cap NDV of the column appropriately. + long ndvCap = params.isSetTable_stats() ? params.table_stats.num_rows : -1; + ColumnStatisticsData colStatsData = ColumnStats.createHiveColStatsData( + ndvCap, entry.getValue(), tableCol.getType()); if (colStatsData == null) continue; if (LOG.isTraceEnabled()) { LOG.trace(String.format("Updating column stats for %s: numDVs=%s numNulls=%s " + @@ -917,57 +920,6 @@ public class CatalogOpExecutor { return colStats; } - private static ColumnStatisticsData createHiveColStatsData( - TAlterTableUpdateStatsParams params, TColumnStats colStats, Type colType) { - ColumnStatisticsData colStatsData = new ColumnStatisticsData(); - long ndv = colStats.getNum_distinct_values(); - // Cap NDV at row count if available. - if (params.isSetTable_stats()) ndv = Math.min(ndv, params.table_stats.num_rows); - - long numNulls = colStats.getNum_nulls(); - switch(colType.getPrimitiveType()) { - case BOOLEAN: - colStatsData.setBooleanStats(new BooleanColumnStatsData(1, -1, numNulls)); - break; - case TINYINT: - ndv = Math.min(ndv, LongMath.pow(2, Byte.SIZE)); - colStatsData.setLongStats(new LongColumnStatsData(numNulls, ndv)); - break; - case SMALLINT: - ndv = Math.min(ndv, LongMath.pow(2, Short.SIZE)); - colStatsData.setLongStats(new LongColumnStatsData(numNulls, ndv)); - break; - case INT: - ndv = Math.min(ndv, LongMath.pow(2, Integer.SIZE)); - colStatsData.setLongStats(new LongColumnStatsData(numNulls, ndv)); - break; - case BIGINT: - case TIMESTAMP: // Hive and Impala use LongColumnStatsData for timestamps. - colStatsData.setLongStats(new LongColumnStatsData(numNulls, ndv)); - break; - case FLOAT: - case DOUBLE: - colStatsData.setDoubleStats(new DoubleColumnStatsData(numNulls, ndv)); - break; - case CHAR: - case VARCHAR: - case STRING: - long maxStrLen = colStats.getMax_size(); - double avgStrLen = colStats.getAvg_size(); - colStatsData.setStringStats( - new StringColumnStatsData(maxStrLen, avgStrLen, numNulls, ndv)); - break; - case DECIMAL: - double decMaxNdv = Math.pow(10, colType.getPrecision()); - ndv = (long) Math.min(ndv, decMaxNdv); - colStatsData.setDecimalStats(new DecimalColumnStatsData(numNulls, ndv)); - break; - default: - return null; - } - return colStatsData; - } - /** * Creates a new database in the metastore and adds the db name to the internal * metadata cache, marking its metadata to be lazily loaded on the next access. http://git-wip-us.apache.org/repos/asf/impala/blob/ef15da08/fe/src/main/java/org/apache/impala/service/FeSupport.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/org/apache/impala/service/FeSupport.java b/fe/src/main/java/org/apache/impala/service/FeSupport.java index 052d496..d64a554 100644 --- a/fe/src/main/java/org/apache/impala/service/FeSupport.java +++ b/fe/src/main/java/org/apache/impala/service/FeSupport.java @@ -103,6 +103,9 @@ public class FeSupport { // using Java Thrift bindings. public native static byte[] NativePrioritizeLoad(byte[] thriftReq); + public native static byte[] NativeGetPartialCatalogObject(byte[] thriftReq) + throws InternalException; + // Parses a string of comma-separated key=value query options ('csvQueryOptions'), // updates the existing query options ('queryOptions') with them and returns the // resulting serialized TQueryOptions object. @@ -349,6 +352,17 @@ public class FeSupport { return MinLogSpaceForBloomFilter(ndv, fpp); } + public static byte[] GetPartialCatalogObject(byte[] thriftReq) + throws InternalException { + try { + return NativeGetPartialCatalogObject(thriftReq); + } catch (UnsatisfiedLinkError e) { + loadLibrary(); + } + return NativeGetPartialCatalogObject(thriftReq); + } + + /** * This function should be called explicitly by the FeSupport to ensure that * native functions are loaded. Tests that depend on JniCatalog or JniFrontend http://git-wip-us.apache.org/repos/asf/impala/blob/ef15da08/fe/src/main/java/org/apache/impala/service/Frontend.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/org/apache/impala/service/Frontend.java b/fe/src/main/java/org/apache/impala/service/Frontend.java index fcb5ce2..e259631 100644 --- a/fe/src/main/java/org/apache/impala/service/Frontend.java +++ b/fe/src/main/java/org/apache/impala/service/Frontend.java @@ -87,6 +87,7 @@ import org.apache.impala.catalog.FeTable; import org.apache.impala.catalog.Function; import org.apache.impala.catalog.ImpaladCatalog; import org.apache.impala.catalog.Type; +import org.apache.impala.catalog.local.InconsistentMetadataFetchException; import org.apache.impala.common.AnalysisException; import org.apache.impala.common.FileSystemUtil; import org.apache.impala.common.ImpalaException; @@ -1019,6 +1020,9 @@ public class Frontend { */ public TExecRequest createExecRequest(TQueryCtx queryCtx, StringBuilder explainString) throws ImpalaException { + // TODO(todd): wrap the planning in a retry loop which catches + // InconsistentMetadataFetchException. + // Timeline of important events in the planning process, used for debugging // and profiling. EventSequence timeline = new EventSequence("Query Compilation"); http://git-wip-us.apache.org/repos/asf/impala/blob/ef15da08/fe/src/main/java/org/apache/impala/service/JniCatalog.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/org/apache/impala/service/JniCatalog.java b/fe/src/main/java/org/apache/impala/service/JniCatalog.java index daa57ab..955f48f 100644 --- a/fe/src/main/java/org/apache/impala/service/JniCatalog.java +++ b/fe/src/main/java/org/apache/impala/service/JniCatalog.java @@ -45,6 +45,7 @@ import org.apache.impala.thrift.TGetDbsParams; import org.apache.impala.thrift.TGetDbsResult; import org.apache.impala.thrift.TGetFunctionsRequest; import org.apache.impala.thrift.TGetFunctionsResponse; +import org.apache.impala.thrift.TGetPartialCatalogObjectRequest; import org.apache.impala.thrift.TGetTablesParams; import org.apache.impala.thrift.TGetTableMetricsParams; import org.apache.impala.thrift.TGetTablesResult; @@ -216,6 +217,15 @@ public class JniCatalog { return serializer.serialize(catalog_.getTCatalogObject(objectDescription)); } + public byte[] getPartialCatalogObject(byte[] thriftParams) throws ImpalaException, + TException { + TGetPartialCatalogObjectRequest req = + new TGetPartialCatalogObjectRequest(); + JniUtil.deserializeThrift(protocolFactory_, req, thriftParams); + TSerializer serializer = new TSerializer(protocolFactory_); + return serializer.serialize(catalog_.getPartialCatalogObject(req)); + } + /** * See comment in CatalogServiceCatalog. */ http://git-wip-us.apache.org/repos/asf/impala/blob/ef15da08/fe/src/test/java/org/apache/impala/catalog/HdfsPartitionTest.java ---------------------------------------------------------------------- diff --git a/fe/src/test/java/org/apache/impala/catalog/HdfsPartitionTest.java b/fe/src/test/java/org/apache/impala/catalog/HdfsPartitionTest.java index 100a513..2e093c1 100644 --- a/fe/src/test/java/org/apache/impala/catalog/HdfsPartitionTest.java +++ b/fe/src/test/java/org/apache/impala/catalog/HdfsPartitionTest.java @@ -17,20 +17,38 @@ package org.apache.impala.catalog; -import static org.junit.Assert.assertTrue; +import static org.apache.impala.catalog.HdfsPartition.comparePartitionKeyValues; +import static org.junit.Assert.*; import java.math.BigDecimal; +import java.util.ArrayList; import java.util.List; -import java.lang.*; -import org.apache.impala.analysis.*; -import com.google.common.collect.Lists; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.LocatedFileStatus; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.RemoteIterator; +import org.apache.impala.analysis.BoolLiteral; +import org.apache.impala.analysis.LiteralExpr; +import org.apache.impala.analysis.NullLiteral; +import org.apache.impala.analysis.NumericLiteral; +import org.apache.impala.analysis.StringLiteral; +import org.apache.impala.catalog.HdfsPartition.FileDescriptor; +import org.apache.impala.catalog.HdfsTable.FileMetadataLoadStats; +import org.apache.impala.service.FeSupport; +import org.apache.impala.thrift.TNetworkAddress; +import org.apache.impala.util.ListMap; import org.junit.Test; -import static org.apache.impala.catalog.HdfsPartition.comparePartitionKeyValues; +import com.google.common.collect.Lists; public class HdfsPartitionTest { + static { + FeSupport.loadLibrary(); + } + private List<LiteralExpr> valuesNull_= Lists.newArrayList(); private List<LiteralExpr> valuesDecimal_ = Lists.newArrayList(); private List<LiteralExpr> valuesDecimal1_ = Lists.newArrayList(); @@ -112,4 +130,45 @@ public class HdfsPartitionTest { Integer.signum(comparePartitionKeyValues(o2, o3))); } } + + /** + * Get the list of all locations of blocks from the given file descriptor. + */ + private static List<TNetworkAddress> getAllReplicaAddresses(FileDescriptor fd, + ListMap<TNetworkAddress> hostIndex) { + List<TNetworkAddress> ret = new ArrayList<>(); + for (int i = 0; i < fd.getNumFileBlocks(); i++) { + for (int j = 0; j < fd.getFbFileBlock(i).replicaHostIdxsLength(); j++) { + int idx = fd.getFbFileBlock(i).replicaHostIdxs(j); + ret.add(hostIndex.getEntry(idx)); + } + } + return ret; + } + + @Test + public void testCloneWithNewHostIndex() throws Exception { + // Fetch some metadata from a directory in HDFS. + Path p = new Path("hdfs://localhost:20500/test-warehouse/schemas"); + FileSystem fs = p.getFileSystem(new Configuration()); + RemoteIterator<LocatedFileStatus> iter = fs.listLocatedStatus(p); + ListMap<TNetworkAddress> origIndex = new ListMap<>(); + List<FileDescriptor> fileDescriptors = HdfsTable.createFileDescriptors(fs, iter, + origIndex, new FileMetadataLoadStats(p)); + assertTrue(!fileDescriptors.isEmpty()); + + FileDescriptor fd = fileDescriptors.get(0); + // Get the list of locations, using the original host index. + List<TNetworkAddress> origAddresses = getAllReplicaAddresses(fd, origIndex); + + // Make a new host index with the hosts in the opposite order. + ListMap<TNetworkAddress> newIndex = new ListMap<>(); + newIndex.populate(Lists.reverse(origIndex.getList())); + + // Clone the FD over to the reversed index. The actual addresses should be the same. + FileDescriptor cloned = fd.cloneWithNewHostIndex(origIndex.getList(), newIndex); + List<TNetworkAddress> newAddresses = getAllReplicaAddresses(cloned, newIndex); + + assertEquals(origAddresses, newAddresses); + } } http://git-wip-us.apache.org/repos/asf/impala/blob/ef15da08/fe/src/test/java/org/apache/impala/catalog/PartialCatalogInfoTest.java ---------------------------------------------------------------------- diff --git a/fe/src/test/java/org/apache/impala/catalog/PartialCatalogInfoTest.java b/fe/src/test/java/org/apache/impala/catalog/PartialCatalogInfoTest.java new file mode 100644 index 0000000..2ff5015 --- /dev/null +++ b/fe/src/test/java/org/apache/impala/catalog/PartialCatalogInfoTest.java @@ -0,0 +1,183 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.impala.catalog; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.util.List; + +import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; +import org.apache.impala.common.InternalException; +import org.apache.impala.testutil.CatalogServiceTestCatalog; +import org.apache.impala.thrift.TCatalogInfoSelector; +import org.apache.impala.thrift.TCatalogObject; +import org.apache.impala.thrift.TCatalogObjectType; +import org.apache.impala.thrift.TDatabase; +import org.apache.impala.thrift.TDbInfoSelector; +import org.apache.impala.thrift.TGetPartialCatalogObjectRequest; +import org.apache.impala.thrift.TGetPartialCatalogObjectResponse; +import org.apache.impala.thrift.TPartialPartitionInfo; +import org.apache.impala.thrift.TTable; +import org.apache.impala.thrift.TTableInfoSelector; +import org.apache.thrift.TDeserializer; +import org.apache.thrift.TException; +import org.apache.thrift.TSerializer; +import org.junit.Test; + +import com.google.common.collect.ImmutableList; + +public class PartialCatalogInfoTest { + private static CatalogServiceCatalog catalog_ = + CatalogServiceTestCatalog.create(); + + private TGetPartialCatalogObjectResponse sendRequest( + TGetPartialCatalogObjectRequest req) + throws CatalogException, InternalException, TException { + System.err.println("req: " + req); + TGetPartialCatalogObjectResponse resp; + resp = catalog_.getPartialCatalogObject(req); + // Round-trip the response through serialization, so if we accidentally forgot to + // set the "isset" flag for any fields, we'll catch that bug. + byte[] respBytes = new TSerializer().serialize(resp); + resp.clear(); + new TDeserializer().deserialize(resp, respBytes); + System.err.println("resp: " + resp); + return resp; + } + + @Test + public void testDbList() throws Exception { + TGetPartialCatalogObjectRequest req = new TGetPartialCatalogObjectRequest(); + req.object_desc = new TCatalogObject(); + req.object_desc.setType(TCatalogObjectType.CATALOG); + req.catalog_info_selector = new TCatalogInfoSelector(); + req.catalog_info_selector.want_db_names = true; + TGetPartialCatalogObjectResponse resp = sendRequest(req); + assertTrue(resp.catalog_info.db_names.contains("functional")); + } + + @Test + public void testDb() throws Exception { + TGetPartialCatalogObjectRequest req = new TGetPartialCatalogObjectRequest(); + req.object_desc = new TCatalogObject(); + req.object_desc.setType(TCatalogObjectType.DATABASE); + req.object_desc.db = new TDatabase("functional"); + req.db_info_selector = new TDbInfoSelector(); + req.db_info_selector.want_hms_database = true; + req.db_info_selector.want_table_names = true; + TGetPartialCatalogObjectResponse resp = sendRequest(req); + assertTrue(resp.isSetObject_version_number()); + assertEquals(resp.db_info.hms_database.getName(), "functional"); + assertTrue(resp.db_info.table_names.contains("alltypes")); + } + + @Test + public void testTable() throws Exception { + TGetPartialCatalogObjectRequest req = new TGetPartialCatalogObjectRequest(); + req.object_desc = new TCatalogObject(); + req.object_desc.setType(TCatalogObjectType.TABLE); + req.object_desc.table = new TTable("functional", "alltypes"); + req.table_info_selector = new TTableInfoSelector(); + req.table_info_selector.want_hms_table = true; + req.table_info_selector.want_partition_names = true; + TGetPartialCatalogObjectResponse resp = sendRequest(req); + assertTrue(resp.isSetObject_version_number()); + assertEquals(resp.table_info.hms_table.getTableName(), "alltypes"); + assertTrue(resp.table_info.partitions.size() > 0); + TPartialPartitionInfo partInfo = resp.table_info.partitions.get(1); + assertTrue("bad part name: " + partInfo.name, + partInfo.name.matches("year=\\d+/month=\\d+")); + + // Fetch again, but specify two specific partitions and ask for metadata. + req.table_info_selector.clear(); + req.table_info_selector.want_partition_metadata = true; + req.table_info_selector.partition_ids = ImmutableList.of( + resp.table_info.partitions.get(1).id, + resp.table_info.partitions.get(3).id); + resp = sendRequest(req); + assertNull(resp.table_info.hms_table); + assertEquals(2, resp.table_info.partitions.size()); + partInfo = resp.table_info.partitions.get(0); + assertNull(partInfo.name); + assertEquals(req.table_info_selector.partition_ids.get(0), (Long)partInfo.id); + assertTrue(partInfo.hms_partition.getSd().getLocation().startsWith( + "hdfs://localhost:20500/test-warehouse/alltypes/year=")); + // TODO(todd): we should probably transfer a compressed descriptor instead + // and refactor the MetaProvider interface to expose those since there is + // a lot of redundant info in partition descriptors. + // TODO(todd): should also filter out the incremental stats. + } + + @Test + public void testFetchMissingPartId() throws Exception { + TGetPartialCatalogObjectRequest req = new TGetPartialCatalogObjectRequest(); + req.object_desc = new TCatalogObject(); + req.object_desc.setType(TCatalogObjectType.TABLE); + req.object_desc.table = new TTable("functional", "alltypes"); + req.table_info_selector = new TTableInfoSelector(); + req.table_info_selector.want_partition_metadata = true; + req.table_info_selector.partition_ids = ImmutableList.of(-12345L); // non-existent + try { + sendRequest(req); + fail("did not throw exception for missing partition"); + } catch (IllegalArgumentException iae) { + assertEquals("Partition id -12345 does not exist", iae.getMessage()); + } + } + + @Test + public void testTableStats() throws Exception { + TGetPartialCatalogObjectRequest req = new TGetPartialCatalogObjectRequest(); + req.object_desc = new TCatalogObject(); + req.object_desc.setType(TCatalogObjectType.TABLE); + req.object_desc.table = new TTable("functional", "alltypes"); + req.table_info_selector = new TTableInfoSelector(); + req.table_info_selector.want_stats_for_column_names = ImmutableList.of( + "year", "month", "id", "bool_col", "tinyint_col", "smallint_col", + "int_col", "bigint_col", "float_col", "double_col", "date_string_col", + "string_col", "timestamp_col"); + TGetPartialCatalogObjectResponse resp = sendRequest(req); + List<ColumnStatisticsObj> stats = resp.table_info.column_stats; + // We have 13 columns, but 2 are the clustering columns which don't have stats. + assertEquals(11, stats.size()); + assertEquals("ColumnStatisticsObj(colName:id, colType:INT, " + + "statsData:<ColumnStatisticsData longStats:LongColumnStatsData(" + + "numNulls:-1, numDVs:7300)>)", stats.get(0).toString()); + } + + @Test + public void testFetchErrorTable() throws Exception { + TGetPartialCatalogObjectRequest req = new TGetPartialCatalogObjectRequest(); + req.object_desc = new TCatalogObject(); + req.object_desc.setType(TCatalogObjectType.TABLE); + req.object_desc.table = new TTable("functional", "bad_serde"); + req.table_info_selector = new TTableInfoSelector(); + req.table_info_selector.want_hms_table = true; + req.table_info_selector.want_partition_names = true; + try { + sendRequest(req); + fail("expected exception"); + } catch (TableLoadingException tle) { + assertEquals("Failed to load metadata for table: functional.bad_serde", + tle.getMessage()); + } + } +} http://git-wip-us.apache.org/repos/asf/impala/blob/ef15da08/fe/src/test/java/org/apache/impala/catalog/local/LocalCatalogTest.java ---------------------------------------------------------------------- diff --git a/fe/src/test/java/org/apache/impala/catalog/local/LocalCatalogTest.java b/fe/src/test/java/org/apache/impala/catalog/local/LocalCatalogTest.java index 4cb2b96..ac15fb2 100644 --- a/fe/src/test/java/org/apache/impala/catalog/local/LocalCatalogTest.java +++ b/fe/src/test/java/org/apache/impala/catalog/local/LocalCatalogTest.java @@ -157,6 +157,24 @@ public class LocalCatalogTest { } } assertEquals(24, totalFds); + assertTrue(t.getHostIndex().size() > 0); + } + + + @Test + public void testLoadFileDescriptorsUnpartitioned() throws Exception { + FeFsTable t = (FeFsTable) catalog_.getTable("tpch", "region"); + int totalFds = 0; + for (FeFsPartition p: FeCatalogUtils.loadAllPartitions(t)) { + List<FileDescriptor> fds = p.getFileDescriptors(); + totalFds += fds.size(); + for (FileDescriptor fd : fds) { + assertTrue(fd.getFileLength() > 0); + assertEquals(fd.getNumFileBlocks(), 1); + assertEquals(3, fd.getFbFileBlock(0).diskIdsLength()); + } + } + assertEquals(1, totalFds); } @Test http://git-wip-us.apache.org/repos/asf/impala/blob/ef15da08/fe/src/test/java/org/apache/impala/common/FrontendTestBase.java ---------------------------------------------------------------------- diff --git a/fe/src/test/java/org/apache/impala/common/FrontendTestBase.java b/fe/src/test/java/org/apache/impala/common/FrontendTestBase.java index cbb48ce..e13e51b 100644 --- a/fe/src/test/java/org/apache/impala/common/FrontendTestBase.java +++ b/fe/src/test/java/org/apache/impala/common/FrontendTestBase.java @@ -97,7 +97,7 @@ public class FrontendTestBase { @AfterClass public static void cleanUp() throws Exception { - RuntimeEnv.INSTANCE.setTestEnv(false); + RuntimeEnv.INSTANCE.reset(); } // Adds a Udf: default.name(args) to the catalog.
