Repository: incubator-impala Updated Branches: refs/heads/master c6fc89913 -> bf1d9677f
IMPALA-4155: Update default partition when table is altered If the table format is changed by the Alter Table statement, the default partition in partitioned tables used to not get updated. This caused a problem because Insert picks up the file format for new partitions from the default partition. This patch fixes the problem by calling addDefaultPartition(). Also removed "drop table if not exists" in tests in alter-table.test because we already have the unique_database fixture. Change-Id: I59bf21caa5c5e7867d07d87cda0c0a5b4b994859 Reviewed-on: http://gerrit.cloudera.org:8080/4750 Reviewed-by: Alex Behm <[email protected]> Tested-by: Internal Jenkins Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/bf1d9677 Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/bf1d9677 Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/bf1d9677 Branch: refs/heads/master Commit: bf1d9677fc09dabc88ab6c71f82ec99ec59ec164 Parents: c6fc899 Author: Taras Bobrovytsky <[email protected]> Authored: Mon Oct 10 12:14:52 2016 -0700 Committer: Internal Jenkins <[email protected]> Committed: Thu Oct 20 23:47:52 2016 +0000 ---------------------------------------------------------------------- .../org/apache/impala/catalog/HdfsTable.java | 29 ++++++++------- .../impala/service/CatalogOpExecutor.java | 3 ++ .../queries/QueryTest/alter-table.test | 38 ++++++++++---------- 3 files changed, 39 insertions(+), 31 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/bf1d9677/fe/src/main/java/org/apache/impala/catalog/HdfsTable.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/org/apache/impala/catalog/HdfsTable.java b/fe/src/main/java/org/apache/impala/catalog/HdfsTable.java index e664f6f..e6206fc 100644 --- a/fe/src/main/java/org/apache/impala/catalog/HdfsTable.java +++ b/fe/src/main/java/org/apache/impala/catalog/HdfsTable.java @@ -965,7 +965,10 @@ public class HdfsTable extends Table { return partition; } - private void addDefaultPartition(StorageDescriptor storageDescriptor) + /** + * Adds or replaces the default partition. + */ + public void addDefaultPartition(StorageDescriptor storageDescriptor) throws CatalogException { // Default partition has no files and is not referred to by scan nodes. Data sinks // refer to this to understand how to create new partitions. @@ -1025,15 +1028,7 @@ public class HdfsTable extends Table { return; } // Load partition and file metadata - if (!reuseMetadata) { - // Load all partitions from Hive Metastore, including file metadata. - LOG.debug("load table from Hive Metastore: " + db_.getName() + "." + name_); - List<org.apache.hadoop.hive.metastore.api.Partition> msPartitions = - Lists.newArrayList(); - msPartitions.addAll(MetaStoreUtil.fetchAllPartitions( - client, db_.getName(), name_, NUM_PARTITION_FETCH_RETRIES)); - loadAllPartitions(msPartitions, msTbl); - } else { + if (reuseMetadata) { // Incrementally update this table's partitions and file metadata LOG.debug("incremental update for table: " + db_.getName() + "." + name_); Preconditions.checkState(partitionsToUpdate == null || loadFileMetadata); @@ -1043,6 +1038,14 @@ public class HdfsTable extends Table { } else { updatePartitionsFromHms(client, partitionsToUpdate, loadFileMetadata); } + } else { + // Load all partitions from Hive Metastore, including file metadata. + LOG.debug("load table from Hive Metastore: " + db_.getName() + "." + name_); + List<org.apache.hadoop.hive.metastore.api.Partition> msPartitions = + Lists.newArrayList(); + msPartitions.addAll(MetaStoreUtil.fetchAllPartitions( + client, db_.getName(), name_, NUM_PARTITION_FETCH_RETRIES)); + loadAllPartitions(msPartitions, msTbl); } if (loadTableSchema) setAvroSchema(client, msTbl); updateStatsFromHmsTable(msTbl); @@ -1420,9 +1423,9 @@ public class HdfsTable extends Table { HdfsStorageDescriptor fileFormatDescriptor = HdfsStorageDescriptor.fromStorageDescriptor(this.name_, msTbl.getSd()); Map<FsKey, FileBlocksInfo> perFsFileBlocks = Maps.newHashMap(); - for (HdfsPartition part: partitions) { + for (HdfsPartition partition: partitions) { org.apache.hadoop.hive.metastore.api.Partition msPart = - part.toHmsPartition(); + partition.toHmsPartition(); StorageDescriptor sd = null; if (msPart == null) { // If this partition is not stored in the Hive Metastore (e.g. default partition @@ -1432,7 +1435,7 @@ public class HdfsTable extends Table { } else { sd = msPart.getSd(); } - loadPartitionFileMetadata(sd, part, fileFormatDescriptor.getFileFormat(), + loadPartitionFileMetadata(sd, partition, fileFormatDescriptor.getFileFormat(), perFsFileBlocks); } loadDiskIds(perFsFileBlocks); http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/bf1d9677/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java b/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java index 780bf13..5743a59 100644 --- a/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java +++ b/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java @@ -1869,6 +1869,9 @@ public class CatalogOpExecutor { org.apache.hadoop.hive.metastore.api.Table msTbl = tbl.getMetaStoreTable().deepCopy(); setStorageDescriptorFileFormat(msTbl.getSd(), fileFormat); + // The default partition must be updated if the file format is changed so that new + // partitions are created with the new file format. + if (tbl instanceof HdfsTable) ((HdfsTable) tbl).addDefaultPartition(msTbl.getSd()); applyAlterTable(msTbl); reloadFileMetadata = true; } else { http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/bf1d9677/testdata/workloads/functional-query/queries/QueryTest/alter-table.test ---------------------------------------------------------------------- diff --git a/testdata/workloads/functional-query/queries/QueryTest/alter-table.test b/testdata/workloads/functional-query/queries/QueryTest/alter-table.test index 1f63c2b..3b1d86b 100644 --- a/testdata/workloads/functional-query/queries/QueryTest/alter-table.test +++ b/testdata/workloads/functional-query/queries/QueryTest/alter-table.test @@ -612,9 +612,6 @@ STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING ==== ---- QUERY # IMPALA-1016: Testing scanning newly added columns -DROP TABLE IF EXISTS imp1016 -==== ----- QUERY CREATE TABLE imp1016 (string1 string) ---- RESULTS ==== @@ -672,9 +669,6 @@ bigint ==== ---- QUERY # Create a larger table to test scanning newly added columns -DROP TABLE IF EXISTS imp1016Large -==== ----- QUERY CREATE TABLE imp1016Large (string1 string) ---- RESULTS ==== @@ -929,30 +923,38 @@ STRING, STRING, STRING, STRING, STRING, STRING, STRING, STRING, STRING, STRING, ==== ---- QUERY # IMPALA-1740: Test setting the skip.header.line.count tblproperty -drop table if exists i1740_alter; -create table i1740_alter (i1 integer); -alter table i1740_alter set tblproperties ('skip.header.line.count'='2'); +create table i1740_alter_1 (i1 integer); +alter table i1740_alter_1 set tblproperties ('skip.header.line.count'='2'); ---- RESULTS ==== ---- QUERY # IMPALA-1740: Test setting the skip.header.line.count tblproperty -drop table if exists i1740_alter; -create table i1740_alter (i1 integer); -alter table i1740_alter set tblproperties ('skip.header.line.count'='-1'); +create table i1740_alter_2 (i1 integer); +alter table i1740_alter_2 set tblproperties ('skip.header.line.count'='-1'); ---- CATCH Invalid value for table property skip.header.line.count: ==== ---- QUERY # IMPALA-1740: Test setting the skip.header.line.count tblproperty -drop table if exists i1740_alter; -create table i1740_alter (i1 integer); -alter table i1740_alter set tblproperties ('skip.header.line.count'='foo'); +create table i1740_alter_3 (i1 integer); +alter table i1740_alter_3 set tblproperties ('skip.header.line.count'='foo'); ---- CATCH Invalid value for table property skip.header.line.count: ==== ---- QUERY # IMPALA-1740: Test setting the skip.header.line.count tblproperty -drop table if exists i1740_alter; -create table i1740_alter (i1 integer) stored as parquet; -alter table i1740_alter set tblproperties ('skip.header.line.count'='2'); +create table i1740_alter_4 (i1 integer) stored as parquet; +alter table i1740_alter_4 set tblproperties ('skip.header.line.count'='2'); +==== +---- QUERY +# IMPALA-4155: Verify that the default partition is updated by the alter table statement +create table i4155_alter (c1 int) +partitioned by (p1 string) row format delimited fields terminated by ','; +alter table i4155_alter set fileformat PARQUET; +insert into i4155_alter partition(p1="abc") values (1); +select * from i4155_alter; +---- RESULTS +1,'abc' +---- TYPES +INT, STRING ====
