This is an automated email from the ASF dual-hosted git repository. stigahuang pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/impala.git
commit a983a347a77af74e1a9bd6156d12a020d6b4df6d Author: Csaba Ringhofer <[email protected]> AuthorDate: Mon Oct 24 20:15:50 2022 +0200 IMPALA-11682: Add tests for minor compacted insert only ACID tables Only test changes. Minor compacted delta dirs are supported in Impala since IMPALA-9512, but at that time Hive supported minor compaction only on full ACID tables. Since that time Hive added support for minor compacting insert only/MM tables (HIVE-22610). Change-Id: I7159283f3658f2119d38bd3393729535edd0a76f Reviewed-on: http://gerrit.cloudera.org:8080/19164 Reviewed-by: Impala Public Jenkins <[email protected]> Tested-by: Impala Public Jenkins <[email protected]> --- .../impala/catalog/FileMetadataLoaderTest.java | 36 +++++++++++++++------ .../functional/functional_schema_template.sql | 37 +++++++++++++++++++++- .../datasets/functional/schema_constraints.csv | 3 ++ .../functional-query/queries/QueryTest/acid.test | 25 +++++++++++++++ 4 files changed, 91 insertions(+), 10 deletions(-) diff --git a/fe/src/test/java/org/apache/impala/catalog/FileMetadataLoaderTest.java b/fe/src/test/java/org/apache/impala/catalog/FileMetadataLoaderTest.java index a92595464..b0ad5f768 100644 --- a/fe/src/test/java/org/apache/impala/catalog/FileMetadataLoaderTest.java +++ b/fe/src/test/java/org/apache/impala/catalog/FileMetadataLoaderTest.java @@ -146,22 +146,40 @@ public class FileMetadataLoaderTest extends FrontendTestBase { relPaths.get(0)); } - @Test - public void testAcidMinorCompactionLoading() throws IOException, CatalogException { - //TODO(IMPALA-9042): Remove "throws CatalogException" + private FileMetadataLoader getLoaderForAcidTable( + String validWriteIdString, String path, HdfsFileFormat format) + throws IOException, CatalogException { ListMap<TNetworkAddress> hostIndex = new ListMap<>(); - ValidWriteIdList writeIds = MetastoreShim.getValidWriteIdListFromString( - "functional_orc_def.complextypestbl_minor_compacted:10:10::"); - Path tablePath = new Path("hdfs://localhost:20500/test-warehouse/managed/" + - "functional_orc_def.db/" + - "complextypestbl_minor_compacted_orc_def/"); + ValidWriteIdList writeIds = + MetastoreShim.getValidWriteIdListFromString(validWriteIdString); + Path tablePath = new Path(path); FileMetadataLoader fml = new FileMetadataLoader(tablePath, /* recursive=*/true, /* oldFds = */ Collections.emptyList(), hostIndex, new ValidReadTxnList(""), - writeIds, HdfsFileFormat.ORC); + writeIds, format); fml.load(); + return fml; + } + + @Test + public void testAcidMinorCompactionLoading() throws IOException, CatalogException { + //TODO(IMPALA-9042): Remove "throws CatalogException" + FileMetadataLoader fml = getLoaderForAcidTable( + "functional_orc_def.complextypestbl_minor_compacted:10:10::", + "hdfs://localhost:20500/test-warehouse/managed/functional_orc_def.db/" + + "complextypestbl_minor_compacted_orc_def/", + HdfsFileFormat.ORC); // Only load the compacted file. assertEquals(1, fml.getStats().loadedFiles); assertEquals(8, fml.getStats().filesSupersededByAcidState); + + fml = getLoaderForAcidTable( + "functional_parquet.insert_only_minor_compacted:6:6::", + "hdfs://localhost:20500/test-warehouse/managed/functional_parquet.db/" + + "insert_only_minor_compacted_parquet/", + HdfsFileFormat.PARQUET); + // Only load files after compaction. + assertEquals(3, fml.getStats().loadedFiles); + assertEquals(2, fml.getStats().filesSupersededByAcidState); } @Test diff --git a/testdata/datasets/functional/functional_schema_template.sql b/testdata/datasets/functional/functional_schema_template.sql index 4f1ff5cd0..d2a0941bd 100644 --- a/testdata/datasets/functional/functional_schema_template.sql +++ b/testdata/datasets/functional/functional_schema_template.sql @@ -3741,5 +3741,40 @@ values ( map(cast("key1" as binary), 1, cast("key2" as binary), 2), map(1, cast("value1" as binary), 2, cast("value2" as binary)), named_struct("i", 0, "b", cast("member" as binary)) - ) + ); +==== +---- DATASET +functional +---- BASE_TABLE_NAME +insert_only_minor_compacted +---- COLUMNS +id bigint +---- DEPENDENT_LOAD_HIVE +INSERT INTO TABLE {db_name}{db_suffix}.{table_name} VALUES (1); +INSERT INTO TABLE {db_name}{db_suffix}.{table_name} VALUES (2); +ALTER TABLE {db_name}{db_suffix}.{table_name} compact 'minor' AND WAIT; +INSERT INTO TABLE {db_name}{db_suffix}.{table_name} VALUES (3); +INSERT INTO TABLE {db_name}{db_suffix}.{table_name} VALUES (4); +---- TABLE_PROPERTIES +transactional=true +transactional_properties=insert_only +==== +---- DATASET +functional +---- BASE_TABLE_NAME +insert_only_major_and_minor_compacted +---- COLUMNS +id bigint +---- DEPENDENT_LOAD_HIVE +INSERT INTO TABLE {db_name}{db_suffix}.{table_name} VALUES (1); +INSERT INTO TABLE {db_name}{db_suffix}.{table_name} VALUES (2); +ALTER TABLE {db_name}{db_suffix}.{table_name} compact 'major' AND WAIT; +INSERT INTO TABLE {db_name}{db_suffix}.{table_name} VALUES (3); +INSERT INTO TABLE {db_name}{db_suffix}.{table_name} VALUES (4); +ALTER TABLE {db_name}{db_suffix}.{table_name} compact 'minor' AND WAIT; +INSERT INTO TABLE {db_name}{db_suffix}.{table_name} VALUES (5); +INSERT INTO TABLE {db_name}{db_suffix}.{table_name} VALUES (6); +---- TABLE_PROPERTIES +transactional=true +transactional_properties=insert_only ==== diff --git a/testdata/datasets/functional/schema_constraints.csv b/testdata/datasets/functional/schema_constraints.csv index 7b3a265be..55e0d17d1 100644 --- a/testdata/datasets/functional/schema_constraints.csv +++ b/testdata/datasets/functional/schema_constraints.csv @@ -352,3 +352,6 @@ table_name:complextypes_maps_view, constraint:restrict_to, table_format:orc/def/ # 'alltypestiny_negative' only used in ORC tests. table_name:alltypestiny_negative, constraint:restrict_to, table_format:orc/def/block + +table_name:insert_only_minor_compacted, constraint:restrict_to, table_format:parquet/none/none +table_name:insert_only_major_and_minor_compacted, constraint:restrict_to, table_format:parquet/none/none diff --git a/testdata/workloads/functional-query/queries/QueryTest/acid.test b/testdata/workloads/functional-query/queries/QueryTest/acid.test index 1b27ee404..1553c9f3c 100644 --- a/testdata/workloads/functional-query/queries/QueryTest/acid.test +++ b/testdata/workloads/functional-query/queries/QueryTest/acid.test @@ -136,3 +136,28 @@ from functional_orc_def.complextypestbl_minor_compacted; ---- TYPES BIGINT,BIGINT ==== +---- QUERY +# Test that Impala sees the compacted delta dir in a minor-compacted insert-only table. +show files in functional_parquet.insert_only_minor_compacted; +---- LABELS +Path,Size,Partition +---- RESULTS +row_regex:'$NAMENODE/test-warehouse/managed/functional_parquet.db/insert_only_minor_compacted_parquet/delta_0000001_0000002_v\d+/000000_0','.+B','' +row_regex:'$NAMENODE/test-warehouse/managed/functional_parquet.db/insert_only_minor_compacted_parquet/delta_0000003_0000003_0000/000000_0','.+B','' +row_regex:'$NAMENODE/test-warehouse/managed/functional_parquet.db/insert_only_minor_compacted_parquet/delta_0000004_0000004_0000/000000_0','.+B','' +---- TYPES +STRING,STRING,STRING +==== +---- QUERY +# Test that Impala sees the compacted delta dir in a first major then minor-compacted insert-only table. +show files in functional_parquet.insert_only_major_and_minor_compacted; +---- LABELS +Path,Size,Partition +---- RESULTS +row_regex:'$NAMENODE/test-warehouse/managed/functional_parquet.db/insert_only_major_and_minor_compacted_parquet/base_0000002_v\d+/000000_0','.+B','' +row_regex:'$NAMENODE/test-warehouse/managed/functional_parquet.db/insert_only_major_and_minor_compacted_parquet/delta_0000003_0000004_v\d+/000000_0','.+B','' +row_regex:'$NAMENODE/test-warehouse/managed/functional_parquet.db/insert_only_major_and_minor_compacted_parquet/delta_0000005_0000005_0000/000000_0','.+B','' +row_regex:'$NAMENODE/test-warehouse/managed/functional_parquet.db/insert_only_major_and_minor_compacted_parquet/delta_0000006_0000006_0000/000000_0','.+B','' +---- TYPES +STRING,STRING,STRING +====
