This is an automated email from the ASF dual-hosted git repository. wzhou pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/impala.git
commit 27955a385e8d442e183cbf22cfc068124f830986 Author: Daniel Becker <[email protected]> AuthorDate: Fri Feb 2 19:04:33 2024 +0100 IMPALA-12783: Nested struct with varlen data crashes If a struct ("main") is within an array and contains two child structs ("s1" ans "s2") which both contain strings (or other varlen data), Impala crashes when this struct is re-materialised (for example in a sort with limit) if codegen is enabled. To reproduce: In Hive: create table nested (arr ARRAY<STRUCT<s1: STRUCT<str1: STRING>, s2: STRUCT<str2: STRING>>>) stored as parquet; insert into nested values (array( named_struct("s1", named_struct("str1", "A string that is long"), "s2", named_struct("str2", "Another string that is long") ))); In Impala: select 1, arr from nested order by 1 limit 1; This is because in the codegen'd code, when checking if the strings ("str1" and "str2" in the example) are NULL, we incorrectly calculate the offset of their null indicator bytes from the memory address of their containing struct, not from the beginning of the "master tuple", which in this case is the item tuple of the array. Note that the null indicators of struct members are always at the end of the tuple containing the struct (recursively), i.e. the master tuple. This change corrects the behaviour, passing the master tuple to functions that need it. Testing: - extended the column 'arr_contains_nested_struct' in table 'collection_struct_mix' to include two nested structs with string members. Updated existing queries, which now cover the problem. Change-Id: Ide2b63f8b18633f38fbe939a17db923606ccb101 Reviewed-on: http://gerrit.cloudera.org:8080/20997 Reviewed-by: Impala Public Jenkins <[email protected]> Tested-by: Impala Public Jenkins <[email protected]> --- be/src/runtime/descriptors.cc | 31 ++++---- be/src/runtime/descriptors.h | 14 ++-- .../functional/functional_schema_template.sql | 16 +++-- .../QueryTest/mixed-collections-and-structs.test | 84 +++++++++++----------- .../queries/QueryTest/sort-complex.test | 8 +-- .../queries/QueryTest/top-n-complex.test | 6 +- 6 files changed, 85 insertions(+), 74 deletions(-) diff --git a/be/src/runtime/descriptors.cc b/be/src/runtime/descriptors.cc index 4d98eeffa..b880f6cac 100644 --- a/be/src/runtime/descriptors.cc +++ b/be/src/runtime/descriptors.cc @@ -1427,13 +1427,14 @@ void SlotDescriptor::CodegenWriteCollectionItemLoopBody(LlvmCodeGen* codegen, llvm::Value* children_tuple = builder->CreateInBoundsGEP(children_tuple_array, item_index, "children_tuple"); - CodegenWriteCollectionIterateOverChildren(codegen, builder, children_tuple, fn, - insert_before, pool_val); + CodegenWriteCollectionIterateOverChildren(codegen, builder, children_tuple, + children_tuple, fn, insert_before, pool_val); } void SlotDescriptor::CodegenWriteCollectionIterateOverChildren(LlvmCodeGen* codegen, - LlvmBuilder* builder, llvm::Value* children_tuple, llvm::Function* fn, - const NonWritableBasicBlock& insert_before, llvm::Value* pool_val) const { + LlvmBuilder* builder, llvm::Value* master_tuple, llvm::Value* children_tuple, + llvm::Function* fn, const NonWritableBasicBlock& insert_before, + llvm::Value* pool_val) const { DCHECK(pool_val != nullptr); const TupleDescriptor* children_tuple_desc = children_tuple_descriptor(); DCHECK(children_tuple_desc != nullptr); @@ -1443,18 +1444,19 @@ void SlotDescriptor::CodegenWriteCollectionIterateOverChildren(LlvmCodeGen* code const ColumnType& child_type = child_slot_desc->type(); if (child_type.IsVarLenStringType() || child_type.IsCollectionType()) { - child_slot_desc->CodegenWriteCollectionVarlenChild(codegen, builder, children_tuple, - fn, insert_before, pool_val); + child_slot_desc->CodegenWriteCollectionVarlenChild(codegen, builder, master_tuple, + children_tuple, fn, insert_before, pool_val); } else if (child_type.IsStructType()) { child_slot_desc->CodegenWriteCollectionStructChild(codegen, builder, - children_tuple, fn, insert_before, pool_val); + master_tuple, children_tuple, fn, insert_before, pool_val); } } } void SlotDescriptor::CodegenWriteCollectionStructChild(LlvmCodeGen* codegen, - LlvmBuilder* builder, llvm::Value* tuple, llvm::Function* fn, - const NonWritableBasicBlock& insert_before, llvm::Value* pool_val) const { + LlvmBuilder* builder, llvm::Value* master_tuple, llvm::Value* tuple, + llvm::Function* fn, const NonWritableBasicBlock& insert_before, + llvm::Value* pool_val) const { DCHECK(type().IsStructType()); const TupleDescriptor* children_tuple_desc = children_tuple_descriptor(); @@ -1464,13 +1466,14 @@ void SlotDescriptor::CodegenWriteCollectionStructChild(LlvmCodeGen* codegen, llvm_field_idx(), "struct_children_tuple"); // TODO IMPALA-12775: Check whether the struct itself is NULL. - CodegenWriteCollectionIterateOverChildren(codegen, builder, children_tuple, fn, - insert_before, pool_val); + CodegenWriteCollectionIterateOverChildren(codegen, builder, master_tuple, + children_tuple, fn, insert_before, pool_val); } void SlotDescriptor::CodegenWriteCollectionVarlenChild(LlvmCodeGen* codegen, - LlvmBuilder* builder, llvm::Value* children_tuple, llvm::Function* fn, - const NonWritableBasicBlock& insert_before, llvm::Value* pool_val) const { + LlvmBuilder* builder, llvm::Value* master_tuple, llvm::Value* children_tuple, + llvm::Function* fn, const NonWritableBasicBlock& insert_before, + llvm::Value* pool_val) const { DCHECK(pool_val != nullptr); DCHECK(type_.IsVarLenStringType() || type_.IsCollectionType()); @@ -1479,7 +1482,7 @@ void SlotDescriptor::CodegenWriteCollectionVarlenChild(LlvmCodeGen* codegen, llvm::BasicBlock* child_written_block = insert_before.CreateBasicBlockBefore( codegen->context(), "next_block_after_child_is_written", fn); - llvm::Value* child_is_null = CodegenIsNull(codegen, builder, children_tuple); + llvm::Value* child_is_null = CodegenIsNull(codegen, builder, master_tuple); builder->CreateCondBr(child_is_null, child_written_block, child_non_null_block); // Note: Although the input of CodegenWriteStringOrCollectionToSlot() is a '*Val', not a diff --git a/be/src/runtime/descriptors.h b/be/src/runtime/descriptors.h index b53b50a6b..b8df42346 100644 --- a/be/src/runtime/descriptors.h +++ b/be/src/runtime/descriptors.h @@ -304,16 +304,20 @@ class SlotDescriptor { llvm::Value* child_index, llvm::Function* fn, const NonWritableBasicBlock& insert_before, llvm::Value* pool_val) const; + // 'master_tuple' differs from 'children_tuple' if this is inside a struct - + // 'master_tuple' is the first non-struct parent tuple. Applies to the below functions + // too. void CodegenWriteCollectionIterateOverChildren(LlvmCodeGen* codegen, - LlvmBuilder* builder, llvm::Value* children_tuple, llvm::Function* fn, - const NonWritableBasicBlock& insert_before, llvm::Value* pool_val) const; + LlvmBuilder* builder, llvm::Value* master_tuple, llvm::Value* children_tuple, + llvm::Function* fn, const NonWritableBasicBlock& insert_before, + llvm::Value* pool_val) const; void CodegenWriteCollectionStructChild(LlvmCodeGen* codegen, LlvmBuilder* builder, - llvm::Value* tuple, llvm::Function* fn, const NonWritableBasicBlock& insert_before, - llvm::Value* pool_val) const; + llvm::Value* master_tuple, llvm::Value* tuple, llvm::Function* fn, + const NonWritableBasicBlock& insert_before, llvm::Value* pool_val) const; void CodegenWriteCollectionVarlenChild(LlvmCodeGen* codegen, LlvmBuilder* builder, - llvm::Value* child_tuple, llvm::Function* fn, + llvm::Value* master_tuple, llvm::Value* child_tuple, llvm::Function* fn, const NonWritableBasicBlock& insert_before, llvm::Value* pool_val) const; static llvm::Value* CodegenToTimestampValue( diff --git a/testdata/datasets/functional/functional_schema_template.sql b/testdata/datasets/functional/functional_schema_template.sql index 62662fa56..319ca97d7 100644 --- a/testdata/datasets/functional/functional_schema_template.sql +++ b/testdata/datasets/functional/functional_schema_template.sql @@ -4126,7 +4126,7 @@ id INT struct_contains_arr STRUCT<arr: ARRAY<INT>> struct_contains_map STRUCT<m: MAP<INT, STRING>> arr_contains_struct ARRAY<STRUCT<i: BIGINT>> -arr_contains_nested_struct ARRAY<STRUCT<inner_struct: STRUCT<str: STRING, l: INT>, small: SMALLINT>> +arr_contains_nested_struct ARRAY<STRUCT<inner_struct1: STRUCT<str: STRING, l: INT>, inner_struct2: STRUCT<str: STRING, l: INT>, small: SMALLINT>> struct_contains_nested_arr STRUCT<arr: ARRAY<ARRAY<DATE>>, i: INT> all_mix MAP<INT, STRUCT<big: STRUCT<arr: ARRAY<STRUCT<inner_arr: ARRAY<ARRAY<INT>>, m: TIMESTAMP>>, n: INT>, small: STRUCT<str: STRING, i: INT>>> ---- DEPENDENT_LOAD_HIVE @@ -4137,8 +4137,10 @@ INSERT OVERWRITE TABLE {db_name}{db_suffix}.{table_name} VALUES named_struct("m", map(1, "one spaceship captain", 2, "two", 0, NULL)), array(named_struct("i", 1L), named_struct("i", 2L), named_struct("i", 3L), named_struct("i", 4L), NULL, named_struct("i", 5L), named_struct("i", NULL)), - array(named_struct("inner_struct", named_struct("str", "", "l", 0), "small", 2S), NULL, - named_struct("inner_struct", named_struct("str", "some spaceship captain", "l", 5), "small", 20S)), + array(named_struct("inner_struct1", named_struct("str", "", "l", 0), + "inner_struct2", named_struct("str", "four spaceship captains", "l", 2), "small", 2S), NULL, + named_struct("inner_struct1", named_struct("str", NULL, "l", 5), + "inner_struct2", named_struct("str", "more spaceship captains", "l", 8), "small", 20S)), named_struct("arr", array(array(to_date("2022-12-05"), to_date("2022-12-06"), NULL, to_date("2022-12-07")), array(to_date("2022-12-08"), to_date("2022-12-09"), NULL)), "i", 2), map( @@ -4170,8 +4172,10 @@ INSERT OVERWRITE TABLE {db_name}{db_suffix}.{table_name} VALUES named_struct("m", if(false, map(1, "one soju distillery"), NULL)), array(named_struct("i", 100L), named_struct("i", 8L), named_struct("i", 35L), named_struct("i", 45L), NULL, named_struct("i", 193L), named_struct("i", NULL)), - array(named_struct("inner_struct", if(false, named_struct("str", "", "l", 0), NULL), "small", 104S), - named_struct("inner_struct", named_struct("str", "a few soju distilleries", "l", 28), "small", 105S), NULL), + array(named_struct("inner_struct1", if(false, named_struct("str", "", "l", 0), NULL), + "inner_struct2", named_struct("str", "very few distilleries", "l", 128), "small", 104S), + named_struct("inner_struct1", named_struct("str", "a few soju distilleries", "l", 28), + "inner_struct2", named_struct("str", "lots of soju distilleries", "l", 228), "small", 105S), NULL), named_struct("arr", array(array(to_date("2022-12-10"), to_date("2022-12-11"), NULL, to_date("2022-12-12")), if(false, array(to_date("2022-12-12")), NULL)), "i", 2754), map( @@ -4493,4 +4497,4 @@ PARTITION BY HASH (id) PARTITIONS 3 STORED AS KUDU; ---- DEPENDENT_LOAD_KUDU INSERT into TABLE {db_name}{db_suffix}.{table_name} SELECT * FROM {db_name}.{table_name}; -==== \ No newline at end of file +==== diff --git a/testdata/workloads/functional-query/queries/QueryTest/mixed-collections-and-structs.test b/testdata/workloads/functional-query/queries/QueryTest/mixed-collections-and-structs.test index 732aac87c..2262d6e92 100644 --- a/testdata/workloads/functional-query/queries/QueryTest/mixed-collections-and-structs.test +++ b/testdata/workloads/functional-query/queries/QueryTest/mixed-collections-and-structs.test @@ -11,8 +11,8 @@ select all_mix from collection_struct_mix; ---- RESULTS -1,'{"arr":[1,2,3,4,null,null,5]}','{"m":{1:"one spaceship captain",2:"two",0:null}}','[{"i":1},{"i":2},{"i":3},{"i":4},null,{"i":5},{"i":null}]','[{"inner_struct":{"str":"","l":0},"small":2},null,{"inner_struct":{"str":"some spaceship captain","l":5},"small":20}]','{"arr":[["2022-12-05","2022-12-06",null,"2022-12-07"],["2022-12-08","2022-12-09",null]],"i":2}','{10:{"big":{"arr":[{"inner_arr":[[0,null,-1,-5,null,8],[20,null]],"m":"2022-12-05 14:30:00"},{"inner_arr":[[12,1024,null],[null,n [...] -2,'{"arr":null}','{"m":null}','[{"i":100},{"i":8},{"i":35},{"i":45},null,{"i":193},{"i":null}]','[{"inner_struct":null,"small":104},{"inner_struct":{"str":"a few soju distilleries","l":28},"small":105},null]','{"arr":[["2022-12-10","2022-12-11",null,"2022-12-12"],null],"i":2754}','{20:{"big":{"arr":[null,{"inner_arr":[[12,1024,null],[null,null,84],[null,15,null]],"m":"2022-12-10 08:15:12"}],"n":95},"small":{"str":"other soju distillery","i":2048}},21:{"big":{"arr":null,"n":8},"small":{"s [...] +1,'{"arr":[1,2,3,4,null,null,5]}','{"m":{1:"one spaceship captain",2:"two",0:null}}','[{"i":1},{"i":2},{"i":3},{"i":4},null,{"i":5},{"i":null}]','[{"inner_struct1":{"str":"","l":0},"inner_struct2":{"str":"four spaceship captains","l":2},"small":2},null,{"inner_struct1":{"str":null,"l":5},"inner_struct2":{"str":"more spaceship captains","l":8},"small":20}]','{"arr":[["2022-12-05","2022-12-06",null,"2022-12-07"],["2022-12-08","2022-12-09",null]],"i":2}','{10:{"big":{"arr":[{"inner_arr":[[0 [...] +2,'{"arr":null}','{"m":null}','[{"i":100},{"i":8},{"i":35},{"i":45},null,{"i":193},{"i":null}]','[{"inner_struct1":null,"inner_struct2":{"str":"very few distilleries","l":128},"small":104},{"inner_struct1":{"str":"a few soju distilleries","l":28},"inner_struct2":{"str":"lots of soju distilleries","l":228},"small":105},null]','{"arr":[["2022-12-10","2022-12-11",null,"2022-12-12"],null],"i":2754}','{20:{"big":{"arr":[null,{"inner_arr":[[12,1024,null],[null,null,84],[null,15,null]],"m":"202 [...] ---- TYPES INT,STRING,STRING,STRING,STRING,STRING,STRING ==== @@ -139,11 +139,11 @@ INT,STRING set CONVERT_LEGACY_HIVE_PARQUET_UTC_TIMESTAMPS=1; select id, item from collection_struct_mix, collection_struct_mix.arr_contains_nested_struct arr; ---- RESULTS -1,'{"inner_struct":{"str":"","l":0},"small":2}' +1,'{"inner_struct1":{"str":"","l":0},"inner_struct2":{"str":"four spaceship captains","l":2},"small":2}' 1,'NULL' -1,'{"inner_struct":{"str":"some spaceship captain","l":5},"small":20}' -2,'{"inner_struct":null,"small":104}' -2,'{"inner_struct":{"str":"a few soju distilleries","l":28},"small":105}' +1,'{"inner_struct1":{"str":null,"l":5},"inner_struct2":{"str":"more spaceship captains","l":8},"small":20}' +2,'{"inner_struct1":null,"inner_struct2":{"str":"very few distilleries","l":128},"small":104}' +2,'{"inner_struct1":{"str":"a few soju distilleries","l":28},"inner_struct2":{"str":"lots of soju distilleries","l":228},"small":105}' 2,'NULL' ---- TYPES INT,STRING @@ -155,11 +155,11 @@ with sub as (select id, arr_contains_nested_struct arr from collection_struct_mi sub2 as (select id, arr arr2 from sub) select id, item from sub2, sub2.arr2 a; ---- RESULTS -1,'{"inner_struct":{"str":"","l":0},"small":2}' +1,'{"inner_struct1":{"str":"","l":0},"inner_struct2":{"str":"four spaceship captains","l":2},"small":2}' 1,'NULL' -1,'{"inner_struct":{"str":"some spaceship captain","l":5},"small":20}' -2,'{"inner_struct":null,"small":104}' -2,'{"inner_struct":{"str":"a few soju distilleries","l":28},"small":105}' +1,'{"inner_struct1":{"str":null,"l":5},"inner_struct2":{"str":"more spaceship captains","l":8},"small":20}' +2,'{"inner_struct1":null,"inner_struct2":{"str":"very few distilleries","l":128},"small":104}' +2,'{"inner_struct1":{"str":"a few soju distilleries","l":28},"inner_struct2":{"str":"lots of soju distilleries","l":228},"small":105}' 2,'NULL' ---- TYPES INT,STRING @@ -170,11 +170,11 @@ set CONVERT_LEGACY_HIVE_PARQUET_UTC_TIMESTAMPS=1; select id, item from collection_struct_mix_view, collection_struct_mix_view.arr_contains_nested_struct a; ---- RESULTS -1,'{"inner_struct":{"str":"","l":0},"small":2}' +1,'{"inner_struct1":{"str":"","l":0},"inner_struct2":{"str":"four spaceship captains","l":2},"small":2}' 1,'NULL' -1,'{"inner_struct":{"str":"some spaceship captain","l":5},"small":20}' -2,'{"inner_struct":null,"small":104}' -2,'{"inner_struct":{"str":"a few soju distilleries","l":28},"small":105}' +1,'{"inner_struct1":{"str":null,"l":5},"inner_struct2":{"str":"more spaceship captains","l":8},"small":20}' +2,'{"inner_struct1":null,"inner_struct2":{"str":"very few distilleries","l":128},"small":104}' +2,'{"inner_struct1":{"str":"a few soju distilleries","l":28},"inner_struct2":{"str":"lots of soju distilleries","l":228},"small":105}' 2,'NULL' ---- TYPES INT,STRING @@ -224,28 +224,28 @@ INT,STRING,DATE ==== ---- QUERY # Join unnest array containing struct and also query struct fields. -select id, a.item, a.item.inner_struct, a.item.small from collection_struct_mix, +select id, a.item, a.item.inner_struct1, a.item.small from collection_struct_mix, collection_struct_mix.arr_contains_nested_struct a; ---- RESULTS -1,'{"inner_struct":{"str":"","l":0},"small":2}','{"str":"","l":0}',2 +1,'{"inner_struct1":{"str":"","l":0},"inner_struct2":{"str":"four spaceship captains","l":2},"small":2}','{"str":"","l":0}',2 1,'NULL','NULL',NULL -1,'{"inner_struct":{"str":"some spaceship captain","l":5},"small":20}','{"str":"some spaceship captain","l":5}',20 -2,'{"inner_struct":null,"small":104}','NULL',104 -2,'{"inner_struct":{"str":"a few soju distilleries","l":28},"small":105}','{"str":"a few soju distilleries","l":28}',105 +1,'{"inner_struct1":{"str":null,"l":5},"inner_struct2":{"str":"more spaceship captains","l":8},"small":20}','{"str":null,"l":5}',20 +2,'{"inner_struct1":null,"inner_struct2":{"str":"very few distilleries","l":128},"small":104}','NULL',104 +2,'{"inner_struct1":{"str":"a few soju distilleries","l":28},"inner_struct2":{"str":"lots of soju distilleries","l":228},"small":105}','{"str":"a few soju distilleries","l":28}',105 2,'NULL','NULL',NULL ---- TYPES INT,STRING,STRING,SMALLINT ==== ---- QUERY # Join unnest array containing struct from HMS view and also query struct fields. -select id, a.item, a.item.inner_struct, a.item.small from collection_struct_mix_view, +select id, a.item, a.item.inner_struct1, a.item.small from collection_struct_mix_view, collection_struct_mix_view.arr_contains_nested_struct a; ---- RESULTS -1,'{"inner_struct":{"str":"","l":0},"small":2}','{"str":"","l":0}',2 +1,'{"inner_struct1":{"str":"","l":0},"inner_struct2":{"str":"four spaceship captains","l":2},"small":2}','{"str":"","l":0}',2 1,'NULL','NULL',NULL -1,'{"inner_struct":{"str":"some spaceship captain","l":5},"small":20}','{"str":"some spaceship captain","l":5}',20 -2,'{"inner_struct":null,"small":104}','NULL',104 -2,'{"inner_struct":{"str":"a few soju distilleries","l":28},"small":105}','{"str":"a few soju distilleries","l":28}',105 +1,'{"inner_struct1":{"str":null,"l":5},"inner_struct2":{"str":"more spaceship captains","l":8},"small":20}','{"str":null,"l":5}',20 +2,'{"inner_struct1":null,"inner_struct2":{"str":"very few distilleries","l":128},"small":104}','NULL',104 +2,'{"inner_struct1":{"str":"a few soju distilleries","l":28},"inner_struct2":{"str":"lots of soju distilleries","l":228},"small":105}','{"str":"a few soju distilleries","l":28}',105 2,'NULL','NULL',NULL ---- TYPES INT,STRING,STRING,SMALLINT @@ -253,14 +253,14 @@ INT,STRING,STRING,SMALLINT ---- QUERY # Join unnest array containing struct from inline view and also query struct fields. with sub as (select id, arr_contains_nested_struct from collection_struct_mix_view) -select id, a.item, a.item.inner_struct, a.item.small from sub, +select id, a.item, a.item.inner_struct1, a.item.small from sub, sub.arr_contains_nested_struct a; ---- RESULTS -1,'{"inner_struct":{"str":"","l":0},"small":2}','{"str":"","l":0}',2 +1,'{"inner_struct1":{"str":"","l":0},"inner_struct2":{"str":"four spaceship captains","l":2},"small":2}','{"str":"","l":0}',2 1,'NULL','NULL',NULL -1,'{"inner_struct":{"str":"some spaceship captain","l":5},"small":20}','{"str":"some spaceship captain","l":5}',20 -2,'{"inner_struct":null,"small":104}','NULL',104 -2,'{"inner_struct":{"str":"a few soju distilleries","l":28},"small":105}','{"str":"a few soju distilleries","l":28}',105 +1,'{"inner_struct1":{"str":null,"l":5},"inner_struct2":{"str":"more spaceship captains","l":8},"small":20}','{"str":null,"l":5}',20 +2,'{"inner_struct1":null,"inner_struct2":{"str":"very few distilleries","l":128},"small":104}','NULL',104 +2,'{"inner_struct1":{"str":"a few soju distilleries","l":28},"inner_struct2":{"str":"lots of soju distilleries","l":228},"small":105}','{"str":"a few soju distilleries","l":28}',105 2,'NULL','NULL',NULL ---- TYPES INT,STRING,STRING,SMALLINT @@ -478,8 +478,8 @@ select * from ( select * from sub ) sub2; ---- RESULTS -1,'{"arr":[1,2,3,4,null,null,5]}','{"m":{1:"one spaceship captain",2:"two",0:null}}','[{"i":1},{"i":2},{"i":3},{"i":4},null,{"i":5},{"i":null}]','[{"inner_struct":{"str":"","l":0},"small":2},null,{"inner_struct":{"str":"some spaceship captain","l":5},"small":20}]','{"arr":[["2022-12-05","2022-12-06",null,"2022-12-07"],["2022-12-08","2022-12-09",null]],"i":2}','{10:{"big":{"arr":[{"inner_arr":[[0,null,-1,-5,null,8],[20,null]],"m":"2022-12-05 14:30:00"},{"inner_arr":[[12,1024,null],[null,n [...] -2,'{"arr":null}','{"m":null}','[{"i":100},{"i":8},{"i":35},{"i":45},null,{"i":193},{"i":null}]','[{"inner_struct":null,"small":104},{"inner_struct":{"str":"a few soju distilleries","l":28},"small":105},null]','{"arr":[["2022-12-10","2022-12-11",null,"2022-12-12"],null],"i":2754}','{20:{"big":{"arr":[null,{"inner_arr":[[12,1024,null],[null,null,84],[null,15,null]],"m":"2022-12-10 08:15:12"}],"n":95},"small":{"str":"other soju distillery","i":2048}},21:{"big":{"arr":null,"n":8},"small":{"s [...] +1,'{"arr":[1,2,3,4,null,null,5]}','{"m":{1:"one spaceship captain",2:"two",0:null}}','[{"i":1},{"i":2},{"i":3},{"i":4},null,{"i":5},{"i":null}]','[{"inner_struct1":{"str":"","l":0},"inner_struct2":{"str":"four spaceship captains","l":2},"small":2},null,{"inner_struct1":{"str":null,"l":5},"inner_struct2":{"str":"more spaceship captains","l":8},"small":20}]','{"arr":[["2022-12-05","2022-12-06",null,"2022-12-07"],["2022-12-08","2022-12-09",null]],"i":2}','{10:{"big":{"arr":[{"inner_arr":[[0 [...] +2,'{"arr":null}','{"m":null}','[{"i":100},{"i":8},{"i":35},{"i":45},null,{"i":193},{"i":null}]','[{"inner_struct1":null,"inner_struct2":{"str":"very few distilleries","l":128},"small":104},{"inner_struct1":{"str":"a few soju distilleries","l":28},"inner_struct2":{"str":"lots of soju distilleries","l":228},"small":105},null]','{"arr":[["2022-12-10","2022-12-11",null,"2022-12-12"],null],"i":2754}','{20:{"big":{"arr":[null,{"inner_arr":[[12,1024,null],[null,null,84],[null,15,null]],"m":"202 [...] ---- TYPES INT,STRING,STRING,STRING,STRING,STRING,STRING ==== @@ -494,8 +494,8 @@ select id, arr_contains_nested_struct, struct_contains_nested_arr.* from ( select id, arr_contains_nested_struct, struct_contains_nested_arr from sub ) sub2; ---- RESULTS -1,'[{"inner_struct":{"str":"","l":0},"small":2},null,{"inner_struct":{"str":"some spaceship captain","l":5},"small":20}]',2 -2,'[{"inner_struct":null,"small":104},{"inner_struct":{"str":"a few soju distilleries","l":28},"small":105},null]',2754 +1,'[{"inner_struct1":{"str":"","l":0},"inner_struct2":{"str":"four spaceship captains","l":2},"small":2},null,{"inner_struct1":{"str":null,"l":5},"inner_struct2":{"str":"more spaceship captains","l":8},"small":20}]',2 +2,'[{"inner_struct1":null,"inner_struct2":{"str":"very few distilleries","l":128},"small":104},{"inner_struct1":{"str":"a few soju distilleries","l":28},"inner_struct2":{"str":"lots of soju distilleries","l":228},"small":105},null]',2754 ---- TYPES INT,STRING,INT ==== @@ -510,8 +510,8 @@ select id, arr_contains_nested_struct, struct_contains_nested_arr.* from ( select id, arr_contains_nested_struct, struct_contains_nested_arr from sub ) sub2; ---- RESULTS -1,'[{"inner_struct":{"str":"","l":0},"small":2},null,{"inner_struct":{"str":"some spaceship captain","l":5},"small":20}]','[["2022-12-05","2022-12-06",null,"2022-12-07"],["2022-12-08","2022-12-09",null]]',2 -2,'[{"inner_struct":null,"small":104},{"inner_struct":{"str":"a few soju distilleries","l":28},"small":105},null]','[["2022-12-10","2022-12-11",null,"2022-12-12"],null]',2754 +1,'[{"inner_struct1":{"str":"","l":0},"inner_struct2":{"str":"four spaceship captains","l":2},"small":2},null,{"inner_struct1":{"str":null,"l":5},"inner_struct2":{"str":"more spaceship captains","l":8},"small":20}]','[["2022-12-05","2022-12-06",null,"2022-12-07"],["2022-12-08","2022-12-09",null]]',2 +2,'[{"inner_struct1":null,"inner_struct2":{"str":"very few distilleries","l":128},"small":104},{"inner_struct1":{"str":"a few soju distilleries","l":28},"inner_struct2":{"str":"lots of soju distilleries","l":228},"small":105},null]','[["2022-12-10","2022-12-11",null,"2022-12-12"],null]',2754 ---- TYPES INT,STRING,STRING,INT ==== @@ -540,14 +540,14 @@ set EXPAND_COMPLEX_TYPES=1; select id, arr.item.* from collection_struct_mix, collection_struct_mix.arr_contains_nested_struct arr; ---- RESULTS -1,'{"str":"","l":0}',2 -1,'NULL',NULL -1,'{"str":"some spaceship captain","l":5}',20 -2,'NULL',104 -2,'{"str":"a few soju distilleries","l":28}',105 -2,'NULL',NULL +1,'{"str":"","l":0}','{"str":"four spaceship captains","l":2}',2 +1,'NULL','NULL',NULL +1,'{"str":null,"l":5}','{"str":"more spaceship captains","l":8}',20 +2,'NULL','{"str":"very few distilleries","l":128}',104 +2,'{"str":"a few soju distilleries","l":28}','{"str":"lots of soju distilleries","l":228}',105 +2,'NULL','NULL',NULL ---- TYPES -INT,STRING,SMALLINT +INT,STRING,STRING,SMALLINT ==== ---- QUERY # Union of types containing structs are not allowed. diff --git a/testdata/workloads/functional-query/queries/QueryTest/sort-complex.test b/testdata/workloads/functional-query/queries/QueryTest/sort-complex.test index eaf8f858c..0693c5b24 100644 --- a/testdata/workloads/functional-query/queries/QueryTest/sort-complex.test +++ b/testdata/workloads/functional-query/queries/QueryTest/sort-complex.test @@ -136,8 +136,8 @@ INT,STRING,STRING # Sort collections containing structs, also containing var-len data. select id, arr_contains_struct, arr_contains_nested_struct from collection_struct_mix order by id desc; ---- RESULTS -2,'[{"i":100},{"i":8},{"i":35},{"i":45},null,{"i":193},{"i":null}]','[{"inner_struct":null,"small":104},{"inner_struct":{"str":"a few soju distilleries","l":28},"small":105},null]' -1,'[{"i":1},{"i":2},{"i":3},{"i":4},null,{"i":5},{"i":null}]','[{"inner_struct":{"str":"","l":0},"small":2},null,{"inner_struct":{"str":"some spaceship captain","l":5},"small":20}]' +2,'[{"i":100},{"i":8},{"i":35},{"i":45},null,{"i":193},{"i":null}]','[{"inner_struct1":null,"inner_struct2":{"str":"very few distilleries","l":128},"small":104},{"inner_struct1":{"str":"a few soju distilleries","l":28},"inner_struct2":{"str":"lots of soju distilleries","l":228},"small":105},null]' +1,'[{"i":1},{"i":2},{"i":3},{"i":4},null,{"i":5},{"i":null}]','[{"inner_struct1":{"str":"","l":0},"inner_struct2":{"str":"four spaceship captains","l":2},"small":2},null,{"inner_struct1":{"str":null,"l":5},"inner_struct2":{"str":"more spaceship captains","l":8},"small":20}]' ---- TYPES INT,STRING,STRING ==== @@ -145,8 +145,8 @@ INT,STRING,STRING # Sorting a var-len struct nested in a collection. select id, arr_contains_nested_struct from collection_struct_mix order by id; ---- RESULTS -1,'[{"inner_struct":{"str":"","l":0},"small":2},null,{"inner_struct":{"str":"some spaceship captain","l":5},"small":20}]' -2,'[{"inner_struct":null,"small":104},{"inner_struct":{"str":"a few soju distilleries","l":28},"small":105},null]' +1,'[{"inner_struct1":{"str":"","l":0},"inner_struct2":{"str":"four spaceship captains","l":2},"small":2},null,{"inner_struct1":{"str":null,"l":5},"inner_struct2":{"str":"more spaceship captains","l":8},"small":20}]' +2,'[{"inner_struct1":null,"inner_struct2":{"str":"very few distilleries","l":128},"small":104},{"inner_struct1":{"str":"a few soju distilleries","l":28},"inner_struct2":{"str":"lots of soju distilleries","l":228},"small":105},null]' ---- TYPES INT,STRING ==== diff --git a/testdata/workloads/functional-query/queries/QueryTest/top-n-complex.test b/testdata/workloads/functional-query/queries/QueryTest/top-n-complex.test index 53b639818..07e5e8cc1 100644 --- a/testdata/workloads/functional-query/queries/QueryTest/top-n-complex.test +++ b/testdata/workloads/functional-query/queries/QueryTest/top-n-complex.test @@ -134,7 +134,7 @@ INT,STRING,STRING # Sort collections containing structs, also containing var-len data. select id, arr_contains_struct, arr_contains_nested_struct from collection_struct_mix order by id desc limit 1; ---- RESULTS -2,'[{"i":100},{"i":8},{"i":35},{"i":45},null,{"i":193},{"i":null}]','[{"inner_struct":null,"small":104},{"inner_struct":{"str":"a few soju distilleries","l":28},"small":105},null]' +2,'[{"i":100},{"i":8},{"i":35},{"i":45},null,{"i":193},{"i":null}]','[{"inner_struct1":null,"inner_struct2":{"str":"very few distilleries","l":128},"small":104},{"inner_struct1":{"str":"a few soju distilleries","l":28},"inner_struct2":{"str":"lots of soju distilleries","l":228},"small":105},null]' ---- TYPES INT,STRING,STRING ==== @@ -142,8 +142,8 @@ INT,STRING,STRING # Sorting a var-len struct nested in a collection. Regression test for IMPALA-12781. select id, arr_contains_nested_struct from collection_struct_mix order by id limit 2; ---- RESULTS -1,'[{"inner_struct":{"str":"","l":0},"small":2},null,{"inner_struct":{"str":"some spaceship captain","l":5},"small":20}]' -2,'[{"inner_struct":null,"small":104},{"inner_struct":{"str":"a few soju distilleries","l":28},"small":105},null]' +1,'[{"inner_struct1":{"str":"","l":0},"inner_struct2":{"str":"four spaceship captains","l":2},"small":2},null,{"inner_struct1":{"str":null,"l":5},"inner_struct2":{"str":"more spaceship captains","l":8},"small":20}]' +2,'[{"inner_struct1":null,"inner_struct2":{"str":"very few distilleries","l":128},"small":104},{"inner_struct1":{"str":"a few soju distilleries","l":28},"inner_struct2":{"str":"lots of soju distilleries","l":228},"small":105},null]' ---- TYPES INT,STRING ====
